1//===- AArch64InstructionSelector.cpp ----------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the InstructionSelector class for
10/// AArch64.
11/// \todo This should be generated by TableGen.
12//===----------------------------------------------------------------------===//
13
14#include "AArch64ExpandImm.h"
15#include "AArch64GlobalISelUtils.h"
16#include "AArch64InstrInfo.h"
17#include "AArch64MachineFunctionInfo.h"
18#include "AArch64RegisterBankInfo.h"
19#include "AArch64RegisterInfo.h"
20#include "AArch64Subtarget.h"
21#include "AArch64TargetMachine.h"
22#include "MCTargetDesc/AArch64AddressingModes.h"
23#include "MCTargetDesc/AArch64MCTargetDesc.h"
24#include "llvm/BinaryFormat/Dwarf.h"
25#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
26#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
27#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
28#include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
29#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
30#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
31#include "llvm/CodeGen/GlobalISel/Utils.h"
32#include "llvm/CodeGen/MachineBasicBlock.h"
33#include "llvm/CodeGen/MachineConstantPool.h"
34#include "llvm/CodeGen/MachineFrameInfo.h"
35#include "llvm/CodeGen/MachineFunction.h"
36#include "llvm/CodeGen/MachineInstr.h"
37#include "llvm/CodeGen/MachineInstrBuilder.h"
38#include "llvm/CodeGen/MachineMemOperand.h"
39#include "llvm/CodeGen/MachineOperand.h"
40#include "llvm/CodeGen/MachineRegisterInfo.h"
41#include "llvm/CodeGen/TargetOpcodes.h"
42#include "llvm/CodeGen/TargetRegisterInfo.h"
43#include "llvm/IR/Constants.h"
44#include "llvm/IR/DerivedTypes.h"
45#include "llvm/IR/Instructions.h"
46#include "llvm/IR/IntrinsicsAArch64.h"
47#include "llvm/IR/Type.h"
48#include "llvm/Pass.h"
49#include "llvm/Support/Debug.h"
50#include "llvm/Support/raw_ostream.h"
51#include <optional>
52
53#define DEBUG_TYPE "aarch64-isel"
54
55using namespace llvm;
56using namespace MIPatternMatch;
57using namespace AArch64GISelUtils;
58
59namespace llvm {
60class BlockFrequencyInfo;
61class ProfileSummaryInfo;
62}
63
64namespace {
65
66#define GET_GLOBALISEL_PREDICATE_BITSET
67#include "AArch64GenGlobalISel.inc"
68#undef GET_GLOBALISEL_PREDICATE_BITSET
69
70
71class AArch64InstructionSelector : public InstructionSelector {
72public:
73 AArch64InstructionSelector(const AArch64TargetMachine &TM,
74 const AArch64Subtarget &STI,
75 const AArch64RegisterBankInfo &RBI);
76
77 bool select(MachineInstr &I) override;
78 static const char *getName() { return DEBUG_TYPE; }
79
80 void setupMF(MachineFunction &MF, GISelValueTracking *VT,
81 CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI,
82 BlockFrequencyInfo *BFI) override {
83 InstructionSelector::setupMF(mf&: MF, vt: VT, covinfo: CoverageInfo, psi: PSI, bfi: BFI);
84 MIB.setMF(MF);
85
86 // hasFnAttribute() is expensive to call on every BRCOND selection, so
87 // cache it here for each run of the selector.
88 ProduceNonFlagSettingCondBr =
89 !MF.getFunction().hasFnAttribute(Kind: Attribute::SpeculativeLoadHardening);
90 MFReturnAddr = Register();
91
92 processPHIs(MF);
93 }
94
95private:
96 /// tblgen-erated 'select' implementation, used as the initial selector for
97 /// the patterns that don't require complex C++.
98 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
99
100 // A lowering phase that runs before any selection attempts.
101 // Returns true if the instruction was modified.
102 bool preISelLower(MachineInstr &I);
103
104 // An early selection function that runs before the selectImpl() call.
105 bool earlySelect(MachineInstr &I);
106
107 /// Save state that is shared between select calls, call select on \p I and
108 /// then restore the saved state. This can be used to recursively call select
109 /// within a select call.
110 bool selectAndRestoreState(MachineInstr &I);
111
112 // Do some preprocessing of G_PHIs before we begin selection.
113 void processPHIs(MachineFunction &MF);
114
115 bool earlySelectSHL(MachineInstr &I, MachineRegisterInfo &MRI);
116
117 /// Eliminate same-sized cross-bank copies into stores before selectImpl().
118 bool contractCrossBankCopyIntoStore(MachineInstr &I,
119 MachineRegisterInfo &MRI);
120
121 bool convertPtrAddToAdd(MachineInstr &I, MachineRegisterInfo &MRI);
122
123 bool selectVaStartAAPCS(MachineInstr &I, MachineFunction &MF,
124 MachineRegisterInfo &MRI) const;
125 bool selectVaStartDarwin(MachineInstr &I, MachineFunction &MF,
126 MachineRegisterInfo &MRI) const;
127
128 ///@{
129 /// Helper functions for selectCompareBranch.
130 bool selectCompareBranchFedByFCmp(MachineInstr &I, MachineInstr &FCmp,
131 MachineIRBuilder &MIB) const;
132 bool selectCompareBranchFedByICmp(MachineInstr &I, MachineInstr &ICmp,
133 MachineIRBuilder &MIB) const;
134 bool tryOptCompareBranchFedByICmp(MachineInstr &I, MachineInstr &ICmp,
135 MachineIRBuilder &MIB) const;
136 bool tryOptAndIntoCompareBranch(MachineInstr &AndInst, bool Invert,
137 MachineBasicBlock *DstMBB,
138 MachineIRBuilder &MIB) const;
139 ///@}
140
141 bool selectCompareBranch(MachineInstr &I, MachineFunction &MF,
142 MachineRegisterInfo &MRI);
143
144 bool selectVectorAshrLshr(MachineInstr &I, MachineRegisterInfo &MRI);
145 bool selectVectorSHL(MachineInstr &I, MachineRegisterInfo &MRI);
146
147 // Helper to generate an equivalent of scalar_to_vector into a new register,
148 // returned via 'Dst'.
149 MachineInstr *emitScalarToVector(unsigned EltSize,
150 const TargetRegisterClass *DstRC,
151 Register Scalar,
152 MachineIRBuilder &MIRBuilder) const;
153 /// Helper to narrow vector that was widened by emitScalarToVector.
154 /// Copy lowest part of 128-bit or 64-bit vector to 64-bit or 32-bit
155 /// vector, correspondingly.
156 MachineInstr *emitNarrowVector(Register DstReg, Register SrcReg,
157 MachineIRBuilder &MIRBuilder,
158 MachineRegisterInfo &MRI) const;
159
160 /// Emit a lane insert into \p DstReg, or a new vector register if
161 /// std::nullopt is provided.
162 ///
163 /// The lane inserted into is defined by \p LaneIdx. The vector source
164 /// register is given by \p SrcReg. The register containing the element is
165 /// given by \p EltReg.
166 MachineInstr *emitLaneInsert(std::optional<Register> DstReg, Register SrcReg,
167 Register EltReg, unsigned LaneIdx,
168 const RegisterBank &RB,
169 MachineIRBuilder &MIRBuilder) const;
170
171 /// Emit a sequence of instructions representing a constant \p Imm for a
172 /// vector register \p Dst via MOVI.
173 MachineInstr *emitConstantMOVVector(DstOp Dst, APInt Imm,
174 MachineRegisterInfo &MRI);
175
176 /// Emit a sequence of instructions representing a constant \p CV for a
177 /// vector register \p Dst. (E.g. a MOV, or a load from a constant pool.)
178 ///
179 /// \returns the last instruction in the sequence on success, and nullptr
180 /// otherwise.
181 MachineInstr *emitConstantVector(Register Dst, Constant *CV, APInt Imm,
182 MachineRegisterInfo &MRI);
183
184 bool tryOptConstantBuildVec(MachineInstr &MI, LLT DstTy,
185 MachineRegisterInfo &MRI);
186 /// \returns true if a G_BUILD_VECTOR instruction \p MI can be selected as a
187 /// SUBREG_TO_REG.
188 bool tryOptBuildVecToSubregToReg(MachineInstr &MI, MachineRegisterInfo &MRI);
189 bool selectBuildVector(MachineInstr &I, MachineRegisterInfo &MRI);
190 bool selectMergeValues(MachineInstr &I, MachineRegisterInfo &MRI);
191 bool selectUnmergeValues(MachineInstr &I, MachineRegisterInfo &MRI);
192
193 bool selectShuffleVector(MachineInstr &I, MachineRegisterInfo &MRI);
194 bool selectExtractElt(MachineInstr &I, MachineRegisterInfo &MRI);
195 bool selectConcatVectors(MachineInstr &I, MachineRegisterInfo &MRI);
196 bool selectSplitVectorUnmerge(MachineInstr &I, MachineRegisterInfo &MRI);
197
198 /// Helper function to select vector load intrinsics like
199 /// @llvm.aarch64.neon.ld2.*, @llvm.aarch64.neon.ld4.*, etc.
200 /// \p Opc is the opcode that the selected instruction should use.
201 /// \p NumVecs is the number of vector destinations for the instruction.
202 /// \p I is the original G_INTRINSIC_W_SIDE_EFFECTS instruction.
203 bool selectVectorLoadIntrinsic(unsigned Opc, unsigned NumVecs,
204 MachineInstr &I);
205 bool selectVectorLoadLaneIntrinsic(unsigned Opc, unsigned NumVecs,
206 MachineInstr &I);
207 void selectVectorStoreIntrinsic(MachineInstr &I, unsigned NumVecs,
208 unsigned Opc);
209 bool selectVectorStoreLaneIntrinsic(MachineInstr &I, unsigned NumVecs,
210 unsigned Opc);
211 bool selectIntrinsicWithSideEffects(MachineInstr &I,
212 MachineRegisterInfo &MRI);
213 bool selectIntrinsic(MachineInstr &I, MachineRegisterInfo &MRI);
214 bool selectJumpTable(MachineInstr &I, MachineRegisterInfo &MRI);
215 bool selectBrJT(MachineInstr &I, MachineRegisterInfo &MRI);
216 bool selectTLSGlobalValueELF(MachineInstr &I, MachineRegisterInfo &MRI);
217 bool selectTLSLocalExecELF(const GlobalValue *GV, MachineInstr &I,
218 MachineRegisterInfo &MRI);
219 bool selectTLSGlobalValueMachO(MachineInstr &I, MachineRegisterInfo &MRI);
220 bool selectTLSGlobalValue(MachineInstr &I, MachineRegisterInfo &MRI);
221 bool selectPtrAuthGlobalValue(MachineInstr &I,
222 MachineRegisterInfo &MRI) const;
223 bool selectMOPS(MachineInstr &I, MachineRegisterInfo &MRI);
224 bool selectUSMovFromExtend(MachineInstr &I, MachineRegisterInfo &MRI);
225 void SelectTable(MachineInstr &I, MachineRegisterInfo &MRI, unsigned NumVecs,
226 unsigned Opc1, unsigned Opc2, bool isExt);
227
228 bool selectIndexedExtLoad(MachineInstr &I, MachineRegisterInfo &MRI);
229 bool selectIndexedLoad(MachineInstr &I, MachineRegisterInfo &MRI);
230 bool selectIndexedStore(GIndexedStore &I, MachineRegisterInfo &MRI);
231
232 unsigned emitConstantPoolEntry(const Constant *CPVal,
233 MachineFunction &MF) const;
234 MachineInstr *emitLoadFromConstantPool(const Constant *CPVal,
235 MachineIRBuilder &MIRBuilder) const;
236
237 // Emit a vector concat operation.
238 MachineInstr *emitVectorConcat(std::optional<Register> Dst, Register Op1,
239 Register Op2,
240 MachineIRBuilder &MIRBuilder) const;
241
242 // Emit an integer compare between LHS and RHS, which checks for Predicate.
243 MachineInstr *emitIntegerCompare(MachineOperand &LHS, MachineOperand &RHS,
244 MachineOperand &Predicate,
245 MachineIRBuilder &MIRBuilder) const;
246
247 /// Emit a floating point comparison between \p LHS and \p RHS.
248 /// \p Pred if given is the intended predicate to use.
249 MachineInstr *
250 emitFPCompare(Register LHS, Register RHS, MachineIRBuilder &MIRBuilder,
251 std::optional<CmpInst::Predicate> = std::nullopt) const;
252
253 MachineInstr *
254 emitInstr(unsigned Opcode, std::initializer_list<llvm::DstOp> DstOps,
255 std::initializer_list<llvm::SrcOp> SrcOps,
256 MachineIRBuilder &MIRBuilder,
257 const ComplexRendererFns &RenderFns = std::nullopt) const;
258 /// Helper function to emit an add or sub instruction.
259 ///
260 /// \p AddrModeAndSizeToOpcode must contain each of the opcode variants above
261 /// in a specific order.
262 ///
263 /// Below is an example of the expected input to \p AddrModeAndSizeToOpcode.
264 ///
265 /// \code
266 /// const std::array<std::array<unsigned, 2>, 4> Table {
267 /// {{AArch64::ADDXri, AArch64::ADDWri},
268 /// {AArch64::ADDXrs, AArch64::ADDWrs},
269 /// {AArch64::ADDXrr, AArch64::ADDWrr},
270 /// {AArch64::SUBXri, AArch64::SUBWri},
271 /// {AArch64::ADDXrx, AArch64::ADDWrx}}};
272 /// \endcode
273 ///
274 /// Each row in the table corresponds to a different addressing mode. Each
275 /// column corresponds to a different register size.
276 ///
277 /// \attention Rows must be structured as follows:
278 /// - Row 0: The ri opcode variants
279 /// - Row 1: The rs opcode variants
280 /// - Row 2: The rr opcode variants
281 /// - Row 3: The ri opcode variants for negative immediates
282 /// - Row 4: The rx opcode variants
283 ///
284 /// \attention Columns must be structured as follows:
285 /// - Column 0: The 64-bit opcode variants
286 /// - Column 1: The 32-bit opcode variants
287 ///
288 /// \p Dst is the destination register of the binop to emit.
289 /// \p LHS is the left-hand operand of the binop to emit.
290 /// \p RHS is the right-hand operand of the binop to emit.
291 MachineInstr *emitAddSub(
292 const std::array<std::array<unsigned, 2>, 5> &AddrModeAndSizeToOpcode,
293 Register Dst, MachineOperand &LHS, MachineOperand &RHS,
294 MachineIRBuilder &MIRBuilder) const;
295 MachineInstr *emitADD(Register DefReg, MachineOperand &LHS,
296 MachineOperand &RHS,
297 MachineIRBuilder &MIRBuilder) const;
298 MachineInstr *emitADDS(Register Dst, MachineOperand &LHS, MachineOperand &RHS,
299 MachineIRBuilder &MIRBuilder) const;
300 MachineInstr *emitSUBS(Register Dst, MachineOperand &LHS, MachineOperand &RHS,
301 MachineIRBuilder &MIRBuilder) const;
302 MachineInstr *emitADCS(Register Dst, MachineOperand &LHS, MachineOperand &RHS,
303 MachineIRBuilder &MIRBuilder) const;
304 MachineInstr *emitSBCS(Register Dst, MachineOperand &LHS, MachineOperand &RHS,
305 MachineIRBuilder &MIRBuilder) const;
306 MachineInstr *emitCMP(MachineOperand &LHS, MachineOperand &RHS,
307 MachineIRBuilder &MIRBuilder) const;
308 MachineInstr *emitCMN(MachineOperand &LHS, MachineOperand &RHS,
309 MachineIRBuilder &MIRBuilder) const;
310 MachineInstr *emitTST(MachineOperand &LHS, MachineOperand &RHS,
311 MachineIRBuilder &MIRBuilder) const;
312 MachineInstr *emitSelect(Register Dst, Register LHS, Register RHS,
313 AArch64CC::CondCode CC,
314 MachineIRBuilder &MIRBuilder) const;
315 MachineInstr *emitExtractVectorElt(std::optional<Register> DstReg,
316 const RegisterBank &DstRB, LLT ScalarTy,
317 Register VecReg, unsigned LaneIdx,
318 MachineIRBuilder &MIRBuilder) const;
319 MachineInstr *emitCSINC(Register Dst, Register Src1, Register Src2,
320 AArch64CC::CondCode Pred,
321 MachineIRBuilder &MIRBuilder) const;
322 /// Emit a CSet for a FP compare.
323 ///
324 /// \p Dst is expected to be a 32-bit scalar register.
325 MachineInstr *emitCSetForFCmp(Register Dst, CmpInst::Predicate Pred,
326 MachineIRBuilder &MIRBuilder) const;
327
328 /// Emit an instruction that sets NZCV to the carry-in expected by \p I.
329 /// Might elide the instruction if the previous instruction already sets NZCV
330 /// correctly.
331 MachineInstr *emitCarryIn(MachineInstr &I, Register CarryReg);
332
333 /// Emit the overflow op for \p Opcode.
334 ///
335 /// \p Opcode is expected to be an overflow op's opcode, e.g. G_UADDO,
336 /// G_USUBO, etc.
337 std::pair<MachineInstr *, AArch64CC::CondCode>
338 emitOverflowOp(unsigned Opcode, Register Dst, MachineOperand &LHS,
339 MachineOperand &RHS, MachineIRBuilder &MIRBuilder) const;
340
341 bool selectOverflowOp(MachineInstr &I, MachineRegisterInfo &MRI);
342
343 /// Emit expression as a conjunction (a series of CCMP/CFCMP ops).
344 /// In some cases this is even possible with OR operations in the expression.
345 MachineInstr *emitConjunction(Register Val, AArch64CC::CondCode &OutCC,
346 MachineIRBuilder &MIB) const;
347 MachineInstr *emitConditionalComparison(Register LHS, Register RHS,
348 CmpInst::Predicate CC,
349 AArch64CC::CondCode Predicate,
350 AArch64CC::CondCode OutCC,
351 MachineIRBuilder &MIB) const;
352 MachineInstr *emitConjunctionRec(Register Val, AArch64CC::CondCode &OutCC,
353 bool Negate, Register CCOp,
354 AArch64CC::CondCode Predicate,
355 MachineIRBuilder &MIB) const;
356
357 /// Emit a TB(N)Z instruction which tests \p Bit in \p TestReg.
358 /// \p IsNegative is true if the test should be "not zero".
359 /// This will also optimize the test bit instruction when possible.
360 MachineInstr *emitTestBit(Register TestReg, uint64_t Bit, bool IsNegative,
361 MachineBasicBlock *DstMBB,
362 MachineIRBuilder &MIB) const;
363
364 /// Emit a CB(N)Z instruction which branches to \p DestMBB.
365 MachineInstr *emitCBZ(Register CompareReg, bool IsNegative,
366 MachineBasicBlock *DestMBB,
367 MachineIRBuilder &MIB) const;
368
369 // Equivalent to the i32shift_a and friends from AArch64InstrInfo.td.
370 // We use these manually instead of using the importer since it doesn't
371 // support SDNodeXForm.
372 ComplexRendererFns selectShiftA_32(const MachineOperand &Root) const;
373 ComplexRendererFns selectShiftB_32(const MachineOperand &Root) const;
374 ComplexRendererFns selectShiftA_64(const MachineOperand &Root) const;
375 ComplexRendererFns selectShiftB_64(const MachineOperand &Root) const;
376
377 template <unsigned ShiftWidth>
378 ComplexRendererFns selectShiftMask(MachineOperand &Root) const;
379 ComplexRendererFns select12BitValueWithLeftShift(uint64_t Immed) const;
380 ComplexRendererFns selectArithImmed(MachineOperand &Root) const;
381 ComplexRendererFns selectNegArithImmed(MachineOperand &Root) const;
382
383 ComplexRendererFns selectAddrModeUnscaled(MachineOperand &Root,
384 unsigned Size) const;
385
386 ComplexRendererFns selectAddrModeUnscaled8(MachineOperand &Root) const {
387 return selectAddrModeUnscaled(Root, Size: 1);
388 }
389 ComplexRendererFns selectAddrModeUnscaled16(MachineOperand &Root) const {
390 return selectAddrModeUnscaled(Root, Size: 2);
391 }
392 ComplexRendererFns selectAddrModeUnscaled32(MachineOperand &Root) const {
393 return selectAddrModeUnscaled(Root, Size: 4);
394 }
395 ComplexRendererFns selectAddrModeUnscaled64(MachineOperand &Root) const {
396 return selectAddrModeUnscaled(Root, Size: 8);
397 }
398 ComplexRendererFns selectAddrModeUnscaled128(MachineOperand &Root) const {
399 return selectAddrModeUnscaled(Root, Size: 16);
400 }
401
402 /// Helper to try to fold in a GISEL_ADD_LOW into an immediate, to be used
403 /// from complex pattern matchers like selectAddrModeIndexed().
404 ComplexRendererFns tryFoldAddLowIntoImm(MachineInstr &RootDef, unsigned Size,
405 MachineRegisterInfo &MRI) const;
406
407 ComplexRendererFns selectAddrModeIndexed(MachineOperand &Root,
408 unsigned Size) const;
409 template <int Width>
410 ComplexRendererFns selectAddrModeIndexed(MachineOperand &Root) const {
411 return selectAddrModeIndexed(Root, Size: Width / 8);
412 }
413
414 std::optional<bool>
415 isWorthFoldingIntoAddrMode(const MachineInstr &MI,
416 const MachineRegisterInfo &MRI) const;
417
418 bool isWorthFoldingIntoExtendedReg(const MachineInstr &MI,
419 const MachineRegisterInfo &MRI,
420 bool IsAddrOperand) const;
421 ComplexRendererFns
422 selectAddrModeShiftedExtendXReg(MachineOperand &Root,
423 unsigned SizeInBytes) const;
424
425 /// Returns a \p ComplexRendererFns which contains a base, offset, and whether
426 /// or not a shift + extend should be folded into an addressing mode. Returns
427 /// None when this is not profitable or possible.
428 ComplexRendererFns
429 selectExtendedSHL(MachineOperand &Root, MachineOperand &Base,
430 MachineOperand &Offset, unsigned SizeInBytes,
431 bool WantsExt) const;
432 ComplexRendererFns selectAddrModeRegisterOffset(MachineOperand &Root) const;
433 ComplexRendererFns selectAddrModeXRO(MachineOperand &Root,
434 unsigned SizeInBytes) const;
435 template <int Width>
436 ComplexRendererFns selectAddrModeXRO(MachineOperand &Root) const {
437 return selectAddrModeXRO(Root, SizeInBytes: Width / 8);
438 }
439
440 ComplexRendererFns selectAddrModeWRO(MachineOperand &Root,
441 unsigned SizeInBytes) const;
442 template <int Width>
443 ComplexRendererFns selectAddrModeWRO(MachineOperand &Root) const {
444 return selectAddrModeWRO(Root, SizeInBytes: Width / 8);
445 }
446
447 ComplexRendererFns selectShiftedRegister(MachineOperand &Root,
448 bool AllowROR = false) const;
449
450 ComplexRendererFns selectArithShiftedRegister(MachineOperand &Root) const {
451 return selectShiftedRegister(Root);
452 }
453
454 ComplexRendererFns selectLogicalShiftedRegister(MachineOperand &Root) const {
455 return selectShiftedRegister(Root, AllowROR: true);
456 }
457
458 /// Given an extend instruction, determine the correct shift-extend type for
459 /// that instruction.
460 ///
461 /// If the instruction is going to be used in a load or store, pass
462 /// \p IsLoadStore = true.
463 AArch64_AM::ShiftExtendType
464 getExtendTypeForInst(MachineInstr &MI, MachineRegisterInfo &MRI,
465 bool IsLoadStore = false) const;
466
467 /// Move \p Reg to \p RC if \p Reg is not already on \p RC.
468 ///
469 /// \returns Either \p Reg if no change was necessary, or the new register
470 /// created by moving \p Reg.
471 ///
472 /// Note: This uses emitCopy right now.
473 Register moveScalarRegClass(Register Reg, const TargetRegisterClass &RC,
474 MachineIRBuilder &MIB) const;
475
476 ComplexRendererFns selectArithExtendedRegister(MachineOperand &Root) const;
477
478 ComplexRendererFns selectExtractHigh(MachineOperand &Root) const;
479 template <unsigned Width>
480 ComplexRendererFns selectCVTFixedPoint(MachineOperand &Root) const;
481 template <unsigned Width>
482 ComplexRendererFns selectCVTFixedPosRecipOperand(MachineOperand &Root) const;
483 ComplexRendererFns selectCVTFixedPointBase(const MachineOperand &Root,
484 unsigned width,
485 bool isReciprocal = false) const;
486 ComplexRendererFns selectCVTFixedPointVec(MachineOperand &Root) const;
487 ComplexRendererFns
488 selectCVTFixedPosRecipOperandVec(MachineOperand &Root) const;
489 void renderFixedPointScalarXForm(MachineInstrBuilder &MIB,
490 const MachineInstr &MI, int OpIdx) const;
491 unsigned getFixedPointWidthFromOperand(const MachineOperand &Root) const;
492 void renderFixedPointXForm(MachineInstrBuilder &MIB, const MachineInstr &MI,
493 int OpIdx = -1) const;
494 void renderFixedPointRecipXForm(MachineInstrBuilder &MIB,
495 const MachineInstr &MI, int OpIdx = -1) const;
496 void renderFixedPointImm(MachineInstrBuilder &MIB, const MachineOperand &Root,
497 unsigned Width, bool isReciprocal) const;
498 void renderTruncImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
499 int OpIdx = -1) const;
500 void renderLogicalImm32(MachineInstrBuilder &MIB, const MachineInstr &I,
501 int OpIdx = -1) const;
502 void renderLogicalImm64(MachineInstrBuilder &MIB, const MachineInstr &I,
503 int OpIdx = -1) const;
504 void renderUbsanTrap(MachineInstrBuilder &MIB, const MachineInstr &MI,
505 int OpIdx) const;
506 void renderFPImm16(MachineInstrBuilder &MIB, const MachineInstr &MI,
507 int OpIdx = -1) const;
508 void renderFPImm32(MachineInstrBuilder &MIB, const MachineInstr &MI,
509 int OpIdx = -1) const;
510 void renderFPImm64(MachineInstrBuilder &MIB, const MachineInstr &MI,
511 int OpIdx = -1) const;
512
513 // Materialize a GlobalValue or BlockAddress using a movz+movk sequence.
514 void materializeLargeCMVal(MachineInstr &I, const Value *V, unsigned OpFlags);
515
516 // Optimization methods.
517 bool tryOptSelect(GSelect &Sel);
518 bool tryOptSelectConjunction(GSelect &Sel, MachineInstr &CondMI);
519 MachineInstr *tryFoldIntegerCompare(MachineOperand &LHS, MachineOperand &RHS,
520 MachineOperand &Predicate,
521 MachineIRBuilder &MIRBuilder) const;
522
523 /// Return true if \p MI is a load or store of \p NumBytes bytes.
524 bool isLoadStoreOfNumBytes(const MachineInstr &MI, unsigned NumBytes) const;
525
526 /// Returns true if \p MI is guaranteed to have the high-half of a 64-bit
527 /// register zeroed out. In other words, the result of MI has been explicitly
528 /// zero extended.
529 bool isDef32(const MachineInstr &MI) const;
530
531 const AArch64TargetMachine &TM;
532 const AArch64Subtarget &STI;
533 const AArch64InstrInfo &TII;
534 const AArch64RegisterInfo &TRI;
535 const AArch64RegisterBankInfo &RBI;
536
537 bool ProduceNonFlagSettingCondBr = false;
538
539 // Some cached values used during selection.
540 // We use LR as a live-in register, and we keep track of it here as it can be
541 // clobbered by calls.
542 Register MFReturnAddr;
543
544 MachineIRBuilder MIB;
545
546#define GET_GLOBALISEL_PREDICATES_DECL
547#include "AArch64GenGlobalISel.inc"
548#undef GET_GLOBALISEL_PREDICATES_DECL
549
550// We declare the temporaries used by selectImpl() in the class to minimize the
551// cost of constructing placeholder values.
552#define GET_GLOBALISEL_TEMPORARIES_DECL
553#include "AArch64GenGlobalISel.inc"
554#undef GET_GLOBALISEL_TEMPORARIES_DECL
555};
556
557} // end anonymous namespace
558
559#define GET_GLOBALISEL_IMPL
560#include "AArch64GenGlobalISel.inc"
561#undef GET_GLOBALISEL_IMPL
562
563AArch64InstructionSelector::AArch64InstructionSelector(
564 const AArch64TargetMachine &TM, const AArch64Subtarget &STI,
565 const AArch64RegisterBankInfo &RBI)
566 : TM(TM), STI(STI), TII(*STI.getInstrInfo()), TRI(*STI.getRegisterInfo()),
567 RBI(RBI),
568#define GET_GLOBALISEL_PREDICATES_INIT
569#include "AArch64GenGlobalISel.inc"
570#undef GET_GLOBALISEL_PREDICATES_INIT
571#define GET_GLOBALISEL_TEMPORARIES_INIT
572#include "AArch64GenGlobalISel.inc"
573#undef GET_GLOBALISEL_TEMPORARIES_INIT
574{
575}
576
577// FIXME: This should be target-independent, inferred from the types declared
578// for each class in the bank.
579//
580/// Given a register bank, and a type, return the smallest register class that
581/// can represent that combination.
582static const TargetRegisterClass *
583getRegClassForTypeOnBank(LLT Ty, const RegisterBank &RB,
584 bool GetAllRegSet = false) {
585 if (RB.getID() == AArch64::GPRRegBankID) {
586 if (Ty.getSizeInBits() <= 32)
587 return GetAllRegSet ? &AArch64::GPR32allRegClass
588 : &AArch64::GPR32RegClass;
589 if (Ty.getSizeInBits() == 64)
590 return GetAllRegSet ? &AArch64::GPR64allRegClass
591 : &AArch64::GPR64RegClass;
592 if (Ty.getSizeInBits() == 128)
593 return &AArch64::XSeqPairsClassRegClass;
594 return nullptr;
595 }
596
597 if (RB.getID() == AArch64::FPRRegBankID) {
598 switch (Ty.getSizeInBits()) {
599 case 8:
600 return &AArch64::FPR8RegClass;
601 case 16:
602 return &AArch64::FPR16RegClass;
603 case 32:
604 return &AArch64::FPR32RegClass;
605 case 64:
606 return &AArch64::FPR64RegClass;
607 case 128:
608 return &AArch64::FPR128RegClass;
609 }
610 return nullptr;
611 }
612
613 return nullptr;
614}
615
616/// Given a register bank, and size in bits, return the smallest register class
617/// that can represent that combination.
618static const TargetRegisterClass *
619getMinClassForRegBank(const RegisterBank &RB, TypeSize SizeInBits,
620 bool GetAllRegSet = false) {
621 if (SizeInBits.isScalable()) {
622 assert(RB.getID() == AArch64::FPRRegBankID &&
623 "Expected FPR regbank for scalable type size");
624 return &AArch64::ZPRRegClass;
625 }
626
627 unsigned RegBankID = RB.getID();
628
629 if (RegBankID == AArch64::GPRRegBankID) {
630 assert(!SizeInBits.isScalable() && "Unexpected scalable register size");
631 if (SizeInBits <= 32)
632 return GetAllRegSet ? &AArch64::GPR32allRegClass
633 : &AArch64::GPR32RegClass;
634 if (SizeInBits == 64)
635 return GetAllRegSet ? &AArch64::GPR64allRegClass
636 : &AArch64::GPR64RegClass;
637 if (SizeInBits == 128)
638 return &AArch64::XSeqPairsClassRegClass;
639 }
640
641 if (RegBankID == AArch64::FPRRegBankID) {
642 if (SizeInBits.isScalable()) {
643 assert(SizeInBits == TypeSize::getScalable(128) &&
644 "Unexpected scalable register size");
645 return &AArch64::ZPRRegClass;
646 }
647
648 switch (SizeInBits) {
649 default:
650 return nullptr;
651 case 8:
652 return &AArch64::FPR8RegClass;
653 case 16:
654 return &AArch64::FPR16RegClass;
655 case 32:
656 return &AArch64::FPR32RegClass;
657 case 64:
658 return &AArch64::FPR64RegClass;
659 case 128:
660 return &AArch64::FPR128RegClass;
661 }
662 }
663
664 return nullptr;
665}
666
667/// Returns the correct subregister to use for a given register class.
668static bool getSubRegForClass(const TargetRegisterClass *RC,
669 const TargetRegisterInfo &TRI, unsigned &SubReg) {
670 switch (TRI.getRegSizeInBits(RC: *RC)) {
671 case 8:
672 SubReg = AArch64::bsub;
673 break;
674 case 16:
675 SubReg = AArch64::hsub;
676 break;
677 case 32:
678 if (RC != &AArch64::FPR32RegClass)
679 SubReg = AArch64::sub_32;
680 else
681 SubReg = AArch64::ssub;
682 break;
683 case 64:
684 SubReg = AArch64::dsub;
685 break;
686 default:
687 LLVM_DEBUG(
688 dbgs() << "Couldn't find appropriate subregister for register class.");
689 return false;
690 }
691
692 return true;
693}
694
695/// Returns the minimum size the given register bank can hold.
696static unsigned getMinSizeForRegBank(const RegisterBank &RB) {
697 switch (RB.getID()) {
698 case AArch64::GPRRegBankID:
699 return 32;
700 case AArch64::FPRRegBankID:
701 return 8;
702 default:
703 llvm_unreachable("Tried to get minimum size for unknown register bank.");
704 }
705}
706
707/// Create a REG_SEQUENCE instruction using the registers in \p Regs.
708/// Helper function for functions like createDTuple and createQTuple.
709///
710/// \p RegClassIDs - The list of register class IDs available for some tuple of
711/// a scalar class. E.g. QQRegClassID, QQQRegClassID, QQQQRegClassID. This is
712/// expected to contain between 2 and 4 tuple classes.
713///
714/// \p SubRegs - The list of subregister classes associated with each register
715/// class ID in \p RegClassIDs. E.g., QQRegClassID should use the qsub0
716/// subregister class. The index of each subregister class is expected to
717/// correspond with the index of each register class.
718///
719/// \returns Either the destination register of REG_SEQUENCE instruction that
720/// was created, or the 0th element of \p Regs if \p Regs contains a single
721/// element.
722static Register createTuple(ArrayRef<Register> Regs,
723 const unsigned RegClassIDs[],
724 const unsigned SubRegs[], MachineIRBuilder &MIB) {
725 unsigned NumRegs = Regs.size();
726 if (NumRegs == 1)
727 return Regs[0];
728 assert(NumRegs >= 2 && NumRegs <= 4 &&
729 "Only support between two and 4 registers in a tuple!");
730 const TargetRegisterInfo *TRI = MIB.getMF().getSubtarget().getRegisterInfo();
731 auto *DesiredClass = TRI->getRegClass(i: RegClassIDs[NumRegs - 2]);
732 auto RegSequence =
733 MIB.buildInstr(Opc: TargetOpcode::REG_SEQUENCE, DstOps: {DesiredClass}, SrcOps: {});
734 for (unsigned I = 0, E = Regs.size(); I < E; ++I) {
735 RegSequence.addUse(RegNo: Regs[I]);
736 RegSequence.addImm(Val: SubRegs[I]);
737 }
738 return RegSequence.getReg(Idx: 0);
739}
740
741/// Create a tuple of D-registers using the registers in \p Regs.
742static Register createDTuple(ArrayRef<Register> Regs, MachineIRBuilder &MIB) {
743 static const unsigned RegClassIDs[] = {
744 AArch64::DDRegClassID, AArch64::DDDRegClassID, AArch64::DDDDRegClassID};
745 static const unsigned SubRegs[] = {AArch64::dsub0, AArch64::dsub1,
746 AArch64::dsub2, AArch64::dsub3};
747 return createTuple(Regs, RegClassIDs, SubRegs, MIB);
748}
749
750/// Create a tuple of Q-registers using the registers in \p Regs.
751static Register createQTuple(ArrayRef<Register> Regs, MachineIRBuilder &MIB) {
752 static const unsigned RegClassIDs[] = {
753 AArch64::QQRegClassID, AArch64::QQQRegClassID, AArch64::QQQQRegClassID};
754 static const unsigned SubRegs[] = {AArch64::qsub0, AArch64::qsub1,
755 AArch64::qsub2, AArch64::qsub3};
756 return createTuple(Regs, RegClassIDs, SubRegs, MIB);
757}
758
759static std::optional<uint64_t> getImmedFromMO(const MachineOperand &Root) {
760 auto &MI = *Root.getParent();
761 auto &MBB = *MI.getParent();
762 auto &MF = *MBB.getParent();
763 auto &MRI = MF.getRegInfo();
764 uint64_t Immed;
765 if (Root.isImm())
766 Immed = Root.getImm();
767 else if (Root.isCImm())
768 Immed = Root.getCImm()->getZExtValue();
769 else if (Root.isReg()) {
770 auto ValAndVReg =
771 getIConstantVRegValWithLookThrough(VReg: Root.getReg(), MRI, LookThroughInstrs: true);
772 if (!ValAndVReg)
773 return std::nullopt;
774 Immed = ValAndVReg->Value.getSExtValue();
775 } else
776 return std::nullopt;
777 return Immed;
778}
779
780/// Select the AArch64 opcode for the basic binary operation \p GenericOpc,
781/// appropriate for the register bank \p RegBankID and of size \p OpSize.
782/// \returns \p GenericOpc if the combination is unsupported.
783static unsigned selectBinaryOp(unsigned GenericOpc, unsigned RegBankID,
784 unsigned OpSize) {
785 if (RegBankID == AArch64::GPRRegBankID) {
786 if (OpSize == 32) {
787 switch (GenericOpc) {
788 case TargetOpcode::G_SHL:
789 return AArch64::LSLVWr;
790 case TargetOpcode::G_LSHR:
791 return AArch64::LSRVWr;
792 case TargetOpcode::G_ASHR:
793 return AArch64::ASRVWr;
794 default:
795 return GenericOpc;
796 }
797 } else if (OpSize == 64) {
798 switch (GenericOpc) {
799 case TargetOpcode::G_SHL:
800 return AArch64::LSLVXr;
801 case TargetOpcode::G_LSHR:
802 return AArch64::LSRVXr;
803 case TargetOpcode::G_ASHR:
804 return AArch64::ASRVXr;
805 default:
806 return GenericOpc;
807 }
808 }
809 }
810 return GenericOpc;
811}
812
813/// Select the AArch64 opcode for the G_LOAD or G_STORE operation \p GenericOpc,
814/// appropriate for the (value) register bank \p RegBankID and of memory access
815/// size \p OpSize. This returns the variant with the base+unsigned-immediate
816/// addressing mode (e.g., LDRXui).
817/// \returns \p GenericOpc if the combination is unsupported.
818static unsigned selectLoadStoreUIOp(unsigned GenericOpc, unsigned RegBankID,
819 unsigned OpSize) {
820 const bool isStore = GenericOpc == TargetOpcode::G_STORE;
821 switch (RegBankID) {
822 case AArch64::GPRRegBankID:
823 switch (OpSize) {
824 case 8:
825 return isStore ? AArch64::STRBBui : AArch64::LDRBBui;
826 case 16:
827 return isStore ? AArch64::STRHHui : AArch64::LDRHHui;
828 case 32:
829 return isStore ? AArch64::STRWui : AArch64::LDRWui;
830 case 64:
831 return isStore ? AArch64::STRXui : AArch64::LDRXui;
832 }
833 break;
834 case AArch64::FPRRegBankID:
835 switch (OpSize) {
836 case 8:
837 return isStore ? AArch64::STRBui : AArch64::LDRBui;
838 case 16:
839 return isStore ? AArch64::STRHui : AArch64::LDRHui;
840 case 32:
841 return isStore ? AArch64::STRSui : AArch64::LDRSui;
842 case 64:
843 return isStore ? AArch64::STRDui : AArch64::LDRDui;
844 case 128:
845 return isStore ? AArch64::STRQui : AArch64::LDRQui;
846 }
847 break;
848 }
849 return GenericOpc;
850}
851
852/// Helper function for selectCopy. Inserts a subregister copy from \p SrcReg
853/// to \p *To.
854///
855/// E.g "To = COPY SrcReg:SubReg"
856static bool copySubReg(MachineInstr &I, MachineRegisterInfo &MRI,
857 const RegisterBankInfo &RBI, Register SrcReg,
858 const TargetRegisterClass *To, unsigned SubReg) {
859 assert(SrcReg.isValid() && "Expected a valid source register?");
860 assert(To && "Destination register class cannot be null");
861 assert(SubReg && "Expected a valid subregister");
862
863 MachineIRBuilder MIB(I);
864 auto SubRegCopy =
865 MIB.buildInstr(Opc: TargetOpcode::COPY, DstOps: {To}, SrcOps: {}).addReg(RegNo: SrcReg, Flags: {}, SubReg);
866 MachineOperand &RegOp = I.getOperand(i: 1);
867 RegOp.setReg(SubRegCopy.getReg(Idx: 0));
868
869 // It's possible that the destination register won't be constrained. Make
870 // sure that happens.
871 if (!I.getOperand(i: 0).getReg().isPhysical())
872 RBI.constrainGenericRegister(Reg: I.getOperand(i: 0).getReg(), RC: *To, MRI);
873
874 return true;
875}
876
877static bool selectCopy(MachineInstr &I, const TargetInstrInfo &TII,
878 MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI,
879 const RegisterBankInfo &RBI) {
880 Register DstReg = I.getOperand(i: 0).getReg();
881 Register SrcReg = I.getOperand(i: 1).getReg();
882 const RegisterBank &DstRegBank = *RBI.getRegBank(Reg: DstReg, MRI, TRI);
883 const RegisterBank &SrcRegBank = *RBI.getRegBank(Reg: SrcReg, MRI, TRI);
884
885 TypeSize DstRegSize = TRI.getRegSizeInBits(Reg: DstReg, MRI);
886 TypeSize SrcRegSize = TRI.getRegSizeInBits(Reg: SrcReg, MRI);
887
888 // Special casing for cross-bank copies of s1s. We can technically represent
889 // a 1-bit value with any size of register. The minimum size for a GPR is 32
890 // bits. So, we need to put the FPR on 32 bits as well.
891 //
892 // FIXME: I'm not sure if this case holds true outside of copies. If it does,
893 // then we can pull it into the helpers that get the appropriate class for a
894 // register bank. Or make a new helper that carries along some constraint
895 // information.
896 if (SrcRegBank != DstRegBank && (DstRegSize == TypeSize::getFixed(ExactSize: 1) &&
897 SrcRegSize == TypeSize::getFixed(ExactSize: 1)))
898 SrcRegSize = DstRegSize = TypeSize::getFixed(ExactSize: 32);
899
900 // Find the correct register classes for the source and destination registers.
901 const TargetRegisterClass *SrcRC =
902 getMinClassForRegBank(RB: SrcRegBank, SizeInBits: SrcRegSize, GetAllRegSet: true);
903 const TargetRegisterClass *DstRC =
904 getMinClassForRegBank(RB: DstRegBank, SizeInBits: DstRegSize, GetAllRegSet: true);
905
906 if (!DstRC) {
907 LLVM_DEBUG(dbgs() << "Unexpected dest size "
908 << TRI.getRegSizeInBits(DstReg, MRI) << '\n');
909 return false;
910 }
911
912 if (I.getOpcode() == TargetOpcode::G_BITCAST &&
913 TRI.getRegSizeInBits(Reg: DstReg, MRI) == TypeSize::getFixed(ExactSize: 16)) {
914 if (DstRegBank.getID() == AArch64::FPRRegBankID &&
915 SrcRegBank.getID() == AArch64::GPRRegBankID) {
916 if (!SrcReg.isPhysical() &&
917 !RBI.constrainGenericRegister(Reg: SrcReg, RC: AArch64::GPR32RegClass, MRI))
918 return false;
919 if (!DstReg.isPhysical() &&
920 !RBI.constrainGenericRegister(Reg: DstReg, RC: AArch64::FPR16RegClass, MRI))
921 return false;
922
923 Register FPR32 = MRI.createVirtualRegister(RegClass: &AArch64::FPR32RegClass);
924 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: AArch64::FMOVWSr))
925 .addDef(RegNo: FPR32)
926 .addUse(RegNo: SrcReg);
927 I.setDesc(TII.get(Opcode: TargetOpcode::COPY));
928 I.getOperand(i: 1).setReg(FPR32);
929 I.getOperand(i: 1).setSubReg(AArch64::hsub);
930 return true;
931 }
932
933 if (DstRegBank.getID() == AArch64::GPRRegBankID &&
934 SrcRegBank.getID() == AArch64::FPRRegBankID) {
935 if (!SrcReg.isPhysical() &&
936 !RBI.constrainGenericRegister(Reg: SrcReg, RC: AArch64::FPR16RegClass, MRI))
937 return false;
938 if (!DstReg.isPhysical() &&
939 !RBI.constrainGenericRegister(Reg: DstReg, RC: AArch64::GPR32RegClass, MRI))
940 return false;
941
942 Register FPR32 = MRI.createVirtualRegister(RegClass: &AArch64::FPR32RegClass);
943 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
944 MCID: TII.get(Opcode: TargetOpcode::SUBREG_TO_REG))
945 .addDef(RegNo: FPR32)
946 .addUse(RegNo: SrcReg)
947 .addImm(Val: AArch64::hsub);
948 I.setDesc(TII.get(Opcode: AArch64::FMOVSWr));
949 I.getOperand(i: 1).setReg(FPR32);
950 return true;
951 }
952 }
953
954 // Is this a copy? If so, then we may need to insert a subregister copy.
955 if (I.isCopy()) {
956 // Yes. Check if there's anything to fix up.
957 if (!SrcRC) {
958 LLVM_DEBUG(dbgs() << "Couldn't determine source register class\n");
959 return false;
960 }
961
962 const TypeSize SrcSize = TRI.getRegSizeInBits(RC: *SrcRC);
963 const TypeSize DstSize = TRI.getRegSizeInBits(RC: *DstRC);
964 unsigned SrcSubReg = I.getOperand(i: 1).getSubReg();
965 unsigned SubReg;
966
967 if (SrcSubReg)
968 return RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI);
969
970 // If the source bank doesn't support a subregister copy small enough,
971 // then we first need to copy to the destination bank.
972 if (getMinSizeForRegBank(RB: SrcRegBank) > DstSize) {
973 const TargetRegisterClass *DstTempRC =
974 getMinClassForRegBank(RB: DstRegBank, SizeInBits: SrcSize, /* GetAllRegSet */ true);
975 getSubRegForClass(RC: DstRC, TRI, SubReg);
976
977 MachineIRBuilder MIB(I);
978 auto Copy = MIB.buildCopy(Res: {DstTempRC}, Op: {SrcReg});
979 copySubReg(I, MRI, RBI, SrcReg: Copy.getReg(Idx: 0), To: DstRC, SubReg);
980 } else if (SrcSize > DstSize) {
981 // If the source register is bigger than the destination we need to
982 // perform a subregister copy.
983 const TargetRegisterClass *SubRegRC =
984 getMinClassForRegBank(RB: SrcRegBank, SizeInBits: DstSize, /* GetAllRegSet */ true);
985 getSubRegForClass(RC: SubRegRC, TRI, SubReg);
986 copySubReg(I, MRI, RBI, SrcReg, To: DstRC, SubReg);
987 } else if (DstSize > SrcSize) {
988 // If the destination register is bigger than the source we need to do
989 // a promotion using SUBREG_TO_REG.
990 const TargetRegisterClass *PromotionRC =
991 getMinClassForRegBank(RB: SrcRegBank, SizeInBits: DstSize, /* GetAllRegSet */ true);
992 getSubRegForClass(RC: SrcRC, TRI, SubReg);
993
994 Register PromoteReg = MRI.createVirtualRegister(RegClass: PromotionRC);
995 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
996 MCID: TII.get(Opcode: AArch64::SUBREG_TO_REG), DestReg: PromoteReg)
997 .addUse(RegNo: SrcReg)
998 .addImm(Val: SubReg);
999 MachineOperand &RegOp = I.getOperand(i: 1);
1000 RegOp.setReg(PromoteReg);
1001 }
1002
1003 // If the destination is a physical register, then there's nothing to
1004 // change, so we're done.
1005 if (DstReg.isPhysical())
1006 return true;
1007 }
1008
1009 // No need to constrain SrcReg. It will get constrained when we hit another
1010 // of its use or its defs. Copies do not have constraints.
1011 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI)) {
1012 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
1013 << " operand\n");
1014 return false;
1015 }
1016
1017 // If this a GPR ZEXT that we want to just reduce down into a copy.
1018 // The sizes will be mismatched with the source < 32b but that's ok.
1019 if (I.getOpcode() == TargetOpcode::G_ZEXT) {
1020 I.setDesc(TII.get(Opcode: AArch64::COPY));
1021 assert(SrcRegBank.getID() == AArch64::GPRRegBankID);
1022 return selectCopy(I, TII, MRI, TRI, RBI);
1023 }
1024
1025 I.setDesc(TII.get(Opcode: AArch64::COPY));
1026 return true;
1027}
1028
1029MachineInstr *
1030AArch64InstructionSelector::emitSelect(Register Dst, Register True,
1031 Register False, AArch64CC::CondCode CC,
1032 MachineIRBuilder &MIB) const {
1033 MachineRegisterInfo &MRI = *MIB.getMRI();
1034 assert(RBI.getRegBank(False, MRI, TRI)->getID() ==
1035 RBI.getRegBank(True, MRI, TRI)->getID() &&
1036 "Expected both select operands to have the same regbank?");
1037 LLT Ty = MRI.getType(Reg: True);
1038 if (Ty.isVector())
1039 return nullptr;
1040 const unsigned Size = Ty.getSizeInBits();
1041 assert((Size == 32 || Size == 64) &&
1042 "Expected 32 bit or 64 bit select only?");
1043 const bool Is32Bit = Size == 32;
1044 if (RBI.getRegBank(Reg: True, MRI, TRI)->getID() != AArch64::GPRRegBankID) {
1045 unsigned Opc = Is32Bit ? AArch64::FCSELSrrr : AArch64::FCSELDrrr;
1046 auto FCSel = MIB.buildInstr(Opc, DstOps: {Dst}, SrcOps: {True, False}).addImm(Val: CC);
1047 constrainSelectedInstRegOperands(I&: *FCSel, TII, TRI, RBI);
1048 return &*FCSel;
1049 }
1050
1051 // By default, we'll try and emit a CSEL.
1052 unsigned Opc = Is32Bit ? AArch64::CSELWr : AArch64::CSELXr;
1053 bool Optimized = false;
1054 auto TryFoldBinOpIntoSelect = [&Opc, Is32Bit, &CC, &MRI,
1055 &Optimized](Register &Reg, Register &OtherReg,
1056 bool Invert) {
1057 if (Optimized)
1058 return false;
1059
1060 // Attempt to fold:
1061 //
1062 // %sub = G_SUB 0, %x
1063 // %select = G_SELECT cc, %reg, %sub
1064 //
1065 // Into:
1066 // %select = CSNEG %reg, %x, cc
1067 Register MatchReg;
1068 if (mi_match(R: Reg, MRI, P: m_Neg(Src: m_Reg(R&: MatchReg)))) {
1069 Opc = Is32Bit ? AArch64::CSNEGWr : AArch64::CSNEGXr;
1070 Reg = MatchReg;
1071 if (Invert) {
1072 CC = AArch64CC::getInvertedCondCode(Code: CC);
1073 std::swap(a&: Reg, b&: OtherReg);
1074 }
1075 return true;
1076 }
1077
1078 // Attempt to fold:
1079 //
1080 // %xor = G_XOR %x, -1
1081 // %select = G_SELECT cc, %reg, %xor
1082 //
1083 // Into:
1084 // %select = CSINV %reg, %x, cc
1085 if (mi_match(R: Reg, MRI, P: m_Not(Src: m_Reg(R&: MatchReg)))) {
1086 Opc = Is32Bit ? AArch64::CSINVWr : AArch64::CSINVXr;
1087 Reg = MatchReg;
1088 if (Invert) {
1089 CC = AArch64CC::getInvertedCondCode(Code: CC);
1090 std::swap(a&: Reg, b&: OtherReg);
1091 }
1092 return true;
1093 }
1094
1095 // Attempt to fold:
1096 //
1097 // %add = G_ADD %x, 1
1098 // %select = G_SELECT cc, %reg, %add
1099 //
1100 // Into:
1101 // %select = CSINC %reg, %x, cc
1102 if (mi_match(R: Reg, MRI,
1103 P: m_any_of(preds: m_GAdd(L: m_Reg(R&: MatchReg), R: m_SpecificICst(RequestedValue: 1)),
1104 preds: m_GPtrAdd(L: m_Reg(R&: MatchReg), R: m_SpecificICst(RequestedValue: 1))))) {
1105 Opc = Is32Bit ? AArch64::CSINCWr : AArch64::CSINCXr;
1106 Reg = MatchReg;
1107 if (Invert) {
1108 CC = AArch64CC::getInvertedCondCode(Code: CC);
1109 std::swap(a&: Reg, b&: OtherReg);
1110 }
1111 return true;
1112 }
1113
1114 return false;
1115 };
1116
1117 // Helper lambda which tries to use CSINC/CSINV for the instruction when its
1118 // true/false values are constants.
1119 // FIXME: All of these patterns already exist in tablegen. We should be
1120 // able to import these.
1121 auto TryOptSelectCst = [&Opc, &True, &False, &CC, Is32Bit, &MRI,
1122 &Optimized]() {
1123 if (Optimized)
1124 return false;
1125 auto TrueCst = getIConstantVRegValWithLookThrough(VReg: True, MRI);
1126 auto FalseCst = getIConstantVRegValWithLookThrough(VReg: False, MRI);
1127 if (!TrueCst && !FalseCst)
1128 return false;
1129
1130 Register ZReg = Is32Bit ? AArch64::WZR : AArch64::XZR;
1131 if (TrueCst && FalseCst) {
1132 int64_t T = TrueCst->Value.getSExtValue();
1133 int64_t F = FalseCst->Value.getSExtValue();
1134
1135 if (T == 0 && F == 1) {
1136 // G_SELECT cc, 0, 1 -> CSINC zreg, zreg, cc
1137 Opc = Is32Bit ? AArch64::CSINCWr : AArch64::CSINCXr;
1138 True = ZReg;
1139 False = ZReg;
1140 return true;
1141 }
1142
1143 if (T == 0 && F == -1) {
1144 // G_SELECT cc 0, -1 -> CSINV zreg, zreg cc
1145 Opc = Is32Bit ? AArch64::CSINVWr : AArch64::CSINVXr;
1146 True = ZReg;
1147 False = ZReg;
1148 return true;
1149 }
1150 }
1151
1152 if (TrueCst) {
1153 int64_t T = TrueCst->Value.getSExtValue();
1154 if (T == 1) {
1155 // G_SELECT cc, 1, f -> CSINC f, zreg, inv_cc
1156 Opc = Is32Bit ? AArch64::CSINCWr : AArch64::CSINCXr;
1157 True = False;
1158 False = ZReg;
1159 CC = AArch64CC::getInvertedCondCode(Code: CC);
1160 return true;
1161 }
1162
1163 if (T == -1) {
1164 // G_SELECT cc, -1, f -> CSINV f, zreg, inv_cc
1165 Opc = Is32Bit ? AArch64::CSINVWr : AArch64::CSINVXr;
1166 True = False;
1167 False = ZReg;
1168 CC = AArch64CC::getInvertedCondCode(Code: CC);
1169 return true;
1170 }
1171 }
1172
1173 if (FalseCst) {
1174 int64_t F = FalseCst->Value.getSExtValue();
1175 if (F == 1) {
1176 // G_SELECT cc, t, 1 -> CSINC t, zreg, cc
1177 Opc = Is32Bit ? AArch64::CSINCWr : AArch64::CSINCXr;
1178 False = ZReg;
1179 return true;
1180 }
1181
1182 if (F == -1) {
1183 // G_SELECT cc, t, -1 -> CSINC t, zreg, cc
1184 Opc = Is32Bit ? AArch64::CSINVWr : AArch64::CSINVXr;
1185 False = ZReg;
1186 return true;
1187 }
1188 }
1189 return false;
1190 };
1191
1192 Optimized |= TryFoldBinOpIntoSelect(False, True, /*Invert = */ false);
1193 Optimized |= TryFoldBinOpIntoSelect(True, False, /*Invert = */ true);
1194 Optimized |= TryOptSelectCst();
1195 auto SelectInst = MIB.buildInstr(Opc, DstOps: {Dst}, SrcOps: {True, False}).addImm(Val: CC);
1196 constrainSelectedInstRegOperands(I&: *SelectInst, TII, TRI, RBI);
1197 return &*SelectInst;
1198}
1199
1200static AArch64CC::CondCode
1201changeICMPPredToAArch64CC(CmpInst::Predicate P, Register RHS = {},
1202 MachineRegisterInfo *MRI = nullptr) {
1203 switch (P) {
1204 default:
1205 llvm_unreachable("Unknown condition code!");
1206 case CmpInst::ICMP_NE:
1207 return AArch64CC::NE;
1208 case CmpInst::ICMP_EQ:
1209 return AArch64CC::EQ;
1210 case CmpInst::ICMP_SGT:
1211 return AArch64CC::GT;
1212 case CmpInst::ICMP_SGE:
1213 if (RHS && MRI) {
1214 auto ValAndVReg = getIConstantVRegValWithLookThrough(VReg: RHS, MRI: *MRI);
1215 if (ValAndVReg && ValAndVReg->Value == 0)
1216 return AArch64CC::PL;
1217 }
1218 return AArch64CC::GE;
1219 case CmpInst::ICMP_SLT:
1220 if (RHS && MRI) {
1221 auto ValAndVReg = getIConstantVRegValWithLookThrough(VReg: RHS, MRI: *MRI);
1222 if (ValAndVReg && ValAndVReg->Value == 0)
1223 return AArch64CC::MI;
1224 }
1225 return AArch64CC::LT;
1226 case CmpInst::ICMP_SLE:
1227 return AArch64CC::LE;
1228 case CmpInst::ICMP_UGT:
1229 return AArch64CC::HI;
1230 case CmpInst::ICMP_UGE:
1231 return AArch64CC::HS;
1232 case CmpInst::ICMP_ULT:
1233 return AArch64CC::LO;
1234 case CmpInst::ICMP_ULE:
1235 return AArch64CC::LS;
1236 }
1237}
1238
1239/// changeFPCCToORAArch64CC - Convert an IR fp condition code to an AArch64 CC.
1240static void changeFPCCToORAArch64CC(CmpInst::Predicate CC,
1241 AArch64CC::CondCode &CondCode,
1242 AArch64CC::CondCode &CondCode2) {
1243 CondCode2 = AArch64CC::AL;
1244 switch (CC) {
1245 default:
1246 llvm_unreachable("Unknown FP condition!");
1247 case CmpInst::FCMP_OEQ:
1248 CondCode = AArch64CC::EQ;
1249 break;
1250 case CmpInst::FCMP_OGT:
1251 CondCode = AArch64CC::GT;
1252 break;
1253 case CmpInst::FCMP_OGE:
1254 CondCode = AArch64CC::GE;
1255 break;
1256 case CmpInst::FCMP_OLT:
1257 CondCode = AArch64CC::MI;
1258 break;
1259 case CmpInst::FCMP_OLE:
1260 CondCode = AArch64CC::LS;
1261 break;
1262 case CmpInst::FCMP_ONE:
1263 CondCode = AArch64CC::MI;
1264 CondCode2 = AArch64CC::GT;
1265 break;
1266 case CmpInst::FCMP_ORD:
1267 CondCode = AArch64CC::VC;
1268 break;
1269 case CmpInst::FCMP_UNO:
1270 CondCode = AArch64CC::VS;
1271 break;
1272 case CmpInst::FCMP_UEQ:
1273 CondCode = AArch64CC::EQ;
1274 CondCode2 = AArch64CC::VS;
1275 break;
1276 case CmpInst::FCMP_UGT:
1277 CondCode = AArch64CC::HI;
1278 break;
1279 case CmpInst::FCMP_UGE:
1280 CondCode = AArch64CC::PL;
1281 break;
1282 case CmpInst::FCMP_ULT:
1283 CondCode = AArch64CC::LT;
1284 break;
1285 case CmpInst::FCMP_ULE:
1286 CondCode = AArch64CC::LE;
1287 break;
1288 case CmpInst::FCMP_UNE:
1289 CondCode = AArch64CC::NE;
1290 break;
1291 }
1292}
1293
1294/// Convert an IR fp condition code to an AArch64 CC.
1295/// This differs from changeFPCCToAArch64CC in that it returns cond codes that
1296/// should be AND'ed instead of OR'ed.
1297static void changeFPCCToANDAArch64CC(CmpInst::Predicate CC,
1298 AArch64CC::CondCode &CondCode,
1299 AArch64CC::CondCode &CondCode2) {
1300 CondCode2 = AArch64CC::AL;
1301 switch (CC) {
1302 default:
1303 changeFPCCToORAArch64CC(CC, CondCode, CondCode2);
1304 assert(CondCode2 == AArch64CC::AL);
1305 break;
1306 case CmpInst::FCMP_ONE:
1307 // (a one b)
1308 // == ((a olt b) || (a ogt b))
1309 // == ((a ord b) && (a une b))
1310 CondCode = AArch64CC::VC;
1311 CondCode2 = AArch64CC::NE;
1312 break;
1313 case CmpInst::FCMP_UEQ:
1314 // (a ueq b)
1315 // == ((a uno b) || (a oeq b))
1316 // == ((a ule b) && (a uge b))
1317 CondCode = AArch64CC::PL;
1318 CondCode2 = AArch64CC::LE;
1319 break;
1320 }
1321}
1322
1323/// Return a register which can be used as a bit to test in a TB(N)Z.
1324static Register getTestBitReg(Register Reg, uint64_t &Bit, bool &Invert,
1325 MachineRegisterInfo &MRI) {
1326 assert(Reg.isValid() && "Expected valid register!");
1327 bool HasZext = false;
1328 while (MachineInstr *MI = getDefIgnoringCopies(Reg, MRI)) {
1329 unsigned Opc = MI->getOpcode();
1330
1331 if (!MI->getOperand(i: 0).isReg() ||
1332 !MRI.hasOneNonDBGUse(RegNo: MI->getOperand(i: 0).getReg()))
1333 break;
1334
1335 // (tbz (any_ext x), b) -> (tbz x, b) and
1336 // (tbz (zext x), b) -> (tbz x, b) if we don't use the extended bits.
1337 //
1338 // (tbz (trunc x), b) -> (tbz x, b) is always safe, because the bit number
1339 // on the truncated x is the same as the bit number on x.
1340 if (Opc == TargetOpcode::G_ANYEXT || Opc == TargetOpcode::G_ZEXT ||
1341 Opc == TargetOpcode::G_TRUNC) {
1342 if (Opc == TargetOpcode::G_ZEXT)
1343 HasZext = true;
1344
1345 Register NextReg = MI->getOperand(i: 1).getReg();
1346 // Did we find something worth folding?
1347 if (!NextReg.isValid() || !MRI.hasOneNonDBGUse(RegNo: NextReg))
1348 break;
1349 TypeSize InSize = MRI.getType(Reg: NextReg).getSizeInBits();
1350 if (Bit >= InSize)
1351 break;
1352
1353 // NextReg is worth folding. Keep looking.
1354 Reg = NextReg;
1355 continue;
1356 }
1357
1358 // Attempt to find a suitable operation with a constant on one side.
1359 std::optional<uint64_t> C;
1360 Register TestReg;
1361 switch (Opc) {
1362 default:
1363 break;
1364 case TargetOpcode::G_AND:
1365 case TargetOpcode::G_XOR: {
1366 TestReg = MI->getOperand(i: 1).getReg();
1367 Register ConstantReg = MI->getOperand(i: 2).getReg();
1368 auto VRegAndVal = getIConstantVRegValWithLookThrough(VReg: ConstantReg, MRI);
1369 if (!VRegAndVal) {
1370 // AND commutes, check the other side for a constant.
1371 // FIXME: Can we canonicalize the constant so that it's always on the
1372 // same side at some point earlier?
1373 std::swap(a&: ConstantReg, b&: TestReg);
1374 VRegAndVal = getIConstantVRegValWithLookThrough(VReg: ConstantReg, MRI);
1375 }
1376 if (VRegAndVal) {
1377 if (HasZext)
1378 C = VRegAndVal->Value.getZExtValue();
1379 else
1380 C = VRegAndVal->Value.getSExtValue();
1381 }
1382 break;
1383 }
1384 case TargetOpcode::G_ASHR:
1385 case TargetOpcode::G_LSHR:
1386 case TargetOpcode::G_SHL: {
1387 TestReg = MI->getOperand(i: 1).getReg();
1388 auto VRegAndVal =
1389 getIConstantVRegValWithLookThrough(VReg: MI->getOperand(i: 2).getReg(), MRI);
1390 if (VRegAndVal)
1391 C = VRegAndVal->Value.getSExtValue();
1392 break;
1393 }
1394 }
1395
1396 // Didn't find a constant or viable register. Bail out of the loop.
1397 if (!C || !TestReg.isValid())
1398 break;
1399
1400 // We found a suitable instruction with a constant. Check to see if we can
1401 // walk through the instruction.
1402 Register NextReg;
1403 unsigned TestRegSize = MRI.getType(Reg: TestReg).getSizeInBits();
1404 switch (Opc) {
1405 default:
1406 break;
1407 case TargetOpcode::G_AND:
1408 // (tbz (and x, m), b) -> (tbz x, b) when the b-th bit of m is set.
1409 if ((*C >> Bit) & 1)
1410 NextReg = TestReg;
1411 break;
1412 case TargetOpcode::G_SHL:
1413 // (tbz (shl x, c), b) -> (tbz x, b-c) when b-c is positive and fits in
1414 // the type of the register.
1415 if (*C <= Bit && (Bit - *C) < TestRegSize) {
1416 NextReg = TestReg;
1417 Bit = Bit - *C;
1418 }
1419 break;
1420 case TargetOpcode::G_ASHR:
1421 // (tbz (ashr x, c), b) -> (tbz x, b+c) or (tbz x, msb) if b+c is > # bits
1422 // in x
1423 NextReg = TestReg;
1424 Bit = Bit + *C;
1425 if (Bit >= TestRegSize)
1426 Bit = TestRegSize - 1;
1427 break;
1428 case TargetOpcode::G_LSHR:
1429 // (tbz (lshr x, c), b) -> (tbz x, b+c) when b + c is < # bits in x
1430 if ((Bit + *C) < TestRegSize) {
1431 NextReg = TestReg;
1432 Bit = Bit + *C;
1433 }
1434 break;
1435 case TargetOpcode::G_XOR:
1436 // We can walk through a G_XOR by inverting whether we use tbz/tbnz when
1437 // appropriate.
1438 //
1439 // e.g. If x' = xor x, c, and the b-th bit is set in c then
1440 //
1441 // tbz x', b -> tbnz x, b
1442 //
1443 // Because x' only has the b-th bit set if x does not.
1444 if ((*C >> Bit) & 1)
1445 Invert = !Invert;
1446 NextReg = TestReg;
1447 break;
1448 }
1449
1450 // Check if we found anything worth folding.
1451 if (!NextReg.isValid())
1452 return Reg;
1453 Reg = NextReg;
1454 }
1455
1456 return Reg;
1457}
1458
1459MachineInstr *AArch64InstructionSelector::emitTestBit(
1460 Register TestReg, uint64_t Bit, bool IsNegative, MachineBasicBlock *DstMBB,
1461 MachineIRBuilder &MIB) const {
1462 assert(TestReg.isValid());
1463 assert(ProduceNonFlagSettingCondBr &&
1464 "Cannot emit TB(N)Z with speculation tracking!");
1465 MachineRegisterInfo &MRI = *MIB.getMRI();
1466
1467 // Attempt to optimize the test bit by walking over instructions.
1468 TestReg = getTestBitReg(Reg: TestReg, Bit, Invert&: IsNegative, MRI);
1469 LLT Ty = MRI.getType(Reg: TestReg);
1470 unsigned Size = Ty.getSizeInBits();
1471 assert(!Ty.isVector() && "Expected a scalar!");
1472 assert(Bit < 64 && "Bit is too large!");
1473
1474 // When the test register is a 64-bit register, we have to narrow to make
1475 // TBNZW work.
1476 bool UseWReg = Bit < 32;
1477 unsigned NecessarySize = UseWReg ? 32 : 64;
1478 if (Size != NecessarySize)
1479 TestReg = moveScalarRegClass(
1480 Reg: TestReg, RC: UseWReg ? AArch64::GPR32RegClass : AArch64::GPR64RegClass,
1481 MIB);
1482
1483 static const unsigned OpcTable[2][2] = {{AArch64::TBZX, AArch64::TBNZX},
1484 {AArch64::TBZW, AArch64::TBNZW}};
1485 unsigned Opc = OpcTable[UseWReg][IsNegative];
1486 auto TestBitMI =
1487 MIB.buildInstr(Opcode: Opc).addReg(RegNo: TestReg).addImm(Val: Bit).addMBB(MBB: DstMBB);
1488 constrainSelectedInstRegOperands(I&: *TestBitMI, TII, TRI, RBI);
1489 return &*TestBitMI;
1490}
1491
1492bool AArch64InstructionSelector::tryOptAndIntoCompareBranch(
1493 MachineInstr &AndInst, bool Invert, MachineBasicBlock *DstMBB,
1494 MachineIRBuilder &MIB) const {
1495 assert(AndInst.getOpcode() == TargetOpcode::G_AND && "Expected G_AND only?");
1496 // Given something like this:
1497 //
1498 // %x = ...Something...
1499 // %one = G_CONSTANT i64 1
1500 // %zero = G_CONSTANT i64 0
1501 // %and = G_AND %x, %one
1502 // %cmp = G_ICMP intpred(ne), %and, %zero
1503 // %cmp_trunc = G_TRUNC %cmp
1504 // G_BRCOND %cmp_trunc, %bb.3
1505 //
1506 // We want to try and fold the AND into the G_BRCOND and produce either a
1507 // TBNZ (when we have intpred(ne)) or a TBZ (when we have intpred(eq)).
1508 //
1509 // In this case, we'd get
1510 //
1511 // TBNZ %x %bb.3
1512 //
1513
1514 // Check if the AND has a constant on its RHS which we can use as a mask.
1515 // If it's a power of 2, then it's the same as checking a specific bit.
1516 // (e.g, ANDing with 8 == ANDing with 000...100 == testing if bit 3 is set)
1517 auto MaybeBit = getIConstantVRegValWithLookThrough(
1518 VReg: AndInst.getOperand(i: 2).getReg(), MRI: *MIB.getMRI());
1519 if (!MaybeBit)
1520 return false;
1521
1522 int32_t Bit = MaybeBit->Value.exactLogBase2();
1523 if (Bit < 0)
1524 return false;
1525
1526 Register TestReg = AndInst.getOperand(i: 1).getReg();
1527
1528 // Emit a TB(N)Z.
1529 emitTestBit(TestReg, Bit, IsNegative: Invert, DstMBB, MIB);
1530 return true;
1531}
1532
1533MachineInstr *AArch64InstructionSelector::emitCBZ(Register CompareReg,
1534 bool IsNegative,
1535 MachineBasicBlock *DestMBB,
1536 MachineIRBuilder &MIB) const {
1537 assert(ProduceNonFlagSettingCondBr && "CBZ does not set flags!");
1538 MachineRegisterInfo &MRI = *MIB.getMRI();
1539 assert(RBI.getRegBank(CompareReg, MRI, TRI)->getID() ==
1540 AArch64::GPRRegBankID &&
1541 "Expected GPRs only?");
1542 auto Ty = MRI.getType(Reg: CompareReg);
1543 unsigned Width = Ty.getSizeInBits();
1544 assert(!Ty.isVector() && "Expected scalar only?");
1545 assert(Width <= 64 && "Expected width to be at most 64?");
1546 static const unsigned OpcTable[2][2] = {{AArch64::CBZW, AArch64::CBZX},
1547 {AArch64::CBNZW, AArch64::CBNZX}};
1548 unsigned Opc = OpcTable[IsNegative][Width == 64];
1549 auto BranchMI = MIB.buildInstr(Opc, DstOps: {}, SrcOps: {CompareReg}).addMBB(MBB: DestMBB);
1550 constrainSelectedInstRegOperands(I&: *BranchMI, TII, TRI, RBI);
1551 return &*BranchMI;
1552}
1553
1554bool AArch64InstructionSelector::selectCompareBranchFedByFCmp(
1555 MachineInstr &I, MachineInstr &FCmp, MachineIRBuilder &MIB) const {
1556 assert(FCmp.getOpcode() == TargetOpcode::G_FCMP);
1557 assert(I.getOpcode() == TargetOpcode::G_BRCOND);
1558 // Unfortunately, the mapping of LLVM FP CC's onto AArch64 CC's isn't
1559 // totally clean. Some of them require two branches to implement.
1560 auto Pred = (CmpInst::Predicate)FCmp.getOperand(i: 1).getPredicate();
1561 emitFPCompare(LHS: FCmp.getOperand(i: 2).getReg(), RHS: FCmp.getOperand(i: 3).getReg(), MIRBuilder&: MIB,
1562 Pred);
1563 AArch64CC::CondCode CC1, CC2;
1564 changeFCMPPredToAArch64CC(P: Pred, CondCode&: CC1, CondCode2&: CC2);
1565 MachineBasicBlock *DestMBB = I.getOperand(i: 1).getMBB();
1566 MIB.buildInstr(Opc: AArch64::Bcc, DstOps: {}, SrcOps: {}).addImm(Val: CC1).addMBB(MBB: DestMBB);
1567 if (CC2 != AArch64CC::AL)
1568 MIB.buildInstr(Opc: AArch64::Bcc, DstOps: {}, SrcOps: {}).addImm(Val: CC2).addMBB(MBB: DestMBB);
1569 I.eraseFromParent();
1570 return true;
1571}
1572
1573bool AArch64InstructionSelector::tryOptCompareBranchFedByICmp(
1574 MachineInstr &I, MachineInstr &ICmp, MachineIRBuilder &MIB) const {
1575 assert(ICmp.getOpcode() == TargetOpcode::G_ICMP);
1576 assert(I.getOpcode() == TargetOpcode::G_BRCOND);
1577 // Attempt to optimize the G_BRCOND + G_ICMP into a TB(N)Z/CB(N)Z.
1578 //
1579 // Speculation tracking/SLH assumes that optimized TB(N)Z/CB(N)Z
1580 // instructions will not be produced, as they are conditional branch
1581 // instructions that do not set flags.
1582 if (!ProduceNonFlagSettingCondBr)
1583 return false;
1584
1585 MachineRegisterInfo &MRI = *MIB.getMRI();
1586 MachineBasicBlock *DestMBB = I.getOperand(i: 1).getMBB();
1587 auto Pred =
1588 static_cast<CmpInst::Predicate>(ICmp.getOperand(i: 1).getPredicate());
1589 Register LHS = ICmp.getOperand(i: 2).getReg();
1590 Register RHS = ICmp.getOperand(i: 3).getReg();
1591
1592 // We're allowed to emit a TB(N)Z/CB(N)Z. Try to do that.
1593 auto VRegAndVal = getIConstantVRegValWithLookThrough(VReg: RHS, MRI);
1594 MachineInstr *AndInst = getOpcodeDef(Opcode: TargetOpcode::G_AND, Reg: LHS, MRI);
1595
1596 // When we can emit a TB(N)Z, prefer that.
1597 //
1598 // Handle non-commutative condition codes first.
1599 // Note that we don't want to do this when we have a G_AND because it can
1600 // become a tst. The tst will make the test bit in the TB(N)Z redundant.
1601 if (VRegAndVal && !AndInst) {
1602 int64_t C = VRegAndVal->Value.getSExtValue();
1603
1604 // When we have a greater-than comparison, we can just test if the msb is
1605 // zero.
1606 if (C == -1 && Pred == CmpInst::ICMP_SGT) {
1607 uint64_t Bit = MRI.getType(Reg: LHS).getSizeInBits() - 1;
1608 emitTestBit(TestReg: LHS, Bit, /*IsNegative = */ false, DstMBB: DestMBB, MIB);
1609 I.eraseFromParent();
1610 return true;
1611 }
1612
1613 // When we have a less than comparison, we can just test if the msb is not
1614 // zero.
1615 if (C == 0 && Pred == CmpInst::ICMP_SLT) {
1616 uint64_t Bit = MRI.getType(Reg: LHS).getSizeInBits() - 1;
1617 emitTestBit(TestReg: LHS, Bit, /*IsNegative = */ true, DstMBB: DestMBB, MIB);
1618 I.eraseFromParent();
1619 return true;
1620 }
1621
1622 // Inversely, if we have a signed greater-than-or-equal comparison to zero,
1623 // we can test if the msb is zero.
1624 if (C == 0 && Pred == CmpInst::ICMP_SGE) {
1625 uint64_t Bit = MRI.getType(Reg: LHS).getSizeInBits() - 1;
1626 emitTestBit(TestReg: LHS, Bit, /*IsNegative = */ false, DstMBB: DestMBB, MIB);
1627 I.eraseFromParent();
1628 return true;
1629 }
1630 }
1631
1632 // Attempt to handle commutative condition codes. Right now, that's only
1633 // eq/ne.
1634 if (ICmpInst::isEquality(P: Pred)) {
1635 if (!VRegAndVal) {
1636 std::swap(a&: RHS, b&: LHS);
1637 VRegAndVal = getIConstantVRegValWithLookThrough(VReg: RHS, MRI);
1638 AndInst = getOpcodeDef(Opcode: TargetOpcode::G_AND, Reg: LHS, MRI);
1639 }
1640
1641 if (VRegAndVal && VRegAndVal->Value == 0) {
1642 // If there's a G_AND feeding into this branch, try to fold it away by
1643 // emitting a TB(N)Z instead.
1644 //
1645 // Note: If we have LT, then it *is* possible to fold, but it wouldn't be
1646 // beneficial. When we have an AND and LT, we need a TST/ANDS, so folding
1647 // would be redundant.
1648 if (AndInst &&
1649 tryOptAndIntoCompareBranch(
1650 AndInst&: *AndInst, /*Invert = */ Pred == CmpInst::ICMP_NE, DstMBB: DestMBB, MIB)) {
1651 I.eraseFromParent();
1652 return true;
1653 }
1654
1655 // Otherwise, try to emit a CB(N)Z instead.
1656 auto LHSTy = MRI.getType(Reg: LHS);
1657 if (!LHSTy.isVector() && LHSTy.getSizeInBits() <= 64) {
1658 emitCBZ(CompareReg: LHS, /*IsNegative = */ Pred == CmpInst::ICMP_NE, DestMBB, MIB);
1659 I.eraseFromParent();
1660 return true;
1661 }
1662 }
1663 }
1664
1665 return false;
1666}
1667
1668bool AArch64InstructionSelector::selectCompareBranchFedByICmp(
1669 MachineInstr &I, MachineInstr &ICmp, MachineIRBuilder &MIB) const {
1670 assert(ICmp.getOpcode() == TargetOpcode::G_ICMP);
1671 assert(I.getOpcode() == TargetOpcode::G_BRCOND);
1672 if (tryOptCompareBranchFedByICmp(I, ICmp, MIB))
1673 return true;
1674
1675 // Couldn't optimize. Emit a compare + a Bcc.
1676 MachineBasicBlock *DestMBB = I.getOperand(i: 1).getMBB();
1677 auto &PredOp = ICmp.getOperand(i: 1);
1678 emitIntegerCompare(LHS&: ICmp.getOperand(i: 2), RHS&: ICmp.getOperand(i: 3), Predicate&: PredOp, MIRBuilder&: MIB);
1679 const AArch64CC::CondCode CC = changeICMPPredToAArch64CC(
1680 P: static_cast<CmpInst::Predicate>(PredOp.getPredicate()),
1681 RHS: ICmp.getOperand(i: 3).getReg(), MRI: MIB.getMRI());
1682 MIB.buildInstr(Opc: AArch64::Bcc, DstOps: {}, SrcOps: {}).addImm(Val: CC).addMBB(MBB: DestMBB);
1683 I.eraseFromParent();
1684 return true;
1685}
1686
1687bool AArch64InstructionSelector::selectCompareBranch(
1688 MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) {
1689 Register CondReg = I.getOperand(i: 0).getReg();
1690 MachineInstr *CCMI = MRI.getVRegDef(Reg: CondReg);
1691 // Try to select the G_BRCOND using whatever is feeding the condition if
1692 // possible.
1693 unsigned CCMIOpc = CCMI->getOpcode();
1694 if (CCMIOpc == TargetOpcode::G_FCMP)
1695 return selectCompareBranchFedByFCmp(I, FCmp&: *CCMI, MIB);
1696 if (CCMIOpc == TargetOpcode::G_ICMP)
1697 return selectCompareBranchFedByICmp(I, ICmp&: *CCMI, MIB);
1698
1699 // Speculation tracking/SLH assumes that optimized TB(N)Z/CB(N)Z
1700 // instructions will not be produced, as they are conditional branch
1701 // instructions that do not set flags.
1702 if (ProduceNonFlagSettingCondBr) {
1703 emitTestBit(TestReg: CondReg, /*Bit = */ 0, /*IsNegative = */ true,
1704 DstMBB: I.getOperand(i: 1).getMBB(), MIB);
1705 I.eraseFromParent();
1706 return true;
1707 }
1708
1709 // Can't emit TB(N)Z/CB(N)Z. Emit a tst + bcc instead.
1710 auto TstMI =
1711 MIB.buildInstr(Opc: AArch64::ANDSWri, DstOps: {LLT::scalar(SizeInBits: 32)}, SrcOps: {CondReg}).addImm(Val: 1);
1712 constrainSelectedInstRegOperands(I&: *TstMI, TII, TRI, RBI);
1713 auto Bcc = MIB.buildInstr(Opcode: AArch64::Bcc)
1714 .addImm(Val: AArch64CC::NE)
1715 .addMBB(MBB: I.getOperand(i: 1).getMBB());
1716 I.eraseFromParent();
1717 constrainSelectedInstRegOperands(I&: *Bcc, TII, TRI, RBI);
1718 return true;
1719}
1720
1721/// Returns the element immediate value of a vector shift operand if found.
1722/// This needs to detect a splat-like operation, e.g. a G_BUILD_VECTOR.
1723static std::optional<APInt> getVectorShiftImm(Register Reg,
1724 MachineRegisterInfo &MRI) {
1725 assert(MRI.getType(Reg).isVector() && "Expected a *vector* shift operand");
1726 MachineInstr *OpMI = MRI.getVRegDef(Reg);
1727 std::optional<Register> Splat = getVectorSplat(MI: *OpMI, MRI);
1728 if (!Splat)
1729 return std::nullopt;
1730 return getIConstantVRegVal(VReg: *Splat, MRI);
1731}
1732
1733/// Matches and returns the shift immediate value for a SHL instruction given
1734/// a shift operand.
1735static std::optional<uint64_t> getVectorSHLImm(LLT SrcTy, Register Reg,
1736 MachineRegisterInfo &MRI) {
1737 std::optional<APInt> ShiftImm = getVectorShiftImm(Reg, MRI);
1738 if (!ShiftImm)
1739 return std::nullopt;
1740 // Check the immediate is in range for a SHL.
1741 uint64_t Imm = ShiftImm->getZExtValue();
1742 if (Imm >= SrcTy.getElementType().getSizeInBits())
1743 return std::nullopt;
1744 return Imm;
1745}
1746
1747bool AArch64InstructionSelector::selectVectorSHL(MachineInstr &I,
1748 MachineRegisterInfo &MRI) {
1749 assert(I.getOpcode() == TargetOpcode::G_SHL);
1750 Register DstReg = I.getOperand(i: 0).getReg();
1751 const LLT Ty = MRI.getType(Reg: DstReg);
1752 Register Src1Reg = I.getOperand(i: 1).getReg();
1753 Register Src2Reg = I.getOperand(i: 2).getReg();
1754
1755 if (!Ty.isVector())
1756 return false;
1757
1758 // Check if we have a vector of constants on RHS that we can select as the
1759 // immediate form.
1760 std::optional<uint64_t> ImmVal = getVectorSHLImm(SrcTy: Ty, Reg: Src2Reg, MRI);
1761
1762 unsigned Opc = 0;
1763 if (Ty == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 64)) {
1764 Opc = ImmVal ? AArch64::SHLv2i64_shift : AArch64::USHLv2i64;
1765 } else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 32)) {
1766 Opc = ImmVal ? AArch64::SHLv4i32_shift : AArch64::USHLv4i32;
1767 } else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 32)) {
1768 Opc = ImmVal ? AArch64::SHLv2i32_shift : AArch64::USHLv2i32;
1769 } else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 16)) {
1770 Opc = ImmVal ? AArch64::SHLv4i16_shift : AArch64::USHLv4i16;
1771 } else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 16)) {
1772 Opc = ImmVal ? AArch64::SHLv8i16_shift : AArch64::USHLv8i16;
1773 } else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 8)) {
1774 Opc = ImmVal ? AArch64::SHLv16i8_shift : AArch64::USHLv16i8;
1775 } else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 8)) {
1776 Opc = ImmVal ? AArch64::SHLv8i8_shift : AArch64::USHLv8i8;
1777 } else {
1778 LLVM_DEBUG(dbgs() << "Unhandled G_SHL type");
1779 return false;
1780 }
1781
1782 auto Shl = MIB.buildInstr(Opc, DstOps: {DstReg}, SrcOps: {Src1Reg});
1783 if (ImmVal)
1784 Shl.addImm(Val: *ImmVal);
1785 else
1786 Shl.addUse(RegNo: Src2Reg);
1787 constrainSelectedInstRegOperands(I&: *Shl, TII, TRI, RBI);
1788 I.eraseFromParent();
1789 return true;
1790}
1791
1792bool AArch64InstructionSelector::selectVectorAshrLshr(
1793 MachineInstr &I, MachineRegisterInfo &MRI) {
1794 assert(I.getOpcode() == TargetOpcode::G_ASHR ||
1795 I.getOpcode() == TargetOpcode::G_LSHR);
1796 Register DstReg = I.getOperand(i: 0).getReg();
1797 const LLT Ty = MRI.getType(Reg: DstReg);
1798 Register Src1Reg = I.getOperand(i: 1).getReg();
1799 Register Src2Reg = I.getOperand(i: 2).getReg();
1800
1801 if (!Ty.isVector())
1802 return false;
1803
1804 bool IsASHR = I.getOpcode() == TargetOpcode::G_ASHR;
1805
1806 // We expect the immediate case to be lowered in the PostLegalCombiner to
1807 // AArch64ISD::VASHR or AArch64ISD::VLSHR equivalents.
1808
1809 // There is not a shift right register instruction, but the shift left
1810 // register instruction takes a signed value, where negative numbers specify a
1811 // right shift.
1812
1813 unsigned Opc = 0;
1814 unsigned NegOpc = 0;
1815 const TargetRegisterClass *RC =
1816 getRegClassForTypeOnBank(Ty, RB: RBI.getRegBank(ID: AArch64::FPRRegBankID));
1817 if (Ty == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 64)) {
1818 Opc = IsASHR ? AArch64::SSHLv2i64 : AArch64::USHLv2i64;
1819 NegOpc = AArch64::NEGv2i64;
1820 } else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 32)) {
1821 Opc = IsASHR ? AArch64::SSHLv4i32 : AArch64::USHLv4i32;
1822 NegOpc = AArch64::NEGv4i32;
1823 } else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 32)) {
1824 Opc = IsASHR ? AArch64::SSHLv2i32 : AArch64::USHLv2i32;
1825 NegOpc = AArch64::NEGv2i32;
1826 } else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 16)) {
1827 Opc = IsASHR ? AArch64::SSHLv4i16 : AArch64::USHLv4i16;
1828 NegOpc = AArch64::NEGv4i16;
1829 } else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 16)) {
1830 Opc = IsASHR ? AArch64::SSHLv8i16 : AArch64::USHLv8i16;
1831 NegOpc = AArch64::NEGv8i16;
1832 } else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 8)) {
1833 Opc = IsASHR ? AArch64::SSHLv16i8 : AArch64::USHLv16i8;
1834 NegOpc = AArch64::NEGv16i8;
1835 } else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 8)) {
1836 Opc = IsASHR ? AArch64::SSHLv8i8 : AArch64::USHLv8i8;
1837 NegOpc = AArch64::NEGv8i8;
1838 } else {
1839 LLVM_DEBUG(dbgs() << "Unhandled G_ASHR type");
1840 return false;
1841 }
1842
1843 auto Neg = MIB.buildInstr(Opc: NegOpc, DstOps: {RC}, SrcOps: {Src2Reg});
1844 constrainSelectedInstRegOperands(I&: *Neg, TII, TRI, RBI);
1845 auto SShl = MIB.buildInstr(Opc, DstOps: {DstReg}, SrcOps: {Src1Reg, Neg});
1846 constrainSelectedInstRegOperands(I&: *SShl, TII, TRI, RBI);
1847 I.eraseFromParent();
1848 return true;
1849}
1850
1851bool AArch64InstructionSelector::selectVaStartAAPCS(
1852 MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) const {
1853
1854 if (STI.isCallingConvWin64(CC: MF.getFunction().getCallingConv(),
1855 IsVarArg: MF.getFunction().isVarArg()))
1856 return false;
1857
1858 // The layout of the va_list struct is specified in the AArch64 Procedure Call
1859 // Standard, section 10.1.5.
1860
1861 const AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
1862 const unsigned PtrSize = STI.isTargetILP32() ? 4 : 8;
1863 const auto *PtrRegClass =
1864 STI.isTargetILP32() ? &AArch64::GPR32RegClass : &AArch64::GPR64RegClass;
1865
1866 const MCInstrDesc &MCIDAddAddr =
1867 TII.get(Opcode: STI.isTargetILP32() ? AArch64::ADDWri : AArch64::ADDXri);
1868 const MCInstrDesc &MCIDStoreAddr =
1869 TII.get(Opcode: STI.isTargetILP32() ? AArch64::STRWui : AArch64::STRXui);
1870
1871 /*
1872 * typedef struct va_list {
1873 * void * stack; // next stack param
1874 * void * gr_top; // end of GP arg reg save area
1875 * void * vr_top; // end of FP/SIMD arg reg save area
1876 * int gr_offs; // offset from gr_top to next GP register arg
1877 * int vr_offs; // offset from vr_top to next FP/SIMD register arg
1878 * } va_list;
1879 */
1880 const auto VAList = I.getOperand(i: 0).getReg();
1881
1882 // Our current offset in bytes from the va_list struct (VAList).
1883 unsigned OffsetBytes = 0;
1884
1885 // Helper function to store (FrameIndex + Imm) to VAList at offset OffsetBytes
1886 // and increment OffsetBytes by PtrSize.
1887 const auto PushAddress = [&](const int FrameIndex, const int64_t Imm) {
1888 const Register Top = MRI.createVirtualRegister(RegClass: PtrRegClass);
1889 auto MIB = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: MCIDAddAddr)
1890 .addDef(RegNo: Top)
1891 .addFrameIndex(Idx: FrameIndex)
1892 .addImm(Val: Imm)
1893 .addImm(Val: 0);
1894 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
1895
1896 const auto *MMO = *I.memoperands_begin();
1897 MIB = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: MCIDStoreAddr)
1898 .addUse(RegNo: Top)
1899 .addUse(RegNo: VAList)
1900 .addImm(Val: OffsetBytes / PtrSize)
1901 .addMemOperand(MMO: MF.getMachineMemOperand(
1902 PtrInfo: MMO->getPointerInfo().getWithOffset(O: OffsetBytes),
1903 F: MachineMemOperand::MOStore, Size: PtrSize, BaseAlignment: MMO->getBaseAlign()));
1904 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
1905
1906 OffsetBytes += PtrSize;
1907 };
1908
1909 // void* stack at offset 0
1910 PushAddress(FuncInfo->getVarArgsStackIndex(), 0);
1911
1912 // void* gr_top at offset 8 (4 on ILP32)
1913 const unsigned GPRSize = FuncInfo->getVarArgsGPRSize();
1914 PushAddress(FuncInfo->getVarArgsGPRIndex(), GPRSize);
1915
1916 // void* vr_top at offset 16 (8 on ILP32)
1917 const unsigned FPRSize = FuncInfo->getVarArgsFPRSize();
1918 PushAddress(FuncInfo->getVarArgsFPRIndex(), FPRSize);
1919
1920 // Helper function to store a 4-byte integer constant to VAList at offset
1921 // OffsetBytes, and increment OffsetBytes by 4.
1922 const auto PushIntConstant = [&](const int32_t Value) {
1923 constexpr int IntSize = 4;
1924 const Register Temp = MRI.createVirtualRegister(RegClass: &AArch64::GPR32RegClass);
1925 auto MIB =
1926 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: AArch64::MOVi32imm))
1927 .addDef(RegNo: Temp)
1928 .addImm(Val: Value);
1929 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
1930
1931 const auto *MMO = *I.memoperands_begin();
1932 MIB = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: AArch64::STRWui))
1933 .addUse(RegNo: Temp)
1934 .addUse(RegNo: VAList)
1935 .addImm(Val: OffsetBytes / IntSize)
1936 .addMemOperand(MMO: MF.getMachineMemOperand(
1937 PtrInfo: MMO->getPointerInfo().getWithOffset(O: OffsetBytes),
1938 F: MachineMemOperand::MOStore, Size: IntSize, BaseAlignment: MMO->getBaseAlign()));
1939 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
1940 OffsetBytes += IntSize;
1941 };
1942
1943 // int gr_offs at offset 24 (12 on ILP32)
1944 PushIntConstant(-static_cast<int32_t>(GPRSize));
1945
1946 // int vr_offs at offset 28 (16 on ILP32)
1947 PushIntConstant(-static_cast<int32_t>(FPRSize));
1948
1949 assert(OffsetBytes == (STI.isTargetILP32() ? 20 : 32) && "Unexpected offset");
1950
1951 I.eraseFromParent();
1952 return true;
1953}
1954
1955bool AArch64InstructionSelector::selectVaStartDarwin(
1956 MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) const {
1957 AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
1958 Register ListReg = I.getOperand(i: 0).getReg();
1959
1960 Register ArgsAddrReg = MRI.createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
1961
1962 int FrameIdx = FuncInfo->getVarArgsStackIndex();
1963 if (MF.getSubtarget<AArch64Subtarget>().isCallingConvWin64(
1964 CC: MF.getFunction().getCallingConv(), IsVarArg: MF.getFunction().isVarArg())) {
1965 FrameIdx = FuncInfo->getVarArgsGPRSize() > 0
1966 ? FuncInfo->getVarArgsGPRIndex()
1967 : FuncInfo->getVarArgsStackIndex();
1968 }
1969
1970 auto MIB =
1971 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: AArch64::ADDXri))
1972 .addDef(RegNo: ArgsAddrReg)
1973 .addFrameIndex(Idx: FrameIdx)
1974 .addImm(Val: 0)
1975 .addImm(Val: 0);
1976
1977 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
1978
1979 MIB = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: AArch64::STRXui))
1980 .addUse(RegNo: ArgsAddrReg)
1981 .addUse(RegNo: ListReg)
1982 .addImm(Val: 0)
1983 .addMemOperand(MMO: *I.memoperands_begin());
1984
1985 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
1986 I.eraseFromParent();
1987 return true;
1988}
1989
1990void AArch64InstructionSelector::materializeLargeCMVal(
1991 MachineInstr &I, const Value *V, unsigned OpFlags) {
1992 MachineBasicBlock &MBB = *I.getParent();
1993 MachineFunction &MF = *MBB.getParent();
1994 MachineRegisterInfo &MRI = MF.getRegInfo();
1995
1996 auto MovZ = MIB.buildInstr(Opc: AArch64::MOVZXi, DstOps: {&AArch64::GPR64RegClass}, SrcOps: {});
1997 MovZ->addOperand(MF, Op: I.getOperand(i: 1));
1998 MovZ->getOperand(i: 1).setTargetFlags(OpFlags | AArch64II::MO_G0 |
1999 AArch64II::MO_NC);
2000 MovZ->addOperand(MF, Op: MachineOperand::CreateImm(Val: 0));
2001 constrainSelectedInstRegOperands(I&: *MovZ, TII, TRI, RBI);
2002
2003 auto BuildMovK = [&](Register SrcReg, unsigned char Flags, unsigned Offset,
2004 Register ForceDstReg) {
2005 Register DstReg = ForceDstReg
2006 ? ForceDstReg
2007 : MRI.createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
2008 auto MovI = MIB.buildInstr(Opcode: AArch64::MOVKXi).addDef(RegNo: DstReg).addUse(RegNo: SrcReg);
2009 if (auto *GV = dyn_cast<GlobalValue>(Val: V)) {
2010 MovI->addOperand(MF, Op: MachineOperand::CreateGA(
2011 GV, Offset: MovZ->getOperand(i: 1).getOffset(), TargetFlags: Flags));
2012 } else {
2013 MovI->addOperand(
2014 MF, Op: MachineOperand::CreateBA(BA: cast<BlockAddress>(Val: V),
2015 Offset: MovZ->getOperand(i: 1).getOffset(), TargetFlags: Flags));
2016 }
2017 MovI->addOperand(MF, Op: MachineOperand::CreateImm(Val: Offset));
2018 constrainSelectedInstRegOperands(I&: *MovI, TII, TRI, RBI);
2019 return DstReg;
2020 };
2021 Register DstReg = BuildMovK(MovZ.getReg(Idx: 0),
2022 AArch64II::MO_G1 | AArch64II::MO_NC, 16, 0);
2023 DstReg = BuildMovK(DstReg, AArch64II::MO_G2 | AArch64II::MO_NC, 32, 0);
2024 BuildMovK(DstReg, AArch64II::MO_G3, 48, I.getOperand(i: 0).getReg());
2025}
2026
2027bool AArch64InstructionSelector::preISelLower(MachineInstr &I) {
2028 MachineBasicBlock &MBB = *I.getParent();
2029 MachineFunction &MF = *MBB.getParent();
2030 MachineRegisterInfo &MRI = MF.getRegInfo();
2031
2032 switch (I.getOpcode()) {
2033 case TargetOpcode::G_CONSTANT: {
2034 Register DefReg = I.getOperand(i: 0).getReg();
2035 const LLT DefTy = MRI.getType(Reg: DefReg);
2036 if (!DefTy.isPointer()) {
2037 if (DefTy.getSizeInBits() >= 32 ||
2038 RBI.getRegBank(Reg: DefReg, MRI, TRI)->getID() != AArch64::GPRRegBankID)
2039 return false;
2040 // Widen narrow GPR constants to s32 so imported patterns can match.
2041 APInt Val = I.getOperand(i: 1).getCImm()->getValue().zext(width: 32);
2042 I.getOperand(i: 1).setCImm(
2043 ConstantInt::get(Context&: MF.getFunction().getContext(), V: Val));
2044
2045 Register WideReg = MRI.createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 32));
2046 MRI.setRegBank(Reg: WideReg, RegBank: RBI.getRegBank(ID: AArch64::GPRRegBankID));
2047 I.getOperand(i: 0).setReg(WideReg);
2048
2049 MIB.setInsertPt(MBB, II: std::next(x: I.getIterator()));
2050 auto Copy = MIB.buildCopy(Res: DefReg, Op: WideReg);
2051 selectCopy(I&: *Copy, TII, MRI, TRI, RBI);
2052 MIB.setInstr(I);
2053 return true;
2054 }
2055 const unsigned PtrSize = DefTy.getSizeInBits();
2056 if (PtrSize != 32 && PtrSize != 64)
2057 return false;
2058 // Convert pointer typed constants to integers so TableGen can select.
2059 MRI.setType(VReg: DefReg, Ty: LLT::integer(SizeInBits: PtrSize));
2060 return true;
2061 }
2062 case TargetOpcode::G_STORE: {
2063 bool Changed = contractCrossBankCopyIntoStore(I, MRI);
2064 MachineOperand &SrcOp = I.getOperand(i: 0);
2065 if (MRI.getType(Reg: SrcOp.getReg()).isPointer()) {
2066 // Allow matching with imported patterns for stores of pointers. Unlike
2067 // G_LOAD/G_PTR_ADD, we may not have selected all users. So, emit a copy
2068 // and constrain.
2069 auto Copy = MIB.buildCopy(Res: LLT::scalar(SizeInBits: 64), Op: SrcOp);
2070 Register NewSrc = Copy.getReg(Idx: 0);
2071 SrcOp.setReg(NewSrc);
2072 RBI.constrainGenericRegister(Reg: NewSrc, RC: AArch64::GPR64RegClass, MRI);
2073 Changed = true;
2074 }
2075 return Changed;
2076 }
2077 case TargetOpcode::G_PTR_ADD: {
2078 // If Checked Pointer Arithmetic (FEAT_CPA) is present, preserve the pointer
2079 // arithmetic semantics instead of falling back to regular arithmetic.
2080 const auto &TL = STI.getTargetLowering();
2081 if (TL->shouldPreservePtrArith(F: MF.getFunction(), PtrVT: EVT()))
2082 return false;
2083 return convertPtrAddToAdd(I, MRI);
2084 }
2085 case TargetOpcode::G_LOAD: {
2086 // For scalar loads of pointers, we try to convert the dest type from p0
2087 // to s64 so that our imported patterns can match. Like with the G_PTR_ADD
2088 // conversion, this should be ok because all users should have been
2089 // selected already, so the type doesn't matter for them.
2090 Register DstReg = I.getOperand(i: 0).getReg();
2091 const LLT DstTy = MRI.getType(Reg: DstReg);
2092 if (!DstTy.isPointer())
2093 return false;
2094 MRI.setType(VReg: DstReg, Ty: LLT::scalar(SizeInBits: 64));
2095 return true;
2096 }
2097 case TargetOpcode::G_VECREDUCE_ADD:
2098 case TargetOpcode::G_VECREDUCE_SMAX:
2099 case TargetOpcode::G_VECREDUCE_SMIN:
2100 case TargetOpcode::G_VECREDUCE_UMAX:
2101 case TargetOpcode::G_VECREDUCE_UMIN: {
2102 // Imported patterns require an FPR result. For a GPR, use a temporary FPR
2103 // and insert a cross-bank copy.
2104 Register DstReg = I.getOperand(i: 0).getReg();
2105 const RegisterBank &DstRB = *RBI.getRegBank(Reg: DstReg, MRI, TRI);
2106 if (DstRB.getID() != AArch64::GPRRegBankID)
2107 return false;
2108
2109 LLT DstTy = MRI.getType(Reg: DstReg);
2110 const TargetRegisterClass *DstRC =
2111 getRegClassForTypeOnBank(Ty: DstTy, RB: DstRB, /*GetAllRegSet=*/true);
2112 if (!DstRC || !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI))
2113 return false;
2114
2115 Register FPRDst = MRI.createGenericVirtualRegister(Ty: DstTy);
2116 MRI.setRegBank(Reg: FPRDst, RegBank: RBI.getRegBank(ID: AArch64::FPRRegBankID));
2117 I.getOperand(i: 0).setReg(FPRDst);
2118
2119 BuildMI(BB&: MBB, I: std::next(x: I.getIterator()), MIMD: MIMetadata(I),
2120 MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: DstReg)
2121 .addReg(RegNo: FPRDst);
2122 return true;
2123 }
2124 case TargetOpcode::G_SPLAT_VECTOR: {
2125 // Convert the type from p0 to s64 to help selection.
2126 LLT DstTy = MRI.getType(Reg: I.getOperand(i: 0).getReg());
2127 if (!DstTy.isPointerVector())
2128 return false;
2129 auto NewSrc = MIB.buildCopy(Res: LLT::scalar(SizeInBits: 64), Op: I.getOperand(i: 1).getReg());
2130 MRI.setType(VReg: I.getOperand(i: 0).getReg(),
2131 Ty: DstTy.changeElementType(NewEltTy: LLT::scalar(SizeInBits: 64)));
2132 MRI.setRegClass(Reg: NewSrc.getReg(Idx: 0), RC: &AArch64::GPR64RegClass);
2133 I.getOperand(i: 1).setReg(NewSrc.getReg(Idx: 0));
2134 return true;
2135 }
2136 case AArch64::G_INSERT_VECTOR_ELT: {
2137 LLT DstTy = MRI.getType(Reg: I.getOperand(i: 0).getReg());
2138 LLT SrcVecTy = MRI.getType(Reg: I.getOperand(i: 1).getReg());
2139 if (SrcVecTy.isPointerVector()) {
2140 // Convert the type from p0 to s64 to help selection.
2141 auto NewSrc = MIB.buildCopy(Res: LLT::scalar(SizeInBits: 64), Op: I.getOperand(i: 2).getReg());
2142 MRI.setType(VReg: I.getOperand(i: 1).getReg(),
2143 Ty: DstTy.changeElementType(NewEltTy: LLT::scalar(SizeInBits: 64)));
2144 MRI.setType(VReg: I.getOperand(i: 0).getReg(),
2145 Ty: DstTy.changeElementType(NewEltTy: LLT::scalar(SizeInBits: 64)));
2146 MRI.setRegClass(Reg: NewSrc.getReg(Idx: 0), RC: &AArch64::GPR64RegClass);
2147 I.getOperand(i: 2).setReg(NewSrc.getReg(Idx: 0));
2148 return true;
2149 }
2150
2151 Register EltReg = I.getOperand(i: 2).getReg();
2152 LLT EltTy = MRI.getType(Reg: EltReg);
2153 if (EltTy.isScalar() &&
2154 (EltTy.getSizeInBits() == 8 || EltTy.getSizeInBits() == 16) &&
2155 RBI.getRegBank(Reg: EltReg, MRI, TRI)->getID() == AArch64::GPRRegBankID) {
2156 // Convert the type from s8/s16 to s32 to help selection.
2157 auto NewElt = MIB.buildCopy(Res: LLT::scalar(SizeInBits: 32), Op: EltReg);
2158 MRI.setRegClass(Reg: NewElt.getReg(Idx: 0), RC: &AArch64::GPR32RegClass);
2159 I.getOperand(i: 2).setReg(NewElt.getReg(Idx: 0));
2160 return true;
2161 }
2162 return false;
2163 }
2164 case TargetOpcode::G_UITOFP:
2165 case TargetOpcode::G_SITOFP: {
2166 // If both source and destination regbanks are FPR, then convert the opcode
2167 // to G_SITOF so that the importer can select it to an fpr variant.
2168 // Otherwise, it ends up matching an fpr/gpr variant and adding a cross-bank
2169 // copy.
2170 Register SrcReg = I.getOperand(i: 1).getReg();
2171 LLT SrcTy = MRI.getType(Reg: SrcReg);
2172 LLT DstTy = MRI.getType(Reg: I.getOperand(i: 0).getReg());
2173 if (SrcTy.isVector() || SrcTy.getSizeInBits() != DstTy.getSizeInBits())
2174 return false;
2175
2176 if (RBI.getRegBank(Reg: SrcReg, MRI, TRI)->getID() == AArch64::FPRRegBankID) {
2177 // Need to add a copy to change the type so that the existing patterns can
2178 // match when there is an integer on an FPR bank.
2179 if (SrcTy.getScalarType().isInteger()) {
2180 auto Copy = MIB.buildCopy(Res: DstTy, Op: SrcReg);
2181 I.getOperand(i: 1).setReg(Copy.getReg(Idx: 0));
2182 MRI.setRegClass(Reg: Copy.getReg(Idx: 0),
2183 RC: getRegClassForTypeOnBank(
2184 Ty: SrcTy, RB: RBI.getRegBank(ID: AArch64::FPRRegBankID)));
2185 }
2186 if (I.getOpcode() == TargetOpcode::G_SITOFP)
2187 I.setDesc(TII.get(Opcode: AArch64::G_SITOF));
2188 else
2189 I.setDesc(TII.get(Opcode: AArch64::G_UITOF));
2190 return true;
2191 }
2192 return false;
2193 }
2194 default:
2195 return false;
2196 }
2197}
2198
2199/// This lowering tries to look for G_PTR_ADD instructions and then converts
2200/// them to a standard G_ADD with a COPY on the source.
2201///
2202/// The motivation behind this is to expose the add semantics to the imported
2203/// tablegen patterns. We shouldn't need to check for uses being loads/stores,
2204/// because the selector works bottom up, uses before defs. By the time we
2205/// end up trying to select a G_PTR_ADD, we should have already attempted to
2206/// fold this into addressing modes and were therefore unsuccessful.
2207bool AArch64InstructionSelector::convertPtrAddToAdd(
2208 MachineInstr &I, MachineRegisterInfo &MRI) {
2209 assert(I.getOpcode() == TargetOpcode::G_PTR_ADD && "Expected G_PTR_ADD");
2210 Register DstReg = I.getOperand(i: 0).getReg();
2211 Register AddOp1Reg = I.getOperand(i: 1).getReg();
2212 const LLT PtrTy = MRI.getType(Reg: DstReg);
2213 if (PtrTy.getAddressSpace() != 0)
2214 return false;
2215
2216 const LLT CastPtrTy = PtrTy.isVector()
2217 ? LLT::fixed_vector(NumElements: 2, ScalarTy: LLT::integer(SizeInBits: 64))
2218 : LLT::integer(SizeInBits: 64);
2219 auto PtrToInt = MIB.buildPtrToInt(Dst: CastPtrTy, Src: AddOp1Reg);
2220 // Set regbanks on the registers.
2221 if (PtrTy.isVector())
2222 MRI.setRegBank(Reg: PtrToInt.getReg(Idx: 0), RegBank: RBI.getRegBank(ID: AArch64::FPRRegBankID));
2223 else
2224 MRI.setRegBank(Reg: PtrToInt.getReg(Idx: 0), RegBank: RBI.getRegBank(ID: AArch64::GPRRegBankID));
2225
2226 // Now turn the %dst(p0) = G_PTR_ADD %base, off into:
2227 // %dst(intty) = G_ADD %intbase, off
2228 I.setDesc(TII.get(Opcode: TargetOpcode::G_ADD));
2229 MRI.setType(VReg: DstReg, Ty: CastPtrTy);
2230 I.getOperand(i: 1).setReg(PtrToInt.getReg(Idx: 0));
2231 if (!select(I&: *PtrToInt)) {
2232 LLVM_DEBUG(dbgs() << "Failed to select G_PTRTOINT in convertPtrAddToAdd");
2233 return false;
2234 }
2235
2236 // Also take the opportunity here to try to do some optimization.
2237 // Try to convert this into a G_SUB if the offset is a 0-x negate idiom.
2238 Register NegatedReg;
2239 if (!mi_match(R: I.getOperand(i: 2).getReg(), MRI, P: m_Neg(Src: m_Reg(R&: NegatedReg))))
2240 return true;
2241 I.getOperand(i: 2).setReg(NegatedReg);
2242 I.setDesc(TII.get(Opcode: TargetOpcode::G_SUB));
2243 return true;
2244}
2245
2246bool AArch64InstructionSelector::earlySelectSHL(MachineInstr &I,
2247 MachineRegisterInfo &MRI) {
2248 // We try to match the immediate variant of LSL, which is actually an alias
2249 // for a special case of UBFM. Otherwise, we fall back to the imported
2250 // selector which will match the register variant.
2251 assert(I.getOpcode() == TargetOpcode::G_SHL && "unexpected op");
2252 const auto &MO = I.getOperand(i: 2);
2253 auto VRegAndVal = getIConstantVRegVal(VReg: MO.getReg(), MRI);
2254 if (!VRegAndVal)
2255 return false;
2256
2257 const LLT DstTy = MRI.getType(Reg: I.getOperand(i: 0).getReg());
2258 if (DstTy.isVector())
2259 return false;
2260 bool Is64Bit = DstTy.getSizeInBits() == 64;
2261 auto Imm1Fn = Is64Bit ? selectShiftA_64(Root: MO) : selectShiftA_32(Root: MO);
2262 auto Imm2Fn = Is64Bit ? selectShiftB_64(Root: MO) : selectShiftB_32(Root: MO);
2263
2264 if (!Imm1Fn || !Imm2Fn)
2265 return false;
2266
2267 auto NewI =
2268 MIB.buildInstr(Opc: Is64Bit ? AArch64::UBFMXri : AArch64::UBFMWri,
2269 DstOps: {I.getOperand(i: 0).getReg()}, SrcOps: {I.getOperand(i: 1).getReg()});
2270
2271 for (auto &RenderFn : *Imm1Fn)
2272 RenderFn(NewI);
2273 for (auto &RenderFn : *Imm2Fn)
2274 RenderFn(NewI);
2275
2276 I.eraseFromParent();
2277 constrainSelectedInstRegOperands(I&: *NewI, TII, TRI, RBI);
2278 return true;
2279}
2280
2281bool AArch64InstructionSelector::contractCrossBankCopyIntoStore(
2282 MachineInstr &I, MachineRegisterInfo &MRI) {
2283 assert(I.getOpcode() == TargetOpcode::G_STORE && "Expected G_STORE");
2284 // If we're storing a scalar, it doesn't matter what register bank that
2285 // scalar is on. All that matters is the size.
2286 //
2287 // So, if we see something like this (with a 32-bit scalar as an example):
2288 //
2289 // %x:gpr(s32) = ... something ...
2290 // %y:fpr(s32) = COPY %x:gpr(s32)
2291 // G_STORE %y:fpr(s32)
2292 //
2293 // We can fix this up into something like this:
2294 //
2295 // G_STORE %x:gpr(s32)
2296 //
2297 // And then continue the selection process normally.
2298 Register DefDstReg = getSrcRegIgnoringCopies(Reg: I.getOperand(i: 0).getReg(), MRI);
2299 if (!DefDstReg.isValid())
2300 return false;
2301 LLT DefDstTy = MRI.getType(Reg: DefDstReg);
2302 Register StoreSrcReg = I.getOperand(i: 0).getReg();
2303 LLT StoreSrcTy = MRI.getType(Reg: StoreSrcReg);
2304
2305 // If we get something strange like a physical register, then we shouldn't
2306 // go any further.
2307 if (!DefDstTy.isValid())
2308 return false;
2309
2310 // Are the source and dst types the same size?
2311 if (DefDstTy.getSizeInBits() != StoreSrcTy.getSizeInBits())
2312 return false;
2313
2314 if (RBI.getRegBank(Reg: StoreSrcReg, MRI, TRI) ==
2315 RBI.getRegBank(Reg: DefDstReg, MRI, TRI))
2316 return false;
2317
2318 // We have a cross-bank copy, which is entering a store. Let's fold it.
2319 I.getOperand(i: 0).setReg(DefDstReg);
2320 return true;
2321}
2322
2323bool AArch64InstructionSelector::earlySelect(MachineInstr &I) {
2324 assert(I.getParent() && "Instruction should be in a basic block!");
2325 assert(I.getParent()->getParent() && "Instruction should be in a function!");
2326
2327 MachineBasicBlock &MBB = *I.getParent();
2328 MachineFunction &MF = *MBB.getParent();
2329 MachineRegisterInfo &MRI = MF.getRegInfo();
2330
2331 switch (I.getOpcode()) {
2332 case TargetOpcode::G_SPLAT_VECTOR: {
2333 Register Dst = I.getOperand(i: 0).getReg();
2334 LLT DstTy = MRI.getType(Reg: Dst);
2335 if (DstTy.isScalableVector())
2336 return false;
2337 // Before selecting a DUP instruction, check if it is better selected as a
2338 // MOV or load from a constant pool.
2339 Register Src = I.getOperand(i: 1).getReg();
2340 auto ValAndVReg = getAnyConstantVRegValWithLookThrough(
2341 VReg: Src, MRI, /*LookThroughInstrs=*/true, /*LookThroughAnyExt=*/true);
2342 if (!ValAndVReg)
2343 return false;
2344 LLVMContext &Ctx = MF.getFunction().getContext();
2345 auto *CV = ConstantDataVector::getSplat(
2346 NumElts: DstTy.getNumElements(),
2347 Elt: ConstantInt::get(Ty: Type::getIntNTy(C&: Ctx, N: DstTy.getScalarSizeInBits()),
2348 V: ValAndVReg->Value.trunc(width: DstTy.getScalarSizeInBits())));
2349 APInt Imm =
2350 APInt::getSplat(NewLen: DstTy.getSizeInBits(),
2351 V: ValAndVReg->Value.trunc(width: DstTy.getScalarSizeInBits()));
2352 if (!emitConstantVector(Dst, CV, Imm, MRI))
2353 return false;
2354 I.eraseFromParent();
2355 return true;
2356 }
2357 case TargetOpcode::G_SEXT:
2358 // Check for i64 sext(i32 vector_extract) prior to tablegen to select SMOV
2359 // over a normal extend.
2360 if (selectUSMovFromExtend(I, MRI))
2361 return true;
2362 return false;
2363 case TargetOpcode::G_BR:
2364 return false;
2365 case TargetOpcode::G_SHL:
2366 return earlySelectSHL(I, MRI);
2367 case TargetOpcode::G_CONSTANT: {
2368 bool IsZero = false;
2369 if (I.getOperand(i: 1).isCImm())
2370 IsZero = I.getOperand(i: 1).getCImm()->isZero();
2371 else if (I.getOperand(i: 1).isImm())
2372 IsZero = I.getOperand(i: 1).getImm() == 0;
2373
2374 if (!IsZero)
2375 return false;
2376
2377 Register DefReg = I.getOperand(i: 0).getReg();
2378 LLT Ty = MRI.getType(Reg: DefReg);
2379 if (Ty.getSizeInBits() == 64) {
2380 I.getOperand(i: 1).ChangeToRegister(Reg: AArch64::XZR, isDef: false);
2381 RBI.constrainGenericRegister(Reg: DefReg, RC: AArch64::GPR64RegClass, MRI);
2382 } else if (Ty.getSizeInBits() <= 32) {
2383 I.getOperand(i: 1).ChangeToRegister(Reg: AArch64::WZR, isDef: false);
2384 RBI.constrainGenericRegister(Reg: DefReg, RC: AArch64::GPR32RegClass, MRI);
2385 } else
2386 return false;
2387
2388 I.setDesc(TII.get(Opcode: TargetOpcode::COPY));
2389 return true;
2390 }
2391
2392 case TargetOpcode::G_ADD: {
2393 // Check if this is being fed by a G_ICMP on either side.
2394 //
2395 // (cmp pred, x, y) + z
2396 //
2397 // In the above case, when the cmp is true, we increment z by 1. So, we can
2398 // fold the add into the cset for the cmp by using cinc.
2399 //
2400 // FIXME: This would probably be a lot nicer in PostLegalizerLowering.
2401 Register AddDst = I.getOperand(i: 0).getReg();
2402 Register AddLHS = I.getOperand(i: 1).getReg();
2403 Register AddRHS = I.getOperand(i: 2).getReg();
2404 // Only handle scalars.
2405 LLT Ty = MRI.getType(Reg: AddLHS);
2406 if (Ty.isVector())
2407 return false;
2408 // Since G_ICMP is modeled as ADDS/SUBS/ANDS, we can handle 32 bits or 64
2409 // bits.
2410 unsigned Size = Ty.getSizeInBits();
2411 if (Size != 32 && Size != 64)
2412 return false;
2413 auto MatchCmp = [&](Register Reg) -> MachineInstr * {
2414 if (!MRI.hasOneNonDBGUse(RegNo: Reg))
2415 return nullptr;
2416 // If the LHS of the add is 32 bits, then we want to fold a 32-bit
2417 // compare.
2418 if (Size == 32)
2419 return getOpcodeDef(Opcode: TargetOpcode::G_ICMP, Reg, MRI);
2420 // We model scalar compares using 32-bit destinations right now.
2421 // If it's a 64-bit compare, it'll have 64-bit sources.
2422 Register ZExt;
2423 if (!mi_match(R: Reg, MRI, P: m_OneUse(SP: m_GZExt(Src: m_OneUse(SP: m_Reg(R&: ZExt))))))
2424 return nullptr;
2425 auto *Cmp = getOpcodeDef(Opcode: TargetOpcode::G_ICMP, Reg: ZExt, MRI);
2426 if (!Cmp ||
2427 MRI.getType(Reg: Cmp->getOperand(i: 2).getReg()).getSizeInBits() != 64)
2428 return nullptr;
2429 return Cmp;
2430 };
2431 // Try to match
2432 // z + (cmp pred, x, y)
2433 MachineInstr *Cmp = MatchCmp(AddRHS);
2434 if (!Cmp) {
2435 // (cmp pred, x, y) + z
2436 std::swap(a&: AddLHS, b&: AddRHS);
2437 Cmp = MatchCmp(AddRHS);
2438 if (!Cmp)
2439 return false;
2440 }
2441 auto &PredOp = Cmp->getOperand(i: 1);
2442 MIB.setInstrAndDebugLoc(I);
2443 emitIntegerCompare(/*LHS=*/Cmp->getOperand(i: 2),
2444 /*RHS=*/Cmp->getOperand(i: 3), Predicate&: PredOp, MIRBuilder&: MIB);
2445 auto Pred = static_cast<CmpInst::Predicate>(PredOp.getPredicate());
2446 const AArch64CC::CondCode InvCC = changeICMPPredToAArch64CC(
2447 P: CmpInst::getInversePredicate(pred: Pred), RHS: Cmp->getOperand(i: 3).getReg(), MRI: &MRI);
2448 emitCSINC(/*Dst=*/AddDst, /*Src =*/Src1: AddLHS, /*Src2=*/AddLHS, Pred: InvCC, MIRBuilder&: MIB);
2449 I.eraseFromParent();
2450 return true;
2451 }
2452 case TargetOpcode::G_OR: {
2453 // Look for operations that take the lower `Width=Size-ShiftImm` bits of
2454 // `ShiftSrc` and insert them into the upper `Width` bits of `MaskSrc` via
2455 // shifting and masking that we can replace with a BFI (encoded as a BFM).
2456 Register Dst = I.getOperand(i: 0).getReg();
2457 LLT Ty = MRI.getType(Reg: Dst);
2458
2459 if (!Ty.isScalar())
2460 return false;
2461
2462 unsigned Size = Ty.getSizeInBits();
2463 if (Size != 32 && Size != 64)
2464 return false;
2465
2466 Register ShiftSrc;
2467 int64_t ShiftImm;
2468 Register MaskSrc;
2469 int64_t MaskImm;
2470 if (!mi_match(R: Dst, MRI,
2471 P: m_GOr(L: m_OneUse(SP: m_GShl(L: m_Reg(R&: ShiftSrc), R: m_ICst(Cst&: ShiftImm))),
2472 R: m_OneUse(SP: m_GAnd(L: m_Reg(R&: MaskSrc), R: m_ICst(Cst&: MaskImm))))))
2473 return false;
2474
2475 if (ShiftImm > Size || ((1ULL << ShiftImm) - 1ULL) != uint64_t(MaskImm))
2476 return false;
2477
2478 int64_t Immr = Size - ShiftImm;
2479 int64_t Imms = Size - ShiftImm - 1;
2480 unsigned Opc = Size == 32 ? AArch64::BFMWri : AArch64::BFMXri;
2481 emitInstr(Opcode: Opc, DstOps: {Dst}, SrcOps: {MaskSrc, ShiftSrc, Immr, Imms}, MIRBuilder&: MIB);
2482 I.eraseFromParent();
2483 return true;
2484 }
2485 case TargetOpcode::G_FENCE: {
2486 if (I.getOperand(i: 1).getImm() == 0)
2487 BuildMI(BB&: MBB, I, MIMD: MIMetadata(I), MCID: TII.get(Opcode: TargetOpcode::MEMBARRIER));
2488 else
2489 BuildMI(BB&: MBB, I, MIMD: MIMetadata(I), MCID: TII.get(Opcode: AArch64::DMB))
2490 .addImm(Val: I.getOperand(i: 0).getImm() == 4 ? 0x9 : 0xb);
2491 I.eraseFromParent();
2492 return true;
2493 }
2494 case TargetOpcode::G_BRINDIRECT: {
2495 const Function &Fn = MF.getFunction();
2496 if (std::optional<uint16_t> BADisc =
2497 STI.getPtrAuthBlockAddressDiscriminatorIfEnabled(ParentFn: Fn)) {
2498 auto MI = MIB.buildInstr(Opc: AArch64::BRA, DstOps: {}, SrcOps: {I.getOperand(i: 0).getReg()});
2499 MI.addImm(Val: AArch64PACKey::IA);
2500 MI.addImm(Val: *BADisc);
2501 MI.addReg(/*AddrDisc=*/RegNo: AArch64::XZR);
2502 I.eraseFromParent();
2503 constrainSelectedInstRegOperands(I&: *MI, TII, TRI, RBI);
2504 return true;
2505 }
2506 // Use table-based selection.
2507 return false;
2508 }
2509
2510 default:
2511 return false;
2512 }
2513}
2514
2515bool AArch64InstructionSelector::select(MachineInstr &I) {
2516 assert(I.getParent() && "Instruction should be in a basic block!");
2517 assert(I.getParent()->getParent() && "Instruction should be in a function!");
2518
2519 MachineBasicBlock &MBB = *I.getParent();
2520 MachineFunction &MF = *MBB.getParent();
2521 MachineRegisterInfo &MRI = MF.getRegInfo();
2522
2523 const AArch64Subtarget *Subtarget = &MF.getSubtarget<AArch64Subtarget>();
2524 if (Subtarget->requiresStrictAlign()) {
2525 // We don't support this feature yet.
2526 LLVM_DEBUG(dbgs() << "AArch64 GISel does not support strict-align yet\n");
2527 return false;
2528 }
2529
2530 MIB.setInstrAndDebugLoc(I);
2531
2532 unsigned Opcode = I.getOpcode();
2533 // G_PHI requires same handling as PHI
2534 if (!I.isPreISelOpcode() || Opcode == TargetOpcode::G_PHI) {
2535 // Certain non-generic instructions also need some special handling.
2536
2537 if (Opcode == TargetOpcode::LOAD_STACK_GUARD) {
2538 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2539 return true;
2540 }
2541
2542 if (Opcode == TargetOpcode::PHI || Opcode == TargetOpcode::G_PHI) {
2543 const Register DefReg = I.getOperand(i: 0).getReg();
2544 const LLT DefTy = MRI.getType(Reg: DefReg);
2545
2546 const RegClassOrRegBank &RegClassOrBank =
2547 MRI.getRegClassOrRegBank(Reg: DefReg);
2548
2549 const TargetRegisterClass *DefRC =
2550 dyn_cast<const TargetRegisterClass *>(Val: RegClassOrBank);
2551 if (!DefRC) {
2552 if (!DefTy.isValid()) {
2553 LLVM_DEBUG(dbgs() << "PHI operand has no type, not a gvreg?\n");
2554 return false;
2555 }
2556 const RegisterBank &RB = *cast<const RegisterBank *>(Val: RegClassOrBank);
2557 DefRC = getRegClassForTypeOnBank(Ty: DefTy, RB);
2558 if (!DefRC) {
2559 LLVM_DEBUG(dbgs() << "PHI operand has unexpected size/bank\n");
2560 return false;
2561 }
2562 }
2563
2564 I.setDesc(TII.get(Opcode: TargetOpcode::PHI));
2565
2566 return RBI.constrainGenericRegister(Reg: DefReg, RC: *DefRC, MRI);
2567 }
2568
2569 if (I.isCopy())
2570 return selectCopy(I, TII, MRI, TRI, RBI);
2571
2572 return true;
2573 }
2574
2575
2576 if (I.getNumOperands() != I.getNumExplicitOperands()) {
2577 LLVM_DEBUG(
2578 dbgs() << "Generic instruction has unexpected implicit operands\n");
2579 return false;
2580 }
2581
2582 // Try to do some lowering before we start instruction selecting. These
2583 // lowerings are purely transformations on the input G_MIR and so selection
2584 // must continue after any modification of the instruction.
2585 if (preISelLower(I)) {
2586 Opcode = I.getOpcode(); // The opcode may have been modified, refresh it.
2587 }
2588
2589 // There may be patterns where the importer can't deal with them optimally,
2590 // but does select it to a suboptimal sequence so our custom C++ selection
2591 // code later never has a chance to work on it. Therefore, we have an early
2592 // selection attempt here to give priority to certain selection routines
2593 // over the imported ones.
2594 if (earlySelect(I))
2595 return true;
2596
2597 if (selectImpl(I, CoverageInfo&: *CoverageInfo))
2598 return true;
2599
2600 LLT Ty =
2601 I.getOperand(i: 0).isReg() ? MRI.getType(Reg: I.getOperand(i: 0).getReg()) : LLT{};
2602
2603 switch (Opcode) {
2604 case TargetOpcode::G_SBFX:
2605 case TargetOpcode::G_UBFX: {
2606 static const unsigned OpcTable[2][2] = {
2607 {AArch64::UBFMWri, AArch64::UBFMXri},
2608 {AArch64::SBFMWri, AArch64::SBFMXri}};
2609 bool IsSigned = Opcode == TargetOpcode::G_SBFX;
2610 unsigned Size = Ty.getSizeInBits();
2611 unsigned Opc = OpcTable[IsSigned][Size == 64];
2612 auto Cst1 =
2613 getIConstantVRegValWithLookThrough(VReg: I.getOperand(i: 2).getReg(), MRI);
2614 assert(Cst1 && "Should have gotten a constant for src 1?");
2615 auto Cst2 =
2616 getIConstantVRegValWithLookThrough(VReg: I.getOperand(i: 3).getReg(), MRI);
2617 assert(Cst2 && "Should have gotten a constant for src 2?");
2618 auto LSB = Cst1->Value.getZExtValue();
2619 auto Width = Cst2->Value.getZExtValue();
2620 auto BitfieldInst =
2621 MIB.buildInstr(Opc, DstOps: {I.getOperand(i: 0)}, SrcOps: {I.getOperand(i: 1)})
2622 .addImm(Val: LSB)
2623 .addImm(Val: LSB + Width - 1);
2624 I.eraseFromParent();
2625 constrainSelectedInstRegOperands(I&: *BitfieldInst, TII, TRI, RBI);
2626 return true;
2627 }
2628 case TargetOpcode::G_BRCOND:
2629 return selectCompareBranch(I, MF, MRI);
2630
2631 case TargetOpcode::G_BRJT:
2632 return selectBrJT(I, MRI);
2633
2634 case AArch64::G_ADD_LOW: {
2635 // This op may have been separated from it's ADRP companion by the localizer
2636 // or some other code motion pass. Given that many CPUs will try to
2637 // macro fuse these operations anyway, select this into a MOVaddr pseudo
2638 // which will later be expanded into an ADRP+ADD pair after scheduling.
2639 MachineInstr *BaseMI = MRI.getVRegDef(Reg: I.getOperand(i: 1).getReg());
2640 if (BaseMI->getOpcode() != AArch64::ADRP) {
2641 I.setDesc(TII.get(Opcode: AArch64::ADDXri));
2642 I.addOperand(Op: MachineOperand::CreateImm(Val: 0));
2643 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2644 return true;
2645 }
2646 assert(TM.getCodeModel() == CodeModel::Small &&
2647 "Expected small code model");
2648 auto Op1 = BaseMI->getOperand(i: 1);
2649 auto Op2 = I.getOperand(i: 2);
2650 auto MovAddr = MIB.buildInstr(Opc: AArch64::MOVaddr, DstOps: {I.getOperand(i: 0)}, SrcOps: {})
2651 .addGlobalAddress(GV: Op1.getGlobal(), Offset: Op1.getOffset(),
2652 TargetFlags: Op1.getTargetFlags())
2653 .addGlobalAddress(GV: Op2.getGlobal(), Offset: Op2.getOffset(),
2654 TargetFlags: Op2.getTargetFlags());
2655 I.eraseFromParent();
2656 constrainSelectedInstRegOperands(I&: *MovAddr, TII, TRI, RBI);
2657 return true;
2658 }
2659
2660 case TargetOpcode::G_FCONSTANT: {
2661 const Register DefReg = I.getOperand(i: 0).getReg();
2662 const LLT DefTy = MRI.getType(Reg: DefReg);
2663 const unsigned DefSize = DefTy.getSizeInBits();
2664 const RegisterBank &RB = *RBI.getRegBank(Reg: DefReg, MRI, TRI);
2665
2666 const TargetRegisterClass &FPRRC = *getRegClassForTypeOnBank(Ty: DefTy, RB);
2667 assert(
2668 (DefSize == 16 || DefSize == 32 || DefSize == 64 || DefSize == 128) &&
2669 "Unexpected destination size for G_FCONSTANT?");
2670
2671 // Try to emit as a vector movi.
2672 const ConstantFP *FPImm = I.getOperand(i: 1).getFPImm();
2673 APInt Imm = DefSize >= 64
2674 ? FPImm->getValue().bitcastToAPInt()
2675 : APInt::getSplat(NewLen: 64, V: FPImm->getValue().bitcastToAPInt());
2676 if (auto *Mov = emitConstantMOVVector(
2677 Dst: DefSize >= 64 ? DstOp(DefReg) : DstOp(&AArch64::FPR64RegClass), Imm,
2678 MRI)) {
2679 if (DefSize < 64) {
2680 MIB.buildInstr(Opc: TargetOpcode::COPY, DstOps: {DefReg}, SrcOps: {})
2681 .addReg(RegNo: Mov->getOperand(i: 0).getReg(), Flags: {},
2682 SubReg: DefSize == 16 ? AArch64::hsub : AArch64::ssub);
2683 RBI.constrainGenericRegister(Reg: DefReg, RC: FPRRC, MRI);
2684 }
2685 I.eraseFromParent();
2686 return true;
2687 }
2688
2689 // If TLI says that this fpimm is legal then prefer a copy from GPR movimm,
2690 // otherwise we'll expand to a constant pool load
2691 bool OptForSize = shouldOptForSize(MF: &MF);
2692 const auto &TLI = MF.getSubtarget().getTargetLowering();
2693 if ((DefSize == 32 || DefSize == 64) &&
2694 TLI->isFPImmLegal(Imm: I.getOperand(i: 1).getFPImm()->getValueAPF(),
2695 VT: EVT::getFloatingPointVT(BitWidth: DefSize), ForCodeSize: OptForSize)) {
2696 // Emit a copy to emit a normal movimm.
2697 const Register DefGPRReg = MRI.createVirtualRegister(
2698 RegClass: DefSize == 64 ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass);
2699 MachineOperand &RegOp = I.getOperand(i: 0);
2700 RegOp.setReg(DefGPRReg);
2701 MIB.setInsertPt(MBB&: MIB.getMBB(), II: std::next(x: I.getIterator()));
2702 MIB.buildCopy(Res: {DefReg}, Op: {DefGPRReg});
2703
2704 if (!RBI.constrainGenericRegister(Reg: DefReg, RC: FPRRC, MRI)) {
2705 LLVM_DEBUG(dbgs() << "Failed to constrain G_FCONSTANT def operand\n");
2706 return false;
2707 }
2708
2709 MachineOperand &ImmOp = I.getOperand(i: 1);
2710 ImmOp.ChangeToImmediate(
2711 ImmVal: ImmOp.getFPImm()->getValueAPF().bitcastToAPInt().getZExtValue());
2712
2713 const unsigned MovOpc =
2714 DefSize == 64 ? AArch64::MOVi64imm : AArch64::MOVi32imm;
2715 I.setDesc(TII.get(Opcode: MovOpc));
2716 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2717 return true;
2718 }
2719
2720 // Expand to a constant pool.
2721 auto *LoadMI = emitLoadFromConstantPool(CPVal: FPImm, MIRBuilder&: MIB);
2722 if (!LoadMI) {
2723 LLVM_DEBUG(dbgs() << "Failed to load double constant pool entry\n");
2724 return false;
2725 }
2726 MIB.buildCopy(Res: {DefReg}, Op: {LoadMI->getOperand(i: 0).getReg()});
2727 I.eraseFromParent();
2728 return RBI.constrainGenericRegister(Reg: DefReg, RC: FPRRC, MRI);
2729 }
2730 case TargetOpcode::G_EXTRACT: {
2731 Register DstReg = I.getOperand(i: 0).getReg();
2732 Register SrcReg = I.getOperand(i: 1).getReg();
2733 LLT SrcTy = MRI.getType(Reg: SrcReg);
2734 LLT DstTy = MRI.getType(Reg: DstReg);
2735 (void)DstTy;
2736 unsigned SrcSize = SrcTy.getSizeInBits();
2737
2738 if (SrcTy.getSizeInBits() > 64) {
2739 // This should be an extract of an s128, which is like a vector extract.
2740 if (SrcTy.getSizeInBits() != 128)
2741 return false;
2742 // Only support extracting 64 bits from an s128 at the moment.
2743 if (DstTy.getSizeInBits() != 64)
2744 return false;
2745
2746 unsigned Offset = I.getOperand(i: 2).getImm();
2747 if (Offset % 64 != 0)
2748 return false;
2749
2750 // Check we have the right regbank always.
2751 const RegisterBank &SrcRB = *RBI.getRegBank(Reg: SrcReg, MRI, TRI);
2752 const RegisterBank &DstRB = *RBI.getRegBank(Reg: DstReg, MRI, TRI);
2753 assert(SrcRB.getID() == DstRB.getID() && "Wrong extract regbank!");
2754
2755 if (SrcRB.getID() == AArch64::GPRRegBankID) {
2756 auto NewI =
2757 MIB.buildInstr(Opc: TargetOpcode::COPY, DstOps: {DstReg}, SrcOps: {})
2758 .addUse(RegNo: SrcReg, Flags: {},
2759 SubReg: Offset == 0 ? AArch64::sube64 : AArch64::subo64);
2760 constrainOperandRegClass(MF, TRI, MRI, TII, RBI, InsertPt&: *NewI,
2761 RegClass: AArch64::GPR64RegClass, RegMO&: NewI->getOperand(i: 0));
2762 I.eraseFromParent();
2763 return true;
2764 }
2765
2766 // Emit the same code as a vector extract.
2767 // Offset must be a multiple of 64.
2768 unsigned LaneIdx = Offset / 64;
2769 MachineInstr *Extract = emitExtractVectorElt(
2770 DstReg, DstRB, ScalarTy: LLT::scalar(SizeInBits: 64), VecReg: SrcReg, LaneIdx, MIRBuilder&: MIB);
2771 if (!Extract)
2772 return false;
2773 I.eraseFromParent();
2774 return true;
2775 }
2776
2777 I.setDesc(TII.get(Opcode: SrcSize == 64 ? AArch64::UBFMXri : AArch64::UBFMWri));
2778 MachineInstrBuilder(MF, I).addImm(Val: I.getOperand(i: 2).getImm() +
2779 Ty.getSizeInBits() - 1);
2780
2781 if (SrcSize < 64) {
2782 assert(SrcSize == 32 && DstTy.getSizeInBits() == 16 &&
2783 "unexpected G_EXTRACT types");
2784 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2785 return true;
2786 }
2787
2788 DstReg = MRI.createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 64));
2789 MIB.setInsertPt(MBB&: MIB.getMBB(), II: std::next(x: I.getIterator()));
2790 MIB.buildInstr(Opc: TargetOpcode::COPY, DstOps: {I.getOperand(i: 0).getReg()}, SrcOps: {})
2791 .addReg(RegNo: DstReg, Flags: {}, SubReg: AArch64::sub_32);
2792 RBI.constrainGenericRegister(Reg: I.getOperand(i: 0).getReg(),
2793 RC: AArch64::GPR32RegClass, MRI);
2794 I.getOperand(i: 0).setReg(DstReg);
2795
2796 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2797 return true;
2798 }
2799
2800 case TargetOpcode::G_INSERT: {
2801 LLT SrcTy = MRI.getType(Reg: I.getOperand(i: 2).getReg());
2802 LLT DstTy = MRI.getType(Reg: I.getOperand(i: 0).getReg());
2803 unsigned DstSize = DstTy.getSizeInBits();
2804 // Larger inserts are vectors, same-size ones should be something else by
2805 // now (split up or turned into COPYs).
2806 if (Ty.getSizeInBits() > 64 || SrcTy.getSizeInBits() > 32)
2807 return false;
2808
2809 I.setDesc(TII.get(Opcode: DstSize == 64 ? AArch64::BFMXri : AArch64::BFMWri));
2810 unsigned LSB = I.getOperand(i: 3).getImm();
2811 unsigned Width = MRI.getType(Reg: I.getOperand(i: 2).getReg()).getSizeInBits();
2812 I.getOperand(i: 3).setImm((DstSize - LSB) % DstSize);
2813 MachineInstrBuilder(MF, I).addImm(Val: Width - 1);
2814
2815 if (DstSize < 64) {
2816 assert(DstSize == 32 && SrcTy.getSizeInBits() == 16 &&
2817 "unexpected G_INSERT types");
2818 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2819 return true;
2820 }
2821
2822 Register SrcReg = MRI.createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 64));
2823 BuildMI(BB&: MBB, I: I.getIterator(), MIMD: I.getDebugLoc(),
2824 MCID: TII.get(Opcode: AArch64::SUBREG_TO_REG))
2825 .addDef(RegNo: SrcReg)
2826 .addUse(RegNo: I.getOperand(i: 2).getReg())
2827 .addImm(Val: AArch64::sub_32);
2828 RBI.constrainGenericRegister(Reg: I.getOperand(i: 2).getReg(),
2829 RC: AArch64::GPR32RegClass, MRI);
2830 I.getOperand(i: 2).setReg(SrcReg);
2831
2832 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2833 return true;
2834 }
2835 case TargetOpcode::G_FRAME_INDEX: {
2836 // allocas and G_FRAME_INDEX are only supported in addrspace(0).
2837 if (Ty != LLT::pointer(AddressSpace: 0, SizeInBits: 64)) {
2838 LLVM_DEBUG(dbgs() << "G_FRAME_INDEX pointer has type: " << Ty
2839 << ", expected: " << LLT::pointer(0, 64) << '\n');
2840 return false;
2841 }
2842 I.setDesc(TII.get(Opcode: AArch64::ADDXri));
2843
2844 // MOs for a #0 shifted immediate.
2845 I.addOperand(Op: MachineOperand::CreateImm(Val: 0));
2846 I.addOperand(Op: MachineOperand::CreateImm(Val: 0));
2847
2848 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2849 return true;
2850 }
2851
2852 case TargetOpcode::G_GLOBAL_VALUE: {
2853 const GlobalValue *GV = nullptr;
2854 unsigned OpFlags;
2855 if (I.getOperand(i: 1).isSymbol()) {
2856 OpFlags = I.getOperand(i: 1).getTargetFlags();
2857 // Currently only used by "RtLibUseGOT".
2858 assert(OpFlags == AArch64II::MO_GOT);
2859 } else {
2860 GV = I.getOperand(i: 1).getGlobal();
2861 if (GV->isThreadLocal())
2862 return selectTLSGlobalValue(I, MRI);
2863
2864 OpFlags = STI.ClassifyGlobalReference(GV, TM);
2865 }
2866
2867 if (OpFlags & AArch64II::MO_GOT) {
2868 bool IsGOTSigned = MF.getInfo<AArch64FunctionInfo>()->hasELFSignedGOT();
2869 I.setDesc(TII.get(Opcode: IsGOTSigned ? AArch64::LOADgotAUTH : AArch64::LOADgot));
2870 I.getOperand(i: 1).setTargetFlags(OpFlags);
2871 I.addImplicitDefUseOperands(MF);
2872 I.setImplicitPhysRegDefsDead();
2873 } else if (TM.getCodeModel() == CodeModel::Large &&
2874 !TM.isPositionIndependent()) {
2875 // Materialize the global using movz/movk instructions.
2876 materializeLargeCMVal(I, V: GV, OpFlags);
2877 I.eraseFromParent();
2878 return true;
2879 } else if (TM.getCodeModel() == CodeModel::Tiny) {
2880 I.setDesc(TII.get(Opcode: AArch64::ADR));
2881 I.getOperand(i: 1).setTargetFlags(OpFlags);
2882 } else {
2883 I.setDesc(TII.get(Opcode: AArch64::MOVaddr));
2884 I.getOperand(i: 1).setTargetFlags(OpFlags | AArch64II::MO_PAGE);
2885 MachineInstrBuilder MIB(MF, I);
2886 MIB.addGlobalAddress(GV, Offset: I.getOperand(i: 1).getOffset(),
2887 TargetFlags: OpFlags | AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
2888 }
2889 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2890 return true;
2891 }
2892
2893 case TargetOpcode::G_PTRAUTH_GLOBAL_VALUE:
2894 return selectPtrAuthGlobalValue(I, MRI);
2895
2896 case TargetOpcode::G_ZEXTLOAD:
2897 case TargetOpcode::G_LOAD:
2898 case TargetOpcode::G_STORE: {
2899 GLoadStore &LdSt = cast<GLoadStore>(Val&: I);
2900 bool IsZExtLoad = I.getOpcode() == TargetOpcode::G_ZEXTLOAD;
2901 LLT PtrTy = MRI.getType(Reg: LdSt.getPointerReg());
2902
2903 // Can only handle AddressSpace 0, 64-bit pointers.
2904 if (PtrTy != LLT::pointer(AddressSpace: 0, SizeInBits: 64)) {
2905 return false;
2906 }
2907
2908 uint64_t MemSizeInBytes = LdSt.getMemSize().getValue();
2909 unsigned MemSizeInBits = LdSt.getMemSizeInBits().getValue();
2910 AtomicOrdering Order = LdSt.getMMO().getSuccessOrdering();
2911
2912 // Need special instructions for atomics that affect ordering.
2913 if (isStrongerThanMonotonic(AO: Order)) {
2914 assert(!isa<GZExtLoad>(LdSt));
2915 assert(MemSizeInBytes <= 8 &&
2916 "128-bit atomics should already be custom-legalized");
2917
2918 if (isa<GLoad>(Val: LdSt)) {
2919 static constexpr unsigned LDAPROpcodes[] = {
2920 AArch64::LDAPRB, AArch64::LDAPRH, AArch64::LDAPRW, AArch64::LDAPRX};
2921 static constexpr unsigned LDAROpcodes[] = {
2922 AArch64::LDARB, AArch64::LDARH, AArch64::LDARW, AArch64::LDARX};
2923 ArrayRef<unsigned> Opcodes =
2924 STI.hasRCPC() && Order != AtomicOrdering::SequentiallyConsistent
2925 ? LDAPROpcodes
2926 : LDAROpcodes;
2927 I.setDesc(TII.get(Opcode: Opcodes[Log2_32(Value: MemSizeInBytes)]));
2928 } else {
2929 static constexpr unsigned Opcodes[] = {AArch64::STLRB, AArch64::STLRH,
2930 AArch64::STLRW, AArch64::STLRX};
2931 Register ValReg = LdSt.getReg(Idx: 0);
2932 if (MRI.getType(Reg: ValReg).getSizeInBits() == 64 && MemSizeInBits != 64) {
2933 // Emit a subreg copy of 32 bits.
2934 Register NewVal = MRI.createVirtualRegister(RegClass: &AArch64::GPR32RegClass);
2935 MIB.buildInstr(Opc: TargetOpcode::COPY, DstOps: {NewVal}, SrcOps: {})
2936 .addReg(RegNo: I.getOperand(i: 0).getReg(), Flags: {}, SubReg: AArch64::sub_32);
2937 I.getOperand(i: 0).setReg(NewVal);
2938 }
2939 I.setDesc(TII.get(Opcode: Opcodes[Log2_32(Value: MemSizeInBytes)]));
2940 }
2941 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2942 return true;
2943 }
2944
2945#ifndef NDEBUG
2946 const Register PtrReg = LdSt.getPointerReg();
2947 const RegisterBank &PtrRB = *RBI.getRegBank(PtrReg, MRI, TRI);
2948 // Check that the pointer register is valid.
2949 assert(PtrRB.getID() == AArch64::GPRRegBankID &&
2950 "Load/Store pointer operand isn't a GPR");
2951 assert(MRI.getType(PtrReg).isPointer() &&
2952 "Load/Store pointer operand isn't a pointer");
2953#endif
2954
2955 const Register ValReg = LdSt.getReg(Idx: 0);
2956 const RegisterBank &RB = *RBI.getRegBank(Reg: ValReg, MRI, TRI);
2957 LLT ValTy = MRI.getType(Reg: ValReg);
2958
2959 // The code below doesn't support truncating stores, so we need to split it
2960 // again.
2961 if (isa<GStore>(Val: LdSt) && ValTy.getSizeInBits() > MemSizeInBits &&
2962 RB.getID() == AArch64::FPRRegBankID) {
2963 unsigned SubReg;
2964 LLT MemTy = LdSt.getMMO().getMemoryType();
2965 auto *RC = getRegClassForTypeOnBank(Ty: MemTy, RB);
2966 if (!getSubRegForClass(RC, TRI, SubReg))
2967 return false;
2968
2969 // Generate a subreg copy.
2970 auto Copy = MIB.buildInstr(Opc: TargetOpcode::COPY, DstOps: {MemTy}, SrcOps: {})
2971 .addReg(RegNo: ValReg, Flags: {}, SubReg)
2972 .getReg(Idx: 0);
2973 RBI.constrainGenericRegister(Reg: Copy, RC: *RC, MRI);
2974 LdSt.getOperand(i: 0).setReg(Copy);
2975 } else if (isa<GLoad>(Val: LdSt) && ValTy.getSizeInBits() > MemSizeInBits) {
2976 // If this is an any-extending load from the FPR bank, split it into a regular
2977 // load + extend.
2978 if (RB.getID() == AArch64::FPRRegBankID) {
2979 unsigned SubReg;
2980 LLT MemTy = LdSt.getMMO().getMemoryType();
2981 auto *RC = getRegClassForTypeOnBank(Ty: MemTy, RB);
2982 if (!getSubRegForClass(RC, TRI, SubReg))
2983 return false;
2984 Register OldDst = LdSt.getReg(Idx: 0);
2985 Register NewDst =
2986 MRI.createGenericVirtualRegister(Ty: LdSt.getMMO().getMemoryType());
2987 LdSt.getOperand(i: 0).setReg(NewDst);
2988 MRI.setRegBank(Reg: NewDst, RegBank: RB);
2989 // Generate a SUBREG_TO_REG to extend it.
2990 MIB.setInsertPt(MBB&: MIB.getMBB(), II: std::next(x: LdSt.getIterator()));
2991 MIB.buildInstr(Opc: AArch64::SUBREG_TO_REG, DstOps: {OldDst}, SrcOps: {})
2992 .addUse(RegNo: NewDst)
2993 .addImm(Val: SubReg);
2994 auto SubRegRC = getRegClassForTypeOnBank(Ty: MRI.getType(Reg: OldDst), RB);
2995 RBI.constrainGenericRegister(Reg: OldDst, RC: *SubRegRC, MRI);
2996 MIB.setInstr(LdSt);
2997 ValTy = MemTy; // This is no longer an extending load.
2998 }
2999 }
3000
3001 // Helper lambda for partially selecting I. Either returns the original
3002 // instruction with an updated opcode, or a new instruction.
3003 auto SelectLoadStoreAddressingMode = [&]() -> MachineInstr * {
3004 bool IsStore = isa<GStore>(Val: I);
3005 const unsigned NewOpc =
3006 selectLoadStoreUIOp(GenericOpc: I.getOpcode(), RegBankID: RB.getID(), OpSize: MemSizeInBits);
3007 if (NewOpc == I.getOpcode())
3008 return nullptr;
3009 // Check if we can fold anything into the addressing mode.
3010 auto AddrModeFns =
3011 selectAddrModeIndexed(Root&: I.getOperand(i: 1), Size: MemSizeInBytes);
3012 if (!AddrModeFns) {
3013 // Can't fold anything. Use the original instruction.
3014 I.setDesc(TII.get(Opcode: NewOpc));
3015 I.addOperand(Op: MachineOperand::CreateImm(Val: 0));
3016 return &I;
3017 }
3018
3019 // Folded something. Create a new instruction and return it.
3020 auto NewInst = MIB.buildInstr(Opc: NewOpc, DstOps: {}, SrcOps: {}, Flags: I.getFlags());
3021 Register CurValReg = I.getOperand(i: 0).getReg();
3022 IsStore ? NewInst.addUse(RegNo: CurValReg) : NewInst.addDef(RegNo: CurValReg);
3023 NewInst.cloneMemRefs(OtherMI: I);
3024 for (auto &Fn : *AddrModeFns)
3025 Fn(NewInst);
3026 I.eraseFromParent();
3027 return &*NewInst;
3028 };
3029
3030 MachineInstr *LoadStore = SelectLoadStoreAddressingMode();
3031 if (!LoadStore)
3032 return false;
3033
3034 // If we're storing a 0, use WZR/XZR.
3035 if (Opcode == TargetOpcode::G_STORE) {
3036 auto CVal = getIConstantVRegValWithLookThrough(
3037 VReg: LoadStore->getOperand(i: 0).getReg(), MRI);
3038 if (CVal && CVal->Value == 0) {
3039 switch (LoadStore->getOpcode()) {
3040 case AArch64::STRWui:
3041 case AArch64::STRHHui:
3042 case AArch64::STRBBui:
3043 LoadStore->getOperand(i: 0).setReg(AArch64::WZR);
3044 break;
3045 case AArch64::STRXui:
3046 LoadStore->getOperand(i: 0).setReg(AArch64::XZR);
3047 break;
3048 }
3049 }
3050 }
3051
3052 if (IsZExtLoad || (Opcode == TargetOpcode::G_LOAD &&
3053 ValTy == LLT::scalar(SizeInBits: 64) && MemSizeInBits == 32)) {
3054 // The any/zextload from a smaller type to i32 should be handled by the
3055 // importer.
3056 if (MRI.getType(Reg: LoadStore->getOperand(i: 0).getReg()).getSizeInBits() != 64)
3057 return false;
3058 // If we have an extending load then change the load's type to be a
3059 // narrower reg and zero_extend with SUBREG_TO_REG.
3060 Register LdReg = MRI.createVirtualRegister(RegClass: &AArch64::GPR32RegClass);
3061 Register DstReg = LoadStore->getOperand(i: 0).getReg();
3062 LoadStore->getOperand(i: 0).setReg(LdReg);
3063
3064 MIB.setInsertPt(MBB&: MIB.getMBB(), II: std::next(x: LoadStore->getIterator()));
3065 MIB.buildInstr(Opc: AArch64::SUBREG_TO_REG, DstOps: {DstReg}, SrcOps: {})
3066 .addUse(RegNo: LdReg)
3067 .addImm(Val: AArch64::sub_32);
3068 constrainSelectedInstRegOperands(I&: *LoadStore, TII, TRI, RBI);
3069 return RBI.constrainGenericRegister(Reg: DstReg, RC: AArch64::GPR64allRegClass,
3070 MRI);
3071 }
3072 constrainSelectedInstRegOperands(I&: *LoadStore, TII, TRI, RBI);
3073 return true;
3074 }
3075
3076 case TargetOpcode::G_INDEXED_ZEXTLOAD:
3077 case TargetOpcode::G_INDEXED_SEXTLOAD:
3078 return selectIndexedExtLoad(I, MRI);
3079 case TargetOpcode::G_INDEXED_LOAD:
3080 return selectIndexedLoad(I, MRI);
3081 case TargetOpcode::G_INDEXED_STORE:
3082 return selectIndexedStore(I&: cast<GIndexedStore>(Val&: I), MRI);
3083
3084 case TargetOpcode::G_LSHR:
3085 case TargetOpcode::G_ASHR:
3086 if (MRI.getType(Reg: I.getOperand(i: 0).getReg()).isVector())
3087 return selectVectorAshrLshr(I, MRI);
3088 [[fallthrough]];
3089 case TargetOpcode::G_SHL: {
3090 if (Opcode == TargetOpcode::G_SHL &&
3091 MRI.getType(Reg: I.getOperand(i: 0).getReg()).isVector())
3092 return selectVectorSHL(I, MRI);
3093
3094 // These shifts were legalized to have 64 bit shift amounts because we
3095 // want to take advantage of the selection patterns that assume the
3096 // immediates are s64s, however, selectBinaryOp will assume both operands
3097 // will have the same bit size.
3098 {
3099 Register SrcReg = I.getOperand(i: 1).getReg();
3100 Register ShiftReg = I.getOperand(i: 2).getReg();
3101 const LLT ShiftTy = MRI.getType(Reg: ShiftReg);
3102 const LLT SrcTy = MRI.getType(Reg: SrcReg);
3103 if (!SrcTy.isVector() && SrcTy.getSizeInBits() == 32 &&
3104 ShiftTy.getSizeInBits() == 64) {
3105 assert(!ShiftTy.isVector() && "unexpected vector shift ty");
3106 // Insert a subregister copy to implement a 64->32 trunc
3107 auto Trunc = MIB.buildInstr(Opc: TargetOpcode::COPY, DstOps: {SrcTy}, SrcOps: {})
3108 .addReg(RegNo: ShiftReg, Flags: {}, SubReg: AArch64::sub_32);
3109 MRI.setRegBank(Reg: Trunc.getReg(Idx: 0), RegBank: RBI.getRegBank(ID: AArch64::GPRRegBankID));
3110 I.getOperand(i: 2).setReg(Trunc.getReg(Idx: 0));
3111 }
3112 }
3113
3114 const unsigned OpSize = Ty.getSizeInBits();
3115 const Register DefReg = I.getOperand(i: 0).getReg();
3116 const RegisterBank &RB = *RBI.getRegBank(Reg: DefReg, MRI, TRI);
3117
3118 const unsigned NewOpc = selectBinaryOp(GenericOpc: I.getOpcode(), RegBankID: RB.getID(), OpSize);
3119 if (NewOpc == I.getOpcode())
3120 return false;
3121
3122 I.setDesc(TII.get(Opcode: NewOpc));
3123 // FIXME: Should the type be always reset in setDesc?
3124
3125 // Now that we selected an opcode, we need to constrain the register
3126 // operands to use appropriate classes.
3127 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
3128 return true;
3129 }
3130 case TargetOpcode::G_PTR_ADD: {
3131 emitADD(DefReg: I.getOperand(i: 0).getReg(), LHS&: I.getOperand(i: 1), RHS&: I.getOperand(i: 2), MIRBuilder&: MIB);
3132 I.eraseFromParent();
3133 return true;
3134 }
3135
3136 case TargetOpcode::G_SADDE:
3137 case TargetOpcode::G_UADDE:
3138 case TargetOpcode::G_SSUBE:
3139 case TargetOpcode::G_USUBE:
3140 case TargetOpcode::G_SADDO:
3141 case TargetOpcode::G_UADDO:
3142 case TargetOpcode::G_SSUBO:
3143 case TargetOpcode::G_USUBO:
3144 return selectOverflowOp(I, MRI);
3145
3146 case TargetOpcode::G_PTRMASK: {
3147 Register MaskReg = I.getOperand(i: 2).getReg();
3148 std::optional<int64_t> MaskVal = getIConstantVRegSExtVal(VReg: MaskReg, MRI);
3149 // TODO: Implement arbitrary cases
3150 if (!MaskVal || !isShiftedMask_64(Value: *MaskVal))
3151 return false;
3152
3153 uint64_t Mask = *MaskVal;
3154 I.setDesc(TII.get(Opcode: AArch64::ANDXri));
3155 I.getOperand(i: 2).ChangeToImmediate(
3156 ImmVal: AArch64_AM::encodeLogicalImmediate(imm: Mask, regSize: 64));
3157
3158 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
3159 return true;
3160 }
3161 case TargetOpcode::G_PTRTOINT:
3162 case TargetOpcode::G_TRUNC: {
3163 const LLT DstTy = MRI.getType(Reg: I.getOperand(i: 0).getReg());
3164 const LLT SrcTy = MRI.getType(Reg: I.getOperand(i: 1).getReg());
3165
3166 const Register DstReg = I.getOperand(i: 0).getReg();
3167 const Register SrcReg = I.getOperand(i: 1).getReg();
3168
3169 const RegisterBank &DstRB = *RBI.getRegBank(Reg: DstReg, MRI, TRI);
3170 const RegisterBank &SrcRB = *RBI.getRegBank(Reg: SrcReg, MRI, TRI);
3171
3172 if (DstRB.getID() != SrcRB.getID()) {
3173 LLVM_DEBUG(
3174 dbgs() << "G_TRUNC/G_PTRTOINT input/output on different banks\n");
3175 return false;
3176 }
3177
3178 if (DstRB.getID() == AArch64::GPRRegBankID) {
3179 const TargetRegisterClass *DstRC = getRegClassForTypeOnBank(Ty: DstTy, RB: DstRB);
3180 if (!DstRC)
3181 return false;
3182
3183 const TargetRegisterClass *SrcRC = getRegClassForTypeOnBank(Ty: SrcTy, RB: SrcRB);
3184 if (!SrcRC)
3185 return false;
3186
3187 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI) ||
3188 !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI)) {
3189 LLVM_DEBUG(dbgs() << "Failed to constrain G_TRUNC/G_PTRTOINT\n");
3190 return false;
3191 }
3192
3193 if (DstRC == SrcRC) {
3194 // Nothing to be done
3195 } else if (Opcode == TargetOpcode::G_TRUNC && DstTy == LLT::scalar(SizeInBits: 32) &&
3196 SrcTy == LLT::scalar(SizeInBits: 64)) {
3197 llvm_unreachable("TableGen can import this case");
3198 return false;
3199 } else if (DstRC == &AArch64::GPR32RegClass &&
3200 SrcRC == &AArch64::GPR64RegClass) {
3201 I.getOperand(i: 1).setSubReg(AArch64::sub_32);
3202 } else {
3203 LLVM_DEBUG(
3204 dbgs() << "Unhandled mismatched classes in G_TRUNC/G_PTRTOINT\n");
3205 return false;
3206 }
3207
3208 I.setDesc(TII.get(Opcode: TargetOpcode::COPY));
3209 return true;
3210 } else if (DstRB.getID() == AArch64::FPRRegBankID) {
3211 if (DstTy == LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 16) &&
3212 SrcTy == LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 32)) {
3213 I.setDesc(TII.get(Opcode: AArch64::XTNv4i16));
3214 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
3215 return true;
3216 }
3217
3218 if (!SrcTy.isVector() && SrcTy.getSizeInBits() == 128) {
3219 MachineInstr *Extract = emitExtractVectorElt(
3220 DstReg, DstRB, ScalarTy: LLT::scalar(SizeInBits: DstTy.getSizeInBits()), VecReg: SrcReg, LaneIdx: 0, MIRBuilder&: MIB);
3221 if (!Extract)
3222 return false;
3223 I.eraseFromParent();
3224 return true;
3225 }
3226
3227 // We might have a vector G_PTRTOINT, in which case just emit a COPY.
3228 if (Opcode == TargetOpcode::G_PTRTOINT) {
3229 assert(DstTy.isVector() && "Expected an FPR ptrtoint to be a vector");
3230 I.setDesc(TII.get(Opcode: TargetOpcode::COPY));
3231 return selectCopy(I, TII, MRI, TRI, RBI);
3232 }
3233 }
3234
3235 return false;
3236 }
3237
3238 case TargetOpcode::G_ANYEXT: {
3239 if (selectUSMovFromExtend(I, MRI))
3240 return true;
3241
3242 const Register DstReg = I.getOperand(i: 0).getReg();
3243 const Register SrcReg = I.getOperand(i: 1).getReg();
3244
3245 const RegisterBank &RBDst = *RBI.getRegBank(Reg: DstReg, MRI, TRI);
3246 if (RBDst.getID() != AArch64::GPRRegBankID) {
3247 LLVM_DEBUG(dbgs() << "G_ANYEXT on bank: " << RBDst
3248 << ", expected: GPR\n");
3249 return false;
3250 }
3251
3252 const RegisterBank &RBSrc = *RBI.getRegBank(Reg: SrcReg, MRI, TRI);
3253 if (RBSrc.getID() != AArch64::GPRRegBankID) {
3254 LLVM_DEBUG(dbgs() << "G_ANYEXT on bank: " << RBSrc
3255 << ", expected: GPR\n");
3256 return false;
3257 }
3258
3259 const unsigned DstSize = MRI.getType(Reg: DstReg).getSizeInBits();
3260
3261 if (DstSize == 0) {
3262 LLVM_DEBUG(dbgs() << "G_ANYEXT operand has no size, not a gvreg?\n");
3263 return false;
3264 }
3265
3266 if (DstSize != 64 && DstSize > 32) {
3267 LLVM_DEBUG(dbgs() << "G_ANYEXT to size: " << DstSize
3268 << ", expected: 32 or 64\n");
3269 return false;
3270 }
3271 // At this point G_ANYEXT is just like a plain COPY, but we need
3272 // to explicitly form the 64-bit value if any.
3273 if (DstSize > 32) {
3274 Register ExtSrc = MRI.createVirtualRegister(RegClass: &AArch64::GPR64allRegClass);
3275 BuildMI(BB&: MBB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: AArch64::SUBREG_TO_REG))
3276 .addDef(RegNo: ExtSrc)
3277 .addUse(RegNo: SrcReg)
3278 .addImm(Val: AArch64::sub_32);
3279 I.getOperand(i: 1).setReg(ExtSrc);
3280 }
3281 return selectCopy(I, TII, MRI, TRI, RBI);
3282 }
3283
3284 case TargetOpcode::G_ZEXT:
3285 case TargetOpcode::G_SEXT_INREG:
3286 case TargetOpcode::G_SEXT: {
3287 if (selectUSMovFromExtend(I, MRI))
3288 return true;
3289
3290 unsigned Opcode = I.getOpcode();
3291 const bool IsSigned = Opcode != TargetOpcode::G_ZEXT;
3292 const Register DefReg = I.getOperand(i: 0).getReg();
3293 Register SrcReg = I.getOperand(i: 1).getReg();
3294 const LLT DstTy = MRI.getType(Reg: DefReg);
3295 const LLT SrcTy = MRI.getType(Reg: SrcReg);
3296 unsigned DstSize = DstTy.getSizeInBits();
3297 unsigned SrcSize = SrcTy.getSizeInBits();
3298
3299 // SEXT_INREG has the same src reg size as dst, the size of the value to be
3300 // extended is encoded in the imm.
3301 if (Opcode == TargetOpcode::G_SEXT_INREG)
3302 SrcSize = I.getOperand(i: 2).getImm();
3303
3304 if (DstTy.isVector())
3305 return false; // Should be handled by imported patterns.
3306
3307 assert((*RBI.getRegBank(DefReg, MRI, TRI)).getID() ==
3308 AArch64::GPRRegBankID &&
3309 "Unexpected ext regbank");
3310
3311 MachineInstr *ExtI;
3312
3313 // First check if we're extending the result of a load which has a dest type
3314 // smaller than 32 bits, then this zext is redundant. GPR32 is the smallest
3315 // GPR register on AArch64 and all loads which are smaller automatically
3316 // zero-extend the upper bits. E.g.
3317 // %v(s8) = G_LOAD %p, :: (load 1)
3318 // %v2(s32) = G_ZEXT %v(s8)
3319 if (!IsSigned) {
3320 auto *LoadMI = getOpcodeDef(Opcode: TargetOpcode::G_LOAD, Reg: SrcReg, MRI);
3321 bool IsGPR =
3322 RBI.getRegBank(Reg: SrcReg, MRI, TRI)->getID() == AArch64::GPRRegBankID;
3323 if (LoadMI && IsGPR) {
3324 const MachineMemOperand *MemOp = *LoadMI->memoperands_begin();
3325 unsigned BytesLoaded = MemOp->getSize().getValue();
3326 if (BytesLoaded < 4 && SrcTy.getSizeInBytes() == BytesLoaded)
3327 return selectCopy(I, TII, MRI, TRI, RBI);
3328 }
3329
3330 // For the 32-bit -> 64-bit case, we can emit a mov (ORRWrs)
3331 // + SUBREG_TO_REG.
3332 if (IsGPR && SrcSize == 32 && DstSize == 64) {
3333 Register SubregToRegSrc =
3334 MRI.createVirtualRegister(RegClass: &AArch64::GPR32RegClass);
3335 const Register ZReg = AArch64::WZR;
3336 MIB.buildInstr(Opc: AArch64::ORRWrs, DstOps: {SubregToRegSrc}, SrcOps: {ZReg, SrcReg})
3337 .addImm(Val: 0);
3338
3339 MIB.buildInstr(Opc: AArch64::SUBREG_TO_REG, DstOps: {DefReg}, SrcOps: {})
3340 .addUse(RegNo: SubregToRegSrc)
3341 .addImm(Val: AArch64::sub_32);
3342
3343 if (!RBI.constrainGenericRegister(Reg: DefReg, RC: AArch64::GPR64RegClass,
3344 MRI)) {
3345 LLVM_DEBUG(dbgs() << "Failed to constrain G_ZEXT destination\n");
3346 return false;
3347 }
3348
3349 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: AArch64::GPR32RegClass,
3350 MRI)) {
3351 LLVM_DEBUG(dbgs() << "Failed to constrain G_ZEXT source\n");
3352 return false;
3353 }
3354
3355 I.eraseFromParent();
3356 return true;
3357 }
3358 }
3359
3360 if (DstSize == 64) {
3361 if (Opcode != TargetOpcode::G_SEXT_INREG) {
3362 // FIXME: Can we avoid manually doing this?
3363 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: AArch64::GPR32RegClass,
3364 MRI)) {
3365 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(Opcode)
3366 << " operand\n");
3367 return false;
3368 }
3369 SrcReg = MIB.buildInstr(Opc: AArch64::SUBREG_TO_REG,
3370 DstOps: {&AArch64::GPR64RegClass}, SrcOps: {})
3371 .addUse(RegNo: SrcReg)
3372 .addImm(Val: AArch64::sub_32)
3373 .getReg(Idx: 0);
3374 }
3375
3376 ExtI = MIB.buildInstr(Opc: IsSigned ? AArch64::SBFMXri : AArch64::UBFMXri,
3377 DstOps: {DefReg}, SrcOps: {SrcReg})
3378 .addImm(Val: 0)
3379 .addImm(Val: SrcSize - 1);
3380 } else if (DstSize <= 32) {
3381 ExtI = MIB.buildInstr(Opc: IsSigned ? AArch64::SBFMWri : AArch64::UBFMWri,
3382 DstOps: {DefReg}, SrcOps: {SrcReg})
3383 .addImm(Val: 0)
3384 .addImm(Val: SrcSize - 1);
3385 } else {
3386 return false;
3387 }
3388
3389 constrainSelectedInstRegOperands(I&: *ExtI, TII, TRI, RBI);
3390 I.eraseFromParent();
3391 return true;
3392 }
3393
3394 case TargetOpcode::G_FREEZE:
3395 return selectCopy(I, TII, MRI, TRI, RBI);
3396
3397 case TargetOpcode::G_INTTOPTR:
3398 // The importer is currently unable to import pointer types since they
3399 // didn't exist in SelectionDAG.
3400 return selectCopy(I, TII, MRI, TRI, RBI);
3401
3402 case TargetOpcode::G_BITCAST:
3403 // Imported SelectionDAG rules can handle every bitcast except those that
3404 // bitcast from a type to the same regbank. The other exception is bitcasts
3405 // involving pointer types, as SelectionDAG has no knowledge of them.
3406 return selectCopy(I, TII, MRI, TRI, RBI);
3407
3408 case TargetOpcode::G_SELECT: {
3409 auto &Sel = cast<GSelect>(Val&: I);
3410 const Register CondReg = Sel.getCondReg();
3411 const Register TReg = Sel.getTrueReg();
3412 const Register FReg = Sel.getFalseReg();
3413
3414 if (tryOptSelect(Sel))
3415 return true;
3416
3417 // Make sure to use an unused vreg instead of wzr, so that the peephole
3418 // optimizations will be able to optimize these.
3419 Register DeadVReg = MRI.createVirtualRegister(RegClass: &AArch64::GPR32RegClass);
3420 auto TstMI = MIB.buildInstr(Opc: AArch64::ANDSWri, DstOps: {DeadVReg}, SrcOps: {CondReg})
3421 .addImm(Val: AArch64_AM::encodeLogicalImmediate(imm: 1, regSize: 32));
3422 constrainSelectedInstRegOperands(I&: *TstMI, TII, TRI, RBI);
3423 if (!emitSelect(Dst: Sel.getReg(Idx: 0), True: TReg, False: FReg, CC: AArch64CC::NE, MIB))
3424 return false;
3425 Sel.eraseFromParent();
3426 return true;
3427 }
3428 case TargetOpcode::G_ICMP: {
3429 if (Ty.isVector())
3430 return false;
3431
3432 if (Ty != LLT::scalar(SizeInBits: 32)) {
3433 LLVM_DEBUG(dbgs() << "G_ICMP result has type: " << Ty
3434 << ", expected: " << LLT::scalar(32) << '\n');
3435 return false;
3436 }
3437
3438 auto &PredOp = I.getOperand(i: 1);
3439 emitIntegerCompare(LHS&: I.getOperand(i: 2), RHS&: I.getOperand(i: 3), Predicate&: PredOp, MIRBuilder&: MIB);
3440 auto Pred = static_cast<CmpInst::Predicate>(PredOp.getPredicate());
3441 const AArch64CC::CondCode InvCC = changeICMPPredToAArch64CC(
3442 P: CmpInst::getInversePredicate(pred: Pred), RHS: I.getOperand(i: 3).getReg(), MRI: &MRI);
3443 emitCSINC(/*Dst=*/I.getOperand(i: 0).getReg(), /*Src1=*/AArch64::WZR,
3444 /*Src2=*/AArch64::WZR, Pred: InvCC, MIRBuilder&: MIB);
3445 I.eraseFromParent();
3446 return true;
3447 }
3448
3449 case TargetOpcode::G_FCMP: {
3450 CmpInst::Predicate Pred =
3451 static_cast<CmpInst::Predicate>(I.getOperand(i: 1).getPredicate());
3452 if (!emitFPCompare(LHS: I.getOperand(i: 2).getReg(), RHS: I.getOperand(i: 3).getReg(), MIRBuilder&: MIB,
3453 Pred) ||
3454 !emitCSetForFCmp(Dst: I.getOperand(i: 0).getReg(), Pred, MIRBuilder&: MIB))
3455 return false;
3456 I.eraseFromParent();
3457 return true;
3458 }
3459 case TargetOpcode::G_VASTART:
3460 return STI.isTargetDarwin() ? selectVaStartDarwin(I, MF, MRI)
3461 : selectVaStartAAPCS(I, MF, MRI);
3462 case TargetOpcode::G_INTRINSIC:
3463 return selectIntrinsic(I, MRI);
3464 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
3465 return selectIntrinsicWithSideEffects(I, MRI);
3466 case TargetOpcode::G_IMPLICIT_DEF: {
3467 I.setDesc(TII.get(Opcode: TargetOpcode::IMPLICIT_DEF));
3468 const LLT DstTy = MRI.getType(Reg: I.getOperand(i: 0).getReg());
3469 const Register DstReg = I.getOperand(i: 0).getReg();
3470 const RegisterBank &DstRB = *RBI.getRegBank(Reg: DstReg, MRI, TRI);
3471 const TargetRegisterClass *DstRC = getRegClassForTypeOnBank(Ty: DstTy, RB: DstRB);
3472 RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI);
3473 return true;
3474 }
3475 case TargetOpcode::G_BLOCK_ADDR: {
3476 Function *BAFn = I.getOperand(i: 1).getBlockAddress()->getFunction();
3477 if (std::optional<uint16_t> BADisc =
3478 STI.getPtrAuthBlockAddressDiscriminatorIfEnabled(ParentFn: *BAFn)) {
3479 MIB.buildInstr(Opcode: AArch64::MOVaddrPAC)
3480 .addBlockAddress(BA: I.getOperand(i: 1).getBlockAddress())
3481 .addImm(Val: AArch64PACKey::IA)
3482 .addReg(/*AddrDisc=*/RegNo: AArch64::XZR)
3483 .addImm(Val: *BADisc)
3484 .constrainAllUses(TII, TRI, RBI);
3485 MIB.buildCopy(Res: I.getOperand(i: 0).getReg(), Op: Register(AArch64::X16));
3486 RBI.constrainGenericRegister(Reg: I.getOperand(i: 0).getReg(),
3487 RC: AArch64::GPR64RegClass, MRI);
3488 I.eraseFromParent();
3489 return true;
3490 }
3491 if (TM.getCodeModel() == CodeModel::Large && !TM.isPositionIndependent()) {
3492 materializeLargeCMVal(I, V: I.getOperand(i: 1).getBlockAddress(), OpFlags: 0);
3493 I.eraseFromParent();
3494 return true;
3495 } else {
3496 I.setDesc(TII.get(Opcode: AArch64::MOVaddrBA));
3497 auto MovMI = BuildMI(BB&: MBB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: AArch64::MOVaddrBA),
3498 DestReg: I.getOperand(i: 0).getReg())
3499 .addBlockAddress(BA: I.getOperand(i: 1).getBlockAddress(),
3500 /* Offset */ 0, TargetFlags: AArch64II::MO_PAGE)
3501 .addBlockAddress(
3502 BA: I.getOperand(i: 1).getBlockAddress(), /* Offset */ 0,
3503 TargetFlags: AArch64II::MO_NC | AArch64II::MO_PAGEOFF);
3504 I.eraseFromParent();
3505 constrainSelectedInstRegOperands(I&: *MovMI, TII, TRI, RBI);
3506 return true;
3507 }
3508 }
3509 case TargetOpcode::G_BUILD_VECTOR:
3510 return selectBuildVector(I, MRI);
3511 case TargetOpcode::G_MERGE_VALUES:
3512 return selectMergeValues(I, MRI);
3513 case TargetOpcode::G_UNMERGE_VALUES:
3514 return selectUnmergeValues(I, MRI);
3515 case TargetOpcode::G_SHUFFLE_VECTOR:
3516 return selectShuffleVector(I, MRI);
3517 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
3518 return selectExtractElt(I, MRI);
3519 case TargetOpcode::G_CONCAT_VECTORS:
3520 return selectConcatVectors(I, MRI);
3521 case TargetOpcode::G_JUMP_TABLE:
3522 return selectJumpTable(I, MRI);
3523 case TargetOpcode::G_MEMCPY:
3524 case TargetOpcode::G_MEMCPY_INLINE:
3525 case TargetOpcode::G_MEMMOVE:
3526 case TargetOpcode::G_MEMSET:
3527 case TargetOpcode::G_MEMSET_INLINE:
3528 assert(STI.hasMOPS() && "Shouldn't get here without +mops feature");
3529 return selectMOPS(I, MRI);
3530 }
3531
3532 return false;
3533}
3534
3535bool AArch64InstructionSelector::selectAndRestoreState(MachineInstr &I) {
3536 MachineIRBuilderState OldMIBState = MIB.getState();
3537 bool Success = select(I);
3538 MIB.setState(OldMIBState);
3539 return Success;
3540}
3541
3542bool AArch64InstructionSelector::selectMOPS(MachineInstr &GI,
3543 MachineRegisterInfo &MRI) {
3544 unsigned Mopcode;
3545 switch (GI.getOpcode()) {
3546 case TargetOpcode::G_MEMCPY:
3547 case TargetOpcode::G_MEMCPY_INLINE:
3548 Mopcode = AArch64::MOPSMemoryCopyPseudo;
3549 break;
3550 case TargetOpcode::G_MEMMOVE:
3551 Mopcode = AArch64::MOPSMemoryMovePseudo;
3552 break;
3553 case TargetOpcode::G_MEMSET:
3554 case TargetOpcode::G_MEMSET_INLINE:
3555 // For tagged memset see llvm.aarch64.mops.memset.tag
3556 Mopcode = AArch64::MOPSMemorySetPseudo;
3557 break;
3558 }
3559
3560 auto &DstPtr = GI.getOperand(i: 0);
3561 auto &SrcOrVal = GI.getOperand(i: 1);
3562 auto &Size = GI.getOperand(i: 2);
3563
3564 // Create copies of the registers that can be clobbered.
3565 const Register DstPtrCopy = MRI.cloneVirtualRegister(VReg: DstPtr.getReg());
3566 const Register SrcValCopy = MRI.cloneVirtualRegister(VReg: SrcOrVal.getReg());
3567 const Register SizeCopy = MRI.cloneVirtualRegister(VReg: Size.getReg());
3568
3569 const bool IsSet = Mopcode == AArch64::MOPSMemorySetPseudo;
3570 const auto &SrcValRegClass =
3571 IsSet ? AArch64::GPR64RegClass : AArch64::GPR64commonRegClass;
3572
3573 // Constrain to specific registers
3574 RBI.constrainGenericRegister(Reg: DstPtrCopy, RC: AArch64::GPR64commonRegClass, MRI);
3575 RBI.constrainGenericRegister(Reg: SrcValCopy, RC: SrcValRegClass, MRI);
3576 RBI.constrainGenericRegister(Reg: SizeCopy, RC: AArch64::GPR64RegClass, MRI);
3577
3578 MIB.buildCopy(Res: DstPtrCopy, Op: DstPtr);
3579 MIB.buildCopy(Res: SrcValCopy, Op: SrcOrVal);
3580 MIB.buildCopy(Res: SizeCopy, Op: Size);
3581
3582 // New instruction uses the copied registers because it must update them.
3583 // The defs are not used since they don't exist in G_MEM*. They are still
3584 // tied.
3585 // Note: order of operands is different from G_MEMSET, G_MEMCPY, G_MEMMOVE
3586 Register DefDstPtr = MRI.createVirtualRegister(RegClass: &AArch64::GPR64commonRegClass);
3587 Register DefSize = MRI.createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
3588 if (IsSet) {
3589 MIB.buildInstr(Opc: Mopcode, DstOps: {DefDstPtr, DefSize},
3590 SrcOps: {DstPtrCopy, SizeCopy, SrcValCopy})
3591 .setOperandDead(5); // implicit-def $nzcv
3592 } else {
3593 Register DefSrcPtr = MRI.createVirtualRegister(RegClass: &SrcValRegClass);
3594 MIB.buildInstr(Opc: Mopcode, DstOps: {DefDstPtr, DefSrcPtr, DefSize},
3595 SrcOps: {DstPtrCopy, SrcValCopy, SizeCopy})
3596 .setOperandDead(6); // implicit-def $nzcv
3597 }
3598
3599 GI.eraseFromParent();
3600 return true;
3601}
3602
3603bool AArch64InstructionSelector::selectBrJT(MachineInstr &I,
3604 MachineRegisterInfo &MRI) {
3605 assert(I.getOpcode() == TargetOpcode::G_BRJT && "Expected G_BRJT");
3606 Register JTAddr = I.getOperand(i: 0).getReg();
3607 unsigned JTI = I.getOperand(i: 1).getIndex();
3608 Register Index = I.getOperand(i: 2).getReg();
3609
3610 MF->getInfo<AArch64FunctionInfo>()->setJumpTableEntryInfo(Idx: JTI, Size: 4, PCRelSym: nullptr);
3611
3612 // With aarch64-jump-table-hardening, we only expand the jump table dispatch
3613 // sequence later, to guarantee the integrity of the intermediate values.
3614 if (MF->getFunction().hasFnAttribute(Kind: "aarch64-jump-table-hardening")) {
3615 CodeModel::Model CM = TM.getCodeModel();
3616 if (STI.isTargetMachO()) {
3617 if (CM != CodeModel::Small && CM != CodeModel::Large)
3618 report_fatal_error(reason: "Unsupported code-model for hardened jump-table");
3619 } else {
3620 // Note that COFF support would likely also need JUMP_TABLE_DEBUG_INFO.
3621 assert(STI.isTargetELF() &&
3622 "jump table hardening only supported on MachO/ELF");
3623 if (CM != CodeModel::Small)
3624 report_fatal_error(reason: "Unsupported code-model for hardened jump-table");
3625 }
3626
3627 MIB.buildCopy(Res: {AArch64::X16}, Op: I.getOperand(i: 2).getReg());
3628 MIB.buildInstr(Opcode: AArch64::BR_JumpTable)
3629 .addJumpTableIndex(Idx: I.getOperand(i: 1).getIndex());
3630 I.eraseFromParent();
3631 return true;
3632 }
3633
3634 Register TargetReg = MRI.createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
3635 Register ScratchReg = MRI.createVirtualRegister(RegClass: &AArch64::GPR64spRegClass);
3636
3637 auto JumpTableInst = MIB.buildInstr(Opc: AArch64::JumpTableDest32,
3638 DstOps: {TargetReg, ScratchReg}, SrcOps: {JTAddr, Index})
3639 .addJumpTableIndex(Idx: JTI);
3640 // Save the jump table info.
3641 MIB.buildInstr(Opc: TargetOpcode::JUMP_TABLE_DEBUG_INFO, DstOps: {},
3642 SrcOps: {static_cast<int64_t>(JTI)});
3643 // Build the indirect branch.
3644 MIB.buildInstr(Opc: AArch64::BR, DstOps: {}, SrcOps: {TargetReg});
3645 I.eraseFromParent();
3646 constrainSelectedInstRegOperands(I&: *JumpTableInst, TII, TRI, RBI);
3647 return true;
3648}
3649
3650bool AArch64InstructionSelector::selectJumpTable(MachineInstr &I,
3651 MachineRegisterInfo &MRI) {
3652 assert(I.getOpcode() == TargetOpcode::G_JUMP_TABLE && "Expected jump table");
3653 assert(I.getOperand(1).isJTI() && "Jump table op should have a JTI!");
3654
3655 Register DstReg = I.getOperand(i: 0).getReg();
3656 unsigned JTI = I.getOperand(i: 1).getIndex();
3657 // We generate a MOVaddrJT which will get expanded to an ADRP + ADD later.
3658 auto MovMI =
3659 MIB.buildInstr(Opc: AArch64::MOVaddrJT, DstOps: {DstReg}, SrcOps: {})
3660 .addJumpTableIndex(Idx: JTI, TargetFlags: AArch64II::MO_PAGE)
3661 .addJumpTableIndex(Idx: JTI, TargetFlags: AArch64II::MO_NC | AArch64II::MO_PAGEOFF);
3662 I.eraseFromParent();
3663 constrainSelectedInstRegOperands(I&: *MovMI, TII, TRI, RBI);
3664 return true;
3665}
3666
3667bool AArch64InstructionSelector::selectTLSLocalExecELF(
3668 const GlobalValue *GV, MachineInstr &I, MachineRegisterInfo &MRI) {
3669 auto ConstrainRegOps = [&](MachineInstrBuilder MIB) {
3670 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
3671 };
3672 Register ThreadBase = MRI.createGenericVirtualRegister(Ty: LLT::pointer(AddressSpace: 0, SizeInBits: 64));
3673 ConstrainRegOps(MIB.buildInstr(Opc: AArch64::MOVbaseTLS, DstOps: {ThreadBase}, SrcOps: {}));
3674
3675 switch (MF->getTarget().Options.TLSSize) {
3676 default:
3677 llvm_unreachable("Unexpected TLS size");
3678 case 12: {
3679 // add x0, x0, :tprel_lo12:a
3680 ConstrainRegOps(
3681 MIB.buildInstr(Opc: AArch64::ADDXri, DstOps: {I.getOperand(i: 0).getReg()},
3682 SrcOps: {ThreadBase})
3683 .addGlobalAddress(GV, Offset: 0, TargetFlags: AArch64II::MO_TLS | AArch64II::MO_PAGEOFF)
3684 .addImm(Val: 0));
3685 break;
3686 }
3687 case 24: {
3688 // add x0, x0, :tprel_hi12:a
3689 // add x0, x0, :tprel_lo12_nc:a
3690 Register Addr = MRI.createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
3691 ConstrainRegOps(
3692 MIB.buildInstr(Opc: AArch64::ADDXri, DstOps: {Addr}, SrcOps: {ThreadBase})
3693 .addGlobalAddress(GV, Offset: 0, TargetFlags: AArch64II::MO_TLS | AArch64II::MO_HI12)
3694 .addImm(Val: 0));
3695 ConstrainRegOps(
3696 MIB.buildInstr(Opc: AArch64::ADDXri, DstOps: {I.getOperand(i: 0).getReg()}, SrcOps: {Addr})
3697 .addGlobalAddress(GV, Offset: 0,
3698 TargetFlags: AArch64II::MO_TLS | AArch64II::MO_PAGEOFF |
3699 AArch64II::MO_NC)
3700 .addImm(Val: 0));
3701 break;
3702 }
3703 case 32: {
3704 // movz x0, #:tprel_g1:a
3705 // movk x0, #:tprel_g0_nc:a
3706 // add x0, x1, x0
3707 Register Addr = MRI.createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
3708 ConstrainRegOps(
3709 MIB.buildInstr(Opc: AArch64::MOVZXi, DstOps: {Addr}, SrcOps: {})
3710 .addGlobalAddress(GV, Offset: 0, TargetFlags: AArch64II::MO_TLS | AArch64II::MO_G1)
3711 .addImm(Val: 16));
3712 Register Addr2 = MRI.createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
3713 ConstrainRegOps(MIB.buildInstr(Opc: AArch64::MOVKXi, DstOps: {Addr2}, SrcOps: {Addr})
3714 .addGlobalAddress(GV, Offset: 0,
3715 TargetFlags: AArch64II::MO_TLS | AArch64II::MO_G0 |
3716 AArch64II::MO_NC)
3717 .addImm(Val: 0));
3718 ConstrainRegOps(MIB.buildInstr(Opc: AArch64::ADDXrr, DstOps: {I.getOperand(i: 0).getReg()},
3719 SrcOps: {ThreadBase, Addr2}));
3720 break;
3721 }
3722 case 48: {
3723 // movz x0, #:tprel_g2:a
3724 // movk x0, #:tprel_g1_nc:a
3725 // movk x0, #:tprel_g0_nc:a
3726 // add x0, x1, x0
3727 Register Addr = MRI.createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
3728 ConstrainRegOps(
3729 MIB.buildInstr(Opc: AArch64::MOVZXi, DstOps: {Addr}, SrcOps: {})
3730 .addGlobalAddress(GV, Offset: 0, TargetFlags: AArch64II::MO_TLS | AArch64II::MO_G2)
3731 .addImm(Val: 32));
3732 Register Addr2 = MRI.createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
3733 ConstrainRegOps(MIB.buildInstr(Opc: AArch64::MOVKXi, DstOps: {Addr2}, SrcOps: {Addr})
3734 .addGlobalAddress(GV, Offset: 0,
3735 TargetFlags: AArch64II::MO_TLS | AArch64II::MO_G1 |
3736 AArch64II::MO_NC)
3737 .addImm(Val: 16));
3738 Register Addr3 = MRI.createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
3739 ConstrainRegOps(MIB.buildInstr(Opc: AArch64::MOVKXi, DstOps: {Addr3}, SrcOps: {Addr2})
3740 .addGlobalAddress(GV, Offset: 0,
3741 TargetFlags: AArch64II::MO_TLS | AArch64II::MO_G0 |
3742 AArch64II::MO_NC)
3743 .addImm(Val: 0));
3744 ConstrainRegOps(MIB.buildInstr(Opc: AArch64::ADDXrr, DstOps: {I.getOperand(i: 0).getReg()},
3745 SrcOps: {ThreadBase, Addr3}));
3746 break;
3747 }
3748 }
3749 I.eraseFromParent();
3750 return true;
3751}
3752
3753// TLS lowering below mirrors the corresponding DAGISel implementation.
3754// See LowerELFTLSModel() for details.
3755bool AArch64InstructionSelector::selectTLSGlobalValueELF(
3756 MachineInstr &I, MachineRegisterInfo &MRI) {
3757 const GlobalValue *GV = I.getOperand(i: 1).getGlobal();
3758 auto *FuncInfo = MF->getInfo<AArch64FunctionInfo>();
3759 TLSModel::Model Model =
3760 AArch64::getELFTLSModel(GV, TM, HasELFSignedGOT: FuncInfo->hasELFSignedGOT());
3761
3762 Register TPOff = MRI.createVirtualRegister(RegClass: &AArch64::GPR64commonRegClass);
3763 switch (Model) {
3764 case TLSModel::LocalExec:
3765 return selectTLSLocalExecELF(GV, I, MRI);
3766 case TLSModel::InitialExec:
3767 MIB.buildInstr(Opc: AArch64::LOADgot, DstOps: {TPOff}, SrcOps: {})
3768 .addGlobalAddress(GV, Offset: 0, TargetFlags: AArch64II::MO_TLS);
3769 break;
3770 case TLSModel::LocalDynamic:
3771 case TLSModel::GeneralDynamic: {
3772#ifndef NDEBUG
3773 SMEAttrs Attrs = MF->getInfo<AArch64FunctionInfo>()->getSMEFnAttrs();
3774 assert(!Attrs.hasZAState() && !Attrs.hasStreamingInterfaceOrBody() &&
3775 !Attrs.hasStreamingCompatibleInterface() &&
3776 "unsupported SME features reached GlobalISel TLS lowering");
3777#endif
3778 unsigned Opcode = FuncInfo->hasELFSignedGOT()
3779 ? AArch64::TLSDESC_AUTH_CALLSEQ
3780 : AArch64::TLSDESC_CALLSEQ;
3781
3782 if (TLSModel::GeneralDynamic == Model) {
3783 MIB.buildInstr(Opc: Opcode, DstOps: {}, SrcOps: {}).addGlobalAddress(GV, Offset: 0, TargetFlags: AArch64II::MO_TLS);
3784 MIB.buildCopy(Res: TPOff, Op: Register(AArch64::X0));
3785 break;
3786 }
3787 assert(TLSModel::LocalDynamic == Model);
3788 // These accesses will need deduplicating if there's more than one.
3789 FuncInfo->incNumLocalDynamicTLSAccesses();
3790
3791 MIB.buildInstr(Opc: Opcode, DstOps: {}, SrcOps: {})
3792 .addExternalSymbol(FnName: "_TLS_MODULE_BASE_", TargetFlags: AArch64II::MO_TLS);
3793 auto Copy = MIB.buildCopy(Res: LLT::scalar(SizeInBits: 64), Op: Register(AArch64::X0));
3794 auto Add1 =
3795 MIB.buildInstr(Opc: AArch64::ADDXri, DstOps: {LLT::scalar(SizeInBits: 64)}, SrcOps: {Copy.getReg(Idx: 0)})
3796 .addGlobalAddress(GV, Offset: 0, TargetFlags: AArch64II::MO_TLS | AArch64II::MO_HI12)
3797 .addImm(Val: 0);
3798 auto Add2 =
3799 MIB.buildInstr(Opc: AArch64::ADDXri, DstOps: {TPOff}, SrcOps: {Add1.getReg(Idx: 0)})
3800 .addGlobalAddress(GV, Offset: 0,
3801 TargetFlags: AArch64II::MO_TLS | AArch64II::MO_PAGEOFF |
3802 AArch64II::MO_NC)
3803 .addImm(Val: 0);
3804 constrainSelectedInstRegOperands(I&: *Add1, TII, TRI, RBI);
3805 constrainSelectedInstRegOperands(I&: *Add2, TII, TRI, RBI);
3806 }
3807 }
3808 Register ThreadBase = MRI.createGenericVirtualRegister(Ty: LLT::pointer(AddressSpace: 0, SizeInBits: 64));
3809 MIB.buildInstr(Opc: AArch64::MOVbaseTLS, DstOps: {ThreadBase}, SrcOps: {});
3810 auto Add = MIB.buildInstr(Opc: AArch64::ADDXrr, DstOps: {I.getOperand(i: 0).getReg()},
3811 SrcOps: {ThreadBase, TPOff});
3812 constrainSelectedInstRegOperands(I&: *Add, TII, TRI, RBI);
3813
3814 I.eraseFromParent();
3815 return true;
3816}
3817
3818bool AArch64InstructionSelector::selectTLSGlobalValueMachO(
3819 MachineInstr &I, MachineRegisterInfo &MRI) {
3820 const auto &GlobalOp = I.getOperand(i: 1);
3821 assert(GlobalOp.getOffset() == 0 &&
3822 "Shouldn't have an offset on TLS globals!");
3823
3824 const GlobalValue &GV = *GlobalOp.getGlobal();
3825 MF->getFrameInfo().setAdjustsStack(true);
3826 auto LoadGOT =
3827 MIB.buildInstr(Opc: AArch64::LOADgot, DstOps: {&AArch64::GPR64commonRegClass}, SrcOps: {})
3828 .addGlobalAddress(GV: &GV, Offset: 0, TargetFlags: AArch64II::MO_TLS);
3829
3830 auto Load = MIB.buildInstr(Opc: AArch64::LDRXui, DstOps: {&AArch64::GPR64commonRegClass},
3831 SrcOps: {LoadGOT.getReg(Idx: 0)})
3832 .addImm(Val: 0);
3833
3834 MIB.buildCopy(Res: Register(AArch64::X0), Op: LoadGOT.getReg(Idx: 0));
3835 // TLS calls preserve all registers except those that absolutely must be
3836 // trashed: X0 (it takes an argument), LR (it's a call) and NZCV (let's not be
3837 // silly).
3838 unsigned Opcode = getBLRCallOpcode(MF: *MF);
3839
3840 // With ptrauth-calls, the tlv access thunk pointer is authenticated (IA, 0).
3841 if (MF->getFunction().hasFnAttribute(Kind: "ptrauth-calls")) {
3842 assert(Opcode == AArch64::BLR);
3843 Opcode = AArch64::BLRAAZ;
3844 }
3845
3846 MIB.buildInstr(Opc: Opcode, DstOps: {}, SrcOps: {Load})
3847 .setOperandDead(1) // implicit-def $lr
3848 .addUse(RegNo: AArch64::X0, Flags: RegState::Implicit)
3849 .addDef(RegNo: AArch64::X0, Flags: RegState::Implicit)
3850 .addRegMask(Mask: TRI.getTLSCallPreservedMask());
3851
3852 MIB.buildCopy(Res: I.getOperand(i: 0).getReg(), Op: Register(AArch64::X0));
3853 RBI.constrainGenericRegister(Reg: I.getOperand(i: 0).getReg(), RC: AArch64::GPR64RegClass,
3854 MRI);
3855 I.eraseFromParent();
3856 return true;
3857}
3858
3859bool AArch64InstructionSelector::selectTLSGlobalValue(
3860 MachineInstr &I, MachineRegisterInfo &MRI) {
3861 // We don't support instructions with emulated TLS variables yet.
3862 if (TM.useEmulatedTLS())
3863 return false;
3864
3865 if (STI.isTargetELF())
3866 return selectTLSGlobalValueELF(I, MRI);
3867
3868 if (STI.isTargetMachO())
3869 return selectTLSGlobalValueMachO(I, MRI);
3870
3871 return false;
3872}
3873
3874MachineInstr *AArch64InstructionSelector::emitScalarToVector(
3875 unsigned EltSize, const TargetRegisterClass *DstRC, Register Scalar,
3876 MachineIRBuilder &MIRBuilder) const {
3877 auto Undef = MIRBuilder.buildInstr(Opc: TargetOpcode::IMPLICIT_DEF, DstOps: {DstRC}, SrcOps: {});
3878
3879 auto BuildFn = [&](unsigned SubregIndex) {
3880 auto Ins =
3881 MIRBuilder
3882 .buildInstr(Opc: TargetOpcode::INSERT_SUBREG, DstOps: {DstRC}, SrcOps: {Undef, Scalar})
3883 .addImm(Val: SubregIndex);
3884 constrainSelectedInstRegOperands(I&: *Undef, TII, TRI, RBI);
3885 constrainSelectedInstRegOperands(I&: *Ins, TII, TRI, RBI);
3886 return &*Ins;
3887 };
3888
3889 switch (EltSize) {
3890 case 8:
3891 return BuildFn(AArch64::bsub);
3892 case 16:
3893 return BuildFn(AArch64::hsub);
3894 case 32:
3895 return BuildFn(AArch64::ssub);
3896 case 64:
3897 return BuildFn(AArch64::dsub);
3898 default:
3899 return nullptr;
3900 }
3901}
3902
3903MachineInstr *
3904AArch64InstructionSelector::emitNarrowVector(Register DstReg, Register SrcReg,
3905 MachineIRBuilder &MIB,
3906 MachineRegisterInfo &MRI) const {
3907 LLT DstTy = MRI.getType(Reg: DstReg);
3908 const TargetRegisterClass *RC =
3909 getRegClassForTypeOnBank(Ty: DstTy, RB: *RBI.getRegBank(Reg: SrcReg, MRI, TRI));
3910 if (RC != &AArch64::FPR32RegClass && RC != &AArch64::FPR64RegClass) {
3911 LLVM_DEBUG(dbgs() << "Unsupported register class!\n");
3912 return nullptr;
3913 }
3914 unsigned SubReg = 0;
3915 if (!getSubRegForClass(RC, TRI, SubReg))
3916 return nullptr;
3917 if (SubReg != AArch64::ssub && SubReg != AArch64::dsub) {
3918 LLVM_DEBUG(dbgs() << "Unsupported destination size! ("
3919 << DstTy.getSizeInBits() << "\n");
3920 return nullptr;
3921 }
3922 auto Copy = MIB.buildInstr(Opc: TargetOpcode::COPY, DstOps: {DstReg}, SrcOps: {})
3923 .addReg(RegNo: SrcReg, Flags: {}, SubReg);
3924 RBI.constrainGenericRegister(Reg: DstReg, RC: *RC, MRI);
3925 return Copy;
3926}
3927
3928bool AArch64InstructionSelector::selectMergeValues(
3929 MachineInstr &I, MachineRegisterInfo &MRI) {
3930 assert(I.getOpcode() == TargetOpcode::G_MERGE_VALUES && "unexpected opcode");
3931 const LLT DstTy = MRI.getType(Reg: I.getOperand(i: 0).getReg());
3932 const LLT SrcTy = MRI.getType(Reg: I.getOperand(i: 1).getReg());
3933 assert(!DstTy.isVector() && !SrcTy.isVector() && "invalid merge operation");
3934 const RegisterBank &RB = *RBI.getRegBank(Reg: I.getOperand(i: 1).getReg(), MRI, TRI);
3935
3936 if (I.getNumOperands() != 3)
3937 return false;
3938
3939 // Merging 2 s64s into an s128.
3940 if (DstTy == LLT::scalar(SizeInBits: 128)) {
3941 if (SrcTy.getSizeInBits() != 64)
3942 return false;
3943 Register DstReg = I.getOperand(i: 0).getReg();
3944 Register Src1Reg = I.getOperand(i: 1).getReg();
3945 Register Src2Reg = I.getOperand(i: 2).getReg();
3946 auto Tmp = MIB.buildInstr(Opc: TargetOpcode::IMPLICIT_DEF, DstOps: {DstTy}, SrcOps: {});
3947 MachineInstr *InsMI = emitLaneInsert(DstReg: std::nullopt, SrcReg: Tmp.getReg(Idx: 0), EltReg: Src1Reg,
3948 /* LaneIdx */ 0, RB, MIRBuilder&: MIB);
3949 if (!InsMI)
3950 return false;
3951 MachineInstr *Ins2MI = emitLaneInsert(DstReg, SrcReg: InsMI->getOperand(i: 0).getReg(),
3952 EltReg: Src2Reg, /* LaneIdx */ 1, RB, MIRBuilder&: MIB);
3953 if (!Ins2MI)
3954 return false;
3955 constrainSelectedInstRegOperands(I&: *InsMI, TII, TRI, RBI);
3956 constrainSelectedInstRegOperands(I&: *Ins2MI, TII, TRI, RBI);
3957 I.eraseFromParent();
3958 return true;
3959 }
3960
3961 if (RB.getID() != AArch64::GPRRegBankID)
3962 return false;
3963
3964 if (DstTy.getSizeInBits() != 64 || SrcTy.getSizeInBits() != 32)
3965 return false;
3966
3967 auto *DstRC = &AArch64::GPR64RegClass;
3968 Register SubToRegDef = MRI.createVirtualRegister(RegClass: DstRC);
3969 MachineInstr &SubRegMI = *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
3970 MCID: TII.get(Opcode: TargetOpcode::SUBREG_TO_REG))
3971 .addDef(RegNo: SubToRegDef)
3972 .addUse(RegNo: I.getOperand(i: 1).getReg())
3973 .addImm(Val: AArch64::sub_32);
3974 Register SubToRegDef2 = MRI.createVirtualRegister(RegClass: DstRC);
3975 // Need to anyext the second scalar before we can use bfm
3976 MachineInstr &SubRegMI2 = *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
3977 MCID: TII.get(Opcode: TargetOpcode::SUBREG_TO_REG))
3978 .addDef(RegNo: SubToRegDef2)
3979 .addUse(RegNo: I.getOperand(i: 2).getReg())
3980 .addImm(Val: AArch64::sub_32);
3981 MachineInstr &BFM =
3982 *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: AArch64::BFMXri))
3983 .addDef(RegNo: I.getOperand(i: 0).getReg())
3984 .addUse(RegNo: SubToRegDef)
3985 .addUse(RegNo: SubToRegDef2)
3986 .addImm(Val: 32)
3987 .addImm(Val: 31);
3988 constrainSelectedInstRegOperands(I&: SubRegMI, TII, TRI, RBI);
3989 constrainSelectedInstRegOperands(I&: SubRegMI2, TII, TRI, RBI);
3990 constrainSelectedInstRegOperands(I&: BFM, TII, TRI, RBI);
3991 I.eraseFromParent();
3992 return true;
3993}
3994
3995static bool getLaneCopyOpcode(unsigned &CopyOpc, unsigned &ExtractSubReg,
3996 const unsigned EltSize) {
3997 // Choose a lane copy opcode and subregister based off of the size of the
3998 // vector's elements.
3999 switch (EltSize) {
4000 case 8:
4001 CopyOpc = AArch64::DUPi8;
4002 ExtractSubReg = AArch64::bsub;
4003 break;
4004 case 16:
4005 CopyOpc = AArch64::DUPi16;
4006 ExtractSubReg = AArch64::hsub;
4007 break;
4008 case 32:
4009 CopyOpc = AArch64::DUPi32;
4010 ExtractSubReg = AArch64::ssub;
4011 break;
4012 case 64:
4013 CopyOpc = AArch64::DUPi64;
4014 ExtractSubReg = AArch64::dsub;
4015 break;
4016 default:
4017 // Unknown size, bail out.
4018 LLVM_DEBUG(dbgs() << "Elt size '" << EltSize << "' unsupported.\n");
4019 return false;
4020 }
4021 return true;
4022}
4023
4024MachineInstr *AArch64InstructionSelector::emitExtractVectorElt(
4025 std::optional<Register> DstReg, const RegisterBank &DstRB, LLT ScalarTy,
4026 Register VecReg, unsigned LaneIdx, MachineIRBuilder &MIRBuilder) const {
4027 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
4028 unsigned CopyOpc = 0;
4029 unsigned ExtractSubReg = 0;
4030 if (!getLaneCopyOpcode(CopyOpc, ExtractSubReg, EltSize: ScalarTy.getSizeInBits())) {
4031 LLVM_DEBUG(
4032 dbgs() << "Couldn't determine lane copy opcode for instruction.\n");
4033 return nullptr;
4034 }
4035
4036 const TargetRegisterClass *DstRC =
4037 getRegClassForTypeOnBank(Ty: ScalarTy, RB: DstRB, GetAllRegSet: true);
4038 if (!DstRC) {
4039 LLVM_DEBUG(dbgs() << "Could not determine destination register class.\n");
4040 return nullptr;
4041 }
4042
4043 const RegisterBank &VecRB = *RBI.getRegBank(Reg: VecReg, MRI, TRI);
4044 const LLT &VecTy = MRI.getType(Reg: VecReg);
4045 const TargetRegisterClass *VecRC =
4046 getRegClassForTypeOnBank(Ty: VecTy, RB: VecRB, GetAllRegSet: true);
4047 if (!VecRC) {
4048 LLVM_DEBUG(dbgs() << "Could not determine source register class.\n");
4049 return nullptr;
4050 }
4051
4052 // The register that we're going to copy into.
4053 Register InsertReg = VecReg;
4054 if (!DstReg)
4055 DstReg = MRI.createVirtualRegister(RegClass: DstRC);
4056 // If the lane index is 0, we just use a subregister COPY.
4057 if (LaneIdx == 0) {
4058 auto Copy = MIRBuilder.buildInstr(Opc: TargetOpcode::COPY, DstOps: {*DstReg}, SrcOps: {})
4059 .addReg(RegNo: VecReg, Flags: {}, SubReg: ExtractSubReg);
4060 RBI.constrainGenericRegister(Reg: *DstReg, RC: *DstRC, MRI);
4061 return &*Copy;
4062 }
4063
4064 // Lane copies require 128-bit wide registers. If we're dealing with an
4065 // unpacked vector, then we need to move up to that width. Insert an implicit
4066 // def and a subregister insert to get us there.
4067 if (VecTy.getSizeInBits() != 128) {
4068 MachineInstr *ScalarToVector = emitScalarToVector(
4069 EltSize: VecTy.getSizeInBits(), DstRC: &AArch64::FPR128RegClass, Scalar: VecReg, MIRBuilder);
4070 if (!ScalarToVector)
4071 return nullptr;
4072 InsertReg = ScalarToVector->getOperand(i: 0).getReg();
4073 }
4074
4075 MachineInstr *LaneCopyMI =
4076 MIRBuilder.buildInstr(Opc: CopyOpc, DstOps: {*DstReg}, SrcOps: {InsertReg}).addImm(Val: LaneIdx);
4077 constrainSelectedInstRegOperands(I&: *LaneCopyMI, TII, TRI, RBI);
4078
4079 // Make sure that we actually constrain the initial copy.
4080 RBI.constrainGenericRegister(Reg: *DstReg, RC: *DstRC, MRI);
4081 return LaneCopyMI;
4082}
4083
4084bool AArch64InstructionSelector::selectExtractElt(
4085 MachineInstr &I, MachineRegisterInfo &MRI) {
4086 assert(I.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT &&
4087 "unexpected opcode!");
4088 Register DstReg = I.getOperand(i: 0).getReg();
4089 const LLT NarrowTy = MRI.getType(Reg: DstReg);
4090 const Register SrcReg = I.getOperand(i: 1).getReg();
4091 const LLT WideTy = MRI.getType(Reg: SrcReg);
4092 assert(WideTy.getSizeInBits() >= NarrowTy.getSizeInBits() &&
4093 "source register size too small!");
4094 assert(!NarrowTy.isVector() && "cannot extract vector into vector!");
4095
4096 // Need the lane index to determine the correct copy opcode.
4097 MachineOperand &LaneIdxOp = I.getOperand(i: 2);
4098 assert(LaneIdxOp.isReg() && "Lane index operand was not a register?");
4099
4100 // Find the index to extract from.
4101 auto VRegAndVal = getIConstantVRegValWithLookThrough(VReg: LaneIdxOp.getReg(), MRI);
4102 if (!VRegAndVal)
4103 return false;
4104 unsigned LaneIdx = VRegAndVal->Value.getSExtValue();
4105
4106 const RegisterBank &DstRB = *RBI.getRegBank(Reg: DstReg, MRI, TRI);
4107 if (DstRB.getID() == AArch64::GPRRegBankID) {
4108 unsigned Opcode;
4109 switch (WideTy.getScalarSizeInBits()) {
4110 case 8:
4111 Opcode = AArch64::UMOVvi8;
4112 break;
4113 case 16:
4114 Opcode = AArch64::UMOVvi16;
4115 break;
4116 case 32:
4117 Opcode = AArch64::UMOVvi32;
4118 break;
4119 default:
4120 return false;
4121 }
4122
4123 if (WideTy.getSizeInBits() != 128) {
4124 MachineInstr *ScalarToVector = emitScalarToVector(
4125 EltSize: WideTy.getSizeInBits(), DstRC: &AArch64::FPR128RegClass, Scalar: SrcReg, MIRBuilder&: MIB);
4126 assert(ScalarToVector && "Didn't expect emitScalarToVector to fail!");
4127 I.getOperand(i: 1).setReg(ScalarToVector->getOperand(i: 0).getReg());
4128 }
4129
4130 I.setDesc(TII.get(Opcode));
4131 I.getOperand(i: 2).ChangeToImmediate(ImmVal: LaneIdx);
4132 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
4133 return true;
4134 }
4135
4136 MachineInstr *Extract = emitExtractVectorElt(DstReg, DstRB, ScalarTy: NarrowTy, VecReg: SrcReg,
4137 LaneIdx, MIRBuilder&: MIB);
4138 if (!Extract)
4139 return false;
4140
4141 I.eraseFromParent();
4142 return true;
4143}
4144
4145bool AArch64InstructionSelector::selectSplitVectorUnmerge(
4146 MachineInstr &I, MachineRegisterInfo &MRI) {
4147 unsigned NumElts = I.getNumOperands() - 1;
4148 Register SrcReg = I.getOperand(i: NumElts).getReg();
4149 const LLT NarrowTy = MRI.getType(Reg: I.getOperand(i: 0).getReg());
4150 const LLT SrcTy = MRI.getType(Reg: SrcReg);
4151
4152 assert(NarrowTy.isVector() && "Expected an unmerge into vectors");
4153 if (SrcTy.getSizeInBits() > 128) {
4154 LLVM_DEBUG(dbgs() << "Unexpected vector type for vec split unmerge");
4155 return false;
4156 }
4157
4158 // We implement a split vector operation by treating the sub-vectors as
4159 // scalars and extracting them.
4160 const RegisterBank &DstRB =
4161 *RBI.getRegBank(Reg: I.getOperand(i: 0).getReg(), MRI, TRI);
4162 for (unsigned OpIdx = 0; OpIdx < NumElts; ++OpIdx) {
4163 Register Dst = I.getOperand(i: OpIdx).getReg();
4164 MachineInstr *Extract =
4165 emitExtractVectorElt(DstReg: Dst, DstRB, ScalarTy: NarrowTy, VecReg: SrcReg, LaneIdx: OpIdx, MIRBuilder&: MIB);
4166 if (!Extract)
4167 return false;
4168 }
4169 I.eraseFromParent();
4170 return true;
4171}
4172
4173bool AArch64InstructionSelector::selectUnmergeValues(MachineInstr &I,
4174 MachineRegisterInfo &MRI) {
4175 assert(I.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
4176 "unexpected opcode");
4177
4178 // The last operand is the vector source register, and every other operand is
4179 // a register to unpack into.
4180 unsigned NumElts = I.getNumOperands() - 1;
4181 Register SrcReg = I.getOperand(i: NumElts).getReg();
4182 Register LoReg = I.getOperand(i: 0).getReg();
4183 Register HiReg = I.getOperand(i: 1).getReg();
4184 const LLT NarrowTy = MRI.getType(Reg: LoReg);
4185 const LLT WideTy = MRI.getType(Reg: SrcReg);
4186 const RegisterBank &LoRB = *RBI.getRegBank(Reg: LoReg, MRI, TRI);
4187 const RegisterBank &HiRB = *RBI.getRegBank(Reg: HiReg, MRI, TRI);
4188 const RegisterBank &SrcRB = *RBI.getRegBank(Reg: SrcReg, MRI, TRI);
4189
4190 // Handle unmerging a 128-bit FPR value into two 64-bit GPR values.
4191 if (NarrowTy == LLT::scalar(SizeInBits: 64) && WideTy == LLT::scalar(SizeInBits: 128) &&
4192 LoRB.getID() == AArch64::GPRRegBankID &&
4193 HiRB.getID() == AArch64::GPRRegBankID &&
4194 SrcRB.getID() == AArch64::FPRRegBankID) {
4195 MachineInstr &Lo = *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
4196 MCID: TII.get(Opcode: AArch64::UMOVvi64), DestReg: LoReg)
4197 .addUse(RegNo: SrcReg)
4198 .addImm(Val: 0);
4199 MachineInstr &Hi = *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
4200 MCID: TII.get(Opcode: AArch64::UMOVvi64), DestReg: HiReg)
4201 .addUse(RegNo: SrcReg)
4202 .addImm(Val: 1);
4203 constrainSelectedInstRegOperands(I&: Lo, TII, TRI, RBI);
4204 constrainSelectedInstRegOperands(I&: Hi, TII, TRI, RBI);
4205 I.eraseFromParent();
4206 return true;
4207 }
4208
4209 // TODO: Handle other unmerges into GPRs and from scalars to scalars.
4210 if (LoRB.getID() != AArch64::FPRRegBankID ||
4211 HiRB.getID() != AArch64::FPRRegBankID) {
4212 LLVM_DEBUG(dbgs() << "Unmerging vector-to-gpr and scalar-to-scalar "
4213 "currently unsupported.\n");
4214 return false;
4215 }
4216
4217 assert(WideTy.getSizeInBits() > NarrowTy.getSizeInBits() &&
4218 "source register size too small!");
4219
4220 if (!NarrowTy.isScalar())
4221 return selectSplitVectorUnmerge(I, MRI);
4222
4223 // Choose a lane copy opcode and subregister based off of the size of the
4224 // vector's elements.
4225 unsigned CopyOpc = 0;
4226 unsigned ExtractSubReg = 0;
4227 if (!getLaneCopyOpcode(CopyOpc, ExtractSubReg, EltSize: NarrowTy.getSizeInBits()))
4228 return false;
4229
4230 // Set up for the lane copies.
4231 MachineBasicBlock &MBB = *I.getParent();
4232
4233 // Stores the registers we'll be copying from.
4234 SmallVector<Register, 4> InsertRegs;
4235
4236 // We'll use the first register twice, so we only need NumElts-1 registers.
4237 unsigned NumInsertRegs = NumElts - 1;
4238
4239 // If our elements fit into exactly 128 bits, then we can copy from the source
4240 // directly. Otherwise, we need to do a bit of setup with some subregister
4241 // inserts.
4242 if (NarrowTy.getSizeInBits() * NumElts == 128) {
4243 InsertRegs.assign(NumElts: NumInsertRegs, Elt: SrcReg);
4244 } else {
4245 // No. We have to perform subregister inserts. For each insert, create an
4246 // implicit def and a subregister insert, and save the register we create.
4247 // For scalar sources, treat as a pseudo-vector of NarrowTy elements.
4248 unsigned EltSize = WideTy.isVector() ? WideTy.getScalarSizeInBits()
4249 : NarrowTy.getSizeInBits();
4250 const TargetRegisterClass *RC = getRegClassForTypeOnBank(
4251 Ty: LLT::fixed_vector(NumElements: NumElts, ScalarSizeInBits: EltSize), RB: *RBI.getRegBank(Reg: SrcReg, MRI, TRI));
4252 unsigned SubReg = 0;
4253 bool Found = getSubRegForClass(RC, TRI, SubReg);
4254 (void)Found;
4255 assert(Found && "expected to find last operand's subeg idx");
4256 for (unsigned Idx = 0; Idx < NumInsertRegs; ++Idx) {
4257 Register ImpDefReg = MRI.createVirtualRegister(RegClass: &AArch64::FPR128RegClass);
4258 MachineInstr &ImpDefMI =
4259 *BuildMI(BB&: MBB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: TargetOpcode::IMPLICIT_DEF),
4260 DestReg: ImpDefReg);
4261
4262 // Now, create the subregister insert from SrcReg.
4263 Register InsertReg = MRI.createVirtualRegister(RegClass: &AArch64::FPR128RegClass);
4264 MachineInstr &InsMI =
4265 *BuildMI(BB&: MBB, I, MIMD: I.getDebugLoc(),
4266 MCID: TII.get(Opcode: TargetOpcode::INSERT_SUBREG), DestReg: InsertReg)
4267 .addUse(RegNo: ImpDefReg)
4268 .addUse(RegNo: SrcReg)
4269 .addImm(Val: SubReg);
4270
4271 constrainSelectedInstRegOperands(I&: ImpDefMI, TII, TRI, RBI);
4272 constrainSelectedInstRegOperands(I&: InsMI, TII, TRI, RBI);
4273
4274 // Save the register so that we can copy from it after.
4275 InsertRegs.push_back(Elt: InsertReg);
4276 }
4277 }
4278
4279 // Now that we've created any necessary subregister inserts, we can
4280 // create the copies.
4281 //
4282 // Perform the first copy separately as a subregister copy.
4283 Register CopyTo = I.getOperand(i: 0).getReg();
4284 auto FirstCopy = MIB.buildInstr(Opc: TargetOpcode::COPY, DstOps: {CopyTo}, SrcOps: {})
4285 .addReg(RegNo: InsertRegs[0], Flags: {}, SubReg: ExtractSubReg);
4286 constrainSelectedInstRegOperands(I&: *FirstCopy, TII, TRI, RBI);
4287
4288 // Now, perform the remaining copies as vector lane copies.
4289 unsigned LaneIdx = 1;
4290 for (Register InsReg : InsertRegs) {
4291 Register CopyTo = I.getOperand(i: LaneIdx).getReg();
4292 MachineInstr &CopyInst =
4293 *BuildMI(BB&: MBB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: CopyOpc), DestReg: CopyTo)
4294 .addUse(RegNo: InsReg)
4295 .addImm(Val: LaneIdx);
4296 constrainSelectedInstRegOperands(I&: CopyInst, TII, TRI, RBI);
4297 ++LaneIdx;
4298 }
4299
4300 // Separately constrain the first copy's destination. Because of the
4301 // limitation in constrainOperandRegClass, we can't guarantee that this will
4302 // actually be constrained. So, do it ourselves using the second operand.
4303 const TargetRegisterClass *RC =
4304 MRI.getRegClassOrNull(Reg: I.getOperand(i: 1).getReg());
4305 if (!RC) {
4306 LLVM_DEBUG(dbgs() << "Couldn't constrain copy destination.\n");
4307 return false;
4308 }
4309
4310 RBI.constrainGenericRegister(Reg: CopyTo, RC: *RC, MRI);
4311 I.eraseFromParent();
4312 return true;
4313}
4314
4315bool AArch64InstructionSelector::selectConcatVectors(
4316 MachineInstr &I, MachineRegisterInfo &MRI) {
4317 assert(I.getOpcode() == TargetOpcode::G_CONCAT_VECTORS &&
4318 "Unexpected opcode");
4319 Register Dst = I.getOperand(i: 0).getReg();
4320 Register Op1 = I.getOperand(i: 1).getReg();
4321 Register Op2 = I.getOperand(i: 2).getReg();
4322 MachineInstr *ConcatMI = emitVectorConcat(Dst, Op1, Op2, MIRBuilder&: MIB);
4323 if (!ConcatMI)
4324 return false;
4325 I.eraseFromParent();
4326 return true;
4327}
4328
4329unsigned
4330AArch64InstructionSelector::emitConstantPoolEntry(const Constant *CPVal,
4331 MachineFunction &MF) const {
4332 Type *CPTy = CPVal->getType();
4333 Align Alignment = MF.getDataLayout().getPrefTypeAlign(Ty: CPTy);
4334
4335 MachineConstantPool *MCP = MF.getConstantPool();
4336 return MCP->getConstantPoolIndex(C: CPVal, Alignment);
4337}
4338
4339MachineInstr *AArch64InstructionSelector::emitLoadFromConstantPool(
4340 const Constant *CPVal, MachineIRBuilder &MIRBuilder) const {
4341 const TargetRegisterClass *RC;
4342 unsigned Opc;
4343 bool IsTiny = TM.getCodeModel() == CodeModel::Tiny;
4344 unsigned Size = MIRBuilder.getDataLayout().getTypeStoreSize(Ty: CPVal->getType());
4345 switch (Size) {
4346 case 16:
4347 RC = &AArch64::FPR128RegClass;
4348 Opc = IsTiny ? AArch64::LDRQl : AArch64::LDRQui;
4349 break;
4350 case 8:
4351 RC = &AArch64::FPR64RegClass;
4352 Opc = IsTiny ? AArch64::LDRDl : AArch64::LDRDui;
4353 break;
4354 case 4:
4355 RC = &AArch64::FPR32RegClass;
4356 Opc = IsTiny ? AArch64::LDRSl : AArch64::LDRSui;
4357 break;
4358 case 2:
4359 RC = &AArch64::FPR16RegClass;
4360 Opc = AArch64::LDRHui;
4361 break;
4362 default:
4363 LLVM_DEBUG(dbgs() << "Could not load from constant pool of type "
4364 << *CPVal->getType());
4365 return nullptr;
4366 }
4367
4368 MachineInstr *LoadMI = nullptr;
4369 auto &MF = MIRBuilder.getMF();
4370 unsigned CPIdx = emitConstantPoolEntry(CPVal, MF);
4371 if (IsTiny && (Size == 16 || Size == 8 || Size == 4)) {
4372 // Use load(literal) for tiny code model.
4373 LoadMI = &*MIRBuilder.buildInstr(Opc, DstOps: {RC}, SrcOps: {}).addConstantPoolIndex(Idx: CPIdx);
4374 } else {
4375 auto Adrp =
4376 MIRBuilder.buildInstr(Opc: AArch64::ADRP, DstOps: {&AArch64::GPR64RegClass}, SrcOps: {})
4377 .addConstantPoolIndex(Idx: CPIdx, Offset: 0, TargetFlags: AArch64II::MO_PAGE);
4378
4379 LoadMI = &*MIRBuilder.buildInstr(Opc, DstOps: {RC}, SrcOps: {Adrp})
4380 .addConstantPoolIndex(
4381 Idx: CPIdx, Offset: 0, TargetFlags: AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
4382
4383 constrainSelectedInstRegOperands(I&: *Adrp, TII, TRI, RBI);
4384 }
4385
4386 MachinePointerInfo PtrInfo = MachinePointerInfo::getConstantPool(MF);
4387 LoadMI->addMemOperand(MF, MO: MF.getMachineMemOperand(PtrInfo,
4388 F: MachineMemOperand::MOLoad,
4389 Size, BaseAlignment: Align(Size)));
4390 constrainSelectedInstRegOperands(I&: *LoadMI, TII, TRI, RBI);
4391 return LoadMI;
4392}
4393
4394/// Return an <Opcode, SubregIndex> pair to do an vector elt insert of a given
4395/// size and RB.
4396static std::pair<unsigned, unsigned>
4397getInsertVecEltOpInfo(const RegisterBank &RB, unsigned EltSize) {
4398 unsigned Opc, SubregIdx;
4399 if (RB.getID() == AArch64::GPRRegBankID) {
4400 if (EltSize == 8) {
4401 Opc = AArch64::INSvi8gpr;
4402 SubregIdx = AArch64::bsub;
4403 } else if (EltSize == 16) {
4404 Opc = AArch64::INSvi16gpr;
4405 SubregIdx = AArch64::ssub;
4406 } else if (EltSize == 32) {
4407 Opc = AArch64::INSvi32gpr;
4408 SubregIdx = AArch64::ssub;
4409 } else if (EltSize == 64) {
4410 Opc = AArch64::INSvi64gpr;
4411 SubregIdx = AArch64::dsub;
4412 } else {
4413 llvm_unreachable("invalid elt size!");
4414 }
4415 } else {
4416 if (EltSize == 8) {
4417 Opc = AArch64::INSvi8lane;
4418 SubregIdx = AArch64::bsub;
4419 } else if (EltSize == 16) {
4420 Opc = AArch64::INSvi16lane;
4421 SubregIdx = AArch64::hsub;
4422 } else if (EltSize == 32) {
4423 Opc = AArch64::INSvi32lane;
4424 SubregIdx = AArch64::ssub;
4425 } else if (EltSize == 64) {
4426 Opc = AArch64::INSvi64lane;
4427 SubregIdx = AArch64::dsub;
4428 } else {
4429 llvm_unreachable("invalid elt size!");
4430 }
4431 }
4432 return std::make_pair(x&: Opc, y&: SubregIdx);
4433}
4434
4435MachineInstr *AArch64InstructionSelector::emitInstr(
4436 unsigned Opcode, std::initializer_list<llvm::DstOp> DstOps,
4437 std::initializer_list<llvm::SrcOp> SrcOps, MachineIRBuilder &MIRBuilder,
4438 const ComplexRendererFns &RenderFns) const {
4439 assert(Opcode && "Expected an opcode?");
4440 assert(!isPreISelGenericOpcode(Opcode) &&
4441 "Function should only be used to produce selected instructions!");
4442 auto MI = MIRBuilder.buildInstr(Opc: Opcode, DstOps, SrcOps);
4443 if (RenderFns)
4444 for (auto &Fn : *RenderFns)
4445 Fn(MI);
4446 constrainSelectedInstRegOperands(I&: *MI, TII, TRI, RBI);
4447 return &*MI;
4448}
4449
4450MachineInstr *AArch64InstructionSelector::emitAddSub(
4451 const std::array<std::array<unsigned, 2>, 5> &AddrModeAndSizeToOpcode,
4452 Register Dst, MachineOperand &LHS, MachineOperand &RHS,
4453 MachineIRBuilder &MIRBuilder) const {
4454 MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo();
4455 assert(LHS.isReg() && RHS.isReg() && "Expected register operands?");
4456 auto Ty = MRI.getType(Reg: LHS.getReg());
4457 assert(!Ty.isVector() && "Expected a scalar or pointer?");
4458 unsigned Size = Ty.getSizeInBits();
4459 assert((Size == 32 || Size == 64) && "Expected a 32-bit or 64-bit type only");
4460 bool Is32Bit = Size == 32;
4461
4462 // INSTRri form with positive arithmetic immediate.
4463 if (auto Fns = selectArithImmed(Root&: RHS))
4464 return emitInstr(Opcode: AddrModeAndSizeToOpcode[0][Is32Bit], DstOps: {Dst}, SrcOps: {LHS},
4465 MIRBuilder, RenderFns: Fns);
4466
4467 // INSTRri form with negative arithmetic immediate.
4468 if (auto Fns = selectNegArithImmed(Root&: RHS))
4469 return emitInstr(Opcode: AddrModeAndSizeToOpcode[3][Is32Bit], DstOps: {Dst}, SrcOps: {LHS},
4470 MIRBuilder, RenderFns: Fns);
4471
4472 // INSTRrx form.
4473 if (auto Fns = selectArithExtendedRegister(Root&: RHS))
4474 return emitInstr(Opcode: AddrModeAndSizeToOpcode[4][Is32Bit], DstOps: {Dst}, SrcOps: {LHS},
4475 MIRBuilder, RenderFns: Fns);
4476
4477 // INSTRrs form.
4478 if (auto Fns = selectShiftedRegister(Root&: RHS))
4479 return emitInstr(Opcode: AddrModeAndSizeToOpcode[1][Is32Bit], DstOps: {Dst}, SrcOps: {LHS},
4480 MIRBuilder, RenderFns: Fns);
4481 return emitInstr(Opcode: AddrModeAndSizeToOpcode[2][Is32Bit], DstOps: {Dst}, SrcOps: {LHS, RHS},
4482 MIRBuilder);
4483}
4484
4485MachineInstr *
4486AArch64InstructionSelector::emitADD(Register DefReg, MachineOperand &LHS,
4487 MachineOperand &RHS,
4488 MachineIRBuilder &MIRBuilder) const {
4489 const std::array<std::array<unsigned, 2>, 5> OpcTable{
4490 ._M_elems: {{AArch64::ADDXri, AArch64::ADDWri},
4491 {AArch64::ADDXrs, AArch64::ADDWrs},
4492 {AArch64::ADDXrr, AArch64::ADDWrr},
4493 {AArch64::SUBXri, AArch64::SUBWri},
4494 {AArch64::ADDXrx, AArch64::ADDWrx}}};
4495 return emitAddSub(AddrModeAndSizeToOpcode: OpcTable, Dst: DefReg, LHS, RHS, MIRBuilder);
4496}
4497
4498MachineInstr *
4499AArch64InstructionSelector::emitADDS(Register Dst, MachineOperand &LHS,
4500 MachineOperand &RHS,
4501 MachineIRBuilder &MIRBuilder) const {
4502 const std::array<std::array<unsigned, 2>, 5> OpcTable{
4503 ._M_elems: {{AArch64::ADDSXri, AArch64::ADDSWri},
4504 {AArch64::ADDSXrs, AArch64::ADDSWrs},
4505 {AArch64::ADDSXrr, AArch64::ADDSWrr},
4506 {AArch64::SUBSXri, AArch64::SUBSWri},
4507 {AArch64::ADDSXrx, AArch64::ADDSWrx}}};
4508 return emitAddSub(AddrModeAndSizeToOpcode: OpcTable, Dst, LHS, RHS, MIRBuilder);
4509}
4510
4511MachineInstr *
4512AArch64InstructionSelector::emitSUBS(Register Dst, MachineOperand &LHS,
4513 MachineOperand &RHS,
4514 MachineIRBuilder &MIRBuilder) const {
4515 const std::array<std::array<unsigned, 2>, 5> OpcTable{
4516 ._M_elems: {{AArch64::SUBSXri, AArch64::SUBSWri},
4517 {AArch64::SUBSXrs, AArch64::SUBSWrs},
4518 {AArch64::SUBSXrr, AArch64::SUBSWrr},
4519 {AArch64::ADDSXri, AArch64::ADDSWri},
4520 {AArch64::SUBSXrx, AArch64::SUBSWrx}}};
4521 return emitAddSub(AddrModeAndSizeToOpcode: OpcTable, Dst, LHS, RHS, MIRBuilder);
4522}
4523
4524MachineInstr *
4525AArch64InstructionSelector::emitADCS(Register Dst, MachineOperand &LHS,
4526 MachineOperand &RHS,
4527 MachineIRBuilder &MIRBuilder) const {
4528 assert(LHS.isReg() && RHS.isReg() && "Expected register operands?");
4529 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
4530 bool Is32Bit = (MRI->getType(Reg: LHS.getReg()).getSizeInBits() == 32);
4531 static const unsigned OpcTable[2] = {AArch64::ADCSXr, AArch64::ADCSWr};
4532 return emitInstr(Opcode: OpcTable[Is32Bit], DstOps: {Dst}, SrcOps: {LHS, RHS}, MIRBuilder);
4533}
4534
4535MachineInstr *
4536AArch64InstructionSelector::emitSBCS(Register Dst, MachineOperand &LHS,
4537 MachineOperand &RHS,
4538 MachineIRBuilder &MIRBuilder) const {
4539 assert(LHS.isReg() && RHS.isReg() && "Expected register operands?");
4540 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
4541 bool Is32Bit = (MRI->getType(Reg: LHS.getReg()).getSizeInBits() == 32);
4542 static const unsigned OpcTable[2] = {AArch64::SBCSXr, AArch64::SBCSWr};
4543 return emitInstr(Opcode: OpcTable[Is32Bit], DstOps: {Dst}, SrcOps: {LHS, RHS}, MIRBuilder);
4544}
4545
4546MachineInstr *
4547AArch64InstructionSelector::emitCMP(MachineOperand &LHS, MachineOperand &RHS,
4548 MachineIRBuilder &MIRBuilder) const {
4549 MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo();
4550 bool Is32Bit = MRI.getType(Reg: LHS.getReg()).getSizeInBits() == 32;
4551 auto RC = Is32Bit ? &AArch64::GPR32RegClass : &AArch64::GPR64RegClass;
4552 return emitSUBS(Dst: MRI.createVirtualRegister(RegClass: RC), LHS, RHS, MIRBuilder);
4553}
4554
4555MachineInstr *
4556AArch64InstructionSelector::emitCMN(MachineOperand &LHS, MachineOperand &RHS,
4557 MachineIRBuilder &MIRBuilder) const {
4558 MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo();
4559 bool Is32Bit = (MRI.getType(Reg: LHS.getReg()).getSizeInBits() == 32);
4560 auto RC = Is32Bit ? &AArch64::GPR32RegClass : &AArch64::GPR64RegClass;
4561 return emitADDS(Dst: MRI.createVirtualRegister(RegClass: RC), LHS, RHS, MIRBuilder);
4562}
4563
4564MachineInstr *
4565AArch64InstructionSelector::emitTST(MachineOperand &LHS, MachineOperand &RHS,
4566 MachineIRBuilder &MIRBuilder) const {
4567 assert(LHS.isReg() && RHS.isReg() && "Expected register operands?");
4568 MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo();
4569 LLT Ty = MRI.getType(Reg: LHS.getReg());
4570 unsigned RegSize = Ty.getSizeInBits();
4571 bool Is32Bit = (RegSize == 32);
4572 const unsigned OpcTable[3][2] = {{AArch64::ANDSXri, AArch64::ANDSWri},
4573 {AArch64::ANDSXrs, AArch64::ANDSWrs},
4574 {AArch64::ANDSXrr, AArch64::ANDSWrr}};
4575 // ANDS needs a logical immediate for its immediate form. Check if we can
4576 // fold one in.
4577 if (auto ValAndVReg = getIConstantVRegValWithLookThrough(VReg: RHS.getReg(), MRI)) {
4578 int64_t Imm = ValAndVReg->Value.getSExtValue();
4579
4580 if (AArch64_AM::isLogicalImmediate(imm: Imm, regSize: RegSize)) {
4581 auto TstMI = MIRBuilder.buildInstr(Opc: OpcTable[0][Is32Bit], DstOps: {Ty}, SrcOps: {LHS});
4582 TstMI.addImm(Val: AArch64_AM::encodeLogicalImmediate(imm: Imm, regSize: RegSize));
4583 constrainSelectedInstRegOperands(I&: *TstMI, TII, TRI, RBI);
4584 return &*TstMI;
4585 }
4586 }
4587
4588 if (auto Fns = selectLogicalShiftedRegister(Root&: RHS))
4589 return emitInstr(Opcode: OpcTable[1][Is32Bit], DstOps: {Ty}, SrcOps: {LHS}, MIRBuilder, RenderFns: Fns);
4590 return emitInstr(Opcode: OpcTable[2][Is32Bit], DstOps: {Ty}, SrcOps: {LHS, RHS}, MIRBuilder);
4591}
4592
4593MachineInstr *AArch64InstructionSelector::emitIntegerCompare(
4594 MachineOperand &LHS, MachineOperand &RHS, MachineOperand &Predicate,
4595 MachineIRBuilder &MIRBuilder) const {
4596 assert(LHS.isReg() && RHS.isReg() && "Expected LHS and RHS to be registers!");
4597 assert(Predicate.isPredicate() && "Expected predicate?");
4598 MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo();
4599 LLT CmpTy = MRI.getType(Reg: LHS.getReg());
4600 assert(!CmpTy.isVector() && "Expected scalar or pointer");
4601 unsigned Size = CmpTy.getSizeInBits();
4602 (void)Size;
4603 assert((Size == 32 || Size == 64) && "Expected a 32-bit or 64-bit LHS/RHS?");
4604 // Fold the compare into a cmn or tst if possible.
4605 if (auto FoldCmp = tryFoldIntegerCompare(LHS, RHS, Predicate, MIRBuilder))
4606 return FoldCmp;
4607 return emitCMP(LHS, RHS, MIRBuilder);
4608}
4609
4610MachineInstr *AArch64InstructionSelector::emitCSetForFCmp(
4611 Register Dst, CmpInst::Predicate Pred, MachineIRBuilder &MIRBuilder) const {
4612 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
4613#ifndef NDEBUG
4614 LLT Ty = MRI.getType(Dst);
4615 assert(!Ty.isVector() && Ty.getSizeInBits() == 32 &&
4616 "Expected a 32-bit scalar register?");
4617#endif
4618 const Register ZReg = AArch64::WZR;
4619 AArch64CC::CondCode CC1, CC2;
4620 changeFCMPPredToAArch64CC(P: Pred, CondCode&: CC1, CondCode2&: CC2);
4621 auto InvCC1 = AArch64CC::getInvertedCondCode(Code: CC1);
4622 if (CC2 == AArch64CC::AL)
4623 return emitCSINC(/*Dst=*/Dst, /*Src1=*/ZReg, /*Src2=*/ZReg, Pred: InvCC1,
4624 MIRBuilder);
4625 const TargetRegisterClass *RC = &AArch64::GPR32RegClass;
4626 Register Def1Reg = MRI.createVirtualRegister(RegClass: RC);
4627 Register Def2Reg = MRI.createVirtualRegister(RegClass: RC);
4628 auto InvCC2 = AArch64CC::getInvertedCondCode(Code: CC2);
4629 emitCSINC(/*Dst=*/Def1Reg, /*Src1=*/ZReg, /*Src2=*/ZReg, Pred: InvCC1, MIRBuilder);
4630 emitCSINC(/*Dst=*/Def2Reg, /*Src1=*/ZReg, /*Src2=*/ZReg, Pred: InvCC2, MIRBuilder);
4631 auto OrMI = MIRBuilder.buildInstr(Opc: AArch64::ORRWrr, DstOps: {Dst}, SrcOps: {Def1Reg, Def2Reg});
4632 constrainSelectedInstRegOperands(I&: *OrMI, TII, TRI, RBI);
4633 return &*OrMI;
4634}
4635
4636MachineInstr *AArch64InstructionSelector::emitFPCompare(
4637 Register LHS, Register RHS, MachineIRBuilder &MIRBuilder,
4638 std::optional<CmpInst::Predicate> Pred) const {
4639 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
4640 LLT Ty = MRI.getType(Reg: LHS);
4641 if (Ty.isVector())
4642 return nullptr;
4643 unsigned OpSize = Ty.getSizeInBits();
4644 assert(OpSize == 16 || OpSize == 32 || OpSize == 64);
4645
4646 // If this is a compare against +0.0, then we don't have
4647 // to explicitly materialize a constant.
4648 bool ShouldUseImm = mi_match(R: RHS, MRI, P: m_PosZeroFP());
4649
4650 auto IsEqualityPred = [](CmpInst::Predicate P) {
4651 return P == CmpInst::FCMP_OEQ || P == CmpInst::FCMP_ONE ||
4652 P == CmpInst::FCMP_UEQ || P == CmpInst::FCMP_UNE;
4653 };
4654 if (!ShouldUseImm && Pred && IsEqualityPred(*Pred)) {
4655 // Try commuting the operands.
4656 if (mi_match(R: LHS, MRI, P: m_PosZeroFP())) {
4657 ShouldUseImm = true;
4658 std::swap(a&: LHS, b&: RHS);
4659 }
4660 }
4661 unsigned CmpOpcTbl[2][3] = {
4662 {AArch64::FCMPHrr, AArch64::FCMPSrr, AArch64::FCMPDrr},
4663 {AArch64::FCMPHri, AArch64::FCMPSri, AArch64::FCMPDri}};
4664 unsigned CmpOpc =
4665 CmpOpcTbl[ShouldUseImm][OpSize == 16 ? 0 : (OpSize == 32 ? 1 : 2)];
4666
4667 // Partially build the compare. Decide if we need to add a use for the
4668 // third operand based off whether or not we're comparing against 0.0.
4669 auto CmpMI = MIRBuilder.buildInstr(Opcode: CmpOpc).addUse(RegNo: LHS);
4670 CmpMI.setMIFlags(MachineInstr::NoFPExcept);
4671 if (!ShouldUseImm)
4672 CmpMI.addUse(RegNo: RHS);
4673 constrainSelectedInstRegOperands(I&: *CmpMI, TII, TRI, RBI);
4674 return &*CmpMI;
4675}
4676
4677MachineInstr *AArch64InstructionSelector::emitVectorConcat(
4678 std::optional<Register> Dst, Register Op1, Register Op2,
4679 MachineIRBuilder &MIRBuilder) const {
4680 // We implement a vector concat by:
4681 // 1. Use scalar_to_vector to insert the lower vector into the larger dest
4682 // 2. Insert the upper vector into the destination's upper element
4683 // TODO: some of this code is common with G_BUILD_VECTOR handling.
4684 MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo();
4685
4686 const LLT Op1Ty = MRI.getType(Reg: Op1);
4687 const LLT Op2Ty = MRI.getType(Reg: Op2);
4688
4689 if (Op1Ty != Op2Ty) {
4690 LLVM_DEBUG(dbgs() << "Could not do vector concat of differing vector tys");
4691 return nullptr;
4692 }
4693 assert(Op1Ty.isVector() && "Expected a vector for vector concat");
4694
4695 if (Op1Ty.getSizeInBits() >= 128) {
4696 LLVM_DEBUG(dbgs() << "Vector concat not supported for full size vectors");
4697 return nullptr;
4698 }
4699
4700 // At the moment we just support 64 bit vector concats.
4701 if (Op1Ty.getSizeInBits() != 64) {
4702 LLVM_DEBUG(dbgs() << "Vector concat supported for 64b vectors");
4703 return nullptr;
4704 }
4705
4706 const LLT ScalarTy = LLT::scalar(SizeInBits: Op1Ty.getSizeInBits());
4707 const RegisterBank &FPRBank = *RBI.getRegBank(Reg: Op1, MRI, TRI);
4708 const TargetRegisterClass *DstRC =
4709 getRegClassForTypeOnBank(Ty: Op1Ty.multiplyElements(Factor: 2), RB: FPRBank);
4710
4711 MachineInstr *WidenedOp1 =
4712 emitScalarToVector(EltSize: ScalarTy.getSizeInBits(), DstRC, Scalar: Op1, MIRBuilder);
4713 MachineInstr *WidenedOp2 =
4714 emitScalarToVector(EltSize: ScalarTy.getSizeInBits(), DstRC, Scalar: Op2, MIRBuilder);
4715 if (!WidenedOp1 || !WidenedOp2) {
4716 LLVM_DEBUG(dbgs() << "Could not emit a vector from scalar value");
4717 return nullptr;
4718 }
4719
4720 // Now do the insert of the upper element.
4721 unsigned InsertOpc, InsSubRegIdx;
4722 std::tie(args&: InsertOpc, args&: InsSubRegIdx) =
4723 getInsertVecEltOpInfo(RB: FPRBank, EltSize: ScalarTy.getSizeInBits());
4724
4725 if (!Dst)
4726 Dst = MRI.createVirtualRegister(RegClass: DstRC);
4727 auto InsElt =
4728 MIRBuilder
4729 .buildInstr(Opc: InsertOpc, DstOps: {*Dst}, SrcOps: {WidenedOp1->getOperand(i: 0).getReg()})
4730 .addImm(Val: 1) /* Lane index */
4731 .addUse(RegNo: WidenedOp2->getOperand(i: 0).getReg())
4732 .addImm(Val: 0);
4733 constrainSelectedInstRegOperands(I&: *InsElt, TII, TRI, RBI);
4734 return &*InsElt;
4735}
4736
4737MachineInstr *
4738AArch64InstructionSelector::emitCSINC(Register Dst, Register Src1,
4739 Register Src2, AArch64CC::CondCode Pred,
4740 MachineIRBuilder &MIRBuilder) const {
4741 auto &MRI = *MIRBuilder.getMRI();
4742 const RegClassOrRegBank &RegClassOrBank = MRI.getRegClassOrRegBank(Reg: Dst);
4743 // If we used a register class, then this won't necessarily have an LLT.
4744 // Compute the size based off whether or not we have a class or bank.
4745 unsigned Size;
4746 if (const auto *RC = dyn_cast<const TargetRegisterClass *>(Val: RegClassOrBank))
4747 Size = TRI.getRegSizeInBits(RC: *RC);
4748 else
4749 Size = MRI.getType(Reg: Dst).getSizeInBits();
4750 // Some opcodes use s1.
4751 assert(Size <= 64 && "Expected 64 bits or less only!");
4752 static const unsigned OpcTable[2] = {AArch64::CSINCWr, AArch64::CSINCXr};
4753 unsigned Opc = OpcTable[Size == 64];
4754 auto CSINC = MIRBuilder.buildInstr(Opc, DstOps: {Dst}, SrcOps: {Src1, Src2}).addImm(Val: Pred);
4755 constrainSelectedInstRegOperands(I&: *CSINC, TII, TRI, RBI);
4756 return &*CSINC;
4757}
4758
4759MachineInstr *AArch64InstructionSelector::emitCarryIn(MachineInstr &I,
4760 Register CarryReg) {
4761 MachineRegisterInfo *MRI = MIB.getMRI();
4762 unsigned Opcode = I.getOpcode();
4763
4764 // If the instruction is a SUB, we need to negate the carry,
4765 // because borrowing is indicated by carry-flag == 0.
4766 bool NeedsNegatedCarry =
4767 (Opcode == TargetOpcode::G_USUBE || Opcode == TargetOpcode::G_SSUBE);
4768
4769 // If the previous instruction will already produce the correct carry, do not
4770 // emit a carry generating instruction. E.g. for G_UADDE/G_USUBE sequences
4771 // generated during legalization of wide add/sub. This optimization depends on
4772 // these sequences not being interrupted by other instructions.
4773 // We have to select the previous instruction before the carry-using
4774 // instruction is deleted by the calling function, otherwise the previous
4775 // instruction might become dead and would get deleted.
4776 MachineInstr *SrcMI = MRI->getVRegDef(Reg: CarryReg);
4777
4778 // Debug instructions (e.g., DBG_VALUE) do not modify NZCV, so skip all that
4779 // immediately precede I.
4780 MachineInstr *PrevMI = I.getPrevNode();
4781 while (PrevMI && PrevMI->isDebugInstr())
4782 PrevMI = PrevMI->getPrevNode();
4783
4784 if (SrcMI == PrevMI) {
4785 if (auto *CarrySrcMI = dyn_cast<GAddSubCarryOut>(Val: SrcMI)) {
4786 bool ProducesNegatedCarry = CarrySrcMI->isSub();
4787 if (NeedsNegatedCarry == ProducesNegatedCarry &&
4788 CarrySrcMI->isUnsigned() &&
4789 CarrySrcMI->getCarryOutReg() == CarryReg &&
4790 selectAndRestoreState(I&: *SrcMI))
4791 return nullptr;
4792 }
4793 }
4794
4795 Register DeadReg = MRI->createVirtualRegister(RegClass: &AArch64::GPR32RegClass);
4796
4797 if (NeedsNegatedCarry) {
4798 // (0 - Carry) sets !C in NZCV when Carry == 1
4799 Register ZReg = AArch64::WZR;
4800 return emitInstr(Opcode: AArch64::SUBSWrr, DstOps: {DeadReg}, SrcOps: {ZReg, CarryReg}, MIRBuilder&: MIB);
4801 }
4802
4803 // (Carry - 1) sets !C in NZCV when Carry == 0
4804 auto Fns = select12BitValueWithLeftShift(Immed: 1);
4805 return emitInstr(Opcode: AArch64::SUBSWri, DstOps: {DeadReg}, SrcOps: {CarryReg}, MIRBuilder&: MIB, RenderFns: Fns);
4806}
4807
4808bool AArch64InstructionSelector::selectOverflowOp(MachineInstr &I,
4809 MachineRegisterInfo &MRI) {
4810 auto &CarryMI = cast<GAddSubCarryOut>(Val&: I);
4811
4812 if (auto *CarryInMI = dyn_cast<GAddSubCarryInOut>(Val: &I)) {
4813 // Set NZCV carry according to carry-in VReg
4814 emitCarryIn(I, CarryReg: CarryInMI->getCarryInReg());
4815 }
4816
4817 // Emit the operation and get the correct condition code.
4818 auto OpAndCC = emitOverflowOp(Opcode: I.getOpcode(), Dst: CarryMI.getDstReg(),
4819 LHS&: CarryMI.getLHS(), RHS&: CarryMI.getRHS(), MIRBuilder&: MIB);
4820
4821 Register CarryOutReg = CarryMI.getCarryOutReg();
4822
4823 // Don't convert carry-out to VReg if it is never used
4824 if (MRI.use_nodbg_empty(RegNo: CarryOutReg)) {
4825 OpAndCC.first->addRegisterDead(Reg: AArch64::NZCV, RegInfo: &TRI);
4826 } else {
4827 // Now, put the overflow result in the register given by the first operand
4828 // to the overflow op. CSINC increments the result when the predicate is
4829 // false, so to get the increment when it's true, we need to use the
4830 // inverse. In this case, we want to increment when carry is set.
4831 Register ZReg = AArch64::WZR;
4832 emitCSINC(/*Dst=*/CarryOutReg, /*Src1=*/ZReg, /*Src2=*/ZReg,
4833 Pred: getInvertedCondCode(Code: OpAndCC.second), MIRBuilder&: MIB);
4834 }
4835
4836 I.eraseFromParent();
4837 return true;
4838}
4839
4840std::pair<MachineInstr *, AArch64CC::CondCode>
4841AArch64InstructionSelector::emitOverflowOp(unsigned Opcode, Register Dst,
4842 MachineOperand &LHS,
4843 MachineOperand &RHS,
4844 MachineIRBuilder &MIRBuilder) const {
4845 switch (Opcode) {
4846 default:
4847 llvm_unreachable("Unexpected opcode!");
4848 case TargetOpcode::G_SADDO:
4849 return std::make_pair(x: emitADDS(Dst, LHS, RHS, MIRBuilder), y: AArch64CC::VS);
4850 case TargetOpcode::G_UADDO:
4851 return std::make_pair(x: emitADDS(Dst, LHS, RHS, MIRBuilder), y: AArch64CC::HS);
4852 case TargetOpcode::G_SSUBO:
4853 return std::make_pair(x: emitSUBS(Dst, LHS, RHS, MIRBuilder), y: AArch64CC::VS);
4854 case TargetOpcode::G_USUBO:
4855 return std::make_pair(x: emitSUBS(Dst, LHS, RHS, MIRBuilder), y: AArch64CC::LO);
4856 case TargetOpcode::G_SADDE:
4857 return std::make_pair(x: emitADCS(Dst, LHS, RHS, MIRBuilder), y: AArch64CC::VS);
4858 case TargetOpcode::G_UADDE:
4859 return std::make_pair(x: emitADCS(Dst, LHS, RHS, MIRBuilder), y: AArch64CC::HS);
4860 case TargetOpcode::G_SSUBE:
4861 return std::make_pair(x: emitSBCS(Dst, LHS, RHS, MIRBuilder), y: AArch64CC::VS);
4862 case TargetOpcode::G_USUBE:
4863 return std::make_pair(x: emitSBCS(Dst, LHS, RHS, MIRBuilder), y: AArch64CC::LO);
4864 }
4865}
4866
4867/// Returns true if @p Val is a tree of AND/OR/CMP operations that can be
4868/// expressed as a conjunction.
4869/// \param CanNegate Set to true if we can negate the whole sub-tree just by
4870/// changing the conditions on the CMP tests.
4871/// (this means we can call emitConjunctionRec() with
4872/// Negate==true on this sub-tree)
4873/// \param MustBeFirst Set to true if this subtree needs to be negated and we
4874/// cannot do the negation naturally. We are required to
4875/// emit the subtree first in this case.
4876/// \param WillNegate Is true if are called when the result of this
4877/// subexpression must be negated. This happens when the
4878/// outer expression is an OR. We can use this fact to know
4879/// that we have a double negation (or (or ...) ...) that
4880/// can be implemented for free.
4881static bool canEmitConjunction(Register Val, bool &CanNegate, bool &MustBeFirst,
4882 bool WillNegate, MachineRegisterInfo &MRI,
4883 unsigned Depth = 0) {
4884 if (!MRI.hasOneNonDBGUse(RegNo: Val))
4885 return false;
4886 MachineInstr *ValDef = MRI.getVRegDef(Reg: Val);
4887 unsigned Opcode = ValDef->getOpcode();
4888 if (isa<GAnyCmp>(Val: ValDef)) {
4889 CanNegate = true;
4890 MustBeFirst = false;
4891 return true;
4892 }
4893 // Protect against exponential runtime and stack overflow.
4894 if (Depth > 6)
4895 return false;
4896 if (Opcode == TargetOpcode::G_AND || Opcode == TargetOpcode::G_OR) {
4897 bool IsOR = Opcode == TargetOpcode::G_OR;
4898 Register O0 = ValDef->getOperand(i: 1).getReg();
4899 Register O1 = ValDef->getOperand(i: 2).getReg();
4900 bool CanNegateL;
4901 bool MustBeFirstL;
4902 if (!canEmitConjunction(Val: O0, CanNegate&: CanNegateL, MustBeFirst&: MustBeFirstL, WillNegate: IsOR, MRI, Depth: Depth + 1))
4903 return false;
4904 bool CanNegateR;
4905 bool MustBeFirstR;
4906 if (!canEmitConjunction(Val: O1, CanNegate&: CanNegateR, MustBeFirst&: MustBeFirstR, WillNegate: IsOR, MRI, Depth: Depth + 1))
4907 return false;
4908
4909 if (MustBeFirstL && MustBeFirstR)
4910 return false;
4911
4912 if (IsOR) {
4913 // For an OR expression we need to be able to naturally negate at least
4914 // one side or we cannot do the transformation at all.
4915 if (!CanNegateL && !CanNegateR)
4916 return false;
4917 // If we the result of the OR will be negated and we can naturally negate
4918 // the leaves, then this sub-tree as a whole negates naturally.
4919 CanNegate = WillNegate && CanNegateL && CanNegateR;
4920 // If we cannot naturally negate the whole sub-tree, then this must be
4921 // emitted first.
4922 MustBeFirst = !CanNegate;
4923 } else {
4924 assert(Opcode == TargetOpcode::G_AND && "Must be G_AND");
4925 // We cannot naturally negate an AND operation.
4926 CanNegate = false;
4927 MustBeFirst = MustBeFirstL || MustBeFirstR;
4928 }
4929 return true;
4930 }
4931 return false;
4932}
4933
4934MachineInstr *AArch64InstructionSelector::emitConditionalComparison(
4935 Register LHS, Register RHS, CmpInst::Predicate CC,
4936 AArch64CC::CondCode Predicate, AArch64CC::CondCode OutCC,
4937 MachineIRBuilder &MIB) const {
4938 auto &MRI = *MIB.getMRI();
4939 LLT OpTy = MRI.getType(Reg: LHS);
4940 unsigned CCmpOpc;
4941 std::optional<ValueAndVReg> C;
4942 if (CmpInst::isIntPredicate(P: CC)) {
4943 assert(OpTy.getSizeInBits() == 32 || OpTy.getSizeInBits() == 64);
4944 C = getIConstantVRegValWithLookThrough(VReg: RHS, MRI);
4945 if (!C || C->Value.sgt(RHS: 31) || C->Value.slt(RHS: -31))
4946 CCmpOpc = OpTy.getSizeInBits() == 32 ? AArch64::CCMPWr : AArch64::CCMPXr;
4947 else if (C->Value.ule(RHS: 31))
4948 CCmpOpc = OpTy.getSizeInBits() == 32 ? AArch64::CCMPWi : AArch64::CCMPXi;
4949 else
4950 CCmpOpc = OpTy.getSizeInBits() == 32 ? AArch64::CCMNWi : AArch64::CCMNXi;
4951 } else {
4952 assert(OpTy.getSizeInBits() == 16 || OpTy.getSizeInBits() == 32 ||
4953 OpTy.getSizeInBits() == 64);
4954 switch (OpTy.getSizeInBits()) {
4955 case 16:
4956 assert(STI.hasFullFP16() && "Expected Full FP16 for fp16 comparisons");
4957 CCmpOpc = AArch64::FCCMPHrr;
4958 break;
4959 case 32:
4960 CCmpOpc = AArch64::FCCMPSrr;
4961 break;
4962 case 64:
4963 CCmpOpc = AArch64::FCCMPDrr;
4964 break;
4965 default:
4966 return nullptr;
4967 }
4968 }
4969 AArch64CC::CondCode InvOutCC = AArch64CC::getInvertedCondCode(Code: OutCC);
4970 unsigned NZCV = AArch64CC::getNZCVToSatisfyCondCode(Code: InvOutCC);
4971 auto CCmp =
4972 MIB.buildInstr(Opc: CCmpOpc, DstOps: {}, SrcOps: {LHS});
4973 if (CCmpOpc == AArch64::CCMPWi || CCmpOpc == AArch64::CCMPXi)
4974 CCmp.addImm(Val: C->Value.getZExtValue());
4975 else if (CCmpOpc == AArch64::CCMNWi || CCmpOpc == AArch64::CCMNXi)
4976 CCmp.addImm(Val: C->Value.abs().getZExtValue());
4977 else
4978 CCmp.addReg(RegNo: RHS);
4979 CCmp.addImm(Val: NZCV).addImm(Val: Predicate);
4980 constrainSelectedInstRegOperands(I&: *CCmp, TII, TRI, RBI);
4981 return &*CCmp;
4982}
4983
4984MachineInstr *AArch64InstructionSelector::emitConjunctionRec(
4985 Register Val, AArch64CC::CondCode &OutCC, bool Negate, Register CCOp,
4986 AArch64CC::CondCode Predicate, MachineIRBuilder &MIB) const {
4987 // We're at a tree leaf, produce a conditional comparison operation.
4988 auto &MRI = *MIB.getMRI();
4989 MachineInstr *ValDef = MRI.getVRegDef(Reg: Val);
4990 unsigned Opcode = ValDef->getOpcode();
4991 if (auto *Cmp = dyn_cast<GAnyCmp>(Val: ValDef)) {
4992 Register LHS = Cmp->getLHSReg();
4993 Register RHS = Cmp->getRHSReg();
4994 CmpInst::Predicate CC = Cmp->getCond();
4995 if (Negate)
4996 CC = CmpInst::getInversePredicate(pred: CC);
4997 if (isa<GICmp>(Val: Cmp)) {
4998 OutCC = changeICMPPredToAArch64CC(P: CC, RHS, MRI: MIB.getMRI());
4999 } else {
5000 // Handle special FP cases.
5001 AArch64CC::CondCode ExtraCC;
5002 changeFPCCToANDAArch64CC(CC, CondCode&: OutCC, CondCode2&: ExtraCC);
5003 // Some floating point conditions can't be tested with a single condition
5004 // code. Construct an additional comparison in this case.
5005 if (ExtraCC != AArch64CC::AL) {
5006 MachineInstr *ExtraCmp;
5007 if (!CCOp)
5008 ExtraCmp = emitFPCompare(LHS, RHS, MIRBuilder&: MIB, Pred: CC);
5009 else
5010 ExtraCmp =
5011 emitConditionalComparison(LHS, RHS, CC, Predicate, OutCC: ExtraCC, MIB);
5012 CCOp = ExtraCmp->getOperand(i: 0).getReg();
5013 Predicate = ExtraCC;
5014 }
5015 }
5016
5017 // Produce a normal comparison if we are first in the chain
5018 if (!CCOp) {
5019 if (isa<GICmp>(Val: Cmp))
5020 return emitCMP(LHS&: Cmp->getOperand(i: 2), RHS&: Cmp->getOperand(i: 3), MIRBuilder&: MIB);
5021 return emitFPCompare(LHS: Cmp->getOperand(i: 2).getReg(),
5022 RHS: Cmp->getOperand(i: 3).getReg(), MIRBuilder&: MIB);
5023 }
5024 // Otherwise produce a ccmp.
5025 return emitConditionalComparison(LHS, RHS, CC, Predicate, OutCC, MIB);
5026 }
5027 assert(MRI.hasOneNonDBGUse(Val) && "Valid conjunction/disjunction tree");
5028
5029 bool IsOR = Opcode == TargetOpcode::G_OR;
5030
5031 Register LHS = ValDef->getOperand(i: 1).getReg();
5032 bool CanNegateL;
5033 bool MustBeFirstL;
5034 bool ValidL = canEmitConjunction(Val: LHS, CanNegate&: CanNegateL, MustBeFirst&: MustBeFirstL, WillNegate: IsOR, MRI);
5035 assert(ValidL && "Valid conjunction/disjunction tree");
5036 (void)ValidL;
5037
5038 Register RHS = ValDef->getOperand(i: 2).getReg();
5039 bool CanNegateR;
5040 bool MustBeFirstR;
5041 bool ValidR = canEmitConjunction(Val: RHS, CanNegate&: CanNegateR, MustBeFirst&: MustBeFirstR, WillNegate: IsOR, MRI);
5042 assert(ValidR && "Valid conjunction/disjunction tree");
5043 (void)ValidR;
5044
5045 // Swap sub-tree that must come first to the right side.
5046 if (MustBeFirstL) {
5047 assert(!MustBeFirstR && "Valid conjunction/disjunction tree");
5048 std::swap(a&: LHS, b&: RHS);
5049 std::swap(a&: CanNegateL, b&: CanNegateR);
5050 std::swap(a&: MustBeFirstL, b&: MustBeFirstR);
5051 }
5052
5053 bool NegateR;
5054 bool NegateAfterR;
5055 bool NegateL;
5056 bool NegateAfterAll;
5057 if (Opcode == TargetOpcode::G_OR) {
5058 // Swap the sub-tree that we can negate naturally to the left.
5059 if (!CanNegateL) {
5060 assert(CanNegateR && "at least one side must be negatable");
5061 assert(!MustBeFirstR && "invalid conjunction/disjunction tree");
5062 assert(!Negate);
5063 std::swap(a&: LHS, b&: RHS);
5064 NegateR = false;
5065 NegateAfterR = true;
5066 } else {
5067 // Negate the left sub-tree if possible, otherwise negate the result.
5068 NegateR = CanNegateR;
5069 NegateAfterR = !CanNegateR;
5070 }
5071 NegateL = true;
5072 NegateAfterAll = !Negate;
5073 } else {
5074 assert(Opcode == TargetOpcode::G_AND &&
5075 "Valid conjunction/disjunction tree");
5076 assert(!Negate && "Valid conjunction/disjunction tree");
5077
5078 NegateL = false;
5079 NegateR = false;
5080 NegateAfterR = false;
5081 NegateAfterAll = false;
5082 }
5083
5084 // Emit sub-trees.
5085 AArch64CC::CondCode RHSCC;
5086 MachineInstr *CmpR =
5087 emitConjunctionRec(Val: RHS, OutCC&: RHSCC, Negate: NegateR, CCOp, Predicate, MIB);
5088 if (NegateAfterR)
5089 RHSCC = AArch64CC::getInvertedCondCode(Code: RHSCC);
5090 MachineInstr *CmpL = emitConjunctionRec(
5091 Val: LHS, OutCC, Negate: NegateL, CCOp: CmpR->getOperand(i: 0).getReg(), Predicate: RHSCC, MIB);
5092 if (NegateAfterAll)
5093 OutCC = AArch64CC::getInvertedCondCode(Code: OutCC);
5094 return CmpL;
5095}
5096
5097MachineInstr *AArch64InstructionSelector::emitConjunction(
5098 Register Val, AArch64CC::CondCode &OutCC, MachineIRBuilder &MIB) const {
5099 bool DummyCanNegate;
5100 bool DummyMustBeFirst;
5101 if (!canEmitConjunction(Val, CanNegate&: DummyCanNegate, MustBeFirst&: DummyMustBeFirst, WillNegate: false,
5102 MRI&: *MIB.getMRI()))
5103 return nullptr;
5104 return emitConjunctionRec(Val, OutCC, Negate: false, CCOp: Register(), Predicate: AArch64CC::AL, MIB);
5105}
5106
5107bool AArch64InstructionSelector::tryOptSelectConjunction(GSelect &SelI,
5108 MachineInstr &CondMI) {
5109 AArch64CC::CondCode AArch64CC;
5110 MachineInstr *ConjMI = emitConjunction(Val: SelI.getCondReg(), OutCC&: AArch64CC, MIB);
5111 if (!ConjMI)
5112 return false;
5113
5114 emitSelect(Dst: SelI.getReg(Idx: 0), True: SelI.getTrueReg(), False: SelI.getFalseReg(), CC: AArch64CC, MIB);
5115 SelI.eraseFromParent();
5116 return true;
5117}
5118
5119bool AArch64InstructionSelector::tryOptSelect(GSelect &I) {
5120 MachineRegisterInfo &MRI = *MIB.getMRI();
5121 // We want to recognize this pattern:
5122 //
5123 // $z = G_FCMP pred, $x, $y
5124 // ...
5125 // $w = G_SELECT $z, $a, $b
5126 //
5127 // Where the value of $z is *only* ever used by the G_SELECT (possibly with
5128 // some copies/truncs in between.)
5129 //
5130 // If we see this, then we can emit something like this:
5131 //
5132 // fcmp $x, $y
5133 // fcsel $w, $a, $b, pred
5134 //
5135 // Rather than emitting both of the rather long sequences in the standard
5136 // G_FCMP/G_SELECT select methods.
5137
5138 // First, check if the condition is defined by a compare.
5139 MachineInstr *CondDef = MRI.getVRegDef(Reg: I.getOperand(i: 1).getReg());
5140
5141 // We can only fold if all of the defs have one use.
5142 Register CondDefReg = CondDef->getOperand(i: 0).getReg();
5143 if (!MRI.hasOneNonDBGUse(RegNo: CondDefReg)) {
5144 // Unless it's another select.
5145 for (const MachineInstr &UI : MRI.use_nodbg_instructions(Reg: CondDefReg)) {
5146 if (CondDef == &UI)
5147 continue;
5148 if (UI.getOpcode() != TargetOpcode::G_SELECT)
5149 return false;
5150 }
5151 }
5152
5153 // Is the condition defined by a compare?
5154 unsigned CondOpc = CondDef->getOpcode();
5155 if (CondOpc != TargetOpcode::G_ICMP && CondOpc != TargetOpcode::G_FCMP) {
5156 if (tryOptSelectConjunction(SelI&: I, CondMI&: *CondDef))
5157 return true;
5158 return false;
5159 }
5160
5161 AArch64CC::CondCode CondCode;
5162 if (CondOpc == TargetOpcode::G_ICMP) {
5163 auto &PredOp = CondDef->getOperand(i: 1);
5164 emitIntegerCompare(LHS&: CondDef->getOperand(i: 2), RHS&: CondDef->getOperand(i: 3), Predicate&: PredOp,
5165 MIRBuilder&: MIB);
5166 auto Pred = static_cast<CmpInst::Predicate>(PredOp.getPredicate());
5167 CondCode =
5168 changeICMPPredToAArch64CC(P: Pred, RHS: CondDef->getOperand(i: 3).getReg(), MRI: &MRI);
5169 } else {
5170 // Get the condition code for the select.
5171 auto Pred =
5172 static_cast<CmpInst::Predicate>(CondDef->getOperand(i: 1).getPredicate());
5173 AArch64CC::CondCode CondCode2;
5174 changeFCMPPredToAArch64CC(P: Pred, CondCode, CondCode2);
5175
5176 // changeFCMPPredToAArch64CC sets CondCode2 to AL when we require two
5177 // instructions to emit the comparison.
5178 // TODO: Handle FCMP_UEQ and FCMP_ONE. After that, this check will be
5179 // unnecessary.
5180 if (CondCode2 != AArch64CC::AL)
5181 return false;
5182
5183 if (!emitFPCompare(LHS: CondDef->getOperand(i: 2).getReg(),
5184 RHS: CondDef->getOperand(i: 3).getReg(), MIRBuilder&: MIB)) {
5185 LLVM_DEBUG(dbgs() << "Couldn't emit compare for select!\n");
5186 return false;
5187 }
5188 }
5189
5190 // Emit the select.
5191 emitSelect(Dst: I.getOperand(i: 0).getReg(), True: I.getOperand(i: 2).getReg(),
5192 False: I.getOperand(i: 3).getReg(), CC: CondCode, MIB);
5193 I.eraseFromParent();
5194 return true;
5195}
5196
5197MachineInstr *AArch64InstructionSelector::tryFoldIntegerCompare(
5198 MachineOperand &LHS, MachineOperand &RHS, MachineOperand &Predicate,
5199 MachineIRBuilder &MIRBuilder) const {
5200 assert(LHS.isReg() && RHS.isReg() && Predicate.isPredicate() &&
5201 "Unexpected MachineOperand");
5202 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
5203 // We want to find this sort of thing:
5204 // x = G_SUB 0, y
5205 // G_ICMP z, x
5206 //
5207 // In this case, we can fold the G_SUB into the G_ICMP using a CMN instead.
5208 // e.g:
5209 //
5210 // cmn z, y
5211
5212 // Check if the RHS or LHS of the G_ICMP is defined by a SUB
5213 MachineInstr *LHSDef = getDefIgnoringCopies(Reg: LHS.getReg(), MRI);
5214 MachineInstr *RHSDef = getDefIgnoringCopies(Reg: RHS.getReg(), MRI);
5215 auto P = static_cast<CmpInst::Predicate>(Predicate.getPredicate());
5216
5217 // Given this:
5218 //
5219 // x = G_SUB 0, y
5220 // G_ICMP z, x
5221 //
5222 // Produce this:
5223 //
5224 // cmn z, y
5225 if (isCMN(MaybeSub: RHSDef, Pred: P, MRI))
5226 return emitCMN(LHS, RHS&: RHSDef->getOperand(i: 2), MIRBuilder);
5227
5228 // Same idea here, but with the LHS of the compare instead:
5229 //
5230 // Given this:
5231 //
5232 // x = G_SUB 0, y
5233 // G_ICMP x, z
5234 //
5235 // Produce this:
5236 //
5237 // cmn y, z
5238 //
5239 // But be careful! We need to swap the predicate!
5240 if (isCMN(MaybeSub: LHSDef, Pred: P, MRI)) {
5241 if (!CmpInst::isEquality(pred: P)) {
5242 P = CmpInst::getSwappedPredicate(pred: P);
5243 Predicate = MachineOperand::CreatePredicate(Pred: P);
5244 }
5245 return emitCMN(LHS&: LHSDef->getOperand(i: 2), RHS, MIRBuilder);
5246 }
5247
5248 // Given this:
5249 //
5250 // z = G_AND x, y
5251 // G_ICMP z, 0
5252 //
5253 // Produce this if the compare is signed:
5254 //
5255 // tst x, y
5256 if (!CmpInst::isUnsigned(Pred: P) && LHSDef &&
5257 LHSDef->getOpcode() == TargetOpcode::G_AND) {
5258 // Make sure that the RHS is 0.
5259 auto ValAndVReg = getIConstantVRegValWithLookThrough(VReg: RHS.getReg(), MRI);
5260 if (!ValAndVReg || ValAndVReg->Value != 0)
5261 return nullptr;
5262
5263 return emitTST(LHS&: LHSDef->getOperand(i: 1),
5264 RHS&: LHSDef->getOperand(i: 2), MIRBuilder);
5265 }
5266
5267 return nullptr;
5268}
5269
5270bool AArch64InstructionSelector::selectShuffleVector(
5271 MachineInstr &I, MachineRegisterInfo &MRI) {
5272 const LLT DstTy = MRI.getType(Reg: I.getOperand(i: 0).getReg());
5273 Register Src1Reg = I.getOperand(i: 1).getReg();
5274 Register Src2Reg = I.getOperand(i: 2).getReg();
5275 ArrayRef<int> Mask = I.getOperand(i: 3).getShuffleMask();
5276 assert(DstTy == MRI.getType(Src1Reg) &&
5277 "Expected equal shuffle types during selection");
5278
5279 MachineBasicBlock &MBB = *I.getParent();
5280 MachineFunction &MF = *MBB.getParent();
5281 LLVMContext &Ctx = MF.getFunction().getContext();
5282
5283 unsigned BytesPerElt = DstTy.getElementType().getSizeInBits() / 8;
5284 int NumElts = DstTy.getNumElements();
5285
5286 SmallVector<int> NewMask;
5287 bool FirstUsed = false;
5288 bool SecondUsed = false;
5289 for (int M : Mask) {
5290 // Map any undef or zero lanes to 255.
5291 if (M < 0 || VT->getKnownBits(R: M < NumElts ? Src1Reg : Src2Reg,
5292 DemandedElts: APInt::getOneBitSet(numBits: NumElts, BitNo: M % NumElts))
5293 .isZero()) {
5294 for (unsigned Byte = 0; Byte < BytesPerElt; ++Byte)
5295 NewMask.push_back(Elt: 255);
5296 continue;
5297 }
5298
5299 FirstUsed |= M < NumElts;
5300 SecondUsed |= M >= NumElts;
5301 for (unsigned Byte = 0; Byte < BytesPerElt; ++Byte) {
5302 unsigned Offset = Byte + M * BytesPerElt;
5303 NewMask.push_back(Elt: Offset);
5304 }
5305 }
5306
5307 // If the first is unused or all zeros, use the second src in a tbl1.
5308 if (!FirstUsed) {
5309 int ByteLanes = DstTy.getSizeInBits() == 128 ? 16 : 8;
5310 for (int &M : NewMask) {
5311 if (M != 255) {
5312 assert(M >= ByteLanes && M < 2 * ByteLanes);
5313 M -= ByteLanes;
5314 }
5315 }
5316 std::swap(a&: Src1Reg, b&: Src2Reg);
5317 std::swap(a&: FirstUsed, b&: SecondUsed);
5318 }
5319
5320 // Use a constant pool to load the index vector for TBL.
5321 SmallVector<Constant *> CstIdxs;
5322 transform(Range&: NewMask, d_first: std::back_inserter(x&: CstIdxs), F: [&Ctx](int M) {
5323 return ConstantInt::get(Ty: Type::getInt8Ty(C&: Ctx), V: M);
5324 });
5325 Constant *CPVal = ConstantVector::get(V: CstIdxs);
5326 MachineInstr *IndexLoad = emitLoadFromConstantPool(CPVal, MIRBuilder&: MIB);
5327 if (!IndexLoad) {
5328 LLVM_DEBUG(dbgs() << "Could not load from a constant pool");
5329 return false;
5330 }
5331
5332 if (DstTy.getSizeInBits() != 128) {
5333 assert(DstTy.getSizeInBits() == 64 && "Unexpected shuffle result ty");
5334 // This case can be done with TBL1.
5335 MachineInstr *Concat =
5336 emitVectorConcat(Dst: std::nullopt, Op1: Src1Reg, Op2: Src2Reg, MIRBuilder&: MIB);
5337 if (!Concat) {
5338 LLVM_DEBUG(dbgs() << "Could not do vector concat for tbl1");
5339 return false;
5340 }
5341
5342 // The constant pool load will be 64 bits, so need to convert to FPR128 reg.
5343 IndexLoad = emitScalarToVector(EltSize: 64, DstRC: &AArch64::FPR128RegClass,
5344 Scalar: IndexLoad->getOperand(i: 0).getReg(), MIRBuilder&: MIB);
5345
5346 auto TBL1 = MIB.buildInstr(
5347 Opc: AArch64::TBLv16i8One, DstOps: {&AArch64::FPR128RegClass},
5348 SrcOps: {Concat->getOperand(i: 0).getReg(), IndexLoad->getOperand(i: 0).getReg()});
5349 constrainSelectedInstRegOperands(I&: *TBL1, TII, TRI, RBI);
5350
5351 auto Copy =
5352 MIB.buildInstr(Opc: TargetOpcode::COPY, DstOps: {I.getOperand(i: 0).getReg()}, SrcOps: {})
5353 .addReg(RegNo: TBL1.getReg(Idx: 0), Flags: {}, SubReg: AArch64::dsub);
5354 RBI.constrainGenericRegister(Reg: Copy.getReg(Idx: 0), RC: AArch64::FPR64RegClass, MRI);
5355 I.eraseFromParent();
5356 return true;
5357 }
5358
5359 if (!SecondUsed) {
5360 auto TBL1 = MIB.buildInstr(Opc: AArch64::TBLv16i8One, DstOps: {I.getOperand(i: 0)},
5361 SrcOps: {Src1Reg, IndexLoad->getOperand(i: 0)});
5362 constrainSelectedInstRegOperands(I&: *TBL1, TII, TRI, RBI);
5363 I.eraseFromParent();
5364 return true;
5365 }
5366
5367 // For TBL2 we need to emit a REG_SEQUENCE to tie together two consecutive
5368 // Q registers for regalloc.
5369 SmallVector<Register, 2> Regs = {Src1Reg, Src2Reg};
5370 auto RegSeq = createQTuple(Regs, MIB);
5371 auto TBL2 = MIB.buildInstr(Opc: AArch64::TBLv16i8Two, DstOps: {I.getOperand(i: 0)},
5372 SrcOps: {RegSeq, IndexLoad->getOperand(i: 0)});
5373 constrainSelectedInstRegOperands(I&: *TBL2, TII, TRI, RBI);
5374 I.eraseFromParent();
5375 return true;
5376}
5377
5378MachineInstr *AArch64InstructionSelector::emitLaneInsert(
5379 std::optional<Register> DstReg, Register SrcReg, Register EltReg,
5380 unsigned LaneIdx, const RegisterBank &RB,
5381 MachineIRBuilder &MIRBuilder) const {
5382 MachineInstr *InsElt = nullptr;
5383 const TargetRegisterClass *DstRC = &AArch64::FPR128RegClass;
5384 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
5385
5386 // Create a register to define with the insert if one wasn't passed in.
5387 if (!DstReg)
5388 DstReg = MRI.createVirtualRegister(RegClass: DstRC);
5389
5390 unsigned EltSize = MRI.getType(Reg: EltReg).getSizeInBits();
5391 unsigned Opc = getInsertVecEltOpInfo(RB, EltSize).first;
5392
5393 if (RB.getID() == AArch64::FPRRegBankID) {
5394 auto InsSub = emitScalarToVector(EltSize, DstRC, Scalar: EltReg, MIRBuilder);
5395 InsElt = MIRBuilder.buildInstr(Opc, DstOps: {*DstReg}, SrcOps: {SrcReg})
5396 .addImm(Val: LaneIdx)
5397 .addUse(RegNo: InsSub->getOperand(i: 0).getReg())
5398 .addImm(Val: 0);
5399 } else {
5400 InsElt = MIRBuilder.buildInstr(Opc, DstOps: {*DstReg}, SrcOps: {SrcReg})
5401 .addImm(Val: LaneIdx)
5402 .addUse(RegNo: EltReg);
5403 }
5404
5405 constrainSelectedInstRegOperands(I&: *InsElt, TII, TRI, RBI);
5406 return InsElt;
5407}
5408
5409bool AArch64InstructionSelector::selectUSMovFromExtend(
5410 MachineInstr &MI, MachineRegisterInfo &MRI) {
5411 if (MI.getOpcode() != TargetOpcode::G_SEXT &&
5412 MI.getOpcode() != TargetOpcode::G_ZEXT &&
5413 MI.getOpcode() != TargetOpcode::G_ANYEXT)
5414 return false;
5415 bool IsSigned = MI.getOpcode() == TargetOpcode::G_SEXT;
5416 const Register DefReg = MI.getOperand(i: 0).getReg();
5417 const LLT DstTy = MRI.getType(Reg: DefReg);
5418 unsigned DstSize = DstTy.getSizeInBits();
5419
5420 if (DstSize != 32 && DstSize != 64)
5421 return false;
5422
5423 MachineInstr *Extract = getOpcodeDef(Opcode: TargetOpcode::G_EXTRACT_VECTOR_ELT,
5424 Reg: MI.getOperand(i: 1).getReg(), MRI);
5425 int64_t Lane;
5426 if (!Extract || !mi_match(R: Extract->getOperand(i: 2).getReg(), MRI, P: m_ICst(Cst&: Lane)))
5427 return false;
5428 Register Src0 = Extract->getOperand(i: 1).getReg();
5429
5430 const LLT VecTy = MRI.getType(Reg: Src0);
5431 if (VecTy.isScalableVector())
5432 return false;
5433
5434 if (VecTy.getSizeInBits() != 128) {
5435 const MachineInstr *ScalarToVector = emitScalarToVector(
5436 EltSize: VecTy.getSizeInBits(), DstRC: &AArch64::FPR128RegClass, Scalar: Src0, MIRBuilder&: MIB);
5437 assert(ScalarToVector && "Didn't expect emitScalarToVector to fail!");
5438 Src0 = ScalarToVector->getOperand(i: 0).getReg();
5439 }
5440
5441 unsigned Opcode;
5442 if (DstSize == 64 && VecTy.getScalarSizeInBits() == 32)
5443 Opcode = IsSigned ? AArch64::SMOVvi32to64 : AArch64::UMOVvi32;
5444 else if (DstSize == 64 && VecTy.getScalarSizeInBits() == 16)
5445 Opcode = IsSigned ? AArch64::SMOVvi16to64 : AArch64::UMOVvi16;
5446 else if (DstSize == 64 && VecTy.getScalarSizeInBits() == 8)
5447 Opcode = IsSigned ? AArch64::SMOVvi8to64 : AArch64::UMOVvi8;
5448 else if (DstSize == 32 && VecTy.getScalarSizeInBits() == 16)
5449 Opcode = IsSigned ? AArch64::SMOVvi16to32 : AArch64::UMOVvi16;
5450 else if (DstSize == 32 && VecTy.getScalarSizeInBits() == 8)
5451 Opcode = IsSigned ? AArch64::SMOVvi8to32 : AArch64::UMOVvi8;
5452 else
5453 llvm_unreachable("Unexpected type combo for S/UMov!");
5454
5455 // We may need to generate one of these, depending on the type and sign of the
5456 // input:
5457 // DstReg = SMOV Src0, Lane;
5458 // NewReg = UMOV Src0, Lane; DstReg = SUBREG_TO_REG NewReg, sub_32;
5459 MachineInstr *ExtI = nullptr;
5460 if (DstSize == 64 && !IsSigned) {
5461 Register NewReg = MRI.createVirtualRegister(RegClass: &AArch64::GPR32RegClass);
5462 MIB.buildInstr(Opc: Opcode, DstOps: {NewReg}, SrcOps: {Src0}).addImm(Val: Lane);
5463 ExtI = MIB.buildInstr(Opc: AArch64::SUBREG_TO_REG, DstOps: {DefReg}, SrcOps: {})
5464 .addUse(RegNo: NewReg)
5465 .addImm(Val: AArch64::sub_32);
5466 RBI.constrainGenericRegister(Reg: DefReg, RC: AArch64::GPR64RegClass, MRI);
5467 } else
5468 ExtI = MIB.buildInstr(Opc: Opcode, DstOps: {DefReg}, SrcOps: {Src0}).addImm(Val: Lane);
5469
5470 constrainSelectedInstRegOperands(I&: *ExtI, TII, TRI, RBI);
5471 MI.eraseFromParent();
5472 return true;
5473}
5474
5475bool AArch64InstructionSelector::selectIndexedExtLoad(
5476 MachineInstr &MI, MachineRegisterInfo &MRI) {
5477 auto &ExtLd = cast<GIndexedAnyExtLoad>(Val&: MI);
5478 Register Dst = ExtLd.getDstReg();
5479 Register WriteBack = ExtLd.getWritebackReg();
5480 Register Base = ExtLd.getBaseReg();
5481 Register Offset = ExtLd.getOffsetReg();
5482 LLT Ty = MRI.getType(Reg: Dst);
5483 assert(Ty.getSizeInBits() <= 64); // Only for scalar GPRs.
5484 unsigned MemSizeBits = ExtLd.getMMO().getMemoryType().getSizeInBits();
5485 bool IsPre = ExtLd.isPre();
5486 bool IsSExt = isa<GIndexedSExtLoad>(Val: ExtLd);
5487 unsigned InsertIntoSubReg = 0;
5488 bool IsDst64 = Ty.getSizeInBits() == 64;
5489
5490 // ZExt/SExt should be on gpr but can handle extload and zextload of fpr, so
5491 // long as they are scalar.
5492 bool IsFPR = RBI.getRegBank(Reg: Dst, MRI, TRI)->getID() == AArch64::FPRRegBankID;
5493 if ((IsSExt && IsFPR) || Ty.isVector())
5494 return false;
5495
5496 unsigned Opc = 0;
5497 LLT NewLdDstTy;
5498 LLT s32 = LLT::scalar(SizeInBits: 32);
5499 LLT s64 = LLT::scalar(SizeInBits: 64);
5500
5501 if (MemSizeBits == 8) {
5502 if (IsSExt) {
5503 if (IsDst64)
5504 Opc = IsPre ? AArch64::LDRSBXpre : AArch64::LDRSBXpost;
5505 else
5506 Opc = IsPre ? AArch64::LDRSBWpre : AArch64::LDRSBWpost;
5507 NewLdDstTy = IsDst64 ? s64 : s32;
5508 } else if (IsFPR) {
5509 Opc = IsPre ? AArch64::LDRBpre : AArch64::LDRBpost;
5510 InsertIntoSubReg = AArch64::bsub;
5511 NewLdDstTy = LLT::scalar(SizeInBits: MemSizeBits);
5512 } else {
5513 Opc = IsPre ? AArch64::LDRBBpre : AArch64::LDRBBpost;
5514 InsertIntoSubReg = IsDst64 ? AArch64::sub_32 : 0;
5515 NewLdDstTy = s32;
5516 }
5517 } else if (MemSizeBits == 16) {
5518 if (IsSExt) {
5519 if (IsDst64)
5520 Opc = IsPre ? AArch64::LDRSHXpre : AArch64::LDRSHXpost;
5521 else
5522 Opc = IsPre ? AArch64::LDRSHWpre : AArch64::LDRSHWpost;
5523 NewLdDstTy = IsDst64 ? s64 : s32;
5524 } else if (IsFPR) {
5525 Opc = IsPre ? AArch64::LDRHpre : AArch64::LDRHpost;
5526 InsertIntoSubReg = AArch64::hsub;
5527 NewLdDstTy = LLT::scalar(SizeInBits: MemSizeBits);
5528 } else {
5529 Opc = IsPre ? AArch64::LDRHHpre : AArch64::LDRHHpost;
5530 InsertIntoSubReg = IsDst64 ? AArch64::sub_32 : 0;
5531 NewLdDstTy = s32;
5532 }
5533 } else if (MemSizeBits == 32) {
5534 if (IsSExt) {
5535 Opc = IsPre ? AArch64::LDRSWpre : AArch64::LDRSWpost;
5536 NewLdDstTy = s64;
5537 } else if (IsFPR) {
5538 Opc = IsPre ? AArch64::LDRSpre : AArch64::LDRSpost;
5539 InsertIntoSubReg = AArch64::ssub;
5540 NewLdDstTy = LLT::scalar(SizeInBits: MemSizeBits);
5541 } else {
5542 Opc = IsPre ? AArch64::LDRWpre : AArch64::LDRWpost;
5543 InsertIntoSubReg = IsDst64 ? AArch64::sub_32 : 0;
5544 NewLdDstTy = s32;
5545 }
5546 } else {
5547 llvm_unreachable("Unexpected size for indexed load");
5548 }
5549
5550 auto Cst = getIConstantVRegVal(VReg: Offset, MRI);
5551 if (!Cst)
5552 return false; // Shouldn't happen, but just in case.
5553
5554 auto LdMI = MIB.buildInstr(Opc, DstOps: {WriteBack, NewLdDstTy}, SrcOps: {Base})
5555 .addImm(Val: Cst->getSExtValue());
5556 LdMI.cloneMemRefs(OtherMI: ExtLd);
5557 constrainSelectedInstRegOperands(I&: *LdMI, TII, TRI, RBI);
5558 // Make sure to select the load with the MemTy as the dest type, and then
5559 // insert into a larger reg if needed.
5560 if (InsertIntoSubReg) {
5561 // Generate a SUBREG_TO_REG.
5562 auto SubToReg = MIB.buildInstr(Opc: TargetOpcode::SUBREG_TO_REG, DstOps: {Dst}, SrcOps: {})
5563 .addUse(RegNo: LdMI.getReg(Idx: 1))
5564 .addImm(Val: InsertIntoSubReg);
5565 RBI.constrainGenericRegister(
5566 Reg: SubToReg.getReg(Idx: 0),
5567 RC: *getRegClassForTypeOnBank(Ty: MRI.getType(Reg: Dst),
5568 RB: *RBI.getRegBank(Reg: Dst, MRI, TRI)),
5569 MRI);
5570 } else {
5571 auto Copy = MIB.buildCopy(Res: Dst, Op: LdMI.getReg(Idx: 1));
5572 selectCopy(I&: *Copy, TII, MRI, TRI, RBI);
5573 }
5574 MI.eraseFromParent();
5575
5576 return true;
5577}
5578
5579bool AArch64InstructionSelector::selectIndexedLoad(MachineInstr &MI,
5580 MachineRegisterInfo &MRI) {
5581 auto &Ld = cast<GIndexedLoad>(Val&: MI);
5582 Register Dst = Ld.getDstReg();
5583 Register WriteBack = Ld.getWritebackReg();
5584 Register Base = Ld.getBaseReg();
5585 Register Offset = Ld.getOffsetReg();
5586 assert(MRI.getType(Dst).getSizeInBits() <= 128 &&
5587 "Unexpected type for indexed load");
5588 unsigned MemSize = Ld.getMMO().getMemoryType().getSizeInBytes();
5589
5590 if (MemSize < MRI.getType(Reg: Dst).getSizeInBytes())
5591 return selectIndexedExtLoad(MI, MRI);
5592
5593 unsigned Opc = 0;
5594 if (Ld.isPre()) {
5595 static constexpr unsigned GPROpcodes[] = {
5596 AArch64::LDRBBpre, AArch64::LDRHHpre, AArch64::LDRWpre,
5597 AArch64::LDRXpre};
5598 static constexpr unsigned FPROpcodes[] = {
5599 AArch64::LDRBpre, AArch64::LDRHpre, AArch64::LDRSpre, AArch64::LDRDpre,
5600 AArch64::LDRQpre};
5601 Opc = (RBI.getRegBank(Reg: Dst, MRI, TRI)->getID() == AArch64::FPRRegBankID)
5602 ? FPROpcodes[Log2_32(Value: MemSize)]
5603 : GPROpcodes[Log2_32(Value: MemSize)];
5604 ;
5605 } else {
5606 static constexpr unsigned GPROpcodes[] = {
5607 AArch64::LDRBBpost, AArch64::LDRHHpost, AArch64::LDRWpost,
5608 AArch64::LDRXpost};
5609 static constexpr unsigned FPROpcodes[] = {
5610 AArch64::LDRBpost, AArch64::LDRHpost, AArch64::LDRSpost,
5611 AArch64::LDRDpost, AArch64::LDRQpost};
5612 Opc = (RBI.getRegBank(Reg: Dst, MRI, TRI)->getID() == AArch64::FPRRegBankID)
5613 ? FPROpcodes[Log2_32(Value: MemSize)]
5614 : GPROpcodes[Log2_32(Value: MemSize)];
5615 ;
5616 }
5617 auto Cst = getIConstantVRegVal(VReg: Offset, MRI);
5618 if (!Cst)
5619 return false; // Shouldn't happen, but just in case.
5620 auto LdMI =
5621 MIB.buildInstr(Opc, DstOps: {WriteBack, Dst}, SrcOps: {Base}).addImm(Val: Cst->getSExtValue());
5622 LdMI.cloneMemRefs(OtherMI: Ld);
5623 constrainSelectedInstRegOperands(I&: *LdMI, TII, TRI, RBI);
5624 MI.eraseFromParent();
5625 return true;
5626}
5627
5628bool AArch64InstructionSelector::selectIndexedStore(GIndexedStore &I,
5629 MachineRegisterInfo &MRI) {
5630 Register Dst = I.getWritebackReg();
5631 Register Val = I.getValueReg();
5632 Register Base = I.getBaseReg();
5633 Register Offset = I.getOffsetReg();
5634 assert(MRI.getType(Val).getSizeInBits() <= 128 &&
5635 "Unexpected type for indexed store");
5636
5637 LocationSize MemSize = I.getMMO().getSize();
5638 unsigned MemSizeInBytes = MemSize.getValue();
5639
5640 assert(MemSizeInBytes && MemSizeInBytes <= 16 &&
5641 "Unexpected indexed store size");
5642 unsigned MemSizeLog2 = Log2_32(Value: MemSizeInBytes);
5643
5644 unsigned Opc = 0;
5645 if (I.isPre()) {
5646 static constexpr unsigned GPROpcodes[] = {
5647 AArch64::STRBBpre, AArch64::STRHHpre, AArch64::STRWpre,
5648 AArch64::STRXpre};
5649 static constexpr unsigned FPROpcodes[] = {
5650 AArch64::STRBpre, AArch64::STRHpre, AArch64::STRSpre, AArch64::STRDpre,
5651 AArch64::STRQpre};
5652
5653 if (RBI.getRegBank(Reg: Val, MRI, TRI)->getID() == AArch64::FPRRegBankID)
5654 Opc = FPROpcodes[MemSizeLog2];
5655 else
5656 Opc = GPROpcodes[MemSizeLog2];
5657 } else {
5658 static constexpr unsigned GPROpcodes[] = {
5659 AArch64::STRBBpost, AArch64::STRHHpost, AArch64::STRWpost,
5660 AArch64::STRXpost};
5661 static constexpr unsigned FPROpcodes[] = {
5662 AArch64::STRBpost, AArch64::STRHpost, AArch64::STRSpost,
5663 AArch64::STRDpost, AArch64::STRQpost};
5664
5665 if (RBI.getRegBank(Reg: Val, MRI, TRI)->getID() == AArch64::FPRRegBankID)
5666 Opc = FPROpcodes[MemSizeLog2];
5667 else
5668 Opc = GPROpcodes[MemSizeLog2];
5669 }
5670
5671 auto Cst = getIConstantVRegVal(VReg: Offset, MRI);
5672 if (!Cst)
5673 return false; // Shouldn't happen, but just in case.
5674 auto Str =
5675 MIB.buildInstr(Opc, DstOps: {Dst}, SrcOps: {Val, Base}).addImm(Val: Cst->getSExtValue());
5676 Str.cloneMemRefs(OtherMI: I);
5677 constrainSelectedInstRegOperands(I&: *Str, TII, TRI, RBI);
5678 I.eraseFromParent();
5679 return true;
5680}
5681
5682MachineInstr *
5683AArch64InstructionSelector::emitConstantMOVVector(DstOp Dst, APInt Imm,
5684 MachineRegisterInfo &MRI) {
5685 unsigned DstSize = Imm.getBitWidth();
5686 assert((DstSize == 64 || DstSize == 128) &&
5687 "Unexpected vector constant size");
5688
5689 SmallVector<AArch64_IMM::ImmInsnModel> Insns;
5690 if (AArch64_IMM::expandVectorMOVImm(Imm, Insn&: Insns, ST: STI)) {
5691 for (AArch64_IMM::ImmInsnModel Insn : Insns) {
5692 switch (Insn.Opcode) {
5693 case AArch64::FMOVD0: {
5694 // Currently emit a MOVIv2d_ns in both cases to match SDAG.
5695 auto Mov =
5696 MIB.buildInstr(
5697 Opc: AArch64::MOVIv2d_ns,
5698 DstOps: {DstSize == 128 ? Dst : DstOp(&AArch64::FPR128RegClass)}, SrcOps: {})
5699 .addImm(Val: 0);
5700 constrainSelectedInstRegOperands(I&: *Mov, TII, TRI, RBI);
5701 if (DstSize != 128) {
5702 Mov = MIB.buildInstr(Opc: TargetOpcode::COPY, DstOps: {Dst}, SrcOps: {})
5703 .addReg(RegNo: Mov.getReg(Idx: 0), Flags: {}, SubReg: AArch64::dsub);
5704 RBI.constrainGenericRegister(Reg: Mov.getReg(Idx: 0), RC: AArch64::FPR64RegClass,
5705 MRI);
5706 }
5707 return &*Mov;
5708 }
5709 case AArch64::FMOVDi:
5710 case AArch64::FMOVv2f32_ns:
5711 case AArch64::FMOVv2f64_ns:
5712 case AArch64::FMOVv4f32_ns:
5713 case AArch64::FMOVv4f16_ns:
5714 case AArch64::FMOVv8f16_ns:
5715 case AArch64::MOVID:
5716 case AArch64::MOVIv16b_ns:
5717 case AArch64::MOVIv2d_ns:
5718 case AArch64::MOVIv2i32:
5719 case AArch64::MOVIv2s_msl:
5720 case AArch64::MOVIv4i16:
5721 case AArch64::MOVIv4i32:
5722 case AArch64::MOVIv4s_msl:
5723 case AArch64::MOVIv8b_ns:
5724 case AArch64::MOVIv8i16:
5725 case AArch64::MVNIv2i32:
5726 case AArch64::MVNIv2s_msl:
5727 case AArch64::MVNIv4i16:
5728 case AArch64::MVNIv4i32:
5729 case AArch64::MVNIv4s_msl:
5730 case AArch64::MVNIv8i16: {
5731 auto Mov = MIB.buildInstr(Opc: Insn.Opcode, DstOps: {Dst}, SrcOps: {});
5732 if (Insn.Op1)
5733 Mov.addImm(Val: *Insn.Op1);
5734 if (Insn.Op2)
5735 Mov.addImm(Val: *Insn.Op2);
5736 constrainSelectedInstRegOperands(I&: *Mov, TII, TRI, RBI);
5737 return &*Mov;
5738 }
5739 case AArch64::DUPM_ZI: {
5740 auto Mov = MIB.buildInstr(Opc: Insn.Opcode, DstOps: {&AArch64::ZPRRegClass}, SrcOps: {})
5741 .addImm(Val: *Insn.Op1);
5742 constrainSelectedInstRegOperands(I&: *Mov, TII, TRI, RBI);
5743 Mov = MIB.buildInstr(Opc: TargetOpcode::COPY, DstOps: {Dst}, SrcOps: {})
5744 .addReg(RegNo: Mov.getReg(Idx: 0), Flags: {},
5745 SubReg: DstSize == 64 ? AArch64::dsub : AArch64::zsub);
5746 RBI.constrainGenericRegister(Reg: Mov.getReg(Idx: 0),
5747 RC: DstSize == 64 ? AArch64::FPR64RegClass
5748 : AArch64::FPR128RegClass,
5749 MRI);
5750 return &*Mov;
5751 }
5752 case AArch64::DUP_ZI_D: {
5753 auto Mov = MIB.buildInstr(Opc: Insn.Opcode, DstOps: {&AArch64::ZPRRegClass}, SrcOps: {})
5754 .addImm(Val: *Insn.Op1)
5755 .addImm(Val: *Insn.Op2);
5756 constrainSelectedInstRegOperands(I&: *Mov, TII, TRI, RBI);
5757 Mov = MIB.buildInstr(Opc: TargetOpcode::COPY, DstOps: {Dst}, SrcOps: {})
5758 .addReg(RegNo: Mov.getReg(Idx: 0), Flags: {},
5759 SubReg: DstSize == 64 ? AArch64::dsub : AArch64::zsub);
5760 RBI.constrainGenericRegister(Reg: Mov.getReg(Idx: 0),
5761 RC: DstSize == 64 ? AArch64::FPR64RegClass
5762 : AArch64::FPR128RegClass,
5763 MRI);
5764 return &*Mov;
5765 }
5766 default:
5767 llvm_unreachable("Unexpected node in expandVectorMOVImm\n");
5768 }
5769 }
5770 }
5771 return nullptr;
5772}
5773
5774MachineInstr *AArch64InstructionSelector::emitConstantVector(
5775 Register Dst, Constant *CV, APInt Imm, MachineRegisterInfo &MRI) {
5776 if (auto *MovI = emitConstantMOVVector(Dst, Imm, MRI))
5777 return MovI;
5778
5779 auto *CPLoad = emitLoadFromConstantPool(CPVal: CV, MIRBuilder&: MIB);
5780 if (!CPLoad) {
5781 LLVM_DEBUG(dbgs() << "Could not generate cp load for constant vector!");
5782 return nullptr;
5783 }
5784
5785 auto Copy = MIB.buildCopy(Res: Dst, Op: CPLoad->getOperand(i: 0));
5786 RBI.constrainGenericRegister(
5787 Reg: Dst, RC: *MRI.getRegClass(Reg: CPLoad->getOperand(i: 0).getReg()), MRI);
5788 return &*Copy;
5789}
5790
5791bool AArch64InstructionSelector::tryOptConstantBuildVec(
5792 MachineInstr &I, LLT DstTy, MachineRegisterInfo &MRI) {
5793 assert(I.getOpcode() == TargetOpcode::G_BUILD_VECTOR);
5794 unsigned DstSize = DstTy.getSizeInBits();
5795 assert(DstSize <= 128 && "Unexpected build_vec type!");
5796 if (DstSize < 32)
5797 return false;
5798 // Check if we're building a constant vector, in which case we want to
5799 // generate a constant pool load instead of a vector insert sequence.
5800 SmallVector<Constant *, 16> Csts;
5801 APInt Imm(DstSize, 0);
5802 for (unsigned Idx = 1; Idx < I.getNumOperands(); ++Idx) {
5803 Register OpReg = I.getOperand(i: Idx).getReg();
5804 if (auto AnyConst = getAnyConstantVRegValWithLookThrough(
5805 VReg: OpReg, MRI, /*LookThroughInstrs=*/true,
5806 /*LookThroughAnyExt=*/true)) {
5807 MachineInstr *DefMI = MRI.getVRegDef(Reg: AnyConst->VReg);
5808 Imm |= AnyConst->Value.trunc(width: DstTy.getScalarSizeInBits())
5809 .zext(width: DstSize)
5810 .shl(shiftAmt: (Idx - 1) * DstTy.getScalarSizeInBits());
5811
5812 if (DefMI->getOpcode() == TargetOpcode::G_CONSTANT) {
5813 Csts.emplace_back(
5814 Args: ConstantInt::get(Context&: MIB.getMF().getFunction().getContext(),
5815 V: std::move(AnyConst->Value)));
5816 continue;
5817 }
5818
5819 if (DefMI->getOpcode() == TargetOpcode::G_FCONSTANT) {
5820 Csts.emplace_back(
5821 Args: const_cast<ConstantFP *>(DefMI->getOperand(i: 1).getFPImm()));
5822 continue;
5823 }
5824 }
5825 return false;
5826 }
5827 Constant *CV = ConstantVector::get(V: Csts);
5828 if (!emitConstantVector(Dst: I.getOperand(i: 0).getReg(), CV, Imm, MRI))
5829 return false;
5830 I.eraseFromParent();
5831 return true;
5832}
5833
5834bool AArch64InstructionSelector::tryOptBuildVecToSubregToReg(
5835 MachineInstr &I, MachineRegisterInfo &MRI) {
5836 // Given:
5837 // %vec = G_BUILD_VECTOR %elt, %undef, %undef, ... %undef
5838 //
5839 // Select the G_BUILD_VECTOR as a SUBREG_TO_REG from %elt.
5840 Register Dst = I.getOperand(i: 0).getReg();
5841 Register EltReg = I.getOperand(i: 1).getReg();
5842 LLT EltTy = MRI.getType(Reg: EltReg);
5843 // If the index isn't on the same bank as its elements, then this can't be a
5844 // SUBREG_TO_REG.
5845 const RegisterBank &EltRB = *RBI.getRegBank(Reg: EltReg, MRI, TRI);
5846 const RegisterBank &DstRB = *RBI.getRegBank(Reg: Dst, MRI, TRI);
5847 if (EltRB != DstRB)
5848 return false;
5849 if (any_of(Range: drop_begin(RangeOrContainer: I.operands(), N: 2), P: [&MRI](const MachineOperand &Op) {
5850 return !getOpcodeDef(Opcode: TargetOpcode::G_IMPLICIT_DEF, Reg: Op.getReg(), MRI);
5851 }))
5852 return false;
5853 unsigned SubReg;
5854 const TargetRegisterClass *EltRC = getRegClassForTypeOnBank(Ty: EltTy, RB: EltRB);
5855 if (!EltRC)
5856 return false;
5857 const TargetRegisterClass *DstRC =
5858 getRegClassForTypeOnBank(Ty: MRI.getType(Reg: Dst), RB: DstRB);
5859 if (!DstRC)
5860 return false;
5861 if (!getSubRegForClass(RC: EltRC, TRI, SubReg))
5862 return false;
5863 auto SubregToReg = MIB.buildInstr(Opc: AArch64::SUBREG_TO_REG, DstOps: {Dst}, SrcOps: {})
5864 .addUse(RegNo: EltReg)
5865 .addImm(Val: SubReg);
5866 I.eraseFromParent();
5867 constrainSelectedInstRegOperands(I&: *SubregToReg, TII, TRI, RBI);
5868 return RBI.constrainGenericRegister(Reg: Dst, RC: *DstRC, MRI);
5869}
5870
5871bool AArch64InstructionSelector::selectBuildVector(MachineInstr &I,
5872 MachineRegisterInfo &MRI) {
5873 assert(I.getOpcode() == TargetOpcode::G_BUILD_VECTOR);
5874 // Until we port more of the optimized selections, for now just use a vector
5875 // insert sequence.
5876 const LLT DstTy = MRI.getType(Reg: I.getOperand(i: 0).getReg());
5877 const LLT EltTy = MRI.getType(Reg: I.getOperand(i: 1).getReg());
5878 unsigned EltSize = EltTy.getSizeInBits();
5879
5880 if (tryOptConstantBuildVec(I, DstTy, MRI))
5881 return true;
5882 if (tryOptBuildVecToSubregToReg(I, MRI))
5883 return true;
5884
5885 if (EltSize != 8 && EltSize != 16 && EltSize != 32 && EltSize != 64)
5886 return false; // Don't support all element types yet.
5887 const RegisterBank &RB = *RBI.getRegBank(Reg: I.getOperand(i: 1).getReg(), MRI, TRI);
5888
5889 const TargetRegisterClass *DstRC = &AArch64::FPR128RegClass;
5890 MachineInstr *ScalarToVec =
5891 emitScalarToVector(EltSize: DstTy.getElementType().getSizeInBits(), DstRC,
5892 Scalar: I.getOperand(i: 1).getReg(), MIRBuilder&: MIB);
5893 if (!ScalarToVec)
5894 return false;
5895
5896 Register DstVec = ScalarToVec->getOperand(i: 0).getReg();
5897 unsigned DstSize = DstTy.getSizeInBits();
5898
5899 // Keep track of the last MI we inserted. Later on, we might be able to save
5900 // a copy using it.
5901 MachineInstr *PrevMI = ScalarToVec;
5902 for (unsigned i = 2, e = DstSize / EltSize + 1; i < e; ++i) {
5903 // Note that if we don't do a subregister copy, we can end up making an
5904 // extra register.
5905 Register OpReg = I.getOperand(i).getReg();
5906 // Do not emit inserts for undefs
5907 if (!getOpcodeDef<GImplicitDef>(Reg: OpReg, MRI)) {
5908 PrevMI = &*emitLaneInsert(DstReg: std::nullopt, SrcReg: DstVec, EltReg: OpReg, LaneIdx: i - 1, RB, MIRBuilder&: MIB);
5909 DstVec = PrevMI->getOperand(i: 0).getReg();
5910 }
5911 }
5912
5913 // If DstTy's size in bits is less than 128, then emit a subregister copy
5914 // from DstVec to the last register we've defined.
5915 if (DstSize < 128) {
5916 // Force this to be FPR using the destination vector.
5917 const TargetRegisterClass *RC =
5918 getRegClassForTypeOnBank(Ty: DstTy, RB: *RBI.getRegBank(Reg: DstVec, MRI, TRI));
5919 if (!RC)
5920 return false;
5921 if (RC != &AArch64::FPR32RegClass && RC != &AArch64::FPR64RegClass) {
5922 LLVM_DEBUG(dbgs() << "Unsupported register class!\n");
5923 return false;
5924 }
5925
5926 unsigned SubReg = 0;
5927 if (!getSubRegForClass(RC, TRI, SubReg))
5928 return false;
5929 if (SubReg != AArch64::ssub && SubReg != AArch64::dsub) {
5930 LLVM_DEBUG(dbgs() << "Unsupported destination size! (" << DstSize
5931 << "\n");
5932 return false;
5933 }
5934
5935 Register Reg = MRI.createVirtualRegister(RegClass: RC);
5936 Register DstReg = I.getOperand(i: 0).getReg();
5937
5938 MIB.buildInstr(Opc: TargetOpcode::COPY, DstOps: {DstReg}, SrcOps: {}).addReg(RegNo: DstVec, Flags: {}, SubReg);
5939 MachineOperand &RegOp = I.getOperand(i: 1);
5940 RegOp.setReg(Reg);
5941 RBI.constrainGenericRegister(Reg: DstReg, RC: *RC, MRI);
5942 } else {
5943 // We either have a vector with all elements (except the first one) undef or
5944 // at least one non-undef non-first element. In the first case, we need to
5945 // constrain the output register ourselves as we may have generated an
5946 // INSERT_SUBREG operation which is a generic operation for which the
5947 // output regclass cannot be automatically chosen.
5948 //
5949 // In the second case, there is no need to do this as it may generate an
5950 // instruction like INSvi32gpr where the regclass can be automatically
5951 // chosen.
5952 //
5953 // Also, we save a copy by re-using the destination register on the final
5954 // insert.
5955 PrevMI->getOperand(i: 0).setReg(I.getOperand(i: 0).getReg());
5956 constrainSelectedInstRegOperands(I&: *PrevMI, TII, TRI, RBI);
5957
5958 Register DstReg = PrevMI->getOperand(i: 0).getReg();
5959 if (PrevMI == ScalarToVec && DstReg.isVirtual()) {
5960 const TargetRegisterClass *RC =
5961 getRegClassForTypeOnBank(Ty: DstTy, RB: *RBI.getRegBank(Reg: DstVec, MRI, TRI));
5962 RBI.constrainGenericRegister(Reg: DstReg, RC: *RC, MRI);
5963 }
5964 }
5965
5966 I.eraseFromParent();
5967 return true;
5968}
5969
5970bool AArch64InstructionSelector::selectVectorLoadIntrinsic(unsigned Opc,
5971 unsigned NumVecs,
5972 MachineInstr &I) {
5973 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS);
5974 assert(Opc && "Expected an opcode?");
5975 assert(NumVecs > 1 && NumVecs < 5 && "Only support 2, 3, or 4 vectors");
5976 auto &MRI = *MIB.getMRI();
5977 LLT Ty = MRI.getType(Reg: I.getOperand(i: 0).getReg());
5978 unsigned Size = Ty.getSizeInBits();
5979 assert((Size == 64 || Size == 128) &&
5980 "Destination must be 64 bits or 128 bits?");
5981 unsigned SubReg = Size == 64 ? AArch64::dsub0 : AArch64::qsub0;
5982 auto Ptr = I.getOperand(i: I.getNumOperands() - 1).getReg();
5983 assert(MRI.getType(Ptr).isPointer() && "Expected a pointer type?");
5984 auto Load = MIB.buildInstr(Opc, DstOps: {Ty}, SrcOps: {Ptr});
5985 Load.cloneMemRefs(OtherMI: I);
5986 constrainSelectedInstRegOperands(I&: *Load, TII, TRI, RBI);
5987 Register SelectedLoadDst = Load->getOperand(i: 0).getReg();
5988 for (unsigned Idx = 0; Idx < NumVecs; ++Idx) {
5989 auto Vec = MIB.buildInstr(Opc: TargetOpcode::COPY, DstOps: {I.getOperand(i: Idx)}, SrcOps: {})
5990 .addReg(RegNo: SelectedLoadDst, Flags: {}, SubReg: SubReg + Idx);
5991 // Emit the subreg copies and immediately select them.
5992 // FIXME: We should refactor our copy code into an emitCopy helper and
5993 // clean up uses of this pattern elsewhere in the selector.
5994 selectCopy(I&: *Vec, TII, MRI, TRI, RBI);
5995 }
5996 return true;
5997}
5998
5999bool AArch64InstructionSelector::selectVectorLoadLaneIntrinsic(
6000 unsigned Opc, unsigned NumVecs, MachineInstr &I) {
6001 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS);
6002 assert(Opc && "Expected an opcode?");
6003 assert(NumVecs > 1 && NumVecs < 5 && "Only support 2, 3, or 4 vectors");
6004 auto &MRI = *MIB.getMRI();
6005 LLT Ty = MRI.getType(Reg: I.getOperand(i: 0).getReg());
6006 bool Narrow = Ty.getSizeInBits() == 64;
6007
6008 auto FirstSrcRegIt = I.operands_begin() + NumVecs + 1;
6009 SmallVector<Register, 4> Regs(NumVecs);
6010 std::transform(first: FirstSrcRegIt, last: FirstSrcRegIt + NumVecs, result: Regs.begin(),
6011 unary_op: [](auto MO) { return MO.getReg(); });
6012
6013 if (Narrow) {
6014 transform(Range&: Regs, d_first: Regs.begin(), F: [this](Register Reg) {
6015 return emitScalarToVector(EltSize: 64, DstRC: &AArch64::FPR128RegClass, Scalar: Reg, MIRBuilder&: MIB)
6016 ->getOperand(i: 0)
6017 .getReg();
6018 });
6019 Ty = Ty.multiplyElements(Factor: 2);
6020 }
6021
6022 Register Tuple = createQTuple(Regs, MIB);
6023 auto LaneNo = getIConstantVRegVal(VReg: (FirstSrcRegIt + NumVecs)->getReg(), MRI);
6024 if (!LaneNo)
6025 return false;
6026
6027 Register Ptr = (FirstSrcRegIt + NumVecs + 1)->getReg();
6028 auto Load = MIB.buildInstr(Opc, DstOps: {Ty}, SrcOps: {})
6029 .addReg(RegNo: Tuple)
6030 .addImm(Val: LaneNo->getZExtValue())
6031 .addReg(RegNo: Ptr);
6032 Load.cloneMemRefs(OtherMI: I);
6033 constrainSelectedInstRegOperands(I&: *Load, TII, TRI, RBI);
6034 Register SelectedLoadDst = Load->getOperand(i: 0).getReg();
6035 unsigned SubReg = AArch64::qsub0;
6036 for (unsigned Idx = 0; Idx < NumVecs; ++Idx) {
6037 auto Vec = MIB.buildInstr(Opc: TargetOpcode::COPY,
6038 DstOps: {Narrow ? DstOp(&AArch64::FPR128RegClass)
6039 : DstOp(I.getOperand(i: Idx).getReg())},
6040 SrcOps: {})
6041 .addReg(RegNo: SelectedLoadDst, Flags: {}, SubReg: SubReg + Idx);
6042 Register WideReg = Vec.getReg(Idx: 0);
6043 // Emit the subreg copies and immediately select them.
6044 selectCopy(I&: *Vec, TII, MRI, TRI, RBI);
6045 if (Narrow &&
6046 !emitNarrowVector(DstReg: I.getOperand(i: Idx).getReg(), SrcReg: WideReg, MIB, MRI))
6047 return false;
6048 }
6049 return true;
6050}
6051
6052void AArch64InstructionSelector::selectVectorStoreIntrinsic(MachineInstr &I,
6053 unsigned NumVecs,
6054 unsigned Opc) {
6055 MachineRegisterInfo &MRI = I.getParent()->getParent()->getRegInfo();
6056 LLT Ty = MRI.getType(Reg: I.getOperand(i: 1).getReg());
6057 Register Ptr = I.getOperand(i: 1 + NumVecs).getReg();
6058
6059 SmallVector<Register, 2> Regs(NumVecs);
6060 std::transform(first: I.operands_begin() + 1, last: I.operands_begin() + 1 + NumVecs,
6061 result: Regs.begin(), unary_op: [](auto MO) { return MO.getReg(); });
6062
6063 Register Tuple = Ty.getSizeInBits() == 128 ? createQTuple(Regs, MIB)
6064 : createDTuple(Regs, MIB);
6065 auto Store = MIB.buildInstr(Opc, DstOps: {}, SrcOps: {Tuple, Ptr});
6066 Store.cloneMemRefs(OtherMI: I);
6067 constrainSelectedInstRegOperands(I&: *Store, TII, TRI, RBI);
6068}
6069
6070bool AArch64InstructionSelector::selectVectorStoreLaneIntrinsic(
6071 MachineInstr &I, unsigned NumVecs, unsigned Opc) {
6072 MachineRegisterInfo &MRI = I.getParent()->getParent()->getRegInfo();
6073 LLT Ty = MRI.getType(Reg: I.getOperand(i: 1).getReg());
6074 bool Narrow = Ty.getSizeInBits() == 64;
6075
6076 SmallVector<Register, 2> Regs(NumVecs);
6077 std::transform(first: I.operands_begin() + 1, last: I.operands_begin() + 1 + NumVecs,
6078 result: Regs.begin(), unary_op: [](auto MO) { return MO.getReg(); });
6079
6080 if (Narrow)
6081 transform(Range&: Regs, d_first: Regs.begin(), F: [this](Register Reg) {
6082 return emitScalarToVector(EltSize: 64, DstRC: &AArch64::FPR128RegClass, Scalar: Reg, MIRBuilder&: MIB)
6083 ->getOperand(i: 0)
6084 .getReg();
6085 });
6086
6087 Register Tuple = createQTuple(Regs, MIB);
6088
6089 auto LaneNo = getIConstantVRegVal(VReg: I.getOperand(i: 1 + NumVecs).getReg(), MRI);
6090 if (!LaneNo)
6091 return false;
6092 Register Ptr = I.getOperand(i: 1 + NumVecs + 1).getReg();
6093 auto Store = MIB.buildInstr(Opc, DstOps: {}, SrcOps: {})
6094 .addReg(RegNo: Tuple)
6095 .addImm(Val: LaneNo->getZExtValue())
6096 .addReg(RegNo: Ptr);
6097 Store.cloneMemRefs(OtherMI: I);
6098 constrainSelectedInstRegOperands(I&: *Store, TII, TRI, RBI);
6099 return true;
6100}
6101
6102bool AArch64InstructionSelector::selectIntrinsicWithSideEffects(
6103 MachineInstr &I, MachineRegisterInfo &MRI) {
6104 // Find the intrinsic ID.
6105 unsigned IntrinID = cast<GIntrinsic>(Val&: I).getIntrinsicID();
6106
6107 const LLT S8 = LLT::scalar(SizeInBits: 8);
6108 const LLT S16 = LLT::scalar(SizeInBits: 16);
6109 const LLT S32 = LLT::scalar(SizeInBits: 32);
6110 const LLT S64 = LLT::scalar(SizeInBits: 64);
6111 const LLT P0 = LLT::pointer(AddressSpace: 0, SizeInBits: 64);
6112 // Select the instruction.
6113 switch (IntrinID) {
6114 default:
6115 return false;
6116 case Intrinsic::aarch64_ldxp:
6117 case Intrinsic::aarch64_ldaxp: {
6118 auto NewI = MIB.buildInstr(
6119 Opc: IntrinID == Intrinsic::aarch64_ldxp ? AArch64::LDXPX : AArch64::LDAXPX,
6120 DstOps: {I.getOperand(i: 0).getReg(), I.getOperand(i: 1).getReg()},
6121 SrcOps: {I.getOperand(i: 3)});
6122 NewI.cloneMemRefs(OtherMI: I);
6123 constrainSelectedInstRegOperands(I&: *NewI, TII, TRI, RBI);
6124 break;
6125 }
6126 case Intrinsic::aarch64_neon_ld1x2: {
6127 LLT Ty = MRI.getType(Reg: I.getOperand(i: 0).getReg());
6128 unsigned Opc = 0;
6129 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6130 Opc = AArch64::LD1Twov8b;
6131 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6132 Opc = AArch64::LD1Twov16b;
6133 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6134 Opc = AArch64::LD1Twov4h;
6135 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6136 Opc = AArch64::LD1Twov8h;
6137 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6138 Opc = AArch64::LD1Twov2s;
6139 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6140 Opc = AArch64::LD1Twov4s;
6141 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6142 Opc = AArch64::LD1Twov2d;
6143 else if (Ty == S64 || Ty == P0)
6144 Opc = AArch64::LD1Twov1d;
6145 else
6146 llvm_unreachable("Unexpected type for ld1x2!");
6147 selectVectorLoadIntrinsic(Opc, NumVecs: 2, I);
6148 break;
6149 }
6150 case Intrinsic::aarch64_neon_ld1x3: {
6151 LLT Ty = MRI.getType(Reg: I.getOperand(i: 0).getReg());
6152 unsigned Opc = 0;
6153 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6154 Opc = AArch64::LD1Threev8b;
6155 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6156 Opc = AArch64::LD1Threev16b;
6157 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6158 Opc = AArch64::LD1Threev4h;
6159 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6160 Opc = AArch64::LD1Threev8h;
6161 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6162 Opc = AArch64::LD1Threev2s;
6163 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6164 Opc = AArch64::LD1Threev4s;
6165 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6166 Opc = AArch64::LD1Threev2d;
6167 else if (Ty == S64 || Ty == P0)
6168 Opc = AArch64::LD1Threev1d;
6169 else
6170 llvm_unreachable("Unexpected type for ld1x3!");
6171 selectVectorLoadIntrinsic(Opc, NumVecs: 3, I);
6172 break;
6173 }
6174 case Intrinsic::aarch64_neon_ld1x4: {
6175 LLT Ty = MRI.getType(Reg: I.getOperand(i: 0).getReg());
6176 unsigned Opc = 0;
6177 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6178 Opc = AArch64::LD1Fourv8b;
6179 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6180 Opc = AArch64::LD1Fourv16b;
6181 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6182 Opc = AArch64::LD1Fourv4h;
6183 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6184 Opc = AArch64::LD1Fourv8h;
6185 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6186 Opc = AArch64::LD1Fourv2s;
6187 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6188 Opc = AArch64::LD1Fourv4s;
6189 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6190 Opc = AArch64::LD1Fourv2d;
6191 else if (Ty == S64 || Ty == P0)
6192 Opc = AArch64::LD1Fourv1d;
6193 else
6194 llvm_unreachable("Unexpected type for ld1x4!");
6195 selectVectorLoadIntrinsic(Opc, NumVecs: 4, I);
6196 break;
6197 }
6198 case Intrinsic::aarch64_neon_ld2: {
6199 LLT Ty = MRI.getType(Reg: I.getOperand(i: 0).getReg());
6200 unsigned Opc = 0;
6201 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6202 Opc = AArch64::LD2Twov8b;
6203 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6204 Opc = AArch64::LD2Twov16b;
6205 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6206 Opc = AArch64::LD2Twov4h;
6207 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6208 Opc = AArch64::LD2Twov8h;
6209 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6210 Opc = AArch64::LD2Twov2s;
6211 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6212 Opc = AArch64::LD2Twov4s;
6213 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6214 Opc = AArch64::LD2Twov2d;
6215 else if (Ty == S64 || Ty == P0)
6216 Opc = AArch64::LD1Twov1d;
6217 else
6218 llvm_unreachable("Unexpected type for ld2!");
6219 selectVectorLoadIntrinsic(Opc, NumVecs: 2, I);
6220 break;
6221 }
6222 case Intrinsic::aarch64_neon_ld2lane: {
6223 LLT Ty = MRI.getType(Reg: I.getOperand(i: 0).getReg());
6224 unsigned Opc;
6225 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8) || Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6226 Opc = AArch64::LD2i8;
6227 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16) || Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6228 Opc = AArch64::LD2i16;
6229 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32) || Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6230 Opc = AArch64::LD2i32;
6231 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) ||
6232 Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0) || Ty == S64 || Ty == P0)
6233 Opc = AArch64::LD2i64;
6234 else
6235 llvm_unreachable("Unexpected type for ld2lane!");
6236 if (!selectVectorLoadLaneIntrinsic(Opc, NumVecs: 2, I))
6237 return false;
6238 break;
6239 }
6240 case Intrinsic::aarch64_neon_ld2r: {
6241 LLT Ty = MRI.getType(Reg: I.getOperand(i: 0).getReg());
6242 unsigned Opc = 0;
6243 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6244 Opc = AArch64::LD2Rv8b;
6245 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6246 Opc = AArch64::LD2Rv16b;
6247 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6248 Opc = AArch64::LD2Rv4h;
6249 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6250 Opc = AArch64::LD2Rv8h;
6251 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6252 Opc = AArch64::LD2Rv2s;
6253 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6254 Opc = AArch64::LD2Rv4s;
6255 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6256 Opc = AArch64::LD2Rv2d;
6257 else if (Ty == S64 || Ty == P0)
6258 Opc = AArch64::LD2Rv1d;
6259 else
6260 llvm_unreachable("Unexpected type for ld2r!");
6261 selectVectorLoadIntrinsic(Opc, NumVecs: 2, I);
6262 break;
6263 }
6264 case Intrinsic::aarch64_neon_ld3: {
6265 LLT Ty = MRI.getType(Reg: I.getOperand(i: 0).getReg());
6266 unsigned Opc = 0;
6267 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6268 Opc = AArch64::LD3Threev8b;
6269 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6270 Opc = AArch64::LD3Threev16b;
6271 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6272 Opc = AArch64::LD3Threev4h;
6273 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6274 Opc = AArch64::LD3Threev8h;
6275 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6276 Opc = AArch64::LD3Threev2s;
6277 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6278 Opc = AArch64::LD3Threev4s;
6279 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6280 Opc = AArch64::LD3Threev2d;
6281 else if (Ty == S64 || Ty == P0)
6282 Opc = AArch64::LD1Threev1d;
6283 else
6284 llvm_unreachable("Unexpected type for ld3!");
6285 selectVectorLoadIntrinsic(Opc, NumVecs: 3, I);
6286 break;
6287 }
6288 case Intrinsic::aarch64_neon_ld3lane: {
6289 LLT Ty = MRI.getType(Reg: I.getOperand(i: 0).getReg());
6290 unsigned Opc;
6291 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8) || Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6292 Opc = AArch64::LD3i8;
6293 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16) || Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6294 Opc = AArch64::LD3i16;
6295 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32) || Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6296 Opc = AArch64::LD3i32;
6297 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) ||
6298 Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0) || Ty == S64 || Ty == P0)
6299 Opc = AArch64::LD3i64;
6300 else
6301 llvm_unreachable("Unexpected type for ld3lane!");
6302 if (!selectVectorLoadLaneIntrinsic(Opc, NumVecs: 3, I))
6303 return false;
6304 break;
6305 }
6306 case Intrinsic::aarch64_neon_ld3r: {
6307 LLT Ty = MRI.getType(Reg: I.getOperand(i: 0).getReg());
6308 unsigned Opc = 0;
6309 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6310 Opc = AArch64::LD3Rv8b;
6311 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6312 Opc = AArch64::LD3Rv16b;
6313 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6314 Opc = AArch64::LD3Rv4h;
6315 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6316 Opc = AArch64::LD3Rv8h;
6317 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6318 Opc = AArch64::LD3Rv2s;
6319 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6320 Opc = AArch64::LD3Rv4s;
6321 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6322 Opc = AArch64::LD3Rv2d;
6323 else if (Ty == S64 || Ty == P0)
6324 Opc = AArch64::LD3Rv1d;
6325 else
6326 llvm_unreachable("Unexpected type for ld3r!");
6327 selectVectorLoadIntrinsic(Opc, NumVecs: 3, I);
6328 break;
6329 }
6330 case Intrinsic::aarch64_neon_ld4: {
6331 LLT Ty = MRI.getType(Reg: I.getOperand(i: 0).getReg());
6332 unsigned Opc = 0;
6333 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6334 Opc = AArch64::LD4Fourv8b;
6335 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6336 Opc = AArch64::LD4Fourv16b;
6337 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6338 Opc = AArch64::LD4Fourv4h;
6339 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6340 Opc = AArch64::LD4Fourv8h;
6341 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6342 Opc = AArch64::LD4Fourv2s;
6343 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6344 Opc = AArch64::LD4Fourv4s;
6345 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6346 Opc = AArch64::LD4Fourv2d;
6347 else if (Ty == S64 || Ty == P0)
6348 Opc = AArch64::LD1Fourv1d;
6349 else
6350 llvm_unreachable("Unexpected type for ld4!");
6351 selectVectorLoadIntrinsic(Opc, NumVecs: 4, I);
6352 break;
6353 }
6354 case Intrinsic::aarch64_neon_ld4lane: {
6355 LLT Ty = MRI.getType(Reg: I.getOperand(i: 0).getReg());
6356 unsigned Opc;
6357 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8) || Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6358 Opc = AArch64::LD4i8;
6359 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16) || Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6360 Opc = AArch64::LD4i16;
6361 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32) || Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6362 Opc = AArch64::LD4i32;
6363 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) ||
6364 Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0) || Ty == S64 || Ty == P0)
6365 Opc = AArch64::LD4i64;
6366 else
6367 llvm_unreachable("Unexpected type for ld4lane!");
6368 if (!selectVectorLoadLaneIntrinsic(Opc, NumVecs: 4, I))
6369 return false;
6370 break;
6371 }
6372 case Intrinsic::aarch64_neon_ld4r: {
6373 LLT Ty = MRI.getType(Reg: I.getOperand(i: 0).getReg());
6374 unsigned Opc = 0;
6375 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6376 Opc = AArch64::LD4Rv8b;
6377 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6378 Opc = AArch64::LD4Rv16b;
6379 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6380 Opc = AArch64::LD4Rv4h;
6381 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6382 Opc = AArch64::LD4Rv8h;
6383 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6384 Opc = AArch64::LD4Rv2s;
6385 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6386 Opc = AArch64::LD4Rv4s;
6387 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6388 Opc = AArch64::LD4Rv2d;
6389 else if (Ty == S64 || Ty == P0)
6390 Opc = AArch64::LD4Rv1d;
6391 else
6392 llvm_unreachable("Unexpected type for ld4r!");
6393 selectVectorLoadIntrinsic(Opc, NumVecs: 4, I);
6394 break;
6395 }
6396 case Intrinsic::aarch64_neon_st1x2: {
6397 LLT Ty = MRI.getType(Reg: I.getOperand(i: 1).getReg());
6398 unsigned Opc;
6399 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6400 Opc = AArch64::ST1Twov8b;
6401 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6402 Opc = AArch64::ST1Twov16b;
6403 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6404 Opc = AArch64::ST1Twov4h;
6405 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6406 Opc = AArch64::ST1Twov8h;
6407 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6408 Opc = AArch64::ST1Twov2s;
6409 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6410 Opc = AArch64::ST1Twov4s;
6411 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6412 Opc = AArch64::ST1Twov2d;
6413 else if (Ty == S64 || Ty == P0)
6414 Opc = AArch64::ST1Twov1d;
6415 else
6416 llvm_unreachable("Unexpected type for st1x2!");
6417 selectVectorStoreIntrinsic(I, NumVecs: 2, Opc);
6418 break;
6419 }
6420 case Intrinsic::aarch64_neon_st1x3: {
6421 LLT Ty = MRI.getType(Reg: I.getOperand(i: 1).getReg());
6422 unsigned Opc;
6423 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6424 Opc = AArch64::ST1Threev8b;
6425 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6426 Opc = AArch64::ST1Threev16b;
6427 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6428 Opc = AArch64::ST1Threev4h;
6429 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6430 Opc = AArch64::ST1Threev8h;
6431 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6432 Opc = AArch64::ST1Threev2s;
6433 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6434 Opc = AArch64::ST1Threev4s;
6435 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6436 Opc = AArch64::ST1Threev2d;
6437 else if (Ty == S64 || Ty == P0)
6438 Opc = AArch64::ST1Threev1d;
6439 else
6440 llvm_unreachable("Unexpected type for st1x3!");
6441 selectVectorStoreIntrinsic(I, NumVecs: 3, Opc);
6442 break;
6443 }
6444 case Intrinsic::aarch64_neon_st1x4: {
6445 LLT Ty = MRI.getType(Reg: I.getOperand(i: 1).getReg());
6446 unsigned Opc;
6447 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6448 Opc = AArch64::ST1Fourv8b;
6449 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6450 Opc = AArch64::ST1Fourv16b;
6451 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6452 Opc = AArch64::ST1Fourv4h;
6453 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6454 Opc = AArch64::ST1Fourv8h;
6455 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6456 Opc = AArch64::ST1Fourv2s;
6457 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6458 Opc = AArch64::ST1Fourv4s;
6459 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6460 Opc = AArch64::ST1Fourv2d;
6461 else if (Ty == S64 || Ty == P0)
6462 Opc = AArch64::ST1Fourv1d;
6463 else
6464 llvm_unreachable("Unexpected type for st1x4!");
6465 selectVectorStoreIntrinsic(I, NumVecs: 4, Opc);
6466 break;
6467 }
6468 case Intrinsic::aarch64_neon_st2: {
6469 LLT Ty = MRI.getType(Reg: I.getOperand(i: 1).getReg());
6470 unsigned Opc;
6471 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6472 Opc = AArch64::ST2Twov8b;
6473 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6474 Opc = AArch64::ST2Twov16b;
6475 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6476 Opc = AArch64::ST2Twov4h;
6477 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6478 Opc = AArch64::ST2Twov8h;
6479 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6480 Opc = AArch64::ST2Twov2s;
6481 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6482 Opc = AArch64::ST2Twov4s;
6483 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6484 Opc = AArch64::ST2Twov2d;
6485 else if (Ty == S64 || Ty == P0)
6486 Opc = AArch64::ST1Twov1d;
6487 else
6488 llvm_unreachable("Unexpected type for st2!");
6489 selectVectorStoreIntrinsic(I, NumVecs: 2, Opc);
6490 break;
6491 }
6492 case Intrinsic::aarch64_neon_st3: {
6493 LLT Ty = MRI.getType(Reg: I.getOperand(i: 1).getReg());
6494 unsigned Opc;
6495 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6496 Opc = AArch64::ST3Threev8b;
6497 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6498 Opc = AArch64::ST3Threev16b;
6499 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6500 Opc = AArch64::ST3Threev4h;
6501 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6502 Opc = AArch64::ST3Threev8h;
6503 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6504 Opc = AArch64::ST3Threev2s;
6505 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6506 Opc = AArch64::ST3Threev4s;
6507 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6508 Opc = AArch64::ST3Threev2d;
6509 else if (Ty == S64 || Ty == P0)
6510 Opc = AArch64::ST1Threev1d;
6511 else
6512 llvm_unreachable("Unexpected type for st3!");
6513 selectVectorStoreIntrinsic(I, NumVecs: 3, Opc);
6514 break;
6515 }
6516 case Intrinsic::aarch64_neon_st4: {
6517 LLT Ty = MRI.getType(Reg: I.getOperand(i: 1).getReg());
6518 unsigned Opc;
6519 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6520 Opc = AArch64::ST4Fourv8b;
6521 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6522 Opc = AArch64::ST4Fourv16b;
6523 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6524 Opc = AArch64::ST4Fourv4h;
6525 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6526 Opc = AArch64::ST4Fourv8h;
6527 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6528 Opc = AArch64::ST4Fourv2s;
6529 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6530 Opc = AArch64::ST4Fourv4s;
6531 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6532 Opc = AArch64::ST4Fourv2d;
6533 else if (Ty == S64 || Ty == P0)
6534 Opc = AArch64::ST1Fourv1d;
6535 else
6536 llvm_unreachable("Unexpected type for st4!");
6537 selectVectorStoreIntrinsic(I, NumVecs: 4, Opc);
6538 break;
6539 }
6540 case Intrinsic::aarch64_neon_st2lane: {
6541 LLT Ty = MRI.getType(Reg: I.getOperand(i: 1).getReg());
6542 unsigned Opc;
6543 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8) || Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6544 Opc = AArch64::ST2i8;
6545 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16) || Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6546 Opc = AArch64::ST2i16;
6547 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32) || Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6548 Opc = AArch64::ST2i32;
6549 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) ||
6550 Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0) || Ty == S64 || Ty == P0)
6551 Opc = AArch64::ST2i64;
6552 else
6553 llvm_unreachable("Unexpected type for st2lane!");
6554 if (!selectVectorStoreLaneIntrinsic(I, NumVecs: 2, Opc))
6555 return false;
6556 break;
6557 }
6558 case Intrinsic::aarch64_neon_st3lane: {
6559 LLT Ty = MRI.getType(Reg: I.getOperand(i: 1).getReg());
6560 unsigned Opc;
6561 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8) || Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6562 Opc = AArch64::ST3i8;
6563 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16) || Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6564 Opc = AArch64::ST3i16;
6565 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32) || Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6566 Opc = AArch64::ST3i32;
6567 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) ||
6568 Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0) || Ty == S64 || Ty == P0)
6569 Opc = AArch64::ST3i64;
6570 else
6571 llvm_unreachable("Unexpected type for st3lane!");
6572 if (!selectVectorStoreLaneIntrinsic(I, NumVecs: 3, Opc))
6573 return false;
6574 break;
6575 }
6576 case Intrinsic::aarch64_neon_st4lane: {
6577 LLT Ty = MRI.getType(Reg: I.getOperand(i: 1).getReg());
6578 unsigned Opc;
6579 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8) || Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6580 Opc = AArch64::ST4i8;
6581 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16) || Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6582 Opc = AArch64::ST4i16;
6583 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32) || Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6584 Opc = AArch64::ST4i32;
6585 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) ||
6586 Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0) || Ty == S64 || Ty == P0)
6587 Opc = AArch64::ST4i64;
6588 else
6589 llvm_unreachable("Unexpected type for st4lane!");
6590 if (!selectVectorStoreLaneIntrinsic(I, NumVecs: 4, Opc))
6591 return false;
6592 break;
6593 }
6594 case Intrinsic::aarch64_mops_memset_tag: {
6595 // Transform
6596 // %dst:gpr(p0) = \
6597 // G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.aarch64.mops.memset.tag),
6598 // \ %dst:gpr(p0), %val:gpr(s64), %n:gpr(s64)
6599 // where %dst is updated, into
6600 // %Rd:GPR64common, %Rn:GPR64) = \
6601 // MOPSMemorySetTaggingPseudo \
6602 // %Rd:GPR64common, %Rn:GPR64, %Rm:GPR64
6603 // where Rd and Rn are tied.
6604 // It is expected that %val has been extended to s64 in legalization.
6605 // Note that the order of the size/value operands are swapped.
6606
6607 Register DstDef = I.getOperand(i: 0).getReg();
6608 // I.getOperand(1) is the intrinsic function
6609 Register DstUse = I.getOperand(i: 2).getReg();
6610 Register ValUse = I.getOperand(i: 3).getReg();
6611 Register SizeUse = I.getOperand(i: 4).getReg();
6612
6613 // MOPSMemorySetTaggingPseudo has two defs; the intrinsic call has only one.
6614 // Therefore an additional virtual register is required for the updated size
6615 // operand. This value is not accessible via the semantics of the intrinsic.
6616 Register SizeDef = MRI.createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 64));
6617
6618 auto Memset = MIB.buildInstr(Opc: AArch64::MOPSMemorySetTaggingPseudo,
6619 DstOps: {DstDef, SizeDef}, SrcOps: {DstUse, SizeUse, ValUse});
6620 Memset.cloneMemRefs(OtherMI: I);
6621 Memset.setOperandDead(5); // implicit-def $nzcv
6622 constrainSelectedInstRegOperands(I&: *Memset, TII, TRI, RBI);
6623 break;
6624 }
6625 case Intrinsic::ptrauth_resign_load_relative: {
6626 Register DstReg = I.getOperand(i: 0).getReg();
6627 Register ValReg = I.getOperand(i: 2).getReg();
6628 uint64_t AUTKey = I.getOperand(i: 3).getImm();
6629 Register AUTDisc = I.getOperand(i: 4).getReg();
6630 uint64_t PACKey = I.getOperand(i: 5).getImm();
6631 Register PACDisc = I.getOperand(i: 6).getReg();
6632 int64_t Addend = I.getOperand(i: 7).getImm();
6633
6634 Register AUTAddrDisc = AUTDisc;
6635 uint16_t AUTConstDiscC = 0;
6636 std::tie(args&: AUTConstDiscC, args&: AUTAddrDisc) =
6637 extractPtrauthBlendDiscriminators(Disc: AUTDisc, MRI);
6638
6639 Register PACAddrDisc = PACDisc;
6640 uint16_t PACConstDiscC = 0;
6641 std::tie(args&: PACConstDiscC, args&: PACAddrDisc) =
6642 extractPtrauthBlendDiscriminators(Disc: PACDisc, MRI);
6643
6644 MIB.buildCopy(Res: {AArch64::X16}, Op: {ValReg});
6645
6646 MIB.buildInstr(Opcode: AArch64::AUTRELLOADPAC)
6647 .addImm(Val: AUTKey)
6648 .addImm(Val: AUTConstDiscC)
6649 .addUse(RegNo: AUTAddrDisc)
6650 .addImm(Val: PACKey)
6651 .addImm(Val: PACConstDiscC)
6652 .addUse(RegNo: PACAddrDisc)
6653 .addImm(Val: Addend)
6654 .setOperandDead(8) // implicit-def $x17
6655 .setOperandDead(9) // implicit-def $nzcv
6656 .constrainAllUses(TII, TRI, RBI);
6657 MIB.buildCopy(Res: {DstReg}, Op: Register(AArch64::X16));
6658
6659 RBI.constrainGenericRegister(Reg: DstReg, RC: AArch64::GPR64RegClass, MRI);
6660 I.eraseFromParent();
6661 return true;
6662 }
6663 }
6664
6665 I.eraseFromParent();
6666 return true;
6667}
6668
6669bool AArch64InstructionSelector::selectIntrinsic(MachineInstr &I,
6670 MachineRegisterInfo &MRI) {
6671 unsigned IntrinID = cast<GIntrinsic>(Val&: I).getIntrinsicID();
6672
6673 switch (IntrinID) {
6674 default:
6675 break;
6676 case Intrinsic::ptrauth_resign: {
6677 Register DstReg = I.getOperand(i: 0).getReg();
6678 Register ValReg = I.getOperand(i: 2).getReg();
6679 uint64_t AUTKey = I.getOperand(i: 3).getImm();
6680 Register AUTDisc = I.getOperand(i: 4).getReg();
6681 uint64_t PACKey = I.getOperand(i: 5).getImm();
6682 Register PACDisc = I.getOperand(i: 6).getReg();
6683
6684 Register AUTAddrDisc = AUTDisc;
6685 uint16_t AUTConstDiscC = 0;
6686 std::tie(args&: AUTConstDiscC, args&: AUTAddrDisc) =
6687 extractPtrauthBlendDiscriminators(Disc: AUTDisc, MRI);
6688
6689 Register PACAddrDisc = PACDisc;
6690 uint16_t PACConstDiscC = 0;
6691 std::tie(args&: PACConstDiscC, args&: PACAddrDisc) =
6692 extractPtrauthBlendDiscriminators(Disc: PACDisc, MRI);
6693
6694 MIB.buildCopy(Res: {AArch64::X16}, Op: {ValReg});
6695 MIB.buildInstr(Opcode: AArch64::AUTPAC)
6696 .addImm(Val: AUTKey)
6697 .addImm(Val: AUTConstDiscC)
6698 .addUse(RegNo: AUTAddrDisc)
6699 .addImm(Val: PACKey)
6700 .addImm(Val: PACConstDiscC)
6701 .addUse(RegNo: PACAddrDisc)
6702 .setOperandDead(7) // implicit-def $x17
6703 .setOperandDead(8) // implicit-def $nzcv
6704 .constrainAllUses(TII, TRI, RBI);
6705 MIB.buildCopy(Res: {DstReg}, Op: Register(AArch64::X16));
6706
6707 RBI.constrainGenericRegister(Reg: DstReg, RC: AArch64::GPR64RegClass, MRI);
6708 I.eraseFromParent();
6709 return true;
6710 }
6711 case Intrinsic::ptrauth_auth_with_pc_and_resign: {
6712 Register DstReg = I.getOperand(i: 0).getReg();
6713 Register ValReg = I.getOperand(i: 2).getReg();
6714 uint64_t AUTKey = I.getOperand(i: 3).getImm();
6715 Register AUTDisc = I.getOperand(i: 4).getReg();
6716 Register AUTPC = I.getOperand(i: 5).getReg();
6717 uint64_t PACKey = I.getOperand(i: 6).getImm();
6718 Register PACDisc = I.getOperand(i: 7).getReg();
6719
6720 assert((AUTKey == AArch64PACKey::IA || AUTKey == AArch64PACKey::IB) &&
6721 "auth_with_pc_and_resign only supports IA and IB keys");
6722
6723 uint16_t PACConstDiscC = 0;
6724 Register PACAddrDisc;
6725 std::tie(args&: PACConstDiscC, args&: PACAddrDisc) =
6726 extractPtrauthBlendDiscriminators(Disc: PACDisc, MRI);
6727
6728 if (!PACAddrDisc.isValid())
6729 PACAddrDisc = AArch64::XZR;
6730
6731 MIB.buildCopy(Res: {AArch64::X17}, Op: {ValReg});
6732 MIB.buildCopy(Res: {AArch64::X16}, Op: {AUTDisc});
6733 MIB.buildCopy(Res: {AArch64::X15}, Op: {AUTPC});
6734
6735 MIB.buildInstr(Opcode: AArch64::AUTPCPAC)
6736 .addImm(Val: AUTKey)
6737 .addImm(Val: PACKey)
6738 .addImm(Val: PACConstDiscC)
6739 .addUse(RegNo: PACAddrDisc)
6740 .setOperandDead(5) // implicit-def $x15
6741 .setOperandDead(6) // implicit-def $x16
6742 .setOperandDead(7) // implicit-def $nzcv
6743 .constrainAllUses(TII, TRI, RBI);
6744
6745 MIB.buildCopy(Res: {DstReg}, Op: Register(AArch64::X17));
6746 RBI.constrainGenericRegister(Reg: DstReg, RC: AArch64::GPR64RegClass, MRI);
6747 I.eraseFromParent();
6748 return true;
6749 }
6750 case Intrinsic::ptrauth_auth: {
6751 Register DstReg = I.getOperand(i: 0).getReg();
6752 Register ValReg = I.getOperand(i: 2).getReg();
6753 uint64_t AUTKey = I.getOperand(i: 3).getImm();
6754 Register AUTDisc = I.getOperand(i: 4).getReg();
6755
6756 Register AUTAddrDisc = AUTDisc;
6757 uint16_t AUTConstDiscC = 0;
6758 std::tie(args&: AUTConstDiscC, args&: AUTAddrDisc) =
6759 extractPtrauthBlendDiscriminators(Disc: AUTDisc, MRI);
6760
6761 if (STI.isX16X17Safer()) {
6762 MIB.buildCopy(Res: {AArch64::X16}, Op: {ValReg});
6763 MIB.buildInstr(Opcode: AArch64::AUTx16x17)
6764 .addImm(Val: AUTKey)
6765 .addImm(Val: AUTConstDiscC)
6766 .addUse(RegNo: AUTAddrDisc)
6767 .setOperandDead(4) // implicit-def $x17
6768 .setOperandDead(5) // implicit-def $nzcv
6769 .constrainAllUses(TII, TRI, RBI);
6770 MIB.buildCopy(Res: {DstReg}, Op: Register(AArch64::X16));
6771 } else {
6772 Register ScratchReg =
6773 MRI.createVirtualRegister(RegClass: &AArch64::GPR64commonRegClass);
6774 auto Auth = MIB.buildInstr(Opcode: AArch64::AUTxMxN)
6775 .addDef(RegNo: DstReg)
6776 .addDef(RegNo: ScratchReg)
6777 .addUse(RegNo: ValReg)
6778 .addImm(Val: AUTKey)
6779 .addImm(Val: AUTConstDiscC)
6780 .addUse(RegNo: AUTAddrDisc);
6781 Auth->setImplicitPhysRegDefsDead();
6782 Auth.constrainAllUses(TII, TRI, RBI);
6783 }
6784
6785 RBI.constrainGenericRegister(Reg: DstReg, RC: AArch64::GPR64RegClass, MRI);
6786 I.eraseFromParent();
6787 return true;
6788 }
6789 case Intrinsic::frameaddress:
6790 case Intrinsic::returnaddress: {
6791 MachineFunction &MF = *I.getParent()->getParent();
6792 MachineFrameInfo &MFI = MF.getFrameInfo();
6793
6794 unsigned Depth = I.getOperand(i: 2).getImm();
6795 Register DstReg = I.getOperand(i: 0).getReg();
6796 RBI.constrainGenericRegister(Reg: DstReg, RC: AArch64::GPR64RegClass, MRI);
6797
6798 if (Depth == 0 && IntrinID == Intrinsic::returnaddress) {
6799 if (!MFReturnAddr) {
6800 // Insert the copy from LR/X30 into the entry block, before it can be
6801 // clobbered by anything.
6802 MFI.setReturnAddressIsTaken(true);
6803 MFReturnAddr = getFunctionLiveInPhysReg(
6804 MF, TII, PhysReg: AArch64::LR, RC: AArch64::GPR64RegClass, DL: I.getDebugLoc());
6805 }
6806
6807 if (STI.hasPAuth()) {
6808 MIB.buildInstr(Opc: AArch64::XPACI, DstOps: {DstReg}, SrcOps: {MFReturnAddr});
6809 } else {
6810 MIB.buildCopy(Res: {Register(AArch64::LR)}, Op: {MFReturnAddr});
6811 MIB.buildInstr(Opcode: AArch64::XPACLRI);
6812 MIB.buildCopy(Res: {DstReg}, Op: {Register(AArch64::LR)});
6813 }
6814
6815 I.eraseFromParent();
6816 return true;
6817 }
6818
6819 MFI.setFrameAddressIsTaken(true);
6820 Register FrameAddr(AArch64::FP);
6821 while (Depth--) {
6822 Register NextFrame = MRI.createVirtualRegister(RegClass: &AArch64::GPR64spRegClass);
6823 auto Ldr =
6824 MIB.buildInstr(Opc: AArch64::LDRXui, DstOps: {NextFrame}, SrcOps: {FrameAddr}).addImm(Val: 0);
6825 constrainSelectedInstRegOperands(I&: *Ldr, TII, TRI, RBI);
6826 FrameAddr = NextFrame;
6827 }
6828
6829 if (IntrinID == Intrinsic::frameaddress)
6830 MIB.buildCopy(Res: {DstReg}, Op: {FrameAddr});
6831 else {
6832 MFI.setReturnAddressIsTaken(true);
6833
6834 if (STI.hasPAuth()) {
6835 Register TmpReg = MRI.createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
6836 MIB.buildInstr(Opc: AArch64::LDRXui, DstOps: {TmpReg}, SrcOps: {FrameAddr}).addImm(Val: 1);
6837 MIB.buildInstr(Opc: AArch64::XPACI, DstOps: {DstReg}, SrcOps: {TmpReg});
6838 } else {
6839 MIB.buildInstr(Opc: AArch64::LDRXui, DstOps: {Register(AArch64::LR)}, SrcOps: {FrameAddr})
6840 .addImm(Val: 1);
6841 MIB.buildInstr(Opcode: AArch64::XPACLRI);
6842 MIB.buildCopy(Res: {DstReg}, Op: {Register(AArch64::LR)});
6843 }
6844 }
6845
6846 I.eraseFromParent();
6847 return true;
6848 }
6849 case Intrinsic::aarch64_neon_tbl2:
6850 SelectTable(I, MRI, NumVecs: 2, Opc1: AArch64::TBLv8i8Two, Opc2: AArch64::TBLv16i8Two, isExt: false);
6851 return true;
6852 case Intrinsic::aarch64_neon_tbl3:
6853 SelectTable(I, MRI, NumVecs: 3, Opc1: AArch64::TBLv8i8Three, Opc2: AArch64::TBLv16i8Three,
6854 isExt: false);
6855 return true;
6856 case Intrinsic::aarch64_neon_tbl4:
6857 SelectTable(I, MRI, NumVecs: 4, Opc1: AArch64::TBLv8i8Four, Opc2: AArch64::TBLv16i8Four, isExt: false);
6858 return true;
6859 case Intrinsic::aarch64_neon_tbx2:
6860 SelectTable(I, MRI, NumVecs: 2, Opc1: AArch64::TBXv8i8Two, Opc2: AArch64::TBXv16i8Two, isExt: true);
6861 return true;
6862 case Intrinsic::aarch64_neon_tbx3:
6863 SelectTable(I, MRI, NumVecs: 3, Opc1: AArch64::TBXv8i8Three, Opc2: AArch64::TBXv16i8Three, isExt: true);
6864 return true;
6865 case Intrinsic::aarch64_neon_tbx4:
6866 SelectTable(I, MRI, NumVecs: 4, Opc1: AArch64::TBXv8i8Four, Opc2: AArch64::TBXv16i8Four, isExt: true);
6867 return true;
6868 case Intrinsic::swift_async_context_addr:
6869 auto Sub = MIB.buildInstr(Opc: AArch64::SUBXri, DstOps: {I.getOperand(i: 0).getReg()},
6870 SrcOps: {Register(AArch64::FP)})
6871 .addImm(Val: 8)
6872 .addImm(Val: 0);
6873 constrainSelectedInstRegOperands(I&: *Sub, TII, TRI, RBI);
6874
6875 MF->getFrameInfo().setFrameAddressIsTaken(true);
6876 MF->getInfo<AArch64FunctionInfo>()->setHasSwiftAsyncContext(true);
6877 I.eraseFromParent();
6878 return true;
6879 }
6880 return false;
6881}
6882
6883// G_PTRAUTH_GLOBAL_VALUE lowering
6884//
6885// We have 3 lowering alternatives to choose from:
6886// - MOVaddrPAC: similar to MOVaddr, with added PAC.
6887// If the GV doesn't need a GOT load (i.e., is locally defined)
6888// materialize the pointer using adrp+add+pac. See LowerMOVaddrPAC.
6889//
6890// - LOADgotPAC: similar to LOADgot, with added PAC.
6891// If the GV needs a GOT load, materialize the pointer using the usual
6892// GOT adrp+ldr, +pac. Pointers in GOT are assumed to be not signed, the GOT
6893// section is assumed to be read-only (for example, via relro mechanism). See
6894// LowerMOVaddrPAC.
6895//
6896// - LOADauthptrstatic: similar to LOADgot, but use a
6897// special stub slot instead of a GOT slot.
6898// Load a signed pointer for symbol 'sym' from a stub slot named
6899// 'sym$auth_ptr$key$disc' filled by dynamic linker during relocation
6900// resolving. This usually lowers to adrp+ldr, but also emits an entry into
6901// .data with an
6902// @AUTH relocation. See LowerLOADauthptrstatic.
6903//
6904// All 3 are pseudos that are expand late to longer sequences: this lets us
6905// provide integrity guarantees on the to-be-signed intermediate values.
6906//
6907// LOADauthptrstatic is undesirable because it requires a large section filled
6908// with often similarly-signed pointers, making it a good harvesting target.
6909// Thus, it's only used for ptrauth references to extern_weak to avoid null
6910// checks.
6911
6912bool AArch64InstructionSelector::selectPtrAuthGlobalValue(
6913 MachineInstr &I, MachineRegisterInfo &MRI) const {
6914 Register DefReg = I.getOperand(i: 0).getReg();
6915 Register Addr = I.getOperand(i: 1).getReg();
6916 uint64_t Key = I.getOperand(i: 2).getImm();
6917 Register AddrDisc = I.getOperand(i: 3).getReg();
6918 uint64_t Disc = I.getOperand(i: 4).getImm();
6919 int64_t Offset = 0;
6920
6921 if (Key > AArch64PACKey::LAST)
6922 report_fatal_error(reason: "key in ptrauth global out of range [0, " +
6923 Twine((int)AArch64PACKey::LAST) + "]");
6924
6925 // Blend only works if the integer discriminator is 16-bit wide.
6926 if (!isUInt<16>(x: Disc))
6927 report_fatal_error(
6928 reason: "constant discriminator in ptrauth global out of range [0, 0xffff]");
6929
6930 // Choosing between 3 lowering alternatives is target-specific.
6931 if (!STI.isTargetELF() && !STI.isTargetMachO())
6932 report_fatal_error(reason: "ptrauth global lowering only supported on MachO/ELF");
6933
6934 if (!MRI.hasOneDef(RegNo: Addr))
6935 return false;
6936
6937 // First match any offset we take from the real global.
6938 const MachineInstr *DefMI = &*MRI.def_instr_begin(RegNo: Addr);
6939 if (DefMI->getOpcode() == TargetOpcode::G_PTR_ADD) {
6940 Register OffsetReg = DefMI->getOperand(i: 2).getReg();
6941 if (!MRI.hasOneDef(RegNo: OffsetReg))
6942 return false;
6943 const MachineInstr &OffsetMI = *MRI.def_instr_begin(RegNo: OffsetReg);
6944 if (OffsetMI.getOpcode() != TargetOpcode::G_CONSTANT)
6945 return false;
6946
6947 Addr = DefMI->getOperand(i: 1).getReg();
6948 if (!MRI.hasOneDef(RegNo: Addr))
6949 return false;
6950
6951 DefMI = &*MRI.def_instr_begin(RegNo: Addr);
6952 Offset = OffsetMI.getOperand(i: 1).getCImm()->getSExtValue();
6953 }
6954
6955 // We should be left with a genuine unauthenticated GlobalValue.
6956 const GlobalValue *GV;
6957 if (DefMI->getOpcode() == TargetOpcode::G_GLOBAL_VALUE) {
6958 GV = DefMI->getOperand(i: 1).getGlobal();
6959 Offset += DefMI->getOperand(i: 1).getOffset();
6960 } else if (DefMI->getOpcode() == AArch64::G_ADD_LOW) {
6961 GV = DefMI->getOperand(i: 2).getGlobal();
6962 Offset += DefMI->getOperand(i: 2).getOffset();
6963 } else {
6964 return false;
6965 }
6966
6967 MachineIRBuilder MIB(I);
6968
6969 // Classify the reference to determine whether it needs a GOT load.
6970 unsigned OpFlags = STI.ClassifyGlobalReference(GV, TM);
6971 const bool NeedsGOTLoad = ((OpFlags & AArch64II::MO_GOT) != 0);
6972 assert(((OpFlags & (~AArch64II::MO_GOT)) == 0) &&
6973 "unsupported non-GOT op flags on ptrauth global reference");
6974 assert((!GV->hasExternalWeakLinkage() || NeedsGOTLoad) &&
6975 "unsupported non-GOT reference to weak ptrauth global");
6976
6977 std::optional<APInt> AddrDiscVal = getIConstantVRegVal(VReg: AddrDisc, MRI);
6978 bool HasAddrDisc = !AddrDiscVal || *AddrDiscVal != 0;
6979
6980 // Non-extern_weak:
6981 // - No GOT load needed -> MOVaddrPAC
6982 // - GOT load for non-extern_weak -> LOADgotPAC
6983 // Note that we disallow extern_weak refs to avoid null checks later.
6984 if (!GV->hasExternalWeakLinkage()) {
6985 MIB.buildInstr(Opcode: NeedsGOTLoad ? AArch64::LOADgotPAC : AArch64::MOVaddrPAC)
6986 .addGlobalAddress(GV, Offset)
6987 .addImm(Val: Key)
6988 .addReg(RegNo: HasAddrDisc ? AddrDisc : AArch64::XZR)
6989 .addImm(Val: Disc)
6990 .constrainAllUses(TII, TRI, RBI);
6991 MIB.buildCopy(Res: DefReg, Op: Register(AArch64::X16));
6992 RBI.constrainGenericRegister(Reg: DefReg, RC: AArch64::GPR64RegClass, MRI);
6993 I.eraseFromParent();
6994 return true;
6995 }
6996
6997 // extern_weak -> LOADauthptrstatic
6998
6999 // Offsets and extern_weak don't mix well: ptrauth aside, you'd get the
7000 // offset alone as a pointer if the symbol wasn't available, which would
7001 // probably break null checks in users. Ptrauth complicates things further:
7002 // error out.
7003 if (Offset != 0)
7004 report_fatal_error(
7005 reason: "unsupported non-zero offset in weak ptrauth global reference");
7006
7007 if (HasAddrDisc)
7008 report_fatal_error(reason: "unsupported weak addr-div ptrauth global");
7009
7010 MIB.buildInstr(Opc: AArch64::LOADauthptrstatic, DstOps: {DefReg}, SrcOps: {})
7011 .addGlobalAddress(GV, Offset)
7012 .addImm(Val: Key)
7013 .addImm(Val: Disc);
7014 RBI.constrainGenericRegister(Reg: DefReg, RC: AArch64::GPR64RegClass, MRI);
7015
7016 I.eraseFromParent();
7017 return true;
7018}
7019
7020void AArch64InstructionSelector::SelectTable(MachineInstr &I,
7021 MachineRegisterInfo &MRI,
7022 unsigned NumVec, unsigned Opc1,
7023 unsigned Opc2, bool isExt) {
7024 Register DstReg = I.getOperand(i: 0).getReg();
7025 unsigned Opc = MRI.getType(Reg: DstReg) == LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 8) ? Opc1 : Opc2;
7026
7027 // Create the REG_SEQUENCE
7028 SmallVector<Register, 4> Regs;
7029 for (unsigned i = 0; i < NumVec; i++)
7030 Regs.push_back(Elt: I.getOperand(i: i + 2 + isExt).getReg());
7031 Register RegSeq = createQTuple(Regs, MIB);
7032
7033 Register IdxReg = I.getOperand(i: 2 + NumVec + isExt).getReg();
7034 MachineInstrBuilder Instr;
7035 if (isExt) {
7036 Register Reg = I.getOperand(i: 2).getReg();
7037 Instr = MIB.buildInstr(Opc, DstOps: {DstReg}, SrcOps: {Reg, RegSeq, IdxReg});
7038 } else
7039 Instr = MIB.buildInstr(Opc, DstOps: {DstReg}, SrcOps: {RegSeq, IdxReg});
7040 constrainSelectedInstRegOperands(I&: *Instr, TII, TRI, RBI);
7041 I.eraseFromParent();
7042}
7043
7044InstructionSelector::ComplexRendererFns
7045AArch64InstructionSelector::selectShiftA_32(const MachineOperand &Root) const {
7046 auto MaybeImmed = getImmedFromMO(Root);
7047 if (MaybeImmed == std::nullopt || *MaybeImmed > 31)
7048 return std::nullopt;
7049 uint64_t Enc = (32 - *MaybeImmed) & 0x1f;
7050 return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Enc); }}};
7051}
7052
7053InstructionSelector::ComplexRendererFns
7054AArch64InstructionSelector::selectShiftB_32(const MachineOperand &Root) const {
7055 auto MaybeImmed = getImmedFromMO(Root);
7056 if (MaybeImmed == std::nullopt || *MaybeImmed > 31)
7057 return std::nullopt;
7058 uint64_t Enc = 31 - *MaybeImmed;
7059 return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Enc); }}};
7060}
7061
7062InstructionSelector::ComplexRendererFns
7063AArch64InstructionSelector::selectShiftA_64(const MachineOperand &Root) const {
7064 auto MaybeImmed = getImmedFromMO(Root);
7065 if (MaybeImmed == std::nullopt || *MaybeImmed > 63)
7066 return std::nullopt;
7067 uint64_t Enc = (64 - *MaybeImmed) & 0x3f;
7068 return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Enc); }}};
7069}
7070
7071InstructionSelector::ComplexRendererFns
7072AArch64InstructionSelector::selectShiftB_64(const MachineOperand &Root) const {
7073 auto MaybeImmed = getImmedFromMO(Root);
7074 if (MaybeImmed == std::nullopt || *MaybeImmed > 63)
7075 return std::nullopt;
7076 uint64_t Enc = 63 - *MaybeImmed;
7077 return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Enc); }}};
7078}
7079
7080template <unsigned ShiftWidth>
7081InstructionSelector::ComplexRendererFns
7082AArch64InstructionSelector::selectShiftMask(MachineOperand &Root) const {
7083 if (!Root.isReg())
7084 return std::nullopt;
7085
7086 MachineRegisterInfo &MRI =
7087 Root.getParent()->getParent()->getParent()->getRegInfo();
7088
7089 Register ShAmtReg = Root.getReg();
7090
7091 // Peek through zext for i32 shifts only. For i64 shifts the zext case
7092 // is already handled by existing patterns in the Shift multiclass.
7093 if (ShiftWidth == 32) {
7094 Register ZExtSrcReg;
7095 if (mi_match(R: ShAmtReg, MRI, P: m_GZExt(Src: m_Reg(R&: ZExtSrcReg))))
7096 ShAmtReg = ZExtSrcReg;
7097 }
7098
7099 // Remove AND if the mask covers at least the low log2(ShiftWidth) bits.
7100 APInt AndMask;
7101 Register AndSrcReg;
7102 if (mi_match(R: ShAmtReg, MRI, P: m_GAnd(L: m_Reg(R&: AndSrcReg), R: m_ICst(Cst&: AndMask))) &&
7103 MRI.getType(Reg: ShAmtReg).getSizeInBits() == ShiftWidth) {
7104 if (AndMask.countr_one() >= Log2_32(Value: ShiftWidth))
7105 ShAmtReg = AndSrcReg;
7106 }
7107
7108 // If shifting by X+/-N where N == 0 mod ShiftWidth, then just shift by X
7109 // to avoid the ADD/SUB. The low log2(ShiftWidth) bits are unchanged, so the
7110 // shift can use X directly; the original ADD/SUB stays for any other users.
7111 Register AddSrcReg;
7112 int64_t AddImm;
7113 if ((mi_match(R: ShAmtReg, MRI,
7114 P: m_GAdd(L: m_Reg(R&: AddSrcReg), R: m_ICstOrSplat(Cst&: AddImm))) ||
7115 mi_match(R: ShAmtReg, MRI,
7116 P: m_GSub(L: m_Reg(R&: AddSrcReg), R: m_ICstOrSplat(Cst&: AddImm)))) &&
7117 (AddImm % ShiftWidth == 0)) {
7118 ShAmtReg = AddSrcReg;
7119 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: ShAmtReg); }}};
7120 }
7121
7122 // If shifting by N-X where N == 0 mod ShiftWidth, then just shift by -X
7123 // to generate a NEG instead of a SUB from a constant.
7124 Register SubSrcReg;
7125 int64_t SubImm;
7126 if (MRI.hasOneNonDBGUse(RegNo: ShAmtReg) &&
7127 mi_match(R: ShAmtReg, MRI, P: m_GSub(L: m_ICst(Cst&: SubImm), R: m_Reg(R&: SubSrcReg))) &&
7128 SubImm != 0 && (SubImm % ShiftWidth == 0)) {
7129 return {{[=](MachineInstrBuilder &MIB) {
7130 MachineInstr *I = MIB.getInstr();
7131 MachineRegisterInfo &MRI2 = I->getMF()->getRegInfo();
7132 const TargetRegisterClass &RC =
7133 ShiftWidth == 32 ? AArch64::GPR32RegClass : AArch64::GPR64RegClass;
7134 unsigned SubOpc = ShiftWidth == 32 ? AArch64::SUBWrr : AArch64::SUBXrr;
7135 Register ZeroReg = ShiftWidth == 32 ? AArch64::WZR : AArch64::XZR;
7136 Register NegReg = MRI2.createVirtualRegister(RegClass: &RC);
7137 auto NegMI = BuildMI(BB&: *I->getParent(), I&: *I, MIMD: I->getDebugLoc(),
7138 MCID: TII.get(Opcode: SubOpc), DestReg: NegReg)
7139 .addReg(RegNo: ZeroReg)
7140 .addReg(RegNo: SubSrcReg);
7141 constrainSelectedInstRegOperands(I&: *NegMI, TII, TRI, RBI);
7142 MIB.addReg(RegNo: NegReg);
7143 }}};
7144 }
7145
7146 // If shifting by N-X where N == -1 mod ShiftWidth, then just shift by ~X
7147 // to generate a NOT (MVN) instead of a SUB from a constant.
7148 Register NotSrcReg;
7149 int64_t NotImm;
7150 if (MRI.hasOneNonDBGUse(RegNo: ShAmtReg) &&
7151 mi_match(R: ShAmtReg, MRI, P: m_GSub(L: m_ICst(Cst&: NotImm), R: m_Reg(R&: NotSrcReg))) &&
7152 (NotImm % ShiftWidth == ShiftWidth - 1)) {
7153 return {{[=](MachineInstrBuilder &MIB) {
7154 MachineInstr *I = MIB.getInstr();
7155 MachineRegisterInfo &MRI2 = I->getMF()->getRegInfo();
7156 const TargetRegisterClass &RC =
7157 ShiftWidth == 32 ? AArch64::GPR32RegClass : AArch64::GPR64RegClass;
7158 unsigned NotOpc = ShiftWidth == 32 ? AArch64::ORNWrr : AArch64::ORNXrr;
7159 Register ZeroReg = ShiftWidth == 32 ? AArch64::WZR : AArch64::XZR;
7160 Register NotReg = MRI2.createVirtualRegister(RegClass: &RC);
7161 auto NotMI = BuildMI(BB&: *I->getParent(), I&: *I, MIMD: I->getDebugLoc(),
7162 MCID: TII.get(Opcode: NotOpc), DestReg: NotReg)
7163 .addReg(RegNo: ZeroReg)
7164 .addReg(RegNo: NotSrcReg);
7165 constrainSelectedInstRegOperands(I&: *NotMI, TII, TRI, RBI);
7166 MIB.addReg(RegNo: NotReg);
7167 }}};
7168 }
7169
7170 // Only succeed if we changed the shift amount; otherwise let other
7171 // patterns (e.g. zext GPR32 -> SUBREG_TO_REG) match instead.
7172 if (ShAmtReg == Root.getReg())
7173 return std::nullopt;
7174
7175 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: ShAmtReg); }}};
7176}
7177
7178/// Helper to select an immediate value that can be represented as a 12-bit
7179/// value shifted left by either 0 or 12. If it is possible to do so, return
7180/// the immediate and shift value. If not, return std::nullopt.
7181///
7182/// Used by selectArithImmed and selectNegArithImmed.
7183InstructionSelector::ComplexRendererFns
7184AArch64InstructionSelector::select12BitValueWithLeftShift(
7185 uint64_t Immed) const {
7186 unsigned ShiftAmt;
7187 if (Immed >> 12 == 0) {
7188 ShiftAmt = 0;
7189 } else if ((Immed & 0xfff) == 0 && Immed >> 24 == 0) {
7190 ShiftAmt = 12;
7191 Immed = Immed >> 12;
7192 } else
7193 return std::nullopt;
7194
7195 unsigned ShVal = AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: ShiftAmt);
7196 return {{
7197 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Immed); },
7198 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: ShVal); },
7199 }};
7200}
7201
7202/// SelectArithImmed - Select an immediate value that can be represented as
7203/// a 12-bit value shifted left by either 0 or 12. If so, return true with
7204/// Val set to the 12-bit value and Shift set to the shifter operand.
7205InstructionSelector::ComplexRendererFns
7206AArch64InstructionSelector::selectArithImmed(MachineOperand &Root) const {
7207 // This function is called from the addsub_shifted_imm ComplexPattern,
7208 // which lists [imm] as the list of opcode it's interested in, however
7209 // we still need to check whether the operand is actually an immediate
7210 // here because the ComplexPattern opcode list is only used in
7211 // root-level opcode matching.
7212 auto MaybeImmed = getImmedFromMO(Root);
7213 if (MaybeImmed == std::nullopt)
7214 return std::nullopt;
7215 return select12BitValueWithLeftShift(Immed: *MaybeImmed);
7216}
7217
7218/// SelectNegArithImmed - As above, but negates the value before trying to
7219/// select it.
7220InstructionSelector::ComplexRendererFns
7221AArch64InstructionSelector::selectNegArithImmed(MachineOperand &Root) const {
7222 // We need a register here, because we need to know if we have a 64 or 32
7223 // bit immediate.
7224 if (!Root.isReg())
7225 return std::nullopt;
7226 auto MaybeImmed = getImmedFromMO(Root);
7227 if (MaybeImmed == std::nullopt)
7228 return std::nullopt;
7229 uint64_t Immed = *MaybeImmed;
7230
7231 // This negation is almost always valid, but "cmp wN, #0" and "cmn wN, #0"
7232 // have the opposite effect on the C flag, so this pattern mustn't match under
7233 // those circumstances.
7234 if (Immed == 0)
7235 return std::nullopt;
7236
7237 // Check if we're dealing with a 32-bit type on the root or a 64-bit type on
7238 // the root.
7239 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
7240 if (MRI.getType(Reg: Root.getReg()).getSizeInBits() == 32)
7241 Immed = ~((uint32_t)Immed) + 1;
7242 else
7243 Immed = ~Immed + 1ULL;
7244
7245 if (Immed & 0xFFFFFFFFFF000000ULL)
7246 return std::nullopt;
7247
7248 Immed &= 0xFFFFFFULL;
7249 return select12BitValueWithLeftShift(Immed);
7250}
7251
7252/// Checks if we are sure that folding MI into load/store addressing mode is
7253/// beneficial or not.
7254///
7255/// Returns:
7256/// - true if folding MI would be beneficial.
7257/// - false if folding MI would be bad.
7258/// - std::nullopt if it is not sure whether folding MI is beneficial.
7259///
7260/// \p MI can be the offset operand of G_PTR_ADD, e.g. G_SHL in the example:
7261///
7262/// %13:gpr(s64) = G_CONSTANT i64 1
7263/// %8:gpr(s64) = G_SHL %6, %13(s64)
7264/// %9:gpr(p0) = G_PTR_ADD %0, %8(s64)
7265/// %12:gpr(s32) = G_LOAD %9(p0) :: (load (s16))
7266std::optional<bool> AArch64InstructionSelector::isWorthFoldingIntoAddrMode(
7267 const MachineInstr &MI, const MachineRegisterInfo &MRI) const {
7268 if (MI.getOpcode() == AArch64::G_SHL) {
7269 // Address operands with shifts are free, except for running on subtargets
7270 // with AddrLSLSlow14.
7271 if (const auto ValAndVeg = getIConstantVRegValWithLookThrough(
7272 VReg: MI.getOperand(i: 2).getReg(), MRI)) {
7273 const APInt ShiftVal = ValAndVeg->Value;
7274
7275 // Don't fold if we know this will be slow.
7276 return !(STI.hasAddrLSLSlow14() && (ShiftVal == 1 || ShiftVal == 4));
7277 }
7278 }
7279 return std::nullopt;
7280}
7281
7282/// Return true if it is worth folding MI into an extended register. That is,
7283/// if it's safe to pull it into the addressing mode of a load or store as a
7284/// shift.
7285/// \p IsAddrOperand whether the def of MI is used as an address operand
7286/// (e.g. feeding into an LDR/STR).
7287bool AArch64InstructionSelector::isWorthFoldingIntoExtendedReg(
7288 const MachineInstr &MI, const MachineRegisterInfo &MRI,
7289 bool IsAddrOperand) const {
7290
7291 // Always fold if there is one use, or if we're optimizing for size.
7292 Register DefReg = MI.getOperand(i: 0).getReg();
7293 if (MRI.hasOneNonDBGUse(RegNo: DefReg) ||
7294 MI.getParent()->getParent()->getFunction().hasOptSize())
7295 return true;
7296
7297 if (IsAddrOperand) {
7298 // If we are already sure that folding MI is good or bad, return the result.
7299 if (const auto Worth = isWorthFoldingIntoAddrMode(MI, MRI))
7300 return *Worth;
7301
7302 // Fold G_PTR_ADD if its offset operand can be folded
7303 if (MI.getOpcode() == AArch64::G_PTR_ADD) {
7304 MachineInstr *OffsetInst =
7305 getDefIgnoringCopies(Reg: MI.getOperand(i: 2).getReg(), MRI);
7306
7307 // Note, we already know G_PTR_ADD is used by at least two instructions.
7308 // If we are also sure about whether folding is beneficial or not,
7309 // return the result.
7310 if (const auto Worth = isWorthFoldingIntoAddrMode(MI: *OffsetInst, MRI))
7311 return *Worth;
7312 }
7313 }
7314
7315 // FIXME: Consider checking HasALULSLFast as appropriate.
7316
7317 // We have a fastpath, so folding a shift in and potentially computing it
7318 // many times may be beneficial. Check if this is only used in memory ops.
7319 // If it is, then we should fold.
7320 return all_of(Range: MRI.use_nodbg_instructions(Reg: DefReg),
7321 P: [](MachineInstr &Use) { return Use.mayLoadOrStore(); });
7322}
7323
7324InstructionSelector::ComplexRendererFns
7325AArch64InstructionSelector::selectExtendedSHL(
7326 MachineOperand &Root, MachineOperand &Base, MachineOperand &Offset,
7327 unsigned SizeInBytes, bool WantsExt) const {
7328 assert(Base.isReg() && "Expected base to be a register operand");
7329 assert(Offset.isReg() && "Expected offset to be a register operand");
7330
7331 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
7332 MachineInstr *OffsetInst = MRI.getVRegDef(Reg: Offset.getReg());
7333
7334 unsigned OffsetOpc = OffsetInst->getOpcode();
7335 bool LookedThroughZExt = false;
7336 if (OffsetOpc != TargetOpcode::G_SHL && OffsetOpc != TargetOpcode::G_MUL) {
7337 // Try to look through a ZEXT.
7338 if (OffsetOpc != TargetOpcode::G_ZEXT || !WantsExt)
7339 return std::nullopt;
7340
7341 OffsetInst = MRI.getVRegDef(Reg: OffsetInst->getOperand(i: 1).getReg());
7342 OffsetOpc = OffsetInst->getOpcode();
7343 LookedThroughZExt = true;
7344
7345 if (OffsetOpc != TargetOpcode::G_SHL && OffsetOpc != TargetOpcode::G_MUL)
7346 return std::nullopt;
7347 }
7348 // Make sure that the memory op is a valid size.
7349 int64_t LegalShiftVal = Log2_32(Value: SizeInBytes);
7350 if (LegalShiftVal == 0)
7351 return std::nullopt;
7352 if (!isWorthFoldingIntoExtendedReg(MI: *OffsetInst, MRI, IsAddrOperand: true))
7353 return std::nullopt;
7354
7355 // Now, try to find the specific G_CONSTANT. Start by assuming that the
7356 // register we will offset is the LHS, and the register containing the
7357 // constant is the RHS.
7358 Register OffsetReg = OffsetInst->getOperand(i: 1).getReg();
7359 Register ConstantReg = OffsetInst->getOperand(i: 2).getReg();
7360 auto ValAndVReg = getIConstantVRegValWithLookThrough(VReg: ConstantReg, MRI);
7361 if (!ValAndVReg) {
7362 // We didn't get a constant on the RHS. If the opcode is a shift, then
7363 // we're done.
7364 if (OffsetOpc == TargetOpcode::G_SHL)
7365 return std::nullopt;
7366
7367 // If we have a G_MUL, we can use either register. Try looking at the RHS.
7368 std::swap(a&: OffsetReg, b&: ConstantReg);
7369 ValAndVReg = getIConstantVRegValWithLookThrough(VReg: ConstantReg, MRI);
7370 if (!ValAndVReg)
7371 return std::nullopt;
7372 }
7373
7374 // The value must fit into 3 bits, and must be positive. Make sure that is
7375 // true.
7376 int64_t ImmVal = ValAndVReg->Value.getSExtValue();
7377
7378 // Since we're going to pull this into a shift, the constant value must be
7379 // a power of 2. If we got a multiply, then we need to check this.
7380 if (OffsetOpc == TargetOpcode::G_MUL) {
7381 if (!llvm::has_single_bit<uint32_t>(Value: ImmVal))
7382 return std::nullopt;
7383
7384 // Got a power of 2. So, the amount we'll shift is the log base-2 of that.
7385 ImmVal = Log2_32(Value: ImmVal);
7386 }
7387
7388 if ((ImmVal & 0x7) != ImmVal)
7389 return std::nullopt;
7390
7391 // We are only allowed to shift by LegalShiftVal. This shift value is built
7392 // into the instruction, so we can't just use whatever we want.
7393 if (ImmVal != LegalShiftVal)
7394 return std::nullopt;
7395
7396 unsigned SignExtend = 0;
7397 if (WantsExt) {
7398 // Check if the offset is defined by an extend, unless we looked through a
7399 // G_ZEXT earlier.
7400 if (!LookedThroughZExt) {
7401 MachineInstr *ExtInst = getDefIgnoringCopies(Reg: OffsetReg, MRI);
7402 auto Ext = getExtendTypeForInst(MI&: *ExtInst, MRI, IsLoadStore: true);
7403 if (Ext == AArch64_AM::InvalidShiftExtend)
7404 return std::nullopt;
7405
7406 SignExtend = AArch64_AM::isSignExtendShiftType(Type: Ext) ? 1 : 0;
7407 // We only support SXTW for signed extension here.
7408 if (SignExtend && Ext != AArch64_AM::SXTW)
7409 return std::nullopt;
7410 OffsetReg = ExtInst->getOperand(i: 1).getReg();
7411 }
7412
7413 // Need a 32-bit wide register here.
7414 MachineIRBuilder MIB(*MRI.getVRegDef(Reg: Root.getReg()));
7415 OffsetReg = moveScalarRegClass(Reg: OffsetReg, RC: AArch64::GPR32RegClass, MIB);
7416 }
7417
7418 // We can use the LHS of the GEP as the base, and the LHS of the shift as an
7419 // offset. Signify that we are shifting by setting the shift flag to 1.
7420 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(RegNo: Base.getReg()); },
7421 [=](MachineInstrBuilder &MIB) { MIB.addUse(RegNo: OffsetReg); },
7422 [=](MachineInstrBuilder &MIB) {
7423 // Need to add both immediates here to make sure that they are both
7424 // added to the instruction.
7425 MIB.addImm(Val: SignExtend);
7426 MIB.addImm(Val: 1);
7427 }}};
7428}
7429
7430/// This is used for computing addresses like this:
7431///
7432/// ldr x1, [x2, x3, lsl #3]
7433///
7434/// Where x2 is the base register, and x3 is an offset register. The shift-left
7435/// is a constant value specific to this load instruction. That is, we'll never
7436/// see anything other than a 3 here (which corresponds to the size of the
7437/// element being loaded.)
7438InstructionSelector::ComplexRendererFns
7439AArch64InstructionSelector::selectAddrModeShiftedExtendXReg(
7440 MachineOperand &Root, unsigned SizeInBytes) const {
7441 if (!Root.isReg())
7442 return std::nullopt;
7443 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
7444
7445 // We want to find something like this:
7446 //
7447 // val = G_CONSTANT LegalShiftVal
7448 // shift = G_SHL off_reg val
7449 // ptr = G_PTR_ADD base_reg shift
7450 // x = G_LOAD ptr
7451 //
7452 // And fold it into this addressing mode:
7453 //
7454 // ldr x, [base_reg, off_reg, lsl #LegalShiftVal]
7455
7456 // Check if we can find the G_PTR_ADD.
7457 MachineInstr *PtrAdd =
7458 getOpcodeDef(Opcode: TargetOpcode::G_PTR_ADD, Reg: Root.getReg(), MRI);
7459 if (!PtrAdd || !isWorthFoldingIntoExtendedReg(MI: *PtrAdd, MRI, IsAddrOperand: true))
7460 return std::nullopt;
7461
7462 // Now, try to match an opcode which will match our specific offset.
7463 // We want a G_SHL or a G_MUL.
7464 MachineInstr *OffsetInst =
7465 getDefIgnoringCopies(Reg: PtrAdd->getOperand(i: 2).getReg(), MRI);
7466 return selectExtendedSHL(Root, Base&: PtrAdd->getOperand(i: 1),
7467 Offset&: OffsetInst->getOperand(i: 0), SizeInBytes,
7468 /*WantsExt=*/false);
7469}
7470
7471/// This is used for computing addresses like this:
7472///
7473/// ldr x1, [x2, x3]
7474///
7475/// Where x2 is the base register, and x3 is an offset register.
7476///
7477/// When possible (or profitable) to fold a G_PTR_ADD into the address
7478/// calculation, this will do so. Otherwise, it will return std::nullopt.
7479InstructionSelector::ComplexRendererFns
7480AArch64InstructionSelector::selectAddrModeRegisterOffset(
7481 MachineOperand &Root) const {
7482 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
7483
7484 // We need a GEP.
7485 Register Base, Offset;
7486 if (!mi_match(R: Root.getReg(), MRI, P: m_GPtrAdd(L: m_Reg(R&: Base), R: m_Reg(R&: Offset))))
7487 return std::nullopt;
7488
7489 // If this is used more than once, let's not bother folding.
7490 // TODO: Check if they are memory ops. If they are, then we can still fold
7491 // without having to recompute anything.
7492 if (!MRI.hasOneNonDBGUse(RegNo: Root.getReg()))
7493 return std::nullopt;
7494
7495 // Base is the GEP's LHS, offset is its RHS.
7496 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(RegNo: Base); },
7497 [=](MachineInstrBuilder &MIB) { MIB.addUse(RegNo: Offset); },
7498 [=](MachineInstrBuilder &MIB) {
7499 // Need to add both immediates here to make sure that they are both
7500 // added to the instruction.
7501 MIB.addImm(Val: 0);
7502 MIB.addImm(Val: 0);
7503 }}};
7504}
7505
7506/// This is intended to be equivalent to selectAddrModeXRO in
7507/// AArch64ISelDAGtoDAG. It's used for selecting X register offset loads.
7508InstructionSelector::ComplexRendererFns
7509AArch64InstructionSelector::selectAddrModeXRO(MachineOperand &Root,
7510 unsigned SizeInBytes) const {
7511 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
7512 if (!Root.isReg())
7513 return std::nullopt;
7514 MachineInstr *PtrAdd =
7515 getOpcodeDef(Opcode: TargetOpcode::G_PTR_ADD, Reg: Root.getReg(), MRI);
7516 if (!PtrAdd)
7517 return std::nullopt;
7518
7519 // Check for an immediates which cannot be encoded in the [base + imm]
7520 // addressing mode, and can't be encoded in an add/sub. If this happens, we'll
7521 // end up with code like:
7522 //
7523 // mov x0, wide
7524 // add x1 base, x0
7525 // ldr x2, [x1, x0]
7526 //
7527 // In this situation, we can use the [base, xreg] addressing mode to save an
7528 // add/sub:
7529 //
7530 // mov x0, wide
7531 // ldr x2, [base, x0]
7532 auto ValAndVReg =
7533 getIConstantVRegValWithLookThrough(VReg: PtrAdd->getOperand(i: 2).getReg(), MRI);
7534 if (ValAndVReg) {
7535 unsigned Scale = Log2_32(Value: SizeInBytes);
7536 int64_t ImmOff = ValAndVReg->Value.getSExtValue();
7537
7538 // Skip immediates that can be selected in the load/store addressing
7539 // mode.
7540 if (ImmOff % SizeInBytes == 0 && ImmOff >= 0 &&
7541 ImmOff < (0x1000 << Scale))
7542 return std::nullopt;
7543
7544 // Helper lambda to decide whether or not it is preferable to emit an add.
7545 auto isPreferredADD = [](int64_t ImmOff) {
7546 // Constants in [0x0, 0xfff] can be encoded in an add.
7547 if ((ImmOff & 0xfffffffffffff000LL) == 0x0LL)
7548 return true;
7549
7550 // Can it be encoded in an add lsl #12?
7551 if ((ImmOff & 0xffffffffff000fffLL) != 0x0LL)
7552 return false;
7553
7554 // It can be encoded in an add lsl #12, but we may not want to. If it is
7555 // possible to select this as a single movz, then prefer that. A single
7556 // movz is faster than an add with a shift.
7557 return (ImmOff & 0xffffffffff00ffffLL) != 0x0LL &&
7558 (ImmOff & 0xffffffffffff0fffLL) != 0x0LL;
7559 };
7560
7561 // If the immediate can be encoded in a single add/sub, then bail out.
7562 if (isPreferredADD(ImmOff) || isPreferredADD(-ImmOff))
7563 return std::nullopt;
7564 }
7565
7566 // Try to fold shifts into the addressing mode.
7567 auto AddrModeFns = selectAddrModeShiftedExtendXReg(Root, SizeInBytes);
7568 if (AddrModeFns)
7569 return AddrModeFns;
7570
7571 // If that doesn't work, see if it's possible to fold in registers from
7572 // a GEP.
7573 return selectAddrModeRegisterOffset(Root);
7574}
7575
7576/// This is used for computing addresses like this:
7577///
7578/// ldr x0, [xBase, wOffset, sxtw #LegalShiftVal]
7579///
7580/// Where we have a 64-bit base register, a 32-bit offset register, and an
7581/// extend (which may or may not be signed).
7582InstructionSelector::ComplexRendererFns
7583AArch64InstructionSelector::selectAddrModeWRO(MachineOperand &Root,
7584 unsigned SizeInBytes) const {
7585 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
7586
7587 MachineInstr *PtrAdd =
7588 getOpcodeDef(Opcode: TargetOpcode::G_PTR_ADD, Reg: Root.getReg(), MRI);
7589 if (!PtrAdd || !isWorthFoldingIntoExtendedReg(MI: *PtrAdd, MRI, IsAddrOperand: true))
7590 return std::nullopt;
7591
7592 MachineOperand &LHS = PtrAdd->getOperand(i: 1);
7593 MachineOperand &RHS = PtrAdd->getOperand(i: 2);
7594 MachineInstr *OffsetInst = getDefIgnoringCopies(Reg: RHS.getReg(), MRI);
7595
7596 // The first case is the same as selectAddrModeXRO, except we need an extend.
7597 // In this case, we try to find a shift and extend, and fold them into the
7598 // addressing mode.
7599 //
7600 // E.g.
7601 //
7602 // off_reg = G_Z/S/ANYEXT ext_reg
7603 // val = G_CONSTANT LegalShiftVal
7604 // shift = G_SHL off_reg val
7605 // ptr = G_PTR_ADD base_reg shift
7606 // x = G_LOAD ptr
7607 //
7608 // In this case we can get a load like this:
7609 //
7610 // ldr x0, [base_reg, ext_reg, sxtw #LegalShiftVal]
7611 auto ExtendedShl = selectExtendedSHL(Root, Base&: LHS, Offset&: OffsetInst->getOperand(i: 0),
7612 SizeInBytes, /*WantsExt=*/true);
7613 if (ExtendedShl)
7614 return ExtendedShl;
7615
7616 // There was no shift. We can try and fold a G_Z/S/ANYEXT in alone though.
7617 //
7618 // e.g.
7619 // ldr something, [base_reg, ext_reg, sxtw]
7620 if (!isWorthFoldingIntoExtendedReg(MI: *OffsetInst, MRI, IsAddrOperand: true))
7621 return std::nullopt;
7622
7623 // Check if this is an extend. We'll get an extend type if it is.
7624 AArch64_AM::ShiftExtendType Ext =
7625 getExtendTypeForInst(MI&: *OffsetInst, MRI, /*IsLoadStore=*/true);
7626 if (Ext == AArch64_AM::InvalidShiftExtend)
7627 return std::nullopt;
7628
7629 // Need a 32-bit wide register.
7630 MachineIRBuilder MIB(*PtrAdd);
7631 Register ExtReg = moveScalarRegClass(Reg: OffsetInst->getOperand(i: 1).getReg(),
7632 RC: AArch64::GPR32RegClass, MIB);
7633 unsigned SignExtend = Ext == AArch64_AM::SXTW;
7634
7635 // Base is LHS, offset is ExtReg.
7636 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(RegNo: LHS.getReg()); },
7637 [=](MachineInstrBuilder &MIB) { MIB.addUse(RegNo: ExtReg); },
7638 [=](MachineInstrBuilder &MIB) {
7639 MIB.addImm(Val: SignExtend);
7640 MIB.addImm(Val: 0);
7641 }}};
7642}
7643
7644/// Select a "register plus unscaled signed 9-bit immediate" address. This
7645/// should only match when there is an offset that is not valid for a scaled
7646/// immediate addressing mode. The "Size" argument is the size in bytes of the
7647/// memory reference, which is needed here to know what is valid for a scaled
7648/// immediate.
7649InstructionSelector::ComplexRendererFns
7650AArch64InstructionSelector::selectAddrModeUnscaled(MachineOperand &Root,
7651 unsigned Size) const {
7652 MachineRegisterInfo &MRI =
7653 Root.getParent()->getParent()->getParent()->getRegInfo();
7654
7655 if (!Root.isReg())
7656 return std::nullopt;
7657
7658 if (!isBaseWithConstantOffset(Root, MRI))
7659 return std::nullopt;
7660
7661 MachineInstr *RootDef = MRI.getVRegDef(Reg: Root.getReg());
7662
7663 MachineOperand &OffImm = RootDef->getOperand(i: 2);
7664 if (!OffImm.isReg())
7665 return std::nullopt;
7666 MachineInstr *RHS = MRI.getVRegDef(Reg: OffImm.getReg());
7667 if (RHS->getOpcode() != TargetOpcode::G_CONSTANT)
7668 return std::nullopt;
7669 int64_t RHSC;
7670 MachineOperand &RHSOp1 = RHS->getOperand(i: 1);
7671 if (!RHSOp1.isCImm() || RHSOp1.getCImm()->getBitWidth() > 64)
7672 return std::nullopt;
7673 RHSC = RHSOp1.getCImm()->getSExtValue();
7674
7675 if (RHSC >= -256 && RHSC < 256) {
7676 MachineOperand &Base = RootDef->getOperand(i: 1);
7677 return {{
7678 [=](MachineInstrBuilder &MIB) { MIB.add(MO: Base); },
7679 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: RHSC); },
7680 }};
7681 }
7682 return std::nullopt;
7683}
7684
7685InstructionSelector::ComplexRendererFns
7686AArch64InstructionSelector::tryFoldAddLowIntoImm(MachineInstr &RootDef,
7687 unsigned Size,
7688 MachineRegisterInfo &MRI) const {
7689 if (RootDef.getOpcode() != AArch64::G_ADD_LOW)
7690 return std::nullopt;
7691 MachineInstr &Adrp = *MRI.getVRegDef(Reg: RootDef.getOperand(i: 1).getReg());
7692 if (Adrp.getOpcode() != AArch64::ADRP)
7693 return std::nullopt;
7694
7695 // TODO: add heuristics like isWorthFoldingADDlow() from SelectionDAG.
7696 auto Offset = Adrp.getOperand(i: 1).getOffset();
7697 if (Offset % Size != 0)
7698 return std::nullopt;
7699
7700 auto GV = Adrp.getOperand(i: 1).getGlobal();
7701 if (GV->isThreadLocal())
7702 return std::nullopt;
7703
7704 auto &MF = *RootDef.getParent()->getParent();
7705 if (GV->getPointerAlignment(DL: MF.getDataLayout()) < Size)
7706 return std::nullopt;
7707
7708 unsigned OpFlags = STI.ClassifyGlobalReference(GV, TM: MF.getTarget());
7709 Register AdrpReg = Adrp.getOperand(i: 0).getReg();
7710 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(RegNo: AdrpReg); },
7711 [=](MachineInstrBuilder &MIB) {
7712 MIB.addGlobalAddress(GV, Offset,
7713 TargetFlags: OpFlags | AArch64II::MO_PAGEOFF |
7714 AArch64II::MO_NC);
7715 }}};
7716}
7717
7718/// Select a "register plus scaled unsigned 12-bit immediate" address. The
7719/// "Size" argument is the size in bytes of the memory reference, which
7720/// determines the scale.
7721InstructionSelector::ComplexRendererFns
7722AArch64InstructionSelector::selectAddrModeIndexed(MachineOperand &Root,
7723 unsigned Size) const {
7724 MachineFunction &MF = *Root.getParent()->getParent()->getParent();
7725 MachineRegisterInfo &MRI = MF.getRegInfo();
7726
7727 if (!Root.isReg())
7728 return std::nullopt;
7729
7730 MachineInstr *RootDef = MRI.getVRegDef(Reg: Root.getReg());
7731 if (RootDef->getOpcode() == TargetOpcode::G_FRAME_INDEX) {
7732 return {{
7733 [=](MachineInstrBuilder &MIB) { MIB.add(MO: RootDef->getOperand(i: 1)); },
7734 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); },
7735 }};
7736 }
7737
7738 CodeModel::Model CM = MF.getTarget().getCodeModel();
7739 // Check if we can fold in the ADD of small code model ADRP + ADD address.
7740 // HACK: ld64 on Darwin doesn't support relocations on PRFM, so we can't fold
7741 // globals into the offset.
7742 MachineInstr *RootParent = Root.getParent();
7743 if (CM == CodeModel::Small &&
7744 !(RootParent->getOpcode() == AArch64::G_AARCH64_PREFETCH &&
7745 STI.isTargetDarwin())) {
7746 auto OpFns = tryFoldAddLowIntoImm(RootDef&: *RootDef, Size, MRI);
7747 if (OpFns)
7748 return OpFns;
7749 }
7750
7751 if (isBaseWithConstantOffset(Root, MRI)) {
7752 MachineOperand &LHS = RootDef->getOperand(i: 1);
7753 MachineOperand &RHS = RootDef->getOperand(i: 2);
7754 MachineInstr *LHSDef = MRI.getVRegDef(Reg: LHS.getReg());
7755 MachineInstr *RHSDef = MRI.getVRegDef(Reg: RHS.getReg());
7756
7757 int64_t RHSC = (int64_t)RHSDef->getOperand(i: 1).getCImm()->getZExtValue();
7758 unsigned Scale = Log2_32(Value: Size);
7759 if ((RHSC & (Size - 1)) == 0 && RHSC >= 0 && RHSC < (0x1000 << Scale)) {
7760 if (LHSDef->getOpcode() == TargetOpcode::G_FRAME_INDEX)
7761 return {{
7762 [=](MachineInstrBuilder &MIB) { MIB.add(MO: LHSDef->getOperand(i: 1)); },
7763 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: RHSC >> Scale); },
7764 }};
7765
7766 return {{
7767 [=](MachineInstrBuilder &MIB) { MIB.add(MO: LHS); },
7768 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: RHSC >> Scale); },
7769 }};
7770 }
7771 }
7772
7773 // Before falling back to our general case, check if the unscaled
7774 // instructions can handle this. If so, that's preferable.
7775 if (selectAddrModeUnscaled(Root, Size))
7776 return std::nullopt;
7777
7778 return {{
7779 [=](MachineInstrBuilder &MIB) { MIB.add(MO: Root); },
7780 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); },
7781 }};
7782}
7783
7784/// Given a shift instruction, return the correct shift type for that
7785/// instruction.
7786static AArch64_AM::ShiftExtendType getShiftTypeForInst(MachineInstr &MI) {
7787 switch (MI.getOpcode()) {
7788 default:
7789 return AArch64_AM::InvalidShiftExtend;
7790 case TargetOpcode::G_SHL:
7791 return AArch64_AM::LSL;
7792 case TargetOpcode::G_LSHR:
7793 return AArch64_AM::LSR;
7794 case TargetOpcode::G_ASHR:
7795 return AArch64_AM::ASR;
7796 case TargetOpcode::G_ROTR:
7797 return AArch64_AM::ROR;
7798 }
7799}
7800
7801/// Select a "shifted register" operand. If the value is not shifted, set the
7802/// shift operand to a default value of "lsl 0".
7803InstructionSelector::ComplexRendererFns
7804AArch64InstructionSelector::selectShiftedRegister(MachineOperand &Root,
7805 bool AllowROR) const {
7806 if (!Root.isReg())
7807 return std::nullopt;
7808 MachineRegisterInfo &MRI =
7809 Root.getParent()->getParent()->getParent()->getRegInfo();
7810
7811 // Check if the operand is defined by an instruction which corresponds to
7812 // a ShiftExtendType. E.g. a G_SHL, G_LSHR, etc.
7813 MachineInstr *ShiftInst = MRI.getVRegDef(Reg: Root.getReg());
7814 AArch64_AM::ShiftExtendType ShType = getShiftTypeForInst(MI&: *ShiftInst);
7815 if (ShType == AArch64_AM::InvalidShiftExtend)
7816 return std::nullopt;
7817 if (ShType == AArch64_AM::ROR && !AllowROR)
7818 return std::nullopt;
7819 if (!isWorthFoldingIntoExtendedReg(MI: *ShiftInst, MRI, IsAddrOperand: false))
7820 return std::nullopt;
7821
7822 // Need an immediate on the RHS.
7823 MachineOperand &ShiftRHS = ShiftInst->getOperand(i: 2);
7824 auto Immed = getImmedFromMO(Root: ShiftRHS);
7825 if (!Immed)
7826 return std::nullopt;
7827
7828 // We have something that we can fold. Fold in the shift's LHS and RHS into
7829 // the instruction.
7830 MachineOperand &ShiftLHS = ShiftInst->getOperand(i: 1);
7831 Register ShiftReg = ShiftLHS.getReg();
7832
7833 unsigned NumBits = MRI.getType(Reg: ShiftReg).getSizeInBits();
7834 unsigned Val = *Immed & (NumBits - 1);
7835 unsigned ShiftVal = AArch64_AM::getShifterImm(ST: ShType, Imm: Val);
7836
7837 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(RegNo: ShiftReg); },
7838 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: ShiftVal); }}};
7839}
7840
7841AArch64_AM::ShiftExtendType AArch64InstructionSelector::getExtendTypeForInst(
7842 MachineInstr &MI, MachineRegisterInfo &MRI, bool IsLoadStore) const {
7843 unsigned Opc = MI.getOpcode();
7844
7845 // Handle explicit extend instructions first.
7846 if (Opc == TargetOpcode::G_SEXT || Opc == TargetOpcode::G_SEXT_INREG) {
7847 unsigned Size;
7848 if (Opc == TargetOpcode::G_SEXT)
7849 Size = MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getSizeInBits();
7850 else
7851 Size = MI.getOperand(i: 2).getImm();
7852 assert(Size != 64 && "Extend from 64 bits?");
7853 switch (Size) {
7854 case 8:
7855 return IsLoadStore ? AArch64_AM::InvalidShiftExtend : AArch64_AM::SXTB;
7856 case 16:
7857 return IsLoadStore ? AArch64_AM::InvalidShiftExtend : AArch64_AM::SXTH;
7858 case 32:
7859 return AArch64_AM::SXTW;
7860 default:
7861 return AArch64_AM::InvalidShiftExtend;
7862 }
7863 }
7864
7865 if (Opc == TargetOpcode::G_ZEXT || Opc == TargetOpcode::G_ANYEXT) {
7866 unsigned Size = MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getSizeInBits();
7867 assert(Size != 64 && "Extend from 64 bits?");
7868 switch (Size) {
7869 case 8:
7870 return IsLoadStore ? AArch64_AM::InvalidShiftExtend : AArch64_AM::UXTB;
7871 case 16:
7872 return IsLoadStore ? AArch64_AM::InvalidShiftExtend : AArch64_AM::UXTH;
7873 case 32:
7874 return AArch64_AM::UXTW;
7875 default:
7876 return AArch64_AM::InvalidShiftExtend;
7877 }
7878 }
7879
7880 // Don't have an explicit extend. Try to handle a G_AND with a constant mask
7881 // on the RHS.
7882 if (Opc != TargetOpcode::G_AND)
7883 return AArch64_AM::InvalidShiftExtend;
7884
7885 std::optional<uint64_t> MaybeAndMask = getImmedFromMO(Root: MI.getOperand(i: 2));
7886 if (!MaybeAndMask)
7887 return AArch64_AM::InvalidShiftExtend;
7888 uint64_t AndMask = *MaybeAndMask;
7889 switch (AndMask) {
7890 default:
7891 return AArch64_AM::InvalidShiftExtend;
7892 case 0xFF:
7893 return !IsLoadStore ? AArch64_AM::UXTB : AArch64_AM::InvalidShiftExtend;
7894 case 0xFFFF:
7895 return !IsLoadStore ? AArch64_AM::UXTH : AArch64_AM::InvalidShiftExtend;
7896 case 0xFFFFFFFF:
7897 return AArch64_AM::UXTW;
7898 }
7899}
7900
7901Register AArch64InstructionSelector::moveScalarRegClass(
7902 Register Reg, const TargetRegisterClass &RC, MachineIRBuilder &MIB) const {
7903 MachineRegisterInfo &MRI = *MIB.getMRI();
7904 auto Ty = MRI.getType(Reg);
7905 assert(!Ty.isVector() && "Expected scalars only!");
7906 if (Ty.getSizeInBits() == TRI.getRegSizeInBits(RC))
7907 return Reg;
7908
7909 // Create a copy and immediately select it.
7910 // FIXME: We should have an emitCopy function?
7911 auto Copy = MIB.buildCopy(Res: {&RC}, Op: {Reg});
7912 selectCopy(I&: *Copy, TII, MRI, TRI, RBI);
7913 return Copy.getReg(Idx: 0);
7914}
7915
7916/// Select an "extended register" operand. This operand folds in an extend
7917/// followed by an optional left shift.
7918InstructionSelector::ComplexRendererFns
7919AArch64InstructionSelector::selectArithExtendedRegister(
7920 MachineOperand &Root) const {
7921 if (!Root.isReg())
7922 return std::nullopt;
7923 MachineRegisterInfo &MRI =
7924 Root.getParent()->getParent()->getParent()->getRegInfo();
7925
7926 uint64_t ShiftVal = 0;
7927 Register ExtReg;
7928 AArch64_AM::ShiftExtendType Ext;
7929 MachineInstr *RootDef = getDefIgnoringCopies(Reg: Root.getReg(), MRI);
7930 if (!RootDef)
7931 return std::nullopt;
7932
7933 if (!isWorthFoldingIntoExtendedReg(MI: *RootDef, MRI, IsAddrOperand: false))
7934 return std::nullopt;
7935
7936 // Check if we can fold a shift and an extend.
7937 if (RootDef->getOpcode() == TargetOpcode::G_SHL) {
7938 // Look for a constant on the RHS of the shift.
7939 MachineOperand &RHS = RootDef->getOperand(i: 2);
7940 std::optional<uint64_t> MaybeShiftVal = getImmedFromMO(Root: RHS);
7941 if (!MaybeShiftVal)
7942 return std::nullopt;
7943 ShiftVal = *MaybeShiftVal;
7944 if (ShiftVal > 4)
7945 return std::nullopt;
7946 // Look for a valid extend instruction on the LHS of the shift.
7947 MachineOperand &LHS = RootDef->getOperand(i: 1);
7948 MachineInstr *ExtDef = getDefIgnoringCopies(Reg: LHS.getReg(), MRI);
7949 if (!ExtDef)
7950 return std::nullopt;
7951 Ext = getExtendTypeForInst(MI&: *ExtDef, MRI);
7952 if (Ext == AArch64_AM::InvalidShiftExtend)
7953 return std::nullopt;
7954 ExtReg = ExtDef->getOperand(i: 1).getReg();
7955 } else {
7956 // Didn't get a shift. Try just folding an extend.
7957 Ext = getExtendTypeForInst(MI&: *RootDef, MRI);
7958 if (Ext == AArch64_AM::InvalidShiftExtend)
7959 return std::nullopt;
7960 ExtReg = RootDef->getOperand(i: 1).getReg();
7961
7962 // If we have a 32 bit instruction which zeroes out the high half of a
7963 // register, we get an implicit zero extend for free. Check if we have one.
7964 // FIXME: We actually emit the extend right now even though we don't have
7965 // to.
7966 if (Ext == AArch64_AM::UXTW && MRI.getType(Reg: ExtReg).getSizeInBits() == 32) {
7967 MachineInstr *ExtInst = MRI.getVRegDef(Reg: ExtReg);
7968 if (isDef32(MI: *ExtInst))
7969 return std::nullopt;
7970 }
7971 }
7972
7973 // We require a GPR32 here. Narrow the ExtReg if needed using a subregister
7974 // copy.
7975 MachineIRBuilder MIB(*RootDef);
7976 ExtReg = moveScalarRegClass(Reg: ExtReg, RC: AArch64::GPR32RegClass, MIB);
7977
7978 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(RegNo: ExtReg); },
7979 [=](MachineInstrBuilder &MIB) {
7980 MIB.addImm(Val: getArithExtendImm(ET: Ext, Imm: ShiftVal));
7981 }}};
7982}
7983
7984InstructionSelector::ComplexRendererFns
7985AArch64InstructionSelector::selectExtractHigh(MachineOperand &Root) const {
7986 if (!Root.isReg())
7987 return std::nullopt;
7988 MachineRegisterInfo &MRI =
7989 Root.getParent()->getParent()->getParent()->getRegInfo();
7990
7991 auto Extract = getDefSrcRegIgnoringCopies(Reg: Root.getReg(), MRI);
7992 while (Extract && Extract->MI->getOpcode() == TargetOpcode::G_BITCAST &&
7993 STI.isLittleEndian())
7994 Extract =
7995 getDefSrcRegIgnoringCopies(Reg: Extract->MI->getOperand(i: 1).getReg(), MRI);
7996 if (!Extract)
7997 return std::nullopt;
7998
7999 if (auto *Unmerge = dyn_cast<GUnmerge>(Val: Extract->MI)) {
8000 if (Unmerge->getNumDefs() == 2 &&
8001 Extract->Reg == Unmerge->getOperand(i: 1).getReg()) {
8002 Register ExtReg = Unmerge->getSourceReg();
8003 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(RegNo: ExtReg); }}};
8004 }
8005 }
8006 if (auto *ExtElt = dyn_cast<GExtractVectorElement>(Val: Extract->MI)) {
8007 LLT SrcTy = MRI.getType(Reg: ExtElt->getVectorReg());
8008 auto LaneIdx =
8009 getIConstantVRegValWithLookThrough(VReg: ExtElt->getIndexReg(), MRI);
8010 if (LaneIdx && SrcTy == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 64) &&
8011 LaneIdx->Value.getSExtValue() == 1) {
8012 Register ExtReg = ExtElt->getVectorReg();
8013 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(RegNo: ExtReg); }}};
8014 }
8015 }
8016 if (auto *Subvec = dyn_cast<GExtractSubvector>(Val: Extract->MI)) {
8017 LLT SrcTy = MRI.getType(Reg: Subvec->getSrcVec());
8018 auto LaneIdx = Subvec->getIndexImm();
8019 if (LaneIdx == SrcTy.getNumElements() / 2) {
8020 Register ExtReg = Subvec->getSrcVec();
8021 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(RegNo: ExtReg); }}};
8022 }
8023 }
8024
8025 return std::nullopt;
8026}
8027
8028InstructionSelector::ComplexRendererFns
8029AArch64InstructionSelector::selectCVTFixedPointBase(const MachineOperand &Root,
8030 unsigned DstElemWidth,
8031 bool isReciprocal) const {
8032 if (!Root.isReg())
8033 return std::nullopt;
8034 const MachineRegisterInfo &MRI =
8035 Root.getParent()->getParent()->getParent()->getRegInfo();
8036
8037 Register Reg = Root.getReg();
8038 MachineInstr *Dup = getDefIgnoringCopies(Reg, MRI);
8039
8040 if (Dup && Dup->getOpcode() == TargetOpcode::G_SPLAT_VECTOR)
8041 Reg = Dup->getOperand(i: 1).getReg();
8042
8043 std::optional<ValueAndVReg> CstVal =
8044 getAnyConstantVRegValWithLookThrough(VReg: Reg, MRI);
8045
8046 if (!CstVal)
8047 return std::nullopt;
8048
8049 unsigned CstElemWidth = MRI.getType(Reg).getScalarSizeInBits();
8050 APFloat FVal(0.0);
8051 switch (CstElemWidth) {
8052 case 16:
8053 FVal = APFloat(APFloat::IEEEhalf(), CstVal->Value.trunc(width: CstElemWidth));
8054 break;
8055 case 32:
8056 FVal = APFloat(APFloat::IEEEsingle(), CstVal->Value);
8057 break;
8058 case 64:
8059 FVal = APFloat(APFloat::IEEEdouble(), CstVal->Value);
8060 break;
8061 default:
8062 return std::nullopt;
8063 };
8064 if (unsigned FBits =
8065 CheckFixedPointOperandConstant(FVal, RegWidth: DstElemWidth, isReciprocal))
8066 return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Val: FBits); }}};
8067
8068 return std::nullopt;
8069}
8070
8071unsigned AArch64InstructionSelector::getFixedPointWidthFromOperand(
8072 const MachineOperand &Root) const {
8073 return Root.getParent()
8074 ->getMF()
8075 ->getRegInfo()
8076 .getType(Reg: Root.getReg())
8077 .getScalarSizeInBits();
8078}
8079
8080template <unsigned Width>
8081InstructionSelector::ComplexRendererFns
8082AArch64InstructionSelector::selectCVTFixedPoint(MachineOperand &Root) const {
8083 return selectCVTFixedPointBase(Root, DstElemWidth: Width, /*isReciprocal*/ false);
8084}
8085
8086template <unsigned Width>
8087InstructionSelector::ComplexRendererFns
8088AArch64InstructionSelector::selectCVTFixedPosRecipOperand(
8089 MachineOperand &Root) const {
8090 return selectCVTFixedPointBase(Root, DstElemWidth: Width, /*isReciprocal*/ true);
8091}
8092
8093InstructionSelector::ComplexRendererFns
8094AArch64InstructionSelector::selectCVTFixedPointVec(MachineOperand &Root) const {
8095 return selectCVTFixedPointBase(Root, DstElemWidth: getFixedPointWidthFromOperand(Root),
8096 /*isReciprocal*/ false);
8097}
8098
8099InstructionSelector::ComplexRendererFns
8100AArch64InstructionSelector::selectCVTFixedPosRecipOperandVec(
8101 MachineOperand &Root) const {
8102 return selectCVTFixedPointBase(Root, DstElemWidth: getFixedPointWidthFromOperand(Root),
8103 /*isReciprocal*/ true);
8104}
8105
8106void AArch64InstructionSelector::renderFixedPointScalarXForm(
8107 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
8108 assert(OpIdx == 3 && MI.getOperand(OpIdx).isImm() &&
8109 "Expected vecshift immediate operand");
8110 MIB.addImm(Val: MI.getOperand(i: OpIdx).getImm());
8111}
8112
8113void AArch64InstructionSelector::renderFixedPointImm(MachineInstrBuilder &MIB,
8114 const MachineOperand &Root,
8115 unsigned Width,
8116 bool isReciprocal) const {
8117 // FIXME: This is only needed to satisfy the type checking in tablegen, and
8118 // should be able to reuse the Renderers already calculated by
8119 // selectCVTFixedPointBase.
8120 InstructionSelector::ComplexRendererFns Renderer =
8121 selectCVTFixedPointBase(Root, DstElemWidth: Width, isReciprocal);
8122 assert((Renderer && Renderer->size() == 1) &&
8123 "Expected selectCVTFixedPointBase to provide a function\n");
8124 (Renderer->front())(MIB);
8125}
8126
8127void AArch64InstructionSelector::renderFixedPointXForm(MachineInstrBuilder &MIB,
8128 const MachineInstr &MI,
8129 int OpIdx) const {
8130 const MachineOperand &Root = MI.getOperand(i: OpIdx);
8131 renderFixedPointImm(MIB, Root, Width: getFixedPointWidthFromOperand(Root),
8132 /*isReciprocal*/ false);
8133}
8134
8135void AArch64InstructionSelector::renderFixedPointRecipXForm(
8136 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
8137 const MachineOperand &Root = MI.getOperand(i: OpIdx);
8138 renderFixedPointImm(MIB, Root, Width: getFixedPointWidthFromOperand(Root),
8139 /*isReciprocal*/ true);
8140}
8141
8142void AArch64InstructionSelector::renderTruncImm(MachineInstrBuilder &MIB,
8143 const MachineInstr &MI,
8144 int OpIdx) const {
8145 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
8146 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
8147 "Expected G_CONSTANT");
8148 std::optional<int64_t> CstVal =
8149 getIConstantVRegSExtVal(VReg: MI.getOperand(i: 0).getReg(), MRI);
8150 assert(CstVal && "Expected constant value");
8151 MIB.addImm(Val: *CstVal);
8152}
8153
8154void AArch64InstructionSelector::renderLogicalImm32(
8155 MachineInstrBuilder &MIB, const MachineInstr &I, int OpIdx) const {
8156 assert(I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
8157 "Expected G_CONSTANT");
8158 uint64_t CstVal = I.getOperand(i: 1).getCImm()->getZExtValue();
8159 uint64_t Enc = AArch64_AM::encodeLogicalImmediate(imm: CstVal, regSize: 32);
8160 MIB.addImm(Val: Enc);
8161}
8162
8163void AArch64InstructionSelector::renderLogicalImm64(
8164 MachineInstrBuilder &MIB, const MachineInstr &I, int OpIdx) const {
8165 assert(I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
8166 "Expected G_CONSTANT");
8167 uint64_t CstVal = I.getOperand(i: 1).getCImm()->getZExtValue();
8168 uint64_t Enc = AArch64_AM::encodeLogicalImmediate(imm: CstVal, regSize: 64);
8169 MIB.addImm(Val: Enc);
8170}
8171
8172void AArch64InstructionSelector::renderUbsanTrap(MachineInstrBuilder &MIB,
8173 const MachineInstr &MI,
8174 int OpIdx) const {
8175 assert(MI.getOpcode() == TargetOpcode::G_UBSANTRAP && OpIdx == 0 &&
8176 "Expected G_UBSANTRAP");
8177 MIB.addImm(Val: MI.getOperand(i: 0).getImm() | ('U' << 8));
8178}
8179
8180void AArch64InstructionSelector::renderFPImm16(MachineInstrBuilder &MIB,
8181 const MachineInstr &MI,
8182 int OpIdx) const {
8183 assert(MI.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 &&
8184 "Expected G_FCONSTANT");
8185 MIB.addImm(
8186 Val: AArch64_AM::getFP16Imm(FPImm: MI.getOperand(i: 1).getFPImm()->getValueAPF()));
8187}
8188
8189void AArch64InstructionSelector::renderFPImm32(MachineInstrBuilder &MIB,
8190 const MachineInstr &MI,
8191 int OpIdx) const {
8192 assert(MI.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 &&
8193 "Expected G_FCONSTANT");
8194 MIB.addImm(
8195 Val: AArch64_AM::getFP32Imm(FPImm: MI.getOperand(i: 1).getFPImm()->getValueAPF()));
8196}
8197
8198void AArch64InstructionSelector::renderFPImm64(MachineInstrBuilder &MIB,
8199 const MachineInstr &MI,
8200 int OpIdx) const {
8201 assert(MI.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 &&
8202 "Expected G_FCONSTANT");
8203 MIB.addImm(
8204 Val: AArch64_AM::getFP64Imm(FPImm: MI.getOperand(i: 1).getFPImm()->getValueAPF()));
8205}
8206
8207bool AArch64InstructionSelector::isLoadStoreOfNumBytes(
8208 const MachineInstr &MI, unsigned NumBytes) const {
8209 if (!MI.mayLoadOrStore())
8210 return false;
8211 assert(MI.hasOneMemOperand() &&
8212 "Expected load/store to have only one mem op!");
8213 return (*MI.memoperands_begin())->getSize() == NumBytes;
8214}
8215
8216bool AArch64InstructionSelector::isDef32(const MachineInstr &MI) const {
8217 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
8218 if (MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits() != 32)
8219 return false;
8220
8221 // Only return true if we know the operation will zero-out the high half of
8222 // the 64-bit register. Truncates can be subregister copies, which don't
8223 // zero out the high bits. Copies and other copy-like instructions can be
8224 // fed by truncates, or could be lowered as subregister copies.
8225 switch (MI.getOpcode()) {
8226 default:
8227 return true;
8228 case TargetOpcode::COPY:
8229 case TargetOpcode::G_BITCAST:
8230 case TargetOpcode::G_TRUNC:
8231 case TargetOpcode::G_PHI:
8232 return false;
8233 }
8234}
8235
8236
8237// Perform fixups on the given PHI instruction's operands to force them all
8238// to be the same as the destination regbank.
8239static void fixupPHIOpBanks(MachineInstr &MI, MachineRegisterInfo &MRI,
8240 const AArch64RegisterBankInfo &RBI) {
8241 assert(MI.getOpcode() == TargetOpcode::G_PHI && "Expected a G_PHI");
8242 Register DstReg = MI.getOperand(i: 0).getReg();
8243 const RegisterBank *DstRB = MRI.getRegBankOrNull(Reg: DstReg);
8244 assert(DstRB && "Expected PHI dst to have regbank assigned");
8245 MachineIRBuilder MIB(MI);
8246
8247 // Go through each operand and ensure it has the same regbank.
8248 for (MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI.operands())) {
8249 if (!MO.isReg())
8250 continue;
8251 Register OpReg = MO.getReg();
8252 const RegisterBank *RB = MRI.getRegBankOrNull(Reg: OpReg);
8253 if (RB != DstRB) {
8254 // Insert a cross-bank copy.
8255 auto *OpDef = MRI.getVRegDef(Reg: OpReg);
8256 const LLT &Ty = MRI.getType(Reg: OpReg);
8257 MachineBasicBlock &OpDefBB = *OpDef->getParent();
8258
8259 // Any instruction we insert must appear after all PHIs in the block
8260 // for the block to be valid MIR.
8261 MachineBasicBlock::iterator InsertPt = std::next(x: OpDef->getIterator());
8262 if (InsertPt != OpDefBB.end() && InsertPt->isPHI())
8263 InsertPt = OpDefBB.getFirstNonPHI();
8264 MIB.setInsertPt(MBB&: *OpDef->getParent(), II: InsertPt);
8265 auto Copy = MIB.buildCopy(Res: Ty, Op: OpReg);
8266 MRI.setRegBank(Reg: Copy.getReg(Idx: 0), RegBank: *DstRB);
8267 MO.setReg(Copy.getReg(Idx: 0));
8268 }
8269 }
8270}
8271
8272void AArch64InstructionSelector::processPHIs(MachineFunction &MF) {
8273 // We're looking for PHIs, build a list so we don't invalidate iterators.
8274 MachineRegisterInfo &MRI = MF.getRegInfo();
8275 SmallVector<MachineInstr *, 32> Phis;
8276 for (auto &BB : MF) {
8277 for (auto &MI : BB) {
8278 if (MI.getOpcode() == TargetOpcode::G_PHI)
8279 Phis.emplace_back(Args: &MI);
8280 }
8281 }
8282
8283 for (auto *MI : Phis) {
8284 // We need to do some work here if the operand types are < 16 bit and they
8285 // are split across fpr/gpr banks. Since all types <32b on gpr
8286 // end up being assigned gpr32 regclasses, we can end up with PHIs here
8287 // which try to select between a gpr32 and an fpr16. Ideally RBS shouldn't
8288 // be selecting heterogenous regbanks for operands if possible, but we
8289 // still need to be able to deal with it here.
8290 //
8291 // To fix this, if we have a gpr-bank operand < 32b in size and at least
8292 // one other operand is on the fpr bank, then we add cross-bank copies
8293 // to homogenize the operand banks. For simplicity the bank that we choose
8294 // to settle on is whatever bank the def operand has. For example:
8295 //
8296 // %endbb:
8297 // %dst:gpr(s16) = G_PHI %in1:gpr(s16), %bb1, %in2:fpr(s16), %bb2
8298 // =>
8299 // %bb2:
8300 // ...
8301 // %in2_copy:gpr(s16) = COPY %in2:fpr(s16)
8302 // ...
8303 // %endbb:
8304 // %dst:gpr(s16) = G_PHI %in1:gpr(s16), %bb1, %in2_copy:gpr(s16), %bb2
8305 bool HasGPROp = false, HasFPROp = false;
8306 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI->operands())) {
8307 if (!MO.isReg())
8308 continue;
8309 const LLT &Ty = MRI.getType(Reg: MO.getReg());
8310 if (!Ty.isValid() || !Ty.isScalar())
8311 break;
8312 if (Ty.getSizeInBits() >= 32)
8313 break;
8314 const RegisterBank *RB = MRI.getRegBankOrNull(Reg: MO.getReg());
8315 // If for some reason we don't have a regbank yet. Don't try anything.
8316 if (!RB)
8317 break;
8318
8319 if (RB->getID() == AArch64::GPRRegBankID)
8320 HasGPROp = true;
8321 else
8322 HasFPROp = true;
8323 }
8324 // We have heterogenous regbanks, need to fixup.
8325 if (HasGPROp && HasFPROp)
8326 fixupPHIOpBanks(MI&: *MI, MRI, RBI);
8327 }
8328}
8329
8330namespace llvm {
8331InstructionSelector *
8332createAArch64InstructionSelector(const AArch64TargetMachine &TM,
8333 const AArch64Subtarget &Subtarget,
8334 const AArch64RegisterBankInfo &RBI) {
8335 return new AArch64InstructionSelector(TM, Subtarget, RBI);
8336}
8337}
8338