1//===- AArch64InstructionSelector.cpp ----------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the InstructionSelector class for
10/// AArch64.
11/// \todo This should be generated by TableGen.
12//===----------------------------------------------------------------------===//
13
14#include "AArch64ExpandImm.h"
15#include "AArch64GlobalISelUtils.h"
16#include "AArch64InstrInfo.h"
17#include "AArch64MachineFunctionInfo.h"
18#include "AArch64RegisterBankInfo.h"
19#include "AArch64RegisterInfo.h"
20#include "AArch64Subtarget.h"
21#include "AArch64TargetMachine.h"
22#include "MCTargetDesc/AArch64AddressingModes.h"
23#include "MCTargetDesc/AArch64MCTargetDesc.h"
24#include "llvm/BinaryFormat/Dwarf.h"
25#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
26#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
27#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
28#include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
29#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
30#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
31#include "llvm/CodeGen/GlobalISel/Utils.h"
32#include "llvm/CodeGen/MachineBasicBlock.h"
33#include "llvm/CodeGen/MachineConstantPool.h"
34#include "llvm/CodeGen/MachineFrameInfo.h"
35#include "llvm/CodeGen/MachineFunction.h"
36#include "llvm/CodeGen/MachineInstr.h"
37#include "llvm/CodeGen/MachineInstrBuilder.h"
38#include "llvm/CodeGen/MachineMemOperand.h"
39#include "llvm/CodeGen/MachineOperand.h"
40#include "llvm/CodeGen/MachineRegisterInfo.h"
41#include "llvm/CodeGen/TargetOpcodes.h"
42#include "llvm/CodeGen/TargetRegisterInfo.h"
43#include "llvm/IR/Constants.h"
44#include "llvm/IR/DerivedTypes.h"
45#include "llvm/IR/Instructions.h"
46#include "llvm/IR/IntrinsicsAArch64.h"
47#include "llvm/IR/Type.h"
48#include "llvm/Pass.h"
49#include "llvm/Support/Debug.h"
50#include "llvm/Support/raw_ostream.h"
51#include <optional>
52
53#define DEBUG_TYPE "aarch64-isel"
54
55using namespace llvm;
56using namespace MIPatternMatch;
57using namespace AArch64GISelUtils;
58
59namespace llvm {
60class BlockFrequencyInfo;
61class ProfileSummaryInfo;
62}
63
64namespace {
65
66#define GET_GLOBALISEL_PREDICATE_BITSET
67#include "AArch64GenGlobalISel.inc"
68#undef GET_GLOBALISEL_PREDICATE_BITSET
69
70
71class AArch64InstructionSelector : public InstructionSelector {
72public:
73 AArch64InstructionSelector(const AArch64TargetMachine &TM,
74 const AArch64Subtarget &STI,
75 const AArch64RegisterBankInfo &RBI);
76
77 bool select(MachineInstr &I) override;
78 static const char *getName() { return DEBUG_TYPE; }
79
80 void setupMF(MachineFunction &MF, GISelValueTracking *VT,
81 CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI,
82 BlockFrequencyInfo *BFI) override {
83 InstructionSelector::setupMF(mf&: MF, vt: VT, covinfo: CoverageInfo, psi: PSI, bfi: BFI);
84 MIB.setMF(MF);
85
86 // hasFnAttribute() is expensive to call on every BRCOND selection, so
87 // cache it here for each run of the selector.
88 ProduceNonFlagSettingCondBr =
89 !MF.getFunction().hasFnAttribute(Kind: Attribute::SpeculativeLoadHardening);
90 MFReturnAddr = Register();
91
92 processPHIs(MF);
93 }
94
95private:
96 /// tblgen-erated 'select' implementation, used as the initial selector for
97 /// the patterns that don't require complex C++.
98 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
99
100 // A lowering phase that runs before any selection attempts.
101 // Returns true if the instruction was modified.
102 bool preISelLower(MachineInstr &I);
103
104 // An early selection function that runs before the selectImpl() call.
105 bool earlySelect(MachineInstr &I);
106
107 /// Save state that is shared between select calls, call select on \p I and
108 /// then restore the saved state. This can be used to recursively call select
109 /// within a select call.
110 bool selectAndRestoreState(MachineInstr &I);
111
112 // Do some preprocessing of G_PHIs before we begin selection.
113 void processPHIs(MachineFunction &MF);
114
115 bool earlySelectSHL(MachineInstr &I, MachineRegisterInfo &MRI);
116
117 /// Eliminate same-sized cross-bank copies into stores before selectImpl().
118 bool contractCrossBankCopyIntoStore(MachineInstr &I,
119 MachineRegisterInfo &MRI);
120
121 bool convertPtrAddToAdd(MachineInstr &I, MachineRegisterInfo &MRI);
122
123 bool selectVaStartAAPCS(MachineInstr &I, MachineFunction &MF,
124 MachineRegisterInfo &MRI) const;
125 bool selectVaStartDarwin(MachineInstr &I, MachineFunction &MF,
126 MachineRegisterInfo &MRI) const;
127
128 ///@{
129 /// Helper functions for selectCompareBranch.
130 bool selectCompareBranchFedByFCmp(MachineInstr &I, MachineInstr &FCmp,
131 MachineIRBuilder &MIB) const;
132 bool selectCompareBranchFedByICmp(MachineInstr &I, MachineInstr &ICmp,
133 MachineIRBuilder &MIB) const;
134 bool tryOptCompareBranchFedByICmp(MachineInstr &I, MachineInstr &ICmp,
135 MachineIRBuilder &MIB) const;
136 bool tryOptAndIntoCompareBranch(MachineInstr &AndInst, bool Invert,
137 MachineBasicBlock *DstMBB,
138 MachineIRBuilder &MIB) const;
139 ///@}
140
141 bool selectCompareBranch(MachineInstr &I, MachineFunction &MF,
142 MachineRegisterInfo &MRI);
143
144 bool selectVectorAshrLshr(MachineInstr &I, MachineRegisterInfo &MRI);
145 bool selectVectorSHL(MachineInstr &I, MachineRegisterInfo &MRI);
146
147 // Helper to generate an equivalent of scalar_to_vector into a new register,
148 // returned via 'Dst'.
149 MachineInstr *emitScalarToVector(unsigned EltSize,
150 const TargetRegisterClass *DstRC,
151 Register Scalar,
152 MachineIRBuilder &MIRBuilder) const;
153 /// Helper to narrow vector that was widened by emitScalarToVector.
154 /// Copy lowest part of 128-bit or 64-bit vector to 64-bit or 32-bit
155 /// vector, correspondingly.
156 MachineInstr *emitNarrowVector(Register DstReg, Register SrcReg,
157 MachineIRBuilder &MIRBuilder,
158 MachineRegisterInfo &MRI) const;
159
160 /// Emit a lane insert into \p DstReg, or a new vector register if
161 /// std::nullopt is provided.
162 ///
163 /// The lane inserted into is defined by \p LaneIdx. The vector source
164 /// register is given by \p SrcReg. The register containing the element is
165 /// given by \p EltReg.
166 MachineInstr *emitLaneInsert(std::optional<Register> DstReg, Register SrcReg,
167 Register EltReg, unsigned LaneIdx,
168 const RegisterBank &RB,
169 MachineIRBuilder &MIRBuilder) const;
170
171 /// Emit a sequence of instructions representing a constant \p Imm for a
172 /// vector register \p Dst via MOVI.
173 MachineInstr *emitConstantMOVVector(DstOp Dst, APInt Imm,
174 MachineRegisterInfo &MRI);
175
176 /// Emit a sequence of instructions representing a constant \p CV for a
177 /// vector register \p Dst. (E.g. a MOV, or a load from a constant pool.)
178 ///
179 /// \returns the last instruction in the sequence on success, and nullptr
180 /// otherwise.
181 MachineInstr *emitConstantVector(Register Dst, Constant *CV, APInt Imm,
182 MachineRegisterInfo &MRI);
183
184 bool tryOptConstantBuildVec(MachineInstr &MI, LLT DstTy,
185 MachineRegisterInfo &MRI);
186 /// \returns true if a G_BUILD_VECTOR instruction \p MI can be selected as a
187 /// SUBREG_TO_REG.
188 bool tryOptBuildVecToSubregToReg(MachineInstr &MI, MachineRegisterInfo &MRI);
189 bool selectBuildVector(MachineInstr &I, MachineRegisterInfo &MRI);
190 bool selectMergeValues(MachineInstr &I, MachineRegisterInfo &MRI);
191 bool selectUnmergeValues(MachineInstr &I, MachineRegisterInfo &MRI);
192
193 bool selectShuffleVector(MachineInstr &I, MachineRegisterInfo &MRI);
194 bool selectExtractElt(MachineInstr &I, MachineRegisterInfo &MRI);
195 bool selectConcatVectors(MachineInstr &I, MachineRegisterInfo &MRI);
196 bool selectSplitVectorUnmerge(MachineInstr &I, MachineRegisterInfo &MRI);
197
198 /// Helper function to select vector load intrinsics like
199 /// @llvm.aarch64.neon.ld2.*, @llvm.aarch64.neon.ld4.*, etc.
200 /// \p Opc is the opcode that the selected instruction should use.
201 /// \p NumVecs is the number of vector destinations for the instruction.
202 /// \p I is the original G_INTRINSIC_W_SIDE_EFFECTS instruction.
203 bool selectVectorLoadIntrinsic(unsigned Opc, unsigned NumVecs,
204 MachineInstr &I);
205 bool selectVectorLoadLaneIntrinsic(unsigned Opc, unsigned NumVecs,
206 MachineInstr &I);
207 void selectVectorStoreIntrinsic(MachineInstr &I, unsigned NumVecs,
208 unsigned Opc);
209 bool selectVectorStoreLaneIntrinsic(MachineInstr &I, unsigned NumVecs,
210 unsigned Opc);
211 bool selectIntrinsicWithSideEffects(MachineInstr &I,
212 MachineRegisterInfo &MRI);
213 bool selectIntrinsic(MachineInstr &I, MachineRegisterInfo &MRI);
214 bool selectJumpTable(MachineInstr &I, MachineRegisterInfo &MRI);
215 bool selectBrJT(MachineInstr &I, MachineRegisterInfo &MRI);
216 bool selectTLSGlobalValueELF(MachineInstr &I, MachineRegisterInfo &MRI);
217 bool selectTLSLocalExecELF(const GlobalValue *GV, MachineInstr &I,
218 MachineRegisterInfo &MRI);
219 bool selectTLSGlobalValueMachO(MachineInstr &I, MachineRegisterInfo &MRI);
220 bool selectTLSGlobalValue(MachineInstr &I, MachineRegisterInfo &MRI);
221 bool selectPtrAuthGlobalValue(MachineInstr &I,
222 MachineRegisterInfo &MRI) const;
223 bool selectMOPS(MachineInstr &I, MachineRegisterInfo &MRI);
224 bool selectUSMovFromExtend(MachineInstr &I, MachineRegisterInfo &MRI);
225 void SelectTable(MachineInstr &I, MachineRegisterInfo &MRI, unsigned NumVecs,
226 unsigned Opc1, unsigned Opc2, bool isExt);
227
228 bool selectIndexedExtLoad(MachineInstr &I, MachineRegisterInfo &MRI);
229 bool selectIndexedLoad(MachineInstr &I, MachineRegisterInfo &MRI);
230 bool selectIndexedStore(GIndexedStore &I, MachineRegisterInfo &MRI);
231
232 unsigned emitConstantPoolEntry(const Constant *CPVal,
233 MachineFunction &MF) const;
234 MachineInstr *emitLoadFromConstantPool(const Constant *CPVal,
235 MachineIRBuilder &MIRBuilder) const;
236
237 // Emit a vector concat operation.
238 MachineInstr *emitVectorConcat(std::optional<Register> Dst, Register Op1,
239 Register Op2,
240 MachineIRBuilder &MIRBuilder) const;
241
242 // Emit an integer compare between LHS and RHS, which checks for Predicate.
243 MachineInstr *emitIntegerCompare(MachineOperand &LHS, MachineOperand &RHS,
244 MachineOperand &Predicate,
245 MachineIRBuilder &MIRBuilder) const;
246
247 /// Emit a floating point comparison between \p LHS and \p RHS.
248 /// \p Pred if given is the intended predicate to use.
249 MachineInstr *
250 emitFPCompare(Register LHS, Register RHS, MachineIRBuilder &MIRBuilder,
251 std::optional<CmpInst::Predicate> = std::nullopt) const;
252
253 MachineInstr *
254 emitInstr(unsigned Opcode, std::initializer_list<llvm::DstOp> DstOps,
255 std::initializer_list<llvm::SrcOp> SrcOps,
256 MachineIRBuilder &MIRBuilder,
257 const ComplexRendererFns &RenderFns = std::nullopt) const;
258 /// Helper function to emit an add or sub instruction.
259 ///
260 /// \p AddrModeAndSizeToOpcode must contain each of the opcode variants above
261 /// in a specific order.
262 ///
263 /// Below is an example of the expected input to \p AddrModeAndSizeToOpcode.
264 ///
265 /// \code
266 /// const std::array<std::array<unsigned, 2>, 4> Table {
267 /// {{AArch64::ADDXri, AArch64::ADDWri},
268 /// {AArch64::ADDXrs, AArch64::ADDWrs},
269 /// {AArch64::ADDXrr, AArch64::ADDWrr},
270 /// {AArch64::SUBXri, AArch64::SUBWri},
271 /// {AArch64::ADDXrx, AArch64::ADDWrx}}};
272 /// \endcode
273 ///
274 /// Each row in the table corresponds to a different addressing mode. Each
275 /// column corresponds to a different register size.
276 ///
277 /// \attention Rows must be structured as follows:
278 /// - Row 0: The ri opcode variants
279 /// - Row 1: The rs opcode variants
280 /// - Row 2: The rr opcode variants
281 /// - Row 3: The ri opcode variants for negative immediates
282 /// - Row 4: The rx opcode variants
283 ///
284 /// \attention Columns must be structured as follows:
285 /// - Column 0: The 64-bit opcode variants
286 /// - Column 1: The 32-bit opcode variants
287 ///
288 /// \p Dst is the destination register of the binop to emit.
289 /// \p LHS is the left-hand operand of the binop to emit.
290 /// \p RHS is the right-hand operand of the binop to emit.
291 MachineInstr *emitAddSub(
292 const std::array<std::array<unsigned, 2>, 5> &AddrModeAndSizeToOpcode,
293 Register Dst, MachineOperand &LHS, MachineOperand &RHS,
294 MachineIRBuilder &MIRBuilder) const;
295 MachineInstr *emitADD(Register DefReg, MachineOperand &LHS,
296 MachineOperand &RHS,
297 MachineIRBuilder &MIRBuilder) const;
298 MachineInstr *emitADDS(Register Dst, MachineOperand &LHS, MachineOperand &RHS,
299 MachineIRBuilder &MIRBuilder) const;
300 MachineInstr *emitSUBS(Register Dst, MachineOperand &LHS, MachineOperand &RHS,
301 MachineIRBuilder &MIRBuilder) const;
302 MachineInstr *emitADCS(Register Dst, MachineOperand &LHS, MachineOperand &RHS,
303 MachineIRBuilder &MIRBuilder) const;
304 MachineInstr *emitSBCS(Register Dst, MachineOperand &LHS, MachineOperand &RHS,
305 MachineIRBuilder &MIRBuilder) const;
306 MachineInstr *emitCMP(MachineOperand &LHS, MachineOperand &RHS,
307 MachineIRBuilder &MIRBuilder) const;
308 MachineInstr *emitCMN(MachineOperand &LHS, MachineOperand &RHS,
309 MachineIRBuilder &MIRBuilder) const;
310 MachineInstr *emitTST(MachineOperand &LHS, MachineOperand &RHS,
311 MachineIRBuilder &MIRBuilder) const;
312 MachineInstr *emitSelect(Register Dst, Register LHS, Register RHS,
313 AArch64CC::CondCode CC,
314 MachineIRBuilder &MIRBuilder) const;
315 MachineInstr *emitExtractVectorElt(std::optional<Register> DstReg,
316 const RegisterBank &DstRB, LLT ScalarTy,
317 Register VecReg, unsigned LaneIdx,
318 MachineIRBuilder &MIRBuilder) const;
319 MachineInstr *emitCSINC(Register Dst, Register Src1, Register Src2,
320 AArch64CC::CondCode Pred,
321 MachineIRBuilder &MIRBuilder) const;
322 /// Emit a CSet for a FP compare.
323 ///
324 /// \p Dst is expected to be a 32-bit scalar register.
325 MachineInstr *emitCSetForFCmp(Register Dst, CmpInst::Predicate Pred,
326 MachineIRBuilder &MIRBuilder) const;
327
328 /// Emit an instruction that sets NZCV to the carry-in expected by \p I.
329 /// Might elide the instruction if the previous instruction already sets NZCV
330 /// correctly.
331 MachineInstr *emitCarryIn(MachineInstr &I, Register CarryReg);
332
333 /// Emit the overflow op for \p Opcode.
334 ///
335 /// \p Opcode is expected to be an overflow op's opcode, e.g. G_UADDO,
336 /// G_USUBO, etc.
337 std::pair<MachineInstr *, AArch64CC::CondCode>
338 emitOverflowOp(unsigned Opcode, Register Dst, MachineOperand &LHS,
339 MachineOperand &RHS, MachineIRBuilder &MIRBuilder) const;
340
341 bool selectOverflowOp(MachineInstr &I, MachineRegisterInfo &MRI);
342
343 /// Emit expression as a conjunction (a series of CCMP/CFCMP ops).
344 /// In some cases this is even possible with OR operations in the expression.
345 MachineInstr *emitConjunction(Register Val, AArch64CC::CondCode &OutCC,
346 MachineIRBuilder &MIB) const;
347 MachineInstr *emitConditionalComparison(Register LHS, Register RHS,
348 CmpInst::Predicate CC,
349 AArch64CC::CondCode Predicate,
350 AArch64CC::CondCode OutCC,
351 MachineIRBuilder &MIB) const;
352 MachineInstr *emitConjunctionRec(Register Val, AArch64CC::CondCode &OutCC,
353 bool Negate, Register CCOp,
354 AArch64CC::CondCode Predicate,
355 MachineIRBuilder &MIB) const;
356
357 /// Emit a TB(N)Z instruction which tests \p Bit in \p TestReg.
358 /// \p IsNegative is true if the test should be "not zero".
359 /// This will also optimize the test bit instruction when possible.
360 MachineInstr *emitTestBit(Register TestReg, uint64_t Bit, bool IsNegative,
361 MachineBasicBlock *DstMBB,
362 MachineIRBuilder &MIB) const;
363
364 /// Emit a CB(N)Z instruction which branches to \p DestMBB.
365 MachineInstr *emitCBZ(Register CompareReg, bool IsNegative,
366 MachineBasicBlock *DestMBB,
367 MachineIRBuilder &MIB) const;
368
369 // Equivalent to the i32shift_a and friends from AArch64InstrInfo.td.
370 // We use these manually instead of using the importer since it doesn't
371 // support SDNodeXForm.
372 ComplexRendererFns selectShiftA_32(const MachineOperand &Root) const;
373 ComplexRendererFns selectShiftB_32(const MachineOperand &Root) const;
374 ComplexRendererFns selectShiftA_64(const MachineOperand &Root) const;
375 ComplexRendererFns selectShiftB_64(const MachineOperand &Root) const;
376
377 template <unsigned ShiftWidth>
378 ComplexRendererFns selectShiftMask(MachineOperand &Root) const;
379 ComplexRendererFns select12BitValueWithLeftShift(uint64_t Immed) const;
380 ComplexRendererFns selectArithImmed(MachineOperand &Root) const;
381 ComplexRendererFns selectNegArithImmed(MachineOperand &Root) const;
382
383 ComplexRendererFns selectAddrModeUnscaled(MachineOperand &Root,
384 unsigned Size) const;
385
386 ComplexRendererFns selectAddrModeUnscaled8(MachineOperand &Root) const {
387 return selectAddrModeUnscaled(Root, Size: 1);
388 }
389 ComplexRendererFns selectAddrModeUnscaled16(MachineOperand &Root) const {
390 return selectAddrModeUnscaled(Root, Size: 2);
391 }
392 ComplexRendererFns selectAddrModeUnscaled32(MachineOperand &Root) const {
393 return selectAddrModeUnscaled(Root, Size: 4);
394 }
395 ComplexRendererFns selectAddrModeUnscaled64(MachineOperand &Root) const {
396 return selectAddrModeUnscaled(Root, Size: 8);
397 }
398 ComplexRendererFns selectAddrModeUnscaled128(MachineOperand &Root) const {
399 return selectAddrModeUnscaled(Root, Size: 16);
400 }
401
402 /// Helper to try to fold in a GISEL_ADD_LOW into an immediate, to be used
403 /// from complex pattern matchers like selectAddrModeIndexed().
404 ComplexRendererFns tryFoldAddLowIntoImm(MachineInstr &RootDef, unsigned Size,
405 MachineRegisterInfo &MRI) const;
406
407 ComplexRendererFns selectAddrModeIndexed(MachineOperand &Root,
408 unsigned Size) const;
409 template <int Width>
410 ComplexRendererFns selectAddrModeIndexed(MachineOperand &Root) const {
411 return selectAddrModeIndexed(Root, Size: Width / 8);
412 }
413
414 std::optional<bool>
415 isWorthFoldingIntoAddrMode(const MachineInstr &MI,
416 const MachineRegisterInfo &MRI) const;
417
418 bool isWorthFoldingIntoExtendedReg(const MachineInstr &MI,
419 const MachineRegisterInfo &MRI,
420 bool IsAddrOperand) const;
421 ComplexRendererFns
422 selectAddrModeShiftedExtendXReg(MachineOperand &Root,
423 unsigned SizeInBytes) const;
424
425 /// Returns a \p ComplexRendererFns which contains a base, offset, and whether
426 /// or not a shift + extend should be folded into an addressing mode. Returns
427 /// None when this is not profitable or possible.
428 ComplexRendererFns
429 selectExtendedSHL(MachineOperand &Root, MachineOperand &Base,
430 MachineOperand &Offset, unsigned SizeInBytes,
431 bool WantsExt) const;
432 ComplexRendererFns selectAddrModeRegisterOffset(MachineOperand &Root) const;
433 ComplexRendererFns selectAddrModeXRO(MachineOperand &Root,
434 unsigned SizeInBytes) const;
435 template <int Width>
436 ComplexRendererFns selectAddrModeXRO(MachineOperand &Root) const {
437 return selectAddrModeXRO(Root, SizeInBytes: Width / 8);
438 }
439
440 ComplexRendererFns selectAddrModeWRO(MachineOperand &Root,
441 unsigned SizeInBytes) const;
442 template <int Width>
443 ComplexRendererFns selectAddrModeWRO(MachineOperand &Root) const {
444 return selectAddrModeWRO(Root, SizeInBytes: Width / 8);
445 }
446
447 ComplexRendererFns selectShiftedRegister(MachineOperand &Root,
448 bool AllowROR = false) const;
449
450 ComplexRendererFns selectArithShiftedRegister(MachineOperand &Root) const {
451 return selectShiftedRegister(Root);
452 }
453
454 ComplexRendererFns selectLogicalShiftedRegister(MachineOperand &Root) const {
455 return selectShiftedRegister(Root, AllowROR: true);
456 }
457
458 /// Given an extend instruction, determine the correct shift-extend type for
459 /// that instruction.
460 ///
461 /// If the instruction is going to be used in a load or store, pass
462 /// \p IsLoadStore = true.
463 AArch64_AM::ShiftExtendType
464 getExtendTypeForInst(MachineInstr &MI, MachineRegisterInfo &MRI,
465 bool IsLoadStore = false) const;
466
467 /// Move \p Reg to \p RC if \p Reg is not already on \p RC.
468 ///
469 /// \returns Either \p Reg if no change was necessary, or the new register
470 /// created by moving \p Reg.
471 ///
472 /// Note: This uses emitCopy right now.
473 Register moveScalarRegClass(Register Reg, const TargetRegisterClass &RC,
474 MachineIRBuilder &MIB) const;
475
476 ComplexRendererFns selectArithExtendedRegister(MachineOperand &Root) const;
477
478 ComplexRendererFns selectExtractHigh(MachineOperand &Root) const;
479 template <unsigned Width>
480 ComplexRendererFns selectCVTFixedPoint(MachineOperand &Root) const;
481 template <unsigned Width>
482 ComplexRendererFns selectCVTFixedPosRecipOperand(MachineOperand &Root) const;
483 ComplexRendererFns selectCVTFixedPointBase(const MachineOperand &Root,
484 unsigned width,
485 bool isReciprocal = false) const;
486 ComplexRendererFns selectCVTFixedPointVec(MachineOperand &Root) const;
487 ComplexRendererFns
488 selectCVTFixedPosRecipOperandVec(MachineOperand &Root) const;
489 void renderFixedPointScalarXForm(MachineInstrBuilder &MIB,
490 const MachineInstr &MI, int OpIdx) const;
491 unsigned getFixedPointWidthFromOperand(const MachineOperand &Root) const;
492 void renderFixedPointXForm(MachineInstrBuilder &MIB, const MachineInstr &MI,
493 int OpIdx = -1) const;
494 void renderFixedPointRecipXForm(MachineInstrBuilder &MIB,
495 const MachineInstr &MI, int OpIdx = -1) const;
496 void renderFixedPointImm(MachineInstrBuilder &MIB, const MachineOperand &Root,
497 unsigned Width, bool isReciprocal) const;
498 void renderTruncImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
499 int OpIdx = -1) const;
500 void renderLogicalImm32(MachineInstrBuilder &MIB, const MachineInstr &I,
501 int OpIdx = -1) const;
502 void renderLogicalImm64(MachineInstrBuilder &MIB, const MachineInstr &I,
503 int OpIdx = -1) const;
504 void renderUbsanTrap(MachineInstrBuilder &MIB, const MachineInstr &MI,
505 int OpIdx) const;
506 void renderFPImm16(MachineInstrBuilder &MIB, const MachineInstr &MI,
507 int OpIdx = -1) const;
508 void renderFPImm32(MachineInstrBuilder &MIB, const MachineInstr &MI,
509 int OpIdx = -1) const;
510 void renderFPImm64(MachineInstrBuilder &MIB, const MachineInstr &MI,
511 int OpIdx = -1) const;
512 void renderFPImm32SIMDModImmType4(MachineInstrBuilder &MIB,
513 const MachineInstr &MI,
514 int OpIdx = -1) const;
515
516 // Materialize a GlobalValue or BlockAddress using a movz+movk sequence.
517 void materializeLargeCMVal(MachineInstr &I, const Value *V, unsigned OpFlags);
518
519 // Optimization methods.
520 bool tryOptSelect(GSelect &Sel);
521 bool tryOptSelectConjunction(GSelect &Sel, MachineInstr &CondMI);
522 MachineInstr *tryFoldIntegerCompare(MachineOperand &LHS, MachineOperand &RHS,
523 MachineOperand &Predicate,
524 MachineIRBuilder &MIRBuilder) const;
525
526 /// Return true if \p MI is a load or store of \p NumBytes bytes.
527 bool isLoadStoreOfNumBytes(const MachineInstr &MI, unsigned NumBytes) const;
528
529 /// Returns true if \p MI is guaranteed to have the high-half of a 64-bit
530 /// register zeroed out. In other words, the result of MI has been explicitly
531 /// zero extended.
532 bool isDef32(const MachineInstr &MI) const;
533
534 const AArch64TargetMachine &TM;
535 const AArch64Subtarget &STI;
536 const AArch64InstrInfo &TII;
537 const AArch64RegisterInfo &TRI;
538 const AArch64RegisterBankInfo &RBI;
539
540 bool ProduceNonFlagSettingCondBr = false;
541
542 // Some cached values used during selection.
543 // We use LR as a live-in register, and we keep track of it here as it can be
544 // clobbered by calls.
545 Register MFReturnAddr;
546
547 MachineIRBuilder MIB;
548
549#define GET_GLOBALISEL_PREDICATES_DECL
550#include "AArch64GenGlobalISel.inc"
551#undef GET_GLOBALISEL_PREDICATES_DECL
552
553// We declare the temporaries used by selectImpl() in the class to minimize the
554// cost of constructing placeholder values.
555#define GET_GLOBALISEL_TEMPORARIES_DECL
556#include "AArch64GenGlobalISel.inc"
557#undef GET_GLOBALISEL_TEMPORARIES_DECL
558};
559
560} // end anonymous namespace
561
562#define GET_GLOBALISEL_IMPL
563#include "AArch64GenGlobalISel.inc"
564#undef GET_GLOBALISEL_IMPL
565
566AArch64InstructionSelector::AArch64InstructionSelector(
567 const AArch64TargetMachine &TM, const AArch64Subtarget &STI,
568 const AArch64RegisterBankInfo &RBI)
569 : TM(TM), STI(STI), TII(*STI.getInstrInfo()), TRI(*STI.getRegisterInfo()),
570 RBI(RBI),
571#define GET_GLOBALISEL_PREDICATES_INIT
572#include "AArch64GenGlobalISel.inc"
573#undef GET_GLOBALISEL_PREDICATES_INIT
574#define GET_GLOBALISEL_TEMPORARIES_INIT
575#include "AArch64GenGlobalISel.inc"
576#undef GET_GLOBALISEL_TEMPORARIES_INIT
577{
578}
579
580// FIXME: This should be target-independent, inferred from the types declared
581// for each class in the bank.
582//
583/// Given a register bank, and a type, return the smallest register class that
584/// can represent that combination.
585static const TargetRegisterClass *
586getRegClassForTypeOnBank(LLT Ty, const RegisterBank &RB,
587 bool GetAllRegSet = false) {
588 if (RB.getID() == AArch64::GPRRegBankID) {
589 if (Ty.getSizeInBits() <= 32)
590 return GetAllRegSet ? &AArch64::GPR32allRegClass
591 : &AArch64::GPR32RegClass;
592 if (Ty.getSizeInBits() == 64)
593 return GetAllRegSet ? &AArch64::GPR64allRegClass
594 : &AArch64::GPR64RegClass;
595 if (Ty.getSizeInBits() == 128)
596 return &AArch64::XSeqPairsClassRegClass;
597 return nullptr;
598 }
599
600 if (RB.getID() == AArch64::FPRRegBankID) {
601 switch (Ty.getSizeInBits()) {
602 case 8:
603 return &AArch64::FPR8RegClass;
604 case 16:
605 return &AArch64::FPR16RegClass;
606 case 32:
607 return &AArch64::FPR32RegClass;
608 case 64:
609 return &AArch64::FPR64RegClass;
610 case 128:
611 return &AArch64::FPR128RegClass;
612 }
613 return nullptr;
614 }
615
616 return nullptr;
617}
618
619/// Given a register bank, and size in bits, return the smallest register class
620/// that can represent that combination.
621static const TargetRegisterClass *
622getMinClassForRegBank(const RegisterBank &RB, TypeSize SizeInBits,
623 bool GetAllRegSet = false) {
624 if (SizeInBits.isScalable()) {
625 assert(RB.getID() == AArch64::FPRRegBankID &&
626 "Expected FPR regbank for scalable type size");
627 return &AArch64::ZPRRegClass;
628 }
629
630 unsigned RegBankID = RB.getID();
631
632 if (RegBankID == AArch64::GPRRegBankID) {
633 assert(!SizeInBits.isScalable() && "Unexpected scalable register size");
634 if (SizeInBits <= 32)
635 return GetAllRegSet ? &AArch64::GPR32allRegClass
636 : &AArch64::GPR32RegClass;
637 if (SizeInBits == 64)
638 return GetAllRegSet ? &AArch64::GPR64allRegClass
639 : &AArch64::GPR64RegClass;
640 if (SizeInBits == 128)
641 return &AArch64::XSeqPairsClassRegClass;
642 }
643
644 if (RegBankID == AArch64::FPRRegBankID) {
645 if (SizeInBits.isScalable()) {
646 assert(SizeInBits == TypeSize::getScalable(128) &&
647 "Unexpected scalable register size");
648 return &AArch64::ZPRRegClass;
649 }
650
651 switch (SizeInBits) {
652 default:
653 return nullptr;
654 case 8:
655 return &AArch64::FPR8RegClass;
656 case 16:
657 return &AArch64::FPR16RegClass;
658 case 32:
659 return &AArch64::FPR32RegClass;
660 case 64:
661 return &AArch64::FPR64RegClass;
662 case 128:
663 return &AArch64::FPR128RegClass;
664 }
665 }
666
667 return nullptr;
668}
669
670/// Returns the correct subregister to use for a given register class.
671static bool getSubRegForClass(const TargetRegisterClass *RC,
672 const TargetRegisterInfo &TRI, unsigned &SubReg) {
673 switch (TRI.getRegSizeInBits(RC: *RC)) {
674 case 8:
675 SubReg = AArch64::bsub;
676 break;
677 case 16:
678 SubReg = AArch64::hsub;
679 break;
680 case 32:
681 if (RC != &AArch64::FPR32RegClass)
682 SubReg = AArch64::sub_32;
683 else
684 SubReg = AArch64::ssub;
685 break;
686 case 64:
687 SubReg = AArch64::dsub;
688 break;
689 default:
690 LLVM_DEBUG(
691 dbgs() << "Couldn't find appropriate subregister for register class.");
692 return false;
693 }
694
695 return true;
696}
697
698/// Returns the minimum size the given register bank can hold.
699static unsigned getMinSizeForRegBank(const RegisterBank &RB) {
700 switch (RB.getID()) {
701 case AArch64::GPRRegBankID:
702 return 32;
703 case AArch64::FPRRegBankID:
704 return 8;
705 default:
706 llvm_unreachable("Tried to get minimum size for unknown register bank.");
707 }
708}
709
710/// Create a REG_SEQUENCE instruction using the registers in \p Regs.
711/// Helper function for functions like createDTuple and createQTuple.
712///
713/// \p RegClassIDs - The list of register class IDs available for some tuple of
714/// a scalar class. E.g. QQRegClassID, QQQRegClassID, QQQQRegClassID. This is
715/// expected to contain between 2 and 4 tuple classes.
716///
717/// \p SubRegs - The list of subregister classes associated with each register
718/// class ID in \p RegClassIDs. E.g., QQRegClassID should use the qsub0
719/// subregister class. The index of each subregister class is expected to
720/// correspond with the index of each register class.
721///
722/// \returns Either the destination register of REG_SEQUENCE instruction that
723/// was created, or the 0th element of \p Regs if \p Regs contains a single
724/// element.
725static Register createTuple(ArrayRef<Register> Regs,
726 const unsigned RegClassIDs[],
727 const unsigned SubRegs[], MachineIRBuilder &MIB) {
728 unsigned NumRegs = Regs.size();
729 if (NumRegs == 1)
730 return Regs[0];
731 assert(NumRegs >= 2 && NumRegs <= 4 &&
732 "Only support between two and 4 registers in a tuple!");
733 const TargetRegisterInfo *TRI = MIB.getMF().getSubtarget().getRegisterInfo();
734 auto *DesiredClass = TRI->getRegClass(i: RegClassIDs[NumRegs - 2]);
735 auto RegSequence =
736 MIB.buildInstr(Opc: TargetOpcode::REG_SEQUENCE, DstOps: {DesiredClass}, SrcOps: {});
737 for (unsigned I = 0, E = Regs.size(); I < E; ++I) {
738 RegSequence.addUse(RegNo: Regs[I]);
739 RegSequence.addImm(Val: SubRegs[I]);
740 }
741 return RegSequence.getReg(Idx: 0);
742}
743
744/// Create a tuple of D-registers using the registers in \p Regs.
745static Register createDTuple(ArrayRef<Register> Regs, MachineIRBuilder &MIB) {
746 static const unsigned RegClassIDs[] = {
747 AArch64::DDRegClassID, AArch64::DDDRegClassID, AArch64::DDDDRegClassID};
748 static const unsigned SubRegs[] = {AArch64::dsub0, AArch64::dsub1,
749 AArch64::dsub2, AArch64::dsub3};
750 return createTuple(Regs, RegClassIDs, SubRegs, MIB);
751}
752
753/// Create a tuple of Q-registers using the registers in \p Regs.
754static Register createQTuple(ArrayRef<Register> Regs, MachineIRBuilder &MIB) {
755 static const unsigned RegClassIDs[] = {
756 AArch64::QQRegClassID, AArch64::QQQRegClassID, AArch64::QQQQRegClassID};
757 static const unsigned SubRegs[] = {AArch64::qsub0, AArch64::qsub1,
758 AArch64::qsub2, AArch64::qsub3};
759 return createTuple(Regs, RegClassIDs, SubRegs, MIB);
760}
761
762static std::optional<uint64_t> getImmedFromMO(const MachineOperand &Root) {
763 auto &MI = *Root.getParent();
764 auto &MBB = *MI.getParent();
765 auto &MF = *MBB.getParent();
766 auto &MRI = MF.getRegInfo();
767 uint64_t Immed;
768 if (Root.isImm())
769 Immed = Root.getImm();
770 else if (Root.isCImm())
771 Immed = Root.getCImm()->getZExtValue();
772 else if (Root.isReg()) {
773 auto ValAndVReg =
774 getIConstantVRegValWithLookThrough(VReg: Root.getReg(), MRI, LookThroughInstrs: true);
775 if (!ValAndVReg)
776 return std::nullopt;
777 Immed = ValAndVReg->Value.getSExtValue();
778 } else
779 return std::nullopt;
780 return Immed;
781}
782
783/// Select the AArch64 opcode for the basic binary operation \p GenericOpc,
784/// appropriate for the register bank \p RegBankID and of size \p OpSize.
785/// \returns \p GenericOpc if the combination is unsupported.
786static unsigned selectBinaryOp(unsigned GenericOpc, unsigned RegBankID,
787 unsigned OpSize) {
788 if (RegBankID == AArch64::GPRRegBankID) {
789 if (OpSize == 32) {
790 switch (GenericOpc) {
791 case TargetOpcode::G_SHL:
792 return AArch64::LSLVWr;
793 case TargetOpcode::G_LSHR:
794 return AArch64::LSRVWr;
795 case TargetOpcode::G_ASHR:
796 return AArch64::ASRVWr;
797 default:
798 return GenericOpc;
799 }
800 } else if (OpSize == 64) {
801 switch (GenericOpc) {
802 case TargetOpcode::G_SHL:
803 return AArch64::LSLVXr;
804 case TargetOpcode::G_LSHR:
805 return AArch64::LSRVXr;
806 case TargetOpcode::G_ASHR:
807 return AArch64::ASRVXr;
808 default:
809 return GenericOpc;
810 }
811 }
812 }
813 return GenericOpc;
814}
815
816/// Select the AArch64 opcode for the G_LOAD or G_STORE operation \p GenericOpc,
817/// appropriate for the (value) register bank \p RegBankID and of memory access
818/// size \p OpSize. This returns the variant with the base+unsigned-immediate
819/// addressing mode (e.g., LDRXui).
820/// \returns \p GenericOpc if the combination is unsupported.
821static unsigned selectLoadStoreUIOp(unsigned GenericOpc, unsigned RegBankID,
822 unsigned OpSize) {
823 const bool isStore = GenericOpc == TargetOpcode::G_STORE;
824 switch (RegBankID) {
825 case AArch64::GPRRegBankID:
826 switch (OpSize) {
827 case 8:
828 return isStore ? AArch64::STRBBui : AArch64::LDRBBui;
829 case 16:
830 return isStore ? AArch64::STRHHui : AArch64::LDRHHui;
831 case 32:
832 return isStore ? AArch64::STRWui : AArch64::LDRWui;
833 case 64:
834 return isStore ? AArch64::STRXui : AArch64::LDRXui;
835 }
836 break;
837 case AArch64::FPRRegBankID:
838 switch (OpSize) {
839 case 8:
840 return isStore ? AArch64::STRBui : AArch64::LDRBui;
841 case 16:
842 return isStore ? AArch64::STRHui : AArch64::LDRHui;
843 case 32:
844 return isStore ? AArch64::STRSui : AArch64::LDRSui;
845 case 64:
846 return isStore ? AArch64::STRDui : AArch64::LDRDui;
847 case 128:
848 return isStore ? AArch64::STRQui : AArch64::LDRQui;
849 }
850 break;
851 }
852 return GenericOpc;
853}
854
855/// Helper function for selectCopy. Inserts a subregister copy from \p SrcReg
856/// to \p *To.
857///
858/// E.g "To = COPY SrcReg:SubReg"
859static bool copySubReg(MachineInstr &I, MachineRegisterInfo &MRI,
860 const RegisterBankInfo &RBI, Register SrcReg,
861 const TargetRegisterClass *To, unsigned SubReg) {
862 assert(SrcReg.isValid() && "Expected a valid source register?");
863 assert(To && "Destination register class cannot be null");
864 assert(SubReg && "Expected a valid subregister");
865
866 MachineIRBuilder MIB(I);
867 auto SubRegCopy =
868 MIB.buildInstr(Opc: TargetOpcode::COPY, DstOps: {To}, SrcOps: {}).addReg(RegNo: SrcReg, Flags: {}, SubReg);
869 MachineOperand &RegOp = I.getOperand(i: 1);
870 RegOp.setReg(SubRegCopy.getReg(Idx: 0));
871
872 // It's possible that the destination register won't be constrained. Make
873 // sure that happens.
874 if (!I.getOperand(i: 0).getReg().isPhysical())
875 RBI.constrainGenericRegister(Reg: I.getOperand(i: 0).getReg(), RC: *To, MRI);
876
877 return true;
878}
879
880// FIXME: We need some sort of API in RBI/TRI to allow generic code to
881// constrain operands of simple instructions given a TargetRegisterClass
882// and LLT
883static bool selectDebugInstr(MachineInstr &I, MachineRegisterInfo &MRI,
884 const RegisterBankInfo &RBI) {
885 for (MachineOperand &MO : I.operands()) {
886 if (!MO.isReg())
887 continue;
888 Register Reg = MO.getReg();
889 if (!Reg)
890 continue;
891 if (Reg.isPhysical())
892 continue;
893 LLT Ty = MRI.getType(Reg);
894 const RegClassOrRegBank &RegClassOrBank = MRI.getRegClassOrRegBank(Reg);
895 const TargetRegisterClass *RC =
896 dyn_cast<const TargetRegisterClass *>(Val: RegClassOrBank);
897 if (!RC) {
898 const RegisterBank &RB = *cast<const RegisterBank *>(Val: RegClassOrBank);
899 RC = getRegClassForTypeOnBank(Ty, RB);
900 if (!RC) {
901 LLVM_DEBUG(
902 dbgs() << "Warning: DBG_VALUE operand has unexpected size/bank\n");
903 break;
904 }
905 }
906 RBI.constrainGenericRegister(Reg, RC: *RC, MRI);
907 }
908
909 return true;
910}
911
912static bool selectCopy(MachineInstr &I, const TargetInstrInfo &TII,
913 MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI,
914 const RegisterBankInfo &RBI) {
915 Register DstReg = I.getOperand(i: 0).getReg();
916 Register SrcReg = I.getOperand(i: 1).getReg();
917 const RegisterBank &DstRegBank = *RBI.getRegBank(Reg: DstReg, MRI, TRI);
918 const RegisterBank &SrcRegBank = *RBI.getRegBank(Reg: SrcReg, MRI, TRI);
919
920 TypeSize DstRegSize = RBI.getSizeInBits(Reg: DstReg, MRI, TRI);
921 TypeSize SrcRegSize = RBI.getSizeInBits(Reg: SrcReg, MRI, TRI);
922
923 // Special casing for cross-bank copies of s1s. We can technically represent
924 // a 1-bit value with any size of register. The minimum size for a GPR is 32
925 // bits. So, we need to put the FPR on 32 bits as well.
926 //
927 // FIXME: I'm not sure if this case holds true outside of copies. If it does,
928 // then we can pull it into the helpers that get the appropriate class for a
929 // register bank. Or make a new helper that carries along some constraint
930 // information.
931 if (SrcRegBank != DstRegBank && (DstRegSize == TypeSize::getFixed(ExactSize: 1) &&
932 SrcRegSize == TypeSize::getFixed(ExactSize: 1)))
933 SrcRegSize = DstRegSize = TypeSize::getFixed(ExactSize: 32);
934
935 // Find the correct register classes for the source and destination registers.
936 const TargetRegisterClass *SrcRC =
937 getMinClassForRegBank(RB: SrcRegBank, SizeInBits: SrcRegSize, GetAllRegSet: true);
938 const TargetRegisterClass *DstRC =
939 getMinClassForRegBank(RB: DstRegBank, SizeInBits: DstRegSize, GetAllRegSet: true);
940
941 if (!DstRC) {
942 LLVM_DEBUG(dbgs() << "Unexpected dest size "
943 << RBI.getSizeInBits(DstReg, MRI, TRI) << '\n');
944 return false;
945 }
946
947 if (I.getOpcode() == TargetOpcode::G_BITCAST &&
948 RBI.getSizeInBits(Reg: DstReg, MRI, TRI) == TypeSize::getFixed(ExactSize: 16)) {
949 if (DstRegBank.getID() == AArch64::FPRRegBankID &&
950 SrcRegBank.getID() == AArch64::GPRRegBankID) {
951 if (!SrcReg.isPhysical() &&
952 !RBI.constrainGenericRegister(Reg: SrcReg, RC: AArch64::GPR32RegClass, MRI))
953 return false;
954 if (!DstReg.isPhysical() &&
955 !RBI.constrainGenericRegister(Reg: DstReg, RC: AArch64::FPR16RegClass, MRI))
956 return false;
957
958 Register FPR32 = MRI.createVirtualRegister(RegClass: &AArch64::FPR32RegClass);
959 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: AArch64::FMOVWSr))
960 .addDef(RegNo: FPR32)
961 .addUse(RegNo: SrcReg);
962 I.setDesc(TII.get(Opcode: TargetOpcode::COPY));
963 I.getOperand(i: 1).setReg(FPR32);
964 I.getOperand(i: 1).setSubReg(AArch64::hsub);
965 return true;
966 }
967
968 if (DstRegBank.getID() == AArch64::GPRRegBankID &&
969 SrcRegBank.getID() == AArch64::FPRRegBankID) {
970 if (!SrcReg.isPhysical() &&
971 !RBI.constrainGenericRegister(Reg: SrcReg, RC: AArch64::FPR16RegClass, MRI))
972 return false;
973 if (!DstReg.isPhysical() &&
974 !RBI.constrainGenericRegister(Reg: DstReg, RC: AArch64::GPR32RegClass, MRI))
975 return false;
976
977 Register FPR32 = MRI.createVirtualRegister(RegClass: &AArch64::FPR32RegClass);
978 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
979 MCID: TII.get(Opcode: TargetOpcode::SUBREG_TO_REG))
980 .addDef(RegNo: FPR32)
981 .addUse(RegNo: SrcReg)
982 .addImm(Val: AArch64::hsub);
983 I.setDesc(TII.get(Opcode: AArch64::FMOVSWr));
984 I.getOperand(i: 1).setReg(FPR32);
985 return true;
986 }
987 }
988
989 // Is this a copy? If so, then we may need to insert a subregister copy.
990 if (I.isCopy()) {
991 // Yes. Check if there's anything to fix up.
992 if (!SrcRC) {
993 LLVM_DEBUG(dbgs() << "Couldn't determine source register class\n");
994 return false;
995 }
996
997 const TypeSize SrcSize = TRI.getRegSizeInBits(RC: *SrcRC);
998 const TypeSize DstSize = TRI.getRegSizeInBits(RC: *DstRC);
999 unsigned SrcSubReg = I.getOperand(i: 1).getSubReg();
1000 unsigned SubReg;
1001
1002 if (SrcSubReg)
1003 return RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI);
1004
1005 // If the source bank doesn't support a subregister copy small enough,
1006 // then we first need to copy to the destination bank.
1007 if (getMinSizeForRegBank(RB: SrcRegBank) > DstSize) {
1008 const TargetRegisterClass *DstTempRC =
1009 getMinClassForRegBank(RB: DstRegBank, SizeInBits: SrcSize, /* GetAllRegSet */ true);
1010 getSubRegForClass(RC: DstRC, TRI, SubReg);
1011
1012 MachineIRBuilder MIB(I);
1013 auto Copy = MIB.buildCopy(Res: {DstTempRC}, Op: {SrcReg});
1014 copySubReg(I, MRI, RBI, SrcReg: Copy.getReg(Idx: 0), To: DstRC, SubReg);
1015 } else if (SrcSize > DstSize) {
1016 // If the source register is bigger than the destination we need to
1017 // perform a subregister copy.
1018 const TargetRegisterClass *SubRegRC =
1019 getMinClassForRegBank(RB: SrcRegBank, SizeInBits: DstSize, /* GetAllRegSet */ true);
1020 getSubRegForClass(RC: SubRegRC, TRI, SubReg);
1021 copySubReg(I, MRI, RBI, SrcReg, To: DstRC, SubReg);
1022 } else if (DstSize > SrcSize) {
1023 // If the destination register is bigger than the source we need to do
1024 // a promotion using SUBREG_TO_REG.
1025 const TargetRegisterClass *PromotionRC =
1026 getMinClassForRegBank(RB: SrcRegBank, SizeInBits: DstSize, /* GetAllRegSet */ true);
1027 getSubRegForClass(RC: SrcRC, TRI, SubReg);
1028
1029 Register PromoteReg = MRI.createVirtualRegister(RegClass: PromotionRC);
1030 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1031 MCID: TII.get(Opcode: AArch64::SUBREG_TO_REG), DestReg: PromoteReg)
1032 .addUse(RegNo: SrcReg)
1033 .addImm(Val: SubReg);
1034 MachineOperand &RegOp = I.getOperand(i: 1);
1035 RegOp.setReg(PromoteReg);
1036 }
1037
1038 // If the destination is a physical register, then there's nothing to
1039 // change, so we're done.
1040 if (DstReg.isPhysical())
1041 return true;
1042 }
1043
1044 // No need to constrain SrcReg. It will get constrained when we hit another
1045 // of its use or its defs. Copies do not have constraints.
1046 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI)) {
1047 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
1048 << " operand\n");
1049 return false;
1050 }
1051
1052 // If this a GPR ZEXT that we want to just reduce down into a copy.
1053 // The sizes will be mismatched with the source < 32b but that's ok.
1054 if (I.getOpcode() == TargetOpcode::G_ZEXT) {
1055 I.setDesc(TII.get(Opcode: AArch64::COPY));
1056 assert(SrcRegBank.getID() == AArch64::GPRRegBankID);
1057 return selectCopy(I, TII, MRI, TRI, RBI);
1058 }
1059
1060 I.setDesc(TII.get(Opcode: AArch64::COPY));
1061 return true;
1062}
1063
1064MachineInstr *
1065AArch64InstructionSelector::emitSelect(Register Dst, Register True,
1066 Register False, AArch64CC::CondCode CC,
1067 MachineIRBuilder &MIB) const {
1068 MachineRegisterInfo &MRI = *MIB.getMRI();
1069 assert(RBI.getRegBank(False, MRI, TRI)->getID() ==
1070 RBI.getRegBank(True, MRI, TRI)->getID() &&
1071 "Expected both select operands to have the same regbank?");
1072 LLT Ty = MRI.getType(Reg: True);
1073 if (Ty.isVector())
1074 return nullptr;
1075 const unsigned Size = Ty.getSizeInBits();
1076 assert((Size == 32 || Size == 64) &&
1077 "Expected 32 bit or 64 bit select only?");
1078 const bool Is32Bit = Size == 32;
1079 if (RBI.getRegBank(Reg: True, MRI, TRI)->getID() != AArch64::GPRRegBankID) {
1080 unsigned Opc = Is32Bit ? AArch64::FCSELSrrr : AArch64::FCSELDrrr;
1081 auto FCSel = MIB.buildInstr(Opc, DstOps: {Dst}, SrcOps: {True, False}).addImm(Val: CC);
1082 constrainSelectedInstRegOperands(I&: *FCSel, TII, TRI, RBI);
1083 return &*FCSel;
1084 }
1085
1086 // By default, we'll try and emit a CSEL.
1087 unsigned Opc = Is32Bit ? AArch64::CSELWr : AArch64::CSELXr;
1088 bool Optimized = false;
1089 auto TryFoldBinOpIntoSelect = [&Opc, Is32Bit, &CC, &MRI,
1090 &Optimized](Register &Reg, Register &OtherReg,
1091 bool Invert) {
1092 if (Optimized)
1093 return false;
1094
1095 // Attempt to fold:
1096 //
1097 // %sub = G_SUB 0, %x
1098 // %select = G_SELECT cc, %reg, %sub
1099 //
1100 // Into:
1101 // %select = CSNEG %reg, %x, cc
1102 Register MatchReg;
1103 if (mi_match(R: Reg, MRI, P: m_Neg(Src: m_Reg(R&: MatchReg)))) {
1104 Opc = Is32Bit ? AArch64::CSNEGWr : AArch64::CSNEGXr;
1105 Reg = MatchReg;
1106 if (Invert) {
1107 CC = AArch64CC::getInvertedCondCode(Code: CC);
1108 std::swap(a&: Reg, b&: OtherReg);
1109 }
1110 return true;
1111 }
1112
1113 // Attempt to fold:
1114 //
1115 // %xor = G_XOR %x, -1
1116 // %select = G_SELECT cc, %reg, %xor
1117 //
1118 // Into:
1119 // %select = CSINV %reg, %x, cc
1120 if (mi_match(R: Reg, MRI, P: m_Not(Src: m_Reg(R&: MatchReg)))) {
1121 Opc = Is32Bit ? AArch64::CSINVWr : AArch64::CSINVXr;
1122 Reg = MatchReg;
1123 if (Invert) {
1124 CC = AArch64CC::getInvertedCondCode(Code: CC);
1125 std::swap(a&: Reg, b&: OtherReg);
1126 }
1127 return true;
1128 }
1129
1130 // Attempt to fold:
1131 //
1132 // %add = G_ADD %x, 1
1133 // %select = G_SELECT cc, %reg, %add
1134 //
1135 // Into:
1136 // %select = CSINC %reg, %x, cc
1137 if (mi_match(R: Reg, MRI,
1138 P: m_any_of(preds: m_GAdd(L: m_Reg(R&: MatchReg), R: m_SpecificICst(RequestedValue: 1)),
1139 preds: m_GPtrAdd(L: m_Reg(R&: MatchReg), R: m_SpecificICst(RequestedValue: 1))))) {
1140 Opc = Is32Bit ? AArch64::CSINCWr : AArch64::CSINCXr;
1141 Reg = MatchReg;
1142 if (Invert) {
1143 CC = AArch64CC::getInvertedCondCode(Code: CC);
1144 std::swap(a&: Reg, b&: OtherReg);
1145 }
1146 return true;
1147 }
1148
1149 return false;
1150 };
1151
1152 // Helper lambda which tries to use CSINC/CSINV for the instruction when its
1153 // true/false values are constants.
1154 // FIXME: All of these patterns already exist in tablegen. We should be
1155 // able to import these.
1156 auto TryOptSelectCst = [&Opc, &True, &False, &CC, Is32Bit, &MRI,
1157 &Optimized]() {
1158 if (Optimized)
1159 return false;
1160 auto TrueCst = getIConstantVRegValWithLookThrough(VReg: True, MRI);
1161 auto FalseCst = getIConstantVRegValWithLookThrough(VReg: False, MRI);
1162 if (!TrueCst && !FalseCst)
1163 return false;
1164
1165 Register ZReg = Is32Bit ? AArch64::WZR : AArch64::XZR;
1166 if (TrueCst && FalseCst) {
1167 int64_t T = TrueCst->Value.getSExtValue();
1168 int64_t F = FalseCst->Value.getSExtValue();
1169
1170 if (T == 0 && F == 1) {
1171 // G_SELECT cc, 0, 1 -> CSINC zreg, zreg, cc
1172 Opc = Is32Bit ? AArch64::CSINCWr : AArch64::CSINCXr;
1173 True = ZReg;
1174 False = ZReg;
1175 return true;
1176 }
1177
1178 if (T == 0 && F == -1) {
1179 // G_SELECT cc 0, -1 -> CSINV zreg, zreg cc
1180 Opc = Is32Bit ? AArch64::CSINVWr : AArch64::CSINVXr;
1181 True = ZReg;
1182 False = ZReg;
1183 return true;
1184 }
1185 }
1186
1187 if (TrueCst) {
1188 int64_t T = TrueCst->Value.getSExtValue();
1189 if (T == 1) {
1190 // G_SELECT cc, 1, f -> CSINC f, zreg, inv_cc
1191 Opc = Is32Bit ? AArch64::CSINCWr : AArch64::CSINCXr;
1192 True = False;
1193 False = ZReg;
1194 CC = AArch64CC::getInvertedCondCode(Code: CC);
1195 return true;
1196 }
1197
1198 if (T == -1) {
1199 // G_SELECT cc, -1, f -> CSINV f, zreg, inv_cc
1200 Opc = Is32Bit ? AArch64::CSINVWr : AArch64::CSINVXr;
1201 True = False;
1202 False = ZReg;
1203 CC = AArch64CC::getInvertedCondCode(Code: CC);
1204 return true;
1205 }
1206 }
1207
1208 if (FalseCst) {
1209 int64_t F = FalseCst->Value.getSExtValue();
1210 if (F == 1) {
1211 // G_SELECT cc, t, 1 -> CSINC t, zreg, cc
1212 Opc = Is32Bit ? AArch64::CSINCWr : AArch64::CSINCXr;
1213 False = ZReg;
1214 return true;
1215 }
1216
1217 if (F == -1) {
1218 // G_SELECT cc, t, -1 -> CSINC t, zreg, cc
1219 Opc = Is32Bit ? AArch64::CSINVWr : AArch64::CSINVXr;
1220 False = ZReg;
1221 return true;
1222 }
1223 }
1224 return false;
1225 };
1226
1227 Optimized |= TryFoldBinOpIntoSelect(False, True, /*Invert = */ false);
1228 Optimized |= TryFoldBinOpIntoSelect(True, False, /*Invert = */ true);
1229 Optimized |= TryOptSelectCst();
1230 auto SelectInst = MIB.buildInstr(Opc, DstOps: {Dst}, SrcOps: {True, False}).addImm(Val: CC);
1231 constrainSelectedInstRegOperands(I&: *SelectInst, TII, TRI, RBI);
1232 return &*SelectInst;
1233}
1234
1235static AArch64CC::CondCode
1236changeICMPPredToAArch64CC(CmpInst::Predicate P, Register RHS = {},
1237 MachineRegisterInfo *MRI = nullptr) {
1238 switch (P) {
1239 default:
1240 llvm_unreachable("Unknown condition code!");
1241 case CmpInst::ICMP_NE:
1242 return AArch64CC::NE;
1243 case CmpInst::ICMP_EQ:
1244 return AArch64CC::EQ;
1245 case CmpInst::ICMP_SGT:
1246 return AArch64CC::GT;
1247 case CmpInst::ICMP_SGE:
1248 if (RHS && MRI) {
1249 auto ValAndVReg = getIConstantVRegValWithLookThrough(VReg: RHS, MRI: *MRI);
1250 if (ValAndVReg && ValAndVReg->Value == 0)
1251 return AArch64CC::PL;
1252 }
1253 return AArch64CC::GE;
1254 case CmpInst::ICMP_SLT:
1255 if (RHS && MRI) {
1256 auto ValAndVReg = getIConstantVRegValWithLookThrough(VReg: RHS, MRI: *MRI);
1257 if (ValAndVReg && ValAndVReg->Value == 0)
1258 return AArch64CC::MI;
1259 }
1260 return AArch64CC::LT;
1261 case CmpInst::ICMP_SLE:
1262 return AArch64CC::LE;
1263 case CmpInst::ICMP_UGT:
1264 return AArch64CC::HI;
1265 case CmpInst::ICMP_UGE:
1266 return AArch64CC::HS;
1267 case CmpInst::ICMP_ULT:
1268 return AArch64CC::LO;
1269 case CmpInst::ICMP_ULE:
1270 return AArch64CC::LS;
1271 }
1272}
1273
1274/// changeFPCCToORAArch64CC - Convert an IR fp condition code to an AArch64 CC.
1275static void changeFPCCToORAArch64CC(CmpInst::Predicate CC,
1276 AArch64CC::CondCode &CondCode,
1277 AArch64CC::CondCode &CondCode2) {
1278 CondCode2 = AArch64CC::AL;
1279 switch (CC) {
1280 default:
1281 llvm_unreachable("Unknown FP condition!");
1282 case CmpInst::FCMP_OEQ:
1283 CondCode = AArch64CC::EQ;
1284 break;
1285 case CmpInst::FCMP_OGT:
1286 CondCode = AArch64CC::GT;
1287 break;
1288 case CmpInst::FCMP_OGE:
1289 CondCode = AArch64CC::GE;
1290 break;
1291 case CmpInst::FCMP_OLT:
1292 CondCode = AArch64CC::MI;
1293 break;
1294 case CmpInst::FCMP_OLE:
1295 CondCode = AArch64CC::LS;
1296 break;
1297 case CmpInst::FCMP_ONE:
1298 CondCode = AArch64CC::MI;
1299 CondCode2 = AArch64CC::GT;
1300 break;
1301 case CmpInst::FCMP_ORD:
1302 CondCode = AArch64CC::VC;
1303 break;
1304 case CmpInst::FCMP_UNO:
1305 CondCode = AArch64CC::VS;
1306 break;
1307 case CmpInst::FCMP_UEQ:
1308 CondCode = AArch64CC::EQ;
1309 CondCode2 = AArch64CC::VS;
1310 break;
1311 case CmpInst::FCMP_UGT:
1312 CondCode = AArch64CC::HI;
1313 break;
1314 case CmpInst::FCMP_UGE:
1315 CondCode = AArch64CC::PL;
1316 break;
1317 case CmpInst::FCMP_ULT:
1318 CondCode = AArch64CC::LT;
1319 break;
1320 case CmpInst::FCMP_ULE:
1321 CondCode = AArch64CC::LE;
1322 break;
1323 case CmpInst::FCMP_UNE:
1324 CondCode = AArch64CC::NE;
1325 break;
1326 }
1327}
1328
1329/// Convert an IR fp condition code to an AArch64 CC.
1330/// This differs from changeFPCCToAArch64CC in that it returns cond codes that
1331/// should be AND'ed instead of OR'ed.
1332static void changeFPCCToANDAArch64CC(CmpInst::Predicate CC,
1333 AArch64CC::CondCode &CondCode,
1334 AArch64CC::CondCode &CondCode2) {
1335 CondCode2 = AArch64CC::AL;
1336 switch (CC) {
1337 default:
1338 changeFPCCToORAArch64CC(CC, CondCode, CondCode2);
1339 assert(CondCode2 == AArch64CC::AL);
1340 break;
1341 case CmpInst::FCMP_ONE:
1342 // (a one b)
1343 // == ((a olt b) || (a ogt b))
1344 // == ((a ord b) && (a une b))
1345 CondCode = AArch64CC::VC;
1346 CondCode2 = AArch64CC::NE;
1347 break;
1348 case CmpInst::FCMP_UEQ:
1349 // (a ueq b)
1350 // == ((a uno b) || (a oeq b))
1351 // == ((a ule b) && (a uge b))
1352 CondCode = AArch64CC::PL;
1353 CondCode2 = AArch64CC::LE;
1354 break;
1355 }
1356}
1357
1358/// Return a register which can be used as a bit to test in a TB(N)Z.
1359static Register getTestBitReg(Register Reg, uint64_t &Bit, bool &Invert,
1360 MachineRegisterInfo &MRI) {
1361 assert(Reg.isValid() && "Expected valid register!");
1362 bool HasZext = false;
1363 while (MachineInstr *MI = getDefIgnoringCopies(Reg, MRI)) {
1364 unsigned Opc = MI->getOpcode();
1365
1366 if (!MI->getOperand(i: 0).isReg() ||
1367 !MRI.hasOneNonDBGUse(RegNo: MI->getOperand(i: 0).getReg()))
1368 break;
1369
1370 // (tbz (any_ext x), b) -> (tbz x, b) and
1371 // (tbz (zext x), b) -> (tbz x, b) if we don't use the extended bits.
1372 //
1373 // (tbz (trunc x), b) -> (tbz x, b) is always safe, because the bit number
1374 // on the truncated x is the same as the bit number on x.
1375 if (Opc == TargetOpcode::G_ANYEXT || Opc == TargetOpcode::G_ZEXT ||
1376 Opc == TargetOpcode::G_TRUNC) {
1377 if (Opc == TargetOpcode::G_ZEXT)
1378 HasZext = true;
1379
1380 Register NextReg = MI->getOperand(i: 1).getReg();
1381 // Did we find something worth folding?
1382 if (!NextReg.isValid() || !MRI.hasOneNonDBGUse(RegNo: NextReg))
1383 break;
1384 TypeSize InSize = MRI.getType(Reg: NextReg).getSizeInBits();
1385 if (Bit >= InSize)
1386 break;
1387
1388 // NextReg is worth folding. Keep looking.
1389 Reg = NextReg;
1390 continue;
1391 }
1392
1393 // Attempt to find a suitable operation with a constant on one side.
1394 std::optional<uint64_t> C;
1395 Register TestReg;
1396 switch (Opc) {
1397 default:
1398 break;
1399 case TargetOpcode::G_AND:
1400 case TargetOpcode::G_XOR: {
1401 TestReg = MI->getOperand(i: 1).getReg();
1402 Register ConstantReg = MI->getOperand(i: 2).getReg();
1403 auto VRegAndVal = getIConstantVRegValWithLookThrough(VReg: ConstantReg, MRI);
1404 if (!VRegAndVal) {
1405 // AND commutes, check the other side for a constant.
1406 // FIXME: Can we canonicalize the constant so that it's always on the
1407 // same side at some point earlier?
1408 std::swap(a&: ConstantReg, b&: TestReg);
1409 VRegAndVal = getIConstantVRegValWithLookThrough(VReg: ConstantReg, MRI);
1410 }
1411 if (VRegAndVal) {
1412 if (HasZext)
1413 C = VRegAndVal->Value.getZExtValue();
1414 else
1415 C = VRegAndVal->Value.getSExtValue();
1416 }
1417 break;
1418 }
1419 case TargetOpcode::G_ASHR:
1420 case TargetOpcode::G_LSHR:
1421 case TargetOpcode::G_SHL: {
1422 TestReg = MI->getOperand(i: 1).getReg();
1423 auto VRegAndVal =
1424 getIConstantVRegValWithLookThrough(VReg: MI->getOperand(i: 2).getReg(), MRI);
1425 if (VRegAndVal)
1426 C = VRegAndVal->Value.getSExtValue();
1427 break;
1428 }
1429 }
1430
1431 // Didn't find a constant or viable register. Bail out of the loop.
1432 if (!C || !TestReg.isValid())
1433 break;
1434
1435 // We found a suitable instruction with a constant. Check to see if we can
1436 // walk through the instruction.
1437 Register NextReg;
1438 unsigned TestRegSize = MRI.getType(Reg: TestReg).getSizeInBits();
1439 switch (Opc) {
1440 default:
1441 break;
1442 case TargetOpcode::G_AND:
1443 // (tbz (and x, m), b) -> (tbz x, b) when the b-th bit of m is set.
1444 if ((*C >> Bit) & 1)
1445 NextReg = TestReg;
1446 break;
1447 case TargetOpcode::G_SHL:
1448 // (tbz (shl x, c), b) -> (tbz x, b-c) when b-c is positive and fits in
1449 // the type of the register.
1450 if (*C <= Bit && (Bit - *C) < TestRegSize) {
1451 NextReg = TestReg;
1452 Bit = Bit - *C;
1453 }
1454 break;
1455 case TargetOpcode::G_ASHR:
1456 // (tbz (ashr x, c), b) -> (tbz x, b+c) or (tbz x, msb) if b+c is > # bits
1457 // in x
1458 NextReg = TestReg;
1459 Bit = Bit + *C;
1460 if (Bit >= TestRegSize)
1461 Bit = TestRegSize - 1;
1462 break;
1463 case TargetOpcode::G_LSHR:
1464 // (tbz (lshr x, c), b) -> (tbz x, b+c) when b + c is < # bits in x
1465 if ((Bit + *C) < TestRegSize) {
1466 NextReg = TestReg;
1467 Bit = Bit + *C;
1468 }
1469 break;
1470 case TargetOpcode::G_XOR:
1471 // We can walk through a G_XOR by inverting whether we use tbz/tbnz when
1472 // appropriate.
1473 //
1474 // e.g. If x' = xor x, c, and the b-th bit is set in c then
1475 //
1476 // tbz x', b -> tbnz x, b
1477 //
1478 // Because x' only has the b-th bit set if x does not.
1479 if ((*C >> Bit) & 1)
1480 Invert = !Invert;
1481 NextReg = TestReg;
1482 break;
1483 }
1484
1485 // Check if we found anything worth folding.
1486 if (!NextReg.isValid())
1487 return Reg;
1488 Reg = NextReg;
1489 }
1490
1491 return Reg;
1492}
1493
1494MachineInstr *AArch64InstructionSelector::emitTestBit(
1495 Register TestReg, uint64_t Bit, bool IsNegative, MachineBasicBlock *DstMBB,
1496 MachineIRBuilder &MIB) const {
1497 assert(TestReg.isValid());
1498 assert(ProduceNonFlagSettingCondBr &&
1499 "Cannot emit TB(N)Z with speculation tracking!");
1500 MachineRegisterInfo &MRI = *MIB.getMRI();
1501
1502 // Attempt to optimize the test bit by walking over instructions.
1503 TestReg = getTestBitReg(Reg: TestReg, Bit, Invert&: IsNegative, MRI);
1504 LLT Ty = MRI.getType(Reg: TestReg);
1505 unsigned Size = Ty.getSizeInBits();
1506 assert(!Ty.isVector() && "Expected a scalar!");
1507 assert(Bit < 64 && "Bit is too large!");
1508
1509 // When the test register is a 64-bit register, we have to narrow to make
1510 // TBNZW work.
1511 bool UseWReg = Bit < 32;
1512 unsigned NecessarySize = UseWReg ? 32 : 64;
1513 if (Size != NecessarySize)
1514 TestReg = moveScalarRegClass(
1515 Reg: TestReg, RC: UseWReg ? AArch64::GPR32RegClass : AArch64::GPR64RegClass,
1516 MIB);
1517
1518 static const unsigned OpcTable[2][2] = {{AArch64::TBZX, AArch64::TBNZX},
1519 {AArch64::TBZW, AArch64::TBNZW}};
1520 unsigned Opc = OpcTable[UseWReg][IsNegative];
1521 auto TestBitMI =
1522 MIB.buildInstr(Opcode: Opc).addReg(RegNo: TestReg).addImm(Val: Bit).addMBB(MBB: DstMBB);
1523 constrainSelectedInstRegOperands(I&: *TestBitMI, TII, TRI, RBI);
1524 return &*TestBitMI;
1525}
1526
1527bool AArch64InstructionSelector::tryOptAndIntoCompareBranch(
1528 MachineInstr &AndInst, bool Invert, MachineBasicBlock *DstMBB,
1529 MachineIRBuilder &MIB) const {
1530 assert(AndInst.getOpcode() == TargetOpcode::G_AND && "Expected G_AND only?");
1531 // Given something like this:
1532 //
1533 // %x = ...Something...
1534 // %one = G_CONSTANT i64 1
1535 // %zero = G_CONSTANT i64 0
1536 // %and = G_AND %x, %one
1537 // %cmp = G_ICMP intpred(ne), %and, %zero
1538 // %cmp_trunc = G_TRUNC %cmp
1539 // G_BRCOND %cmp_trunc, %bb.3
1540 //
1541 // We want to try and fold the AND into the G_BRCOND and produce either a
1542 // TBNZ (when we have intpred(ne)) or a TBZ (when we have intpred(eq)).
1543 //
1544 // In this case, we'd get
1545 //
1546 // TBNZ %x %bb.3
1547 //
1548
1549 // Check if the AND has a constant on its RHS which we can use as a mask.
1550 // If it's a power of 2, then it's the same as checking a specific bit.
1551 // (e.g, ANDing with 8 == ANDing with 000...100 == testing if bit 3 is set)
1552 auto MaybeBit = getIConstantVRegValWithLookThrough(
1553 VReg: AndInst.getOperand(i: 2).getReg(), MRI: *MIB.getMRI());
1554 if (!MaybeBit)
1555 return false;
1556
1557 int32_t Bit = MaybeBit->Value.exactLogBase2();
1558 if (Bit < 0)
1559 return false;
1560
1561 Register TestReg = AndInst.getOperand(i: 1).getReg();
1562
1563 // Emit a TB(N)Z.
1564 emitTestBit(TestReg, Bit, IsNegative: Invert, DstMBB, MIB);
1565 return true;
1566}
1567
1568MachineInstr *AArch64InstructionSelector::emitCBZ(Register CompareReg,
1569 bool IsNegative,
1570 MachineBasicBlock *DestMBB,
1571 MachineIRBuilder &MIB) const {
1572 assert(ProduceNonFlagSettingCondBr && "CBZ does not set flags!");
1573 MachineRegisterInfo &MRI = *MIB.getMRI();
1574 assert(RBI.getRegBank(CompareReg, MRI, TRI)->getID() ==
1575 AArch64::GPRRegBankID &&
1576 "Expected GPRs only?");
1577 auto Ty = MRI.getType(Reg: CompareReg);
1578 unsigned Width = Ty.getSizeInBits();
1579 assert(!Ty.isVector() && "Expected scalar only?");
1580 assert(Width <= 64 && "Expected width to be at most 64?");
1581 static const unsigned OpcTable[2][2] = {{AArch64::CBZW, AArch64::CBZX},
1582 {AArch64::CBNZW, AArch64::CBNZX}};
1583 unsigned Opc = OpcTable[IsNegative][Width == 64];
1584 auto BranchMI = MIB.buildInstr(Opc, DstOps: {}, SrcOps: {CompareReg}).addMBB(MBB: DestMBB);
1585 constrainSelectedInstRegOperands(I&: *BranchMI, TII, TRI, RBI);
1586 return &*BranchMI;
1587}
1588
1589bool AArch64InstructionSelector::selectCompareBranchFedByFCmp(
1590 MachineInstr &I, MachineInstr &FCmp, MachineIRBuilder &MIB) const {
1591 assert(FCmp.getOpcode() == TargetOpcode::G_FCMP);
1592 assert(I.getOpcode() == TargetOpcode::G_BRCOND);
1593 // Unfortunately, the mapping of LLVM FP CC's onto AArch64 CC's isn't
1594 // totally clean. Some of them require two branches to implement.
1595 auto Pred = (CmpInst::Predicate)FCmp.getOperand(i: 1).getPredicate();
1596 emitFPCompare(LHS: FCmp.getOperand(i: 2).getReg(), RHS: FCmp.getOperand(i: 3).getReg(), MIRBuilder&: MIB,
1597 Pred);
1598 AArch64CC::CondCode CC1, CC2;
1599 changeFCMPPredToAArch64CC(P: Pred, CondCode&: CC1, CondCode2&: CC2);
1600 MachineBasicBlock *DestMBB = I.getOperand(i: 1).getMBB();
1601 MIB.buildInstr(Opc: AArch64::Bcc, DstOps: {}, SrcOps: {}).addImm(Val: CC1).addMBB(MBB: DestMBB);
1602 if (CC2 != AArch64CC::AL)
1603 MIB.buildInstr(Opc: AArch64::Bcc, DstOps: {}, SrcOps: {}).addImm(Val: CC2).addMBB(MBB: DestMBB);
1604 I.eraseFromParent();
1605 return true;
1606}
1607
1608bool AArch64InstructionSelector::tryOptCompareBranchFedByICmp(
1609 MachineInstr &I, MachineInstr &ICmp, MachineIRBuilder &MIB) const {
1610 assert(ICmp.getOpcode() == TargetOpcode::G_ICMP);
1611 assert(I.getOpcode() == TargetOpcode::G_BRCOND);
1612 // Attempt to optimize the G_BRCOND + G_ICMP into a TB(N)Z/CB(N)Z.
1613 //
1614 // Speculation tracking/SLH assumes that optimized TB(N)Z/CB(N)Z
1615 // instructions will not be produced, as they are conditional branch
1616 // instructions that do not set flags.
1617 if (!ProduceNonFlagSettingCondBr)
1618 return false;
1619
1620 MachineRegisterInfo &MRI = *MIB.getMRI();
1621 MachineBasicBlock *DestMBB = I.getOperand(i: 1).getMBB();
1622 auto Pred =
1623 static_cast<CmpInst::Predicate>(ICmp.getOperand(i: 1).getPredicate());
1624 Register LHS = ICmp.getOperand(i: 2).getReg();
1625 Register RHS = ICmp.getOperand(i: 3).getReg();
1626
1627 // We're allowed to emit a TB(N)Z/CB(N)Z. Try to do that.
1628 auto VRegAndVal = getIConstantVRegValWithLookThrough(VReg: RHS, MRI);
1629 MachineInstr *AndInst = getOpcodeDef(Opcode: TargetOpcode::G_AND, Reg: LHS, MRI);
1630
1631 // When we can emit a TB(N)Z, prefer that.
1632 //
1633 // Handle non-commutative condition codes first.
1634 // Note that we don't want to do this when we have a G_AND because it can
1635 // become a tst. The tst will make the test bit in the TB(N)Z redundant.
1636 if (VRegAndVal && !AndInst) {
1637 int64_t C = VRegAndVal->Value.getSExtValue();
1638
1639 // When we have a greater-than comparison, we can just test if the msb is
1640 // zero.
1641 if (C == -1 && Pred == CmpInst::ICMP_SGT) {
1642 uint64_t Bit = MRI.getType(Reg: LHS).getSizeInBits() - 1;
1643 emitTestBit(TestReg: LHS, Bit, /*IsNegative = */ false, DstMBB: DestMBB, MIB);
1644 I.eraseFromParent();
1645 return true;
1646 }
1647
1648 // When we have a less than comparison, we can just test if the msb is not
1649 // zero.
1650 if (C == 0 && Pred == CmpInst::ICMP_SLT) {
1651 uint64_t Bit = MRI.getType(Reg: LHS).getSizeInBits() - 1;
1652 emitTestBit(TestReg: LHS, Bit, /*IsNegative = */ true, DstMBB: DestMBB, MIB);
1653 I.eraseFromParent();
1654 return true;
1655 }
1656
1657 // Inversely, if we have a signed greater-than-or-equal comparison to zero,
1658 // we can test if the msb is zero.
1659 if (C == 0 && Pred == CmpInst::ICMP_SGE) {
1660 uint64_t Bit = MRI.getType(Reg: LHS).getSizeInBits() - 1;
1661 emitTestBit(TestReg: LHS, Bit, /*IsNegative = */ false, DstMBB: DestMBB, MIB);
1662 I.eraseFromParent();
1663 return true;
1664 }
1665 }
1666
1667 // Attempt to handle commutative condition codes. Right now, that's only
1668 // eq/ne.
1669 if (ICmpInst::isEquality(P: Pred)) {
1670 if (!VRegAndVal) {
1671 std::swap(a&: RHS, b&: LHS);
1672 VRegAndVal = getIConstantVRegValWithLookThrough(VReg: RHS, MRI);
1673 AndInst = getOpcodeDef(Opcode: TargetOpcode::G_AND, Reg: LHS, MRI);
1674 }
1675
1676 if (VRegAndVal && VRegAndVal->Value == 0) {
1677 // If there's a G_AND feeding into this branch, try to fold it away by
1678 // emitting a TB(N)Z instead.
1679 //
1680 // Note: If we have LT, then it *is* possible to fold, but it wouldn't be
1681 // beneficial. When we have an AND and LT, we need a TST/ANDS, so folding
1682 // would be redundant.
1683 if (AndInst &&
1684 tryOptAndIntoCompareBranch(
1685 AndInst&: *AndInst, /*Invert = */ Pred == CmpInst::ICMP_NE, DstMBB: DestMBB, MIB)) {
1686 I.eraseFromParent();
1687 return true;
1688 }
1689
1690 // Otherwise, try to emit a CB(N)Z instead.
1691 auto LHSTy = MRI.getType(Reg: LHS);
1692 if (!LHSTy.isVector() && LHSTy.getSizeInBits() <= 64) {
1693 emitCBZ(CompareReg: LHS, /*IsNegative = */ Pred == CmpInst::ICMP_NE, DestMBB, MIB);
1694 I.eraseFromParent();
1695 return true;
1696 }
1697 }
1698 }
1699
1700 return false;
1701}
1702
1703bool AArch64InstructionSelector::selectCompareBranchFedByICmp(
1704 MachineInstr &I, MachineInstr &ICmp, MachineIRBuilder &MIB) const {
1705 assert(ICmp.getOpcode() == TargetOpcode::G_ICMP);
1706 assert(I.getOpcode() == TargetOpcode::G_BRCOND);
1707 if (tryOptCompareBranchFedByICmp(I, ICmp, MIB))
1708 return true;
1709
1710 // Couldn't optimize. Emit a compare + a Bcc.
1711 MachineBasicBlock *DestMBB = I.getOperand(i: 1).getMBB();
1712 auto &PredOp = ICmp.getOperand(i: 1);
1713 emitIntegerCompare(LHS&: ICmp.getOperand(i: 2), RHS&: ICmp.getOperand(i: 3), Predicate&: PredOp, MIRBuilder&: MIB);
1714 const AArch64CC::CondCode CC = changeICMPPredToAArch64CC(
1715 P: static_cast<CmpInst::Predicate>(PredOp.getPredicate()),
1716 RHS: ICmp.getOperand(i: 3).getReg(), MRI: MIB.getMRI());
1717 MIB.buildInstr(Opc: AArch64::Bcc, DstOps: {}, SrcOps: {}).addImm(Val: CC).addMBB(MBB: DestMBB);
1718 I.eraseFromParent();
1719 return true;
1720}
1721
1722bool AArch64InstructionSelector::selectCompareBranch(
1723 MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) {
1724 Register CondReg = I.getOperand(i: 0).getReg();
1725 MachineInstr *CCMI = MRI.getVRegDef(Reg: CondReg);
1726 // Try to select the G_BRCOND using whatever is feeding the condition if
1727 // possible.
1728 unsigned CCMIOpc = CCMI->getOpcode();
1729 if (CCMIOpc == TargetOpcode::G_FCMP)
1730 return selectCompareBranchFedByFCmp(I, FCmp&: *CCMI, MIB);
1731 if (CCMIOpc == TargetOpcode::G_ICMP)
1732 return selectCompareBranchFedByICmp(I, ICmp&: *CCMI, MIB);
1733
1734 // Speculation tracking/SLH assumes that optimized TB(N)Z/CB(N)Z
1735 // instructions will not be produced, as they are conditional branch
1736 // instructions that do not set flags.
1737 if (ProduceNonFlagSettingCondBr) {
1738 emitTestBit(TestReg: CondReg, /*Bit = */ 0, /*IsNegative = */ true,
1739 DstMBB: I.getOperand(i: 1).getMBB(), MIB);
1740 I.eraseFromParent();
1741 return true;
1742 }
1743
1744 // Can't emit TB(N)Z/CB(N)Z. Emit a tst + bcc instead.
1745 auto TstMI =
1746 MIB.buildInstr(Opc: AArch64::ANDSWri, DstOps: {LLT::scalar(SizeInBits: 32)}, SrcOps: {CondReg}).addImm(Val: 1);
1747 constrainSelectedInstRegOperands(I&: *TstMI, TII, TRI, RBI);
1748 auto Bcc = MIB.buildInstr(Opcode: AArch64::Bcc)
1749 .addImm(Val: AArch64CC::NE)
1750 .addMBB(MBB: I.getOperand(i: 1).getMBB());
1751 I.eraseFromParent();
1752 constrainSelectedInstRegOperands(I&: *Bcc, TII, TRI, RBI);
1753 return true;
1754}
1755
1756/// Returns the element immediate value of a vector shift operand if found.
1757/// This needs to detect a splat-like operation, e.g. a G_BUILD_VECTOR.
1758static std::optional<APInt> getVectorShiftImm(Register Reg,
1759 MachineRegisterInfo &MRI) {
1760 assert(MRI.getType(Reg).isVector() && "Expected a *vector* shift operand");
1761 MachineInstr *OpMI = MRI.getVRegDef(Reg);
1762 std::optional<Register> Splat = getVectorSplat(MI: *OpMI, MRI);
1763 if (!Splat)
1764 return std::nullopt;
1765 return getIConstantVRegVal(VReg: *Splat, MRI);
1766}
1767
1768/// Matches and returns the shift immediate value for a SHL instruction given
1769/// a shift operand.
1770static std::optional<uint64_t> getVectorSHLImm(LLT SrcTy, Register Reg,
1771 MachineRegisterInfo &MRI) {
1772 std::optional<APInt> ShiftImm = getVectorShiftImm(Reg, MRI);
1773 if (!ShiftImm)
1774 return std::nullopt;
1775 // Check the immediate is in range for a SHL.
1776 uint64_t Imm = ShiftImm->getZExtValue();
1777 if (Imm >= SrcTy.getElementType().getSizeInBits())
1778 return std::nullopt;
1779 return Imm;
1780}
1781
1782bool AArch64InstructionSelector::selectVectorSHL(MachineInstr &I,
1783 MachineRegisterInfo &MRI) {
1784 assert(I.getOpcode() == TargetOpcode::G_SHL);
1785 Register DstReg = I.getOperand(i: 0).getReg();
1786 const LLT Ty = MRI.getType(Reg: DstReg);
1787 Register Src1Reg = I.getOperand(i: 1).getReg();
1788 Register Src2Reg = I.getOperand(i: 2).getReg();
1789
1790 if (!Ty.isVector())
1791 return false;
1792
1793 // Check if we have a vector of constants on RHS that we can select as the
1794 // immediate form.
1795 std::optional<uint64_t> ImmVal = getVectorSHLImm(SrcTy: Ty, Reg: Src2Reg, MRI);
1796
1797 unsigned Opc = 0;
1798 if (Ty == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 64)) {
1799 Opc = ImmVal ? AArch64::SHLv2i64_shift : AArch64::USHLv2i64;
1800 } else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 32)) {
1801 Opc = ImmVal ? AArch64::SHLv4i32_shift : AArch64::USHLv4i32;
1802 } else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 32)) {
1803 Opc = ImmVal ? AArch64::SHLv2i32_shift : AArch64::USHLv2i32;
1804 } else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 16)) {
1805 Opc = ImmVal ? AArch64::SHLv4i16_shift : AArch64::USHLv4i16;
1806 } else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 16)) {
1807 Opc = ImmVal ? AArch64::SHLv8i16_shift : AArch64::USHLv8i16;
1808 } else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 8)) {
1809 Opc = ImmVal ? AArch64::SHLv16i8_shift : AArch64::USHLv16i8;
1810 } else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 8)) {
1811 Opc = ImmVal ? AArch64::SHLv8i8_shift : AArch64::USHLv8i8;
1812 } else {
1813 LLVM_DEBUG(dbgs() << "Unhandled G_SHL type");
1814 return false;
1815 }
1816
1817 auto Shl = MIB.buildInstr(Opc, DstOps: {DstReg}, SrcOps: {Src1Reg});
1818 if (ImmVal)
1819 Shl.addImm(Val: *ImmVal);
1820 else
1821 Shl.addUse(RegNo: Src2Reg);
1822 constrainSelectedInstRegOperands(I&: *Shl, TII, TRI, RBI);
1823 I.eraseFromParent();
1824 return true;
1825}
1826
1827bool AArch64InstructionSelector::selectVectorAshrLshr(
1828 MachineInstr &I, MachineRegisterInfo &MRI) {
1829 assert(I.getOpcode() == TargetOpcode::G_ASHR ||
1830 I.getOpcode() == TargetOpcode::G_LSHR);
1831 Register DstReg = I.getOperand(i: 0).getReg();
1832 const LLT Ty = MRI.getType(Reg: DstReg);
1833 Register Src1Reg = I.getOperand(i: 1).getReg();
1834 Register Src2Reg = I.getOperand(i: 2).getReg();
1835
1836 if (!Ty.isVector())
1837 return false;
1838
1839 bool IsASHR = I.getOpcode() == TargetOpcode::G_ASHR;
1840
1841 // We expect the immediate case to be lowered in the PostLegalCombiner to
1842 // AArch64ISD::VASHR or AArch64ISD::VLSHR equivalents.
1843
1844 // There is not a shift right register instruction, but the shift left
1845 // register instruction takes a signed value, where negative numbers specify a
1846 // right shift.
1847
1848 unsigned Opc = 0;
1849 unsigned NegOpc = 0;
1850 const TargetRegisterClass *RC =
1851 getRegClassForTypeOnBank(Ty, RB: RBI.getRegBank(ID: AArch64::FPRRegBankID));
1852 if (Ty == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 64)) {
1853 Opc = IsASHR ? AArch64::SSHLv2i64 : AArch64::USHLv2i64;
1854 NegOpc = AArch64::NEGv2i64;
1855 } else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 32)) {
1856 Opc = IsASHR ? AArch64::SSHLv4i32 : AArch64::USHLv4i32;
1857 NegOpc = AArch64::NEGv4i32;
1858 } else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 32)) {
1859 Opc = IsASHR ? AArch64::SSHLv2i32 : AArch64::USHLv2i32;
1860 NegOpc = AArch64::NEGv2i32;
1861 } else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 16)) {
1862 Opc = IsASHR ? AArch64::SSHLv4i16 : AArch64::USHLv4i16;
1863 NegOpc = AArch64::NEGv4i16;
1864 } else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 16)) {
1865 Opc = IsASHR ? AArch64::SSHLv8i16 : AArch64::USHLv8i16;
1866 NegOpc = AArch64::NEGv8i16;
1867 } else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 8)) {
1868 Opc = IsASHR ? AArch64::SSHLv16i8 : AArch64::USHLv16i8;
1869 NegOpc = AArch64::NEGv16i8;
1870 } else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 8)) {
1871 Opc = IsASHR ? AArch64::SSHLv8i8 : AArch64::USHLv8i8;
1872 NegOpc = AArch64::NEGv8i8;
1873 } else {
1874 LLVM_DEBUG(dbgs() << "Unhandled G_ASHR type");
1875 return false;
1876 }
1877
1878 auto Neg = MIB.buildInstr(Opc: NegOpc, DstOps: {RC}, SrcOps: {Src2Reg});
1879 constrainSelectedInstRegOperands(I&: *Neg, TII, TRI, RBI);
1880 auto SShl = MIB.buildInstr(Opc, DstOps: {DstReg}, SrcOps: {Src1Reg, Neg});
1881 constrainSelectedInstRegOperands(I&: *SShl, TII, TRI, RBI);
1882 I.eraseFromParent();
1883 return true;
1884}
1885
1886bool AArch64InstructionSelector::selectVaStartAAPCS(
1887 MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) const {
1888
1889 if (STI.isCallingConvWin64(CC: MF.getFunction().getCallingConv(),
1890 IsVarArg: MF.getFunction().isVarArg()))
1891 return false;
1892
1893 // The layout of the va_list struct is specified in the AArch64 Procedure Call
1894 // Standard, section 10.1.5.
1895
1896 const AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
1897 const unsigned PtrSize = STI.isTargetILP32() ? 4 : 8;
1898 const auto *PtrRegClass =
1899 STI.isTargetILP32() ? &AArch64::GPR32RegClass : &AArch64::GPR64RegClass;
1900
1901 const MCInstrDesc &MCIDAddAddr =
1902 TII.get(Opcode: STI.isTargetILP32() ? AArch64::ADDWri : AArch64::ADDXri);
1903 const MCInstrDesc &MCIDStoreAddr =
1904 TII.get(Opcode: STI.isTargetILP32() ? AArch64::STRWui : AArch64::STRXui);
1905
1906 /*
1907 * typedef struct va_list {
1908 * void * stack; // next stack param
1909 * void * gr_top; // end of GP arg reg save area
1910 * void * vr_top; // end of FP/SIMD arg reg save area
1911 * int gr_offs; // offset from gr_top to next GP register arg
1912 * int vr_offs; // offset from vr_top to next FP/SIMD register arg
1913 * } va_list;
1914 */
1915 const auto VAList = I.getOperand(i: 0).getReg();
1916
1917 // Our current offset in bytes from the va_list struct (VAList).
1918 unsigned OffsetBytes = 0;
1919
1920 // Helper function to store (FrameIndex + Imm) to VAList at offset OffsetBytes
1921 // and increment OffsetBytes by PtrSize.
1922 const auto PushAddress = [&](const int FrameIndex, const int64_t Imm) {
1923 const Register Top = MRI.createVirtualRegister(RegClass: PtrRegClass);
1924 auto MIB = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: MCIDAddAddr)
1925 .addDef(RegNo: Top)
1926 .addFrameIndex(Idx: FrameIndex)
1927 .addImm(Val: Imm)
1928 .addImm(Val: 0);
1929 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
1930
1931 const auto *MMO = *I.memoperands_begin();
1932 MIB = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: MCIDStoreAddr)
1933 .addUse(RegNo: Top)
1934 .addUse(RegNo: VAList)
1935 .addImm(Val: OffsetBytes / PtrSize)
1936 .addMemOperand(MMO: MF.getMachineMemOperand(
1937 PtrInfo: MMO->getPointerInfo().getWithOffset(O: OffsetBytes),
1938 F: MachineMemOperand::MOStore, Size: PtrSize, BaseAlignment: MMO->getBaseAlign()));
1939 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
1940
1941 OffsetBytes += PtrSize;
1942 };
1943
1944 // void* stack at offset 0
1945 PushAddress(FuncInfo->getVarArgsStackIndex(), 0);
1946
1947 // void* gr_top at offset 8 (4 on ILP32)
1948 const unsigned GPRSize = FuncInfo->getVarArgsGPRSize();
1949 PushAddress(FuncInfo->getVarArgsGPRIndex(), GPRSize);
1950
1951 // void* vr_top at offset 16 (8 on ILP32)
1952 const unsigned FPRSize = FuncInfo->getVarArgsFPRSize();
1953 PushAddress(FuncInfo->getVarArgsFPRIndex(), FPRSize);
1954
1955 // Helper function to store a 4-byte integer constant to VAList at offset
1956 // OffsetBytes, and increment OffsetBytes by 4.
1957 const auto PushIntConstant = [&](const int32_t Value) {
1958 constexpr int IntSize = 4;
1959 const Register Temp = MRI.createVirtualRegister(RegClass: &AArch64::GPR32RegClass);
1960 auto MIB =
1961 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: AArch64::MOVi32imm))
1962 .addDef(RegNo: Temp)
1963 .addImm(Val: Value);
1964 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
1965
1966 const auto *MMO = *I.memoperands_begin();
1967 MIB = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: AArch64::STRWui))
1968 .addUse(RegNo: Temp)
1969 .addUse(RegNo: VAList)
1970 .addImm(Val: OffsetBytes / IntSize)
1971 .addMemOperand(MMO: MF.getMachineMemOperand(
1972 PtrInfo: MMO->getPointerInfo().getWithOffset(O: OffsetBytes),
1973 F: MachineMemOperand::MOStore, Size: IntSize, BaseAlignment: MMO->getBaseAlign()));
1974 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
1975 OffsetBytes += IntSize;
1976 };
1977
1978 // int gr_offs at offset 24 (12 on ILP32)
1979 PushIntConstant(-static_cast<int32_t>(GPRSize));
1980
1981 // int vr_offs at offset 28 (16 on ILP32)
1982 PushIntConstant(-static_cast<int32_t>(FPRSize));
1983
1984 assert(OffsetBytes == (STI.isTargetILP32() ? 20 : 32) && "Unexpected offset");
1985
1986 I.eraseFromParent();
1987 return true;
1988}
1989
1990bool AArch64InstructionSelector::selectVaStartDarwin(
1991 MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) const {
1992 AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
1993 Register ListReg = I.getOperand(i: 0).getReg();
1994
1995 Register ArgsAddrReg = MRI.createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
1996
1997 int FrameIdx = FuncInfo->getVarArgsStackIndex();
1998 if (MF.getSubtarget<AArch64Subtarget>().isCallingConvWin64(
1999 CC: MF.getFunction().getCallingConv(), IsVarArg: MF.getFunction().isVarArg())) {
2000 FrameIdx = FuncInfo->getVarArgsGPRSize() > 0
2001 ? FuncInfo->getVarArgsGPRIndex()
2002 : FuncInfo->getVarArgsStackIndex();
2003 }
2004
2005 auto MIB =
2006 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: AArch64::ADDXri))
2007 .addDef(RegNo: ArgsAddrReg)
2008 .addFrameIndex(Idx: FrameIdx)
2009 .addImm(Val: 0)
2010 .addImm(Val: 0);
2011
2012 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
2013
2014 MIB = BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: AArch64::STRXui))
2015 .addUse(RegNo: ArgsAddrReg)
2016 .addUse(RegNo: ListReg)
2017 .addImm(Val: 0)
2018 .addMemOperand(MMO: *I.memoperands_begin());
2019
2020 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
2021 I.eraseFromParent();
2022 return true;
2023}
2024
2025void AArch64InstructionSelector::materializeLargeCMVal(
2026 MachineInstr &I, const Value *V, unsigned OpFlags) {
2027 MachineBasicBlock &MBB = *I.getParent();
2028 MachineFunction &MF = *MBB.getParent();
2029 MachineRegisterInfo &MRI = MF.getRegInfo();
2030
2031 auto MovZ = MIB.buildInstr(Opc: AArch64::MOVZXi, DstOps: {&AArch64::GPR64RegClass}, SrcOps: {});
2032 MovZ->addOperand(MF, Op: I.getOperand(i: 1));
2033 MovZ->getOperand(i: 1).setTargetFlags(OpFlags | AArch64II::MO_G0 |
2034 AArch64II::MO_NC);
2035 MovZ->addOperand(MF, Op: MachineOperand::CreateImm(Val: 0));
2036 constrainSelectedInstRegOperands(I&: *MovZ, TII, TRI, RBI);
2037
2038 auto BuildMovK = [&](Register SrcReg, unsigned char Flags, unsigned Offset,
2039 Register ForceDstReg) {
2040 Register DstReg = ForceDstReg
2041 ? ForceDstReg
2042 : MRI.createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
2043 auto MovI = MIB.buildInstr(Opcode: AArch64::MOVKXi).addDef(RegNo: DstReg).addUse(RegNo: SrcReg);
2044 if (auto *GV = dyn_cast<GlobalValue>(Val: V)) {
2045 MovI->addOperand(MF, Op: MachineOperand::CreateGA(
2046 GV, Offset: MovZ->getOperand(i: 1).getOffset(), TargetFlags: Flags));
2047 } else {
2048 MovI->addOperand(
2049 MF, Op: MachineOperand::CreateBA(BA: cast<BlockAddress>(Val: V),
2050 Offset: MovZ->getOperand(i: 1).getOffset(), TargetFlags: Flags));
2051 }
2052 MovI->addOperand(MF, Op: MachineOperand::CreateImm(Val: Offset));
2053 constrainSelectedInstRegOperands(I&: *MovI, TII, TRI, RBI);
2054 return DstReg;
2055 };
2056 Register DstReg = BuildMovK(MovZ.getReg(Idx: 0),
2057 AArch64II::MO_G1 | AArch64II::MO_NC, 16, 0);
2058 DstReg = BuildMovK(DstReg, AArch64II::MO_G2 | AArch64II::MO_NC, 32, 0);
2059 BuildMovK(DstReg, AArch64II::MO_G3, 48, I.getOperand(i: 0).getReg());
2060}
2061
2062bool AArch64InstructionSelector::preISelLower(MachineInstr &I) {
2063 MachineBasicBlock &MBB = *I.getParent();
2064 MachineFunction &MF = *MBB.getParent();
2065 MachineRegisterInfo &MRI = MF.getRegInfo();
2066
2067 switch (I.getOpcode()) {
2068 case TargetOpcode::G_CONSTANT: {
2069 Register DefReg = I.getOperand(i: 0).getReg();
2070 const LLT DefTy = MRI.getType(Reg: DefReg);
2071 if (!DefTy.isPointer()) {
2072 if (DefTy.getSizeInBits() >= 32 ||
2073 RBI.getRegBank(Reg: DefReg, MRI, TRI)->getID() != AArch64::GPRRegBankID)
2074 return false;
2075 // Widen narrow GPR constants to s32 so imported patterns can match.
2076 APInt Val = I.getOperand(i: 1).getCImm()->getValue().zext(width: 32);
2077 I.getOperand(i: 1).setCImm(
2078 ConstantInt::get(Context&: MF.getFunction().getContext(), V: Val));
2079
2080 Register WideReg = MRI.createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 32));
2081 MRI.setRegBank(Reg: WideReg, RegBank: RBI.getRegBank(ID: AArch64::GPRRegBankID));
2082 I.getOperand(i: 0).setReg(WideReg);
2083
2084 MIB.setInsertPt(MBB, II: std::next(x: I.getIterator()));
2085 auto Copy = MIB.buildCopy(Res: DefReg, Op: WideReg);
2086 selectCopy(I&: *Copy, TII, MRI, TRI, RBI);
2087 MIB.setInstr(I);
2088 return true;
2089 }
2090 const unsigned PtrSize = DefTy.getSizeInBits();
2091 if (PtrSize != 32 && PtrSize != 64)
2092 return false;
2093 // Convert pointer typed constants to integers so TableGen can select.
2094 MRI.setType(VReg: DefReg, Ty: LLT::integer(SizeInBits: PtrSize));
2095 return true;
2096 }
2097 case TargetOpcode::G_STORE: {
2098 bool Changed = contractCrossBankCopyIntoStore(I, MRI);
2099 MachineOperand &SrcOp = I.getOperand(i: 0);
2100 if (MRI.getType(Reg: SrcOp.getReg()).isPointer()) {
2101 // Allow matching with imported patterns for stores of pointers. Unlike
2102 // G_LOAD/G_PTR_ADD, we may not have selected all users. So, emit a copy
2103 // and constrain.
2104 auto Copy = MIB.buildCopy(Res: LLT::scalar(SizeInBits: 64), Op: SrcOp);
2105 Register NewSrc = Copy.getReg(Idx: 0);
2106 SrcOp.setReg(NewSrc);
2107 RBI.constrainGenericRegister(Reg: NewSrc, RC: AArch64::GPR64RegClass, MRI);
2108 Changed = true;
2109 }
2110 return Changed;
2111 }
2112 case TargetOpcode::G_PTR_ADD: {
2113 // If Checked Pointer Arithmetic (FEAT_CPA) is present, preserve the pointer
2114 // arithmetic semantics instead of falling back to regular arithmetic.
2115 const auto &TL = STI.getTargetLowering();
2116 if (TL->shouldPreservePtrArith(F: MF.getFunction(), PtrVT: EVT()))
2117 return false;
2118 return convertPtrAddToAdd(I, MRI);
2119 }
2120 case TargetOpcode::G_LOAD: {
2121 // For scalar loads of pointers, we try to convert the dest type from p0
2122 // to s64 so that our imported patterns can match. Like with the G_PTR_ADD
2123 // conversion, this should be ok because all users should have been
2124 // selected already, so the type doesn't matter for them.
2125 Register DstReg = I.getOperand(i: 0).getReg();
2126 const LLT DstTy = MRI.getType(Reg: DstReg);
2127 if (!DstTy.isPointer())
2128 return false;
2129 MRI.setType(VReg: DstReg, Ty: LLT::scalar(SizeInBits: 64));
2130 return true;
2131 }
2132 case TargetOpcode::G_VECREDUCE_ADD:
2133 case TargetOpcode::G_VECREDUCE_SMAX:
2134 case TargetOpcode::G_VECREDUCE_SMIN:
2135 case TargetOpcode::G_VECREDUCE_UMAX:
2136 case TargetOpcode::G_VECREDUCE_UMIN: {
2137 // Imported patterns require an FPR result. For a GPR, use a temporary FPR
2138 // and insert a cross-bank copy.
2139 Register DstReg = I.getOperand(i: 0).getReg();
2140 const RegisterBank &DstRB = *RBI.getRegBank(Reg: DstReg, MRI, TRI);
2141 if (DstRB.getID() != AArch64::GPRRegBankID)
2142 return false;
2143
2144 LLT DstTy = MRI.getType(Reg: DstReg);
2145 const TargetRegisterClass *DstRC =
2146 getRegClassForTypeOnBank(Ty: DstTy, RB: DstRB, /*GetAllRegSet=*/true);
2147 if (!DstRC || !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI))
2148 return false;
2149
2150 Register FPRDst = MRI.createGenericVirtualRegister(Ty: DstTy);
2151 MRI.setRegBank(Reg: FPRDst, RegBank: RBI.getRegBank(ID: AArch64::FPRRegBankID));
2152 I.getOperand(i: 0).setReg(FPRDst);
2153
2154 BuildMI(BB&: MBB, I: std::next(x: I.getIterator()), MIMD: MIMetadata(I),
2155 MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: DstReg)
2156 .addReg(RegNo: FPRDst);
2157 return true;
2158 }
2159 case TargetOpcode::G_SPLAT_VECTOR: {
2160 // Convert the type from p0 to s64 to help selection.
2161 LLT DstTy = MRI.getType(Reg: I.getOperand(i: 0).getReg());
2162 if (!DstTy.isPointerVector())
2163 return false;
2164 auto NewSrc = MIB.buildCopy(Res: LLT::scalar(SizeInBits: 64), Op: I.getOperand(i: 1).getReg());
2165 MRI.setType(VReg: I.getOperand(i: 0).getReg(),
2166 Ty: DstTy.changeElementType(NewEltTy: LLT::scalar(SizeInBits: 64)));
2167 MRI.setRegClass(Reg: NewSrc.getReg(Idx: 0), RC: &AArch64::GPR64RegClass);
2168 I.getOperand(i: 1).setReg(NewSrc.getReg(Idx: 0));
2169 return true;
2170 }
2171 case AArch64::G_INSERT_VECTOR_ELT: {
2172 LLT DstTy = MRI.getType(Reg: I.getOperand(i: 0).getReg());
2173 LLT SrcVecTy = MRI.getType(Reg: I.getOperand(i: 1).getReg());
2174 if (SrcVecTy.isPointerVector()) {
2175 // Convert the type from p0 to s64 to help selection.
2176 auto NewSrc = MIB.buildCopy(Res: LLT::scalar(SizeInBits: 64), Op: I.getOperand(i: 2).getReg());
2177 MRI.setType(VReg: I.getOperand(i: 1).getReg(),
2178 Ty: DstTy.changeElementType(NewEltTy: LLT::scalar(SizeInBits: 64)));
2179 MRI.setType(VReg: I.getOperand(i: 0).getReg(),
2180 Ty: DstTy.changeElementType(NewEltTy: LLT::scalar(SizeInBits: 64)));
2181 MRI.setRegClass(Reg: NewSrc.getReg(Idx: 0), RC: &AArch64::GPR64RegClass);
2182 I.getOperand(i: 2).setReg(NewSrc.getReg(Idx: 0));
2183 return true;
2184 }
2185
2186 Register EltReg = I.getOperand(i: 2).getReg();
2187 LLT EltTy = MRI.getType(Reg: EltReg);
2188 if (EltTy.isScalar() &&
2189 (EltTy.getSizeInBits() == 8 || EltTy.getSizeInBits() == 16) &&
2190 RBI.getRegBank(Reg: EltReg, MRI, TRI)->getID() == AArch64::GPRRegBankID) {
2191 // Convert the type from s8/s16 to s32 to help selection.
2192 auto NewElt = MIB.buildCopy(Res: LLT::scalar(SizeInBits: 32), Op: EltReg);
2193 MRI.setRegClass(Reg: NewElt.getReg(Idx: 0), RC: &AArch64::GPR32RegClass);
2194 I.getOperand(i: 2).setReg(NewElt.getReg(Idx: 0));
2195 return true;
2196 }
2197 return false;
2198 }
2199 case TargetOpcode::G_UITOFP:
2200 case TargetOpcode::G_SITOFP: {
2201 // If both source and destination regbanks are FPR, then convert the opcode
2202 // to G_SITOF so that the importer can select it to an fpr variant.
2203 // Otherwise, it ends up matching an fpr/gpr variant and adding a cross-bank
2204 // copy.
2205 Register SrcReg = I.getOperand(i: 1).getReg();
2206 LLT SrcTy = MRI.getType(Reg: SrcReg);
2207 LLT DstTy = MRI.getType(Reg: I.getOperand(i: 0).getReg());
2208 if (SrcTy.isVector() || SrcTy.getSizeInBits() != DstTy.getSizeInBits())
2209 return false;
2210
2211 if (RBI.getRegBank(Reg: SrcReg, MRI, TRI)->getID() == AArch64::FPRRegBankID) {
2212 // Need to add a copy to change the type so that the existing patterns can
2213 // match when there is an integer on an FPR bank.
2214 if (SrcTy.getScalarType().isInteger()) {
2215 auto Copy = MIB.buildCopy(Res: DstTy, Op: SrcReg);
2216 I.getOperand(i: 1).setReg(Copy.getReg(Idx: 0));
2217 MRI.setRegClass(Reg: Copy.getReg(Idx: 0),
2218 RC: getRegClassForTypeOnBank(
2219 Ty: SrcTy, RB: RBI.getRegBank(ID: AArch64::FPRRegBankID)));
2220 }
2221 if (I.getOpcode() == TargetOpcode::G_SITOFP)
2222 I.setDesc(TII.get(Opcode: AArch64::G_SITOF));
2223 else
2224 I.setDesc(TII.get(Opcode: AArch64::G_UITOF));
2225 return true;
2226 }
2227 return false;
2228 }
2229 default:
2230 return false;
2231 }
2232}
2233
2234/// This lowering tries to look for G_PTR_ADD instructions and then converts
2235/// them to a standard G_ADD with a COPY on the source.
2236///
2237/// The motivation behind this is to expose the add semantics to the imported
2238/// tablegen patterns. We shouldn't need to check for uses being loads/stores,
2239/// because the selector works bottom up, uses before defs. By the time we
2240/// end up trying to select a G_PTR_ADD, we should have already attempted to
2241/// fold this into addressing modes and were therefore unsuccessful.
2242bool AArch64InstructionSelector::convertPtrAddToAdd(
2243 MachineInstr &I, MachineRegisterInfo &MRI) {
2244 assert(I.getOpcode() == TargetOpcode::G_PTR_ADD && "Expected G_PTR_ADD");
2245 Register DstReg = I.getOperand(i: 0).getReg();
2246 Register AddOp1Reg = I.getOperand(i: 1).getReg();
2247 const LLT PtrTy = MRI.getType(Reg: DstReg);
2248 if (PtrTy.getAddressSpace() != 0)
2249 return false;
2250
2251 const LLT CastPtrTy = PtrTy.isVector()
2252 ? LLT::fixed_vector(NumElements: 2, ScalarTy: LLT::integer(SizeInBits: 64))
2253 : LLT::integer(SizeInBits: 64);
2254 auto PtrToInt = MIB.buildPtrToInt(Dst: CastPtrTy, Src: AddOp1Reg);
2255 // Set regbanks on the registers.
2256 if (PtrTy.isVector())
2257 MRI.setRegBank(Reg: PtrToInt.getReg(Idx: 0), RegBank: RBI.getRegBank(ID: AArch64::FPRRegBankID));
2258 else
2259 MRI.setRegBank(Reg: PtrToInt.getReg(Idx: 0), RegBank: RBI.getRegBank(ID: AArch64::GPRRegBankID));
2260
2261 // Now turn the %dst(p0) = G_PTR_ADD %base, off into:
2262 // %dst(intty) = G_ADD %intbase, off
2263 I.setDesc(TII.get(Opcode: TargetOpcode::G_ADD));
2264 MRI.setType(VReg: DstReg, Ty: CastPtrTy);
2265 I.getOperand(i: 1).setReg(PtrToInt.getReg(Idx: 0));
2266 if (!select(I&: *PtrToInt)) {
2267 LLVM_DEBUG(dbgs() << "Failed to select G_PTRTOINT in convertPtrAddToAdd");
2268 return false;
2269 }
2270
2271 // Also take the opportunity here to try to do some optimization.
2272 // Try to convert this into a G_SUB if the offset is a 0-x negate idiom.
2273 Register NegatedReg;
2274 if (!mi_match(R: I.getOperand(i: 2).getReg(), MRI, P: m_Neg(Src: m_Reg(R&: NegatedReg))))
2275 return true;
2276 I.getOperand(i: 2).setReg(NegatedReg);
2277 I.setDesc(TII.get(Opcode: TargetOpcode::G_SUB));
2278 return true;
2279}
2280
2281bool AArch64InstructionSelector::earlySelectSHL(MachineInstr &I,
2282 MachineRegisterInfo &MRI) {
2283 // We try to match the immediate variant of LSL, which is actually an alias
2284 // for a special case of UBFM. Otherwise, we fall back to the imported
2285 // selector which will match the register variant.
2286 assert(I.getOpcode() == TargetOpcode::G_SHL && "unexpected op");
2287 const auto &MO = I.getOperand(i: 2);
2288 auto VRegAndVal = getIConstantVRegVal(VReg: MO.getReg(), MRI);
2289 if (!VRegAndVal)
2290 return false;
2291
2292 const LLT DstTy = MRI.getType(Reg: I.getOperand(i: 0).getReg());
2293 if (DstTy.isVector())
2294 return false;
2295 bool Is64Bit = DstTy.getSizeInBits() == 64;
2296 auto Imm1Fn = Is64Bit ? selectShiftA_64(Root: MO) : selectShiftA_32(Root: MO);
2297 auto Imm2Fn = Is64Bit ? selectShiftB_64(Root: MO) : selectShiftB_32(Root: MO);
2298
2299 if (!Imm1Fn || !Imm2Fn)
2300 return false;
2301
2302 auto NewI =
2303 MIB.buildInstr(Opc: Is64Bit ? AArch64::UBFMXri : AArch64::UBFMWri,
2304 DstOps: {I.getOperand(i: 0).getReg()}, SrcOps: {I.getOperand(i: 1).getReg()});
2305
2306 for (auto &RenderFn : *Imm1Fn)
2307 RenderFn(NewI);
2308 for (auto &RenderFn : *Imm2Fn)
2309 RenderFn(NewI);
2310
2311 I.eraseFromParent();
2312 constrainSelectedInstRegOperands(I&: *NewI, TII, TRI, RBI);
2313 return true;
2314}
2315
2316bool AArch64InstructionSelector::contractCrossBankCopyIntoStore(
2317 MachineInstr &I, MachineRegisterInfo &MRI) {
2318 assert(I.getOpcode() == TargetOpcode::G_STORE && "Expected G_STORE");
2319 // If we're storing a scalar, it doesn't matter what register bank that
2320 // scalar is on. All that matters is the size.
2321 //
2322 // So, if we see something like this (with a 32-bit scalar as an example):
2323 //
2324 // %x:gpr(s32) = ... something ...
2325 // %y:fpr(s32) = COPY %x:gpr(s32)
2326 // G_STORE %y:fpr(s32)
2327 //
2328 // We can fix this up into something like this:
2329 //
2330 // G_STORE %x:gpr(s32)
2331 //
2332 // And then continue the selection process normally.
2333 Register DefDstReg = getSrcRegIgnoringCopies(Reg: I.getOperand(i: 0).getReg(), MRI);
2334 if (!DefDstReg.isValid())
2335 return false;
2336 LLT DefDstTy = MRI.getType(Reg: DefDstReg);
2337 Register StoreSrcReg = I.getOperand(i: 0).getReg();
2338 LLT StoreSrcTy = MRI.getType(Reg: StoreSrcReg);
2339
2340 // If we get something strange like a physical register, then we shouldn't
2341 // go any further.
2342 if (!DefDstTy.isValid())
2343 return false;
2344
2345 // Are the source and dst types the same size?
2346 if (DefDstTy.getSizeInBits() != StoreSrcTy.getSizeInBits())
2347 return false;
2348
2349 if (RBI.getRegBank(Reg: StoreSrcReg, MRI, TRI) ==
2350 RBI.getRegBank(Reg: DefDstReg, MRI, TRI))
2351 return false;
2352
2353 // We have a cross-bank copy, which is entering a store. Let's fold it.
2354 I.getOperand(i: 0).setReg(DefDstReg);
2355 return true;
2356}
2357
2358bool AArch64InstructionSelector::earlySelect(MachineInstr &I) {
2359 assert(I.getParent() && "Instruction should be in a basic block!");
2360 assert(I.getParent()->getParent() && "Instruction should be in a function!");
2361
2362 MachineBasicBlock &MBB = *I.getParent();
2363 MachineFunction &MF = *MBB.getParent();
2364 MachineRegisterInfo &MRI = MF.getRegInfo();
2365
2366 switch (I.getOpcode()) {
2367 case TargetOpcode::G_SPLAT_VECTOR: {
2368 Register Dst = I.getOperand(i: 0).getReg();
2369 LLT DstTy = MRI.getType(Reg: Dst);
2370 if (DstTy.isScalableVector())
2371 return false;
2372 // Before selecting a DUP instruction, check if it is better selected as a
2373 // MOV or load from a constant pool.
2374 Register Src = I.getOperand(i: 1).getReg();
2375 auto ValAndVReg = getAnyConstantVRegValWithLookThrough(
2376 VReg: Src, MRI, /*LookThroughInstrs=*/true, /*LookThroughAnyExt=*/true);
2377 if (!ValAndVReg)
2378 return false;
2379 LLVMContext &Ctx = MF.getFunction().getContext();
2380 auto *CV = ConstantDataVector::getSplat(
2381 NumElts: DstTy.getNumElements(),
2382 Elt: ConstantInt::get(Ty: Type::getIntNTy(C&: Ctx, N: DstTy.getScalarSizeInBits()),
2383 V: ValAndVReg->Value.trunc(width: DstTy.getScalarSizeInBits())));
2384 APInt Imm =
2385 APInt::getSplat(NewLen: DstTy.getSizeInBits(),
2386 V: ValAndVReg->Value.trunc(width: DstTy.getScalarSizeInBits()));
2387 if (!emitConstantVector(Dst, CV, Imm, MRI))
2388 return false;
2389 I.eraseFromParent();
2390 return true;
2391 }
2392 case TargetOpcode::G_SEXT:
2393 // Check for i64 sext(i32 vector_extract) prior to tablegen to select SMOV
2394 // over a normal extend.
2395 if (selectUSMovFromExtend(I, MRI))
2396 return true;
2397 return false;
2398 case TargetOpcode::G_BR:
2399 return false;
2400 case TargetOpcode::G_SHL:
2401 return earlySelectSHL(I, MRI);
2402 case TargetOpcode::G_CONSTANT: {
2403 bool IsZero = false;
2404 if (I.getOperand(i: 1).isCImm())
2405 IsZero = I.getOperand(i: 1).getCImm()->isZero();
2406 else if (I.getOperand(i: 1).isImm())
2407 IsZero = I.getOperand(i: 1).getImm() == 0;
2408
2409 if (!IsZero)
2410 return false;
2411
2412 Register DefReg = I.getOperand(i: 0).getReg();
2413 LLT Ty = MRI.getType(Reg: DefReg);
2414 if (Ty.getSizeInBits() == 64) {
2415 I.getOperand(i: 1).ChangeToRegister(Reg: AArch64::XZR, isDef: false);
2416 RBI.constrainGenericRegister(Reg: DefReg, RC: AArch64::GPR64RegClass, MRI);
2417 } else if (Ty.getSizeInBits() <= 32) {
2418 I.getOperand(i: 1).ChangeToRegister(Reg: AArch64::WZR, isDef: false);
2419 RBI.constrainGenericRegister(Reg: DefReg, RC: AArch64::GPR32RegClass, MRI);
2420 } else
2421 return false;
2422
2423 I.setDesc(TII.get(Opcode: TargetOpcode::COPY));
2424 return true;
2425 }
2426
2427 case TargetOpcode::G_ADD: {
2428 // Check if this is being fed by a G_ICMP on either side.
2429 //
2430 // (cmp pred, x, y) + z
2431 //
2432 // In the above case, when the cmp is true, we increment z by 1. So, we can
2433 // fold the add into the cset for the cmp by using cinc.
2434 //
2435 // FIXME: This would probably be a lot nicer in PostLegalizerLowering.
2436 Register AddDst = I.getOperand(i: 0).getReg();
2437 Register AddLHS = I.getOperand(i: 1).getReg();
2438 Register AddRHS = I.getOperand(i: 2).getReg();
2439 // Only handle scalars.
2440 LLT Ty = MRI.getType(Reg: AddLHS);
2441 if (Ty.isVector())
2442 return false;
2443 // Since G_ICMP is modeled as ADDS/SUBS/ANDS, we can handle 32 bits or 64
2444 // bits.
2445 unsigned Size = Ty.getSizeInBits();
2446 if (Size != 32 && Size != 64)
2447 return false;
2448 auto MatchCmp = [&](Register Reg) -> MachineInstr * {
2449 if (!MRI.hasOneNonDBGUse(RegNo: Reg))
2450 return nullptr;
2451 // If the LHS of the add is 32 bits, then we want to fold a 32-bit
2452 // compare.
2453 if (Size == 32)
2454 return getOpcodeDef(Opcode: TargetOpcode::G_ICMP, Reg, MRI);
2455 // We model scalar compares using 32-bit destinations right now.
2456 // If it's a 64-bit compare, it'll have 64-bit sources.
2457 Register ZExt;
2458 if (!mi_match(R: Reg, MRI,
2459 P: m_OneNonDBGUse(SP: m_GZExt(Src: m_OneNonDBGUse(SP: m_Reg(R&: ZExt))))))
2460 return nullptr;
2461 auto *Cmp = getOpcodeDef(Opcode: TargetOpcode::G_ICMP, Reg: ZExt, MRI);
2462 if (!Cmp ||
2463 MRI.getType(Reg: Cmp->getOperand(i: 2).getReg()).getSizeInBits() != 64)
2464 return nullptr;
2465 return Cmp;
2466 };
2467 // Try to match
2468 // z + (cmp pred, x, y)
2469 MachineInstr *Cmp = MatchCmp(AddRHS);
2470 if (!Cmp) {
2471 // (cmp pred, x, y) + z
2472 std::swap(a&: AddLHS, b&: AddRHS);
2473 Cmp = MatchCmp(AddRHS);
2474 if (!Cmp)
2475 return false;
2476 }
2477 auto &PredOp = Cmp->getOperand(i: 1);
2478 MIB.setInstrAndDebugLoc(I);
2479 emitIntegerCompare(/*LHS=*/Cmp->getOperand(i: 2),
2480 /*RHS=*/Cmp->getOperand(i: 3), Predicate&: PredOp, MIRBuilder&: MIB);
2481 auto Pred = static_cast<CmpInst::Predicate>(PredOp.getPredicate());
2482 const AArch64CC::CondCode InvCC = changeICMPPredToAArch64CC(
2483 P: CmpInst::getInversePredicate(pred: Pred), RHS: Cmp->getOperand(i: 3).getReg(), MRI: &MRI);
2484 emitCSINC(/*Dst=*/AddDst, /*Src =*/Src1: AddLHS, /*Src2=*/AddLHS, Pred: InvCC, MIRBuilder&: MIB);
2485 I.eraseFromParent();
2486 return true;
2487 }
2488 case TargetOpcode::G_OR: {
2489 // Look for operations that take the lower `Width=Size-ShiftImm` bits of
2490 // `ShiftSrc` and insert them into the upper `Width` bits of `MaskSrc` via
2491 // shifting and masking that we can replace with a BFI (encoded as a BFM).
2492 Register Dst = I.getOperand(i: 0).getReg();
2493 LLT Ty = MRI.getType(Reg: Dst);
2494
2495 if (!Ty.isScalar())
2496 return false;
2497
2498 unsigned Size = Ty.getSizeInBits();
2499 if (Size != 32 && Size != 64)
2500 return false;
2501
2502 Register ShiftSrc;
2503 int64_t ShiftImm;
2504 Register MaskSrc;
2505 int64_t MaskImm;
2506 if (!mi_match(
2507 R: Dst, MRI,
2508 P: m_GOr(L: m_OneNonDBGUse(SP: m_GShl(L: m_Reg(R&: ShiftSrc), R: m_ICst(Cst&: ShiftImm))),
2509 R: m_OneNonDBGUse(SP: m_GAnd(L: m_Reg(R&: MaskSrc), R: m_ICst(Cst&: MaskImm))))))
2510 return false;
2511
2512 if (ShiftImm > Size || ((1ULL << ShiftImm) - 1ULL) != uint64_t(MaskImm))
2513 return false;
2514
2515 int64_t Immr = Size - ShiftImm;
2516 int64_t Imms = Size - ShiftImm - 1;
2517 unsigned Opc = Size == 32 ? AArch64::BFMWri : AArch64::BFMXri;
2518 emitInstr(Opcode: Opc, DstOps: {Dst}, SrcOps: {MaskSrc, ShiftSrc, Immr, Imms}, MIRBuilder&: MIB);
2519 I.eraseFromParent();
2520 return true;
2521 }
2522 case TargetOpcode::G_FENCE: {
2523 if (I.getOperand(i: 1).getImm() == 0)
2524 BuildMI(BB&: MBB, I, MIMD: MIMetadata(I), MCID: TII.get(Opcode: TargetOpcode::MEMBARRIER));
2525 else
2526 BuildMI(BB&: MBB, I, MIMD: MIMetadata(I), MCID: TII.get(Opcode: AArch64::DMB))
2527 .addImm(Val: I.getOperand(i: 0).getImm() == 4 ? 0x9 : 0xb);
2528 I.eraseFromParent();
2529 return true;
2530 }
2531 case TargetOpcode::G_BRINDIRECT: {
2532 const Function &Fn = MF.getFunction();
2533 if (std::optional<uint16_t> BADisc =
2534 STI.getPtrAuthBlockAddressDiscriminatorIfEnabled(ParentFn: Fn)) {
2535 auto MI = MIB.buildInstr(Opc: AArch64::BRA, DstOps: {}, SrcOps: {I.getOperand(i: 0).getReg()});
2536 MI.addImm(Val: AArch64PACKey::IA);
2537 MI.addImm(Val: *BADisc);
2538 MI.addReg(/*AddrDisc=*/RegNo: AArch64::XZR);
2539 I.eraseFromParent();
2540 constrainSelectedInstRegOperands(I&: *MI, TII, TRI, RBI);
2541 return true;
2542 }
2543 // Use table-based selection.
2544 return false;
2545 }
2546
2547 default:
2548 return false;
2549 }
2550}
2551
2552bool AArch64InstructionSelector::select(MachineInstr &I) {
2553 assert(I.getParent() && "Instruction should be in a basic block!");
2554 assert(I.getParent()->getParent() && "Instruction should be in a function!");
2555
2556 MachineBasicBlock &MBB = *I.getParent();
2557 MachineFunction &MF = *MBB.getParent();
2558 MachineRegisterInfo &MRI = MF.getRegInfo();
2559
2560 const AArch64Subtarget *Subtarget = &MF.getSubtarget<AArch64Subtarget>();
2561 if (Subtarget->requiresStrictAlign()) {
2562 // We don't support this feature yet.
2563 LLVM_DEBUG(dbgs() << "AArch64 GISel does not support strict-align yet\n");
2564 return false;
2565 }
2566
2567 MIB.setInstrAndDebugLoc(I);
2568
2569 unsigned Opcode = I.getOpcode();
2570 // G_PHI requires same handling as PHI
2571 if (!I.isPreISelOpcode() || Opcode == TargetOpcode::G_PHI) {
2572 // Certain non-generic instructions also need some special handling.
2573
2574 if (Opcode == TargetOpcode::LOAD_STACK_GUARD) {
2575 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2576 return true;
2577 }
2578
2579 if (Opcode == TargetOpcode::PHI || Opcode == TargetOpcode::G_PHI) {
2580 const Register DefReg = I.getOperand(i: 0).getReg();
2581 const LLT DefTy = MRI.getType(Reg: DefReg);
2582
2583 const RegClassOrRegBank &RegClassOrBank =
2584 MRI.getRegClassOrRegBank(Reg: DefReg);
2585
2586 const TargetRegisterClass *DefRC =
2587 dyn_cast<const TargetRegisterClass *>(Val: RegClassOrBank);
2588 if (!DefRC) {
2589 if (!DefTy.isValid()) {
2590 LLVM_DEBUG(dbgs() << "PHI operand has no type, not a gvreg?\n");
2591 return false;
2592 }
2593 const RegisterBank &RB = *cast<const RegisterBank *>(Val: RegClassOrBank);
2594 DefRC = getRegClassForTypeOnBank(Ty: DefTy, RB);
2595 if (!DefRC) {
2596 LLVM_DEBUG(dbgs() << "PHI operand has unexpected size/bank\n");
2597 return false;
2598 }
2599 }
2600
2601 I.setDesc(TII.get(Opcode: TargetOpcode::PHI));
2602
2603 return RBI.constrainGenericRegister(Reg: DefReg, RC: *DefRC, MRI);
2604 }
2605
2606 if (I.isCopy())
2607 return selectCopy(I, TII, MRI, TRI, RBI);
2608
2609 if (I.isDebugInstr())
2610 return selectDebugInstr(I, MRI, RBI);
2611
2612 return true;
2613 }
2614
2615
2616 if (I.getNumOperands() != I.getNumExplicitOperands()) {
2617 LLVM_DEBUG(
2618 dbgs() << "Generic instruction has unexpected implicit operands\n");
2619 return false;
2620 }
2621
2622 // Try to do some lowering before we start instruction selecting. These
2623 // lowerings are purely transformations on the input G_MIR and so selection
2624 // must continue after any modification of the instruction.
2625 if (preISelLower(I)) {
2626 Opcode = I.getOpcode(); // The opcode may have been modified, refresh it.
2627 }
2628
2629 // There may be patterns where the importer can't deal with them optimally,
2630 // but does select it to a suboptimal sequence so our custom C++ selection
2631 // code later never has a chance to work on it. Therefore, we have an early
2632 // selection attempt here to give priority to certain selection routines
2633 // over the imported ones.
2634 if (earlySelect(I))
2635 return true;
2636
2637 if (selectImpl(I, CoverageInfo&: *CoverageInfo))
2638 return true;
2639
2640 LLT Ty =
2641 I.getOperand(i: 0).isReg() ? MRI.getType(Reg: I.getOperand(i: 0).getReg()) : LLT{};
2642
2643 switch (Opcode) {
2644 case TargetOpcode::G_SBFX:
2645 case TargetOpcode::G_UBFX: {
2646 static const unsigned OpcTable[2][2] = {
2647 {AArch64::UBFMWri, AArch64::UBFMXri},
2648 {AArch64::SBFMWri, AArch64::SBFMXri}};
2649 bool IsSigned = Opcode == TargetOpcode::G_SBFX;
2650 unsigned Size = Ty.getSizeInBits();
2651 unsigned Opc = OpcTable[IsSigned][Size == 64];
2652 auto Cst1 =
2653 getIConstantVRegValWithLookThrough(VReg: I.getOperand(i: 2).getReg(), MRI);
2654 assert(Cst1 && "Should have gotten a constant for src 1?");
2655 auto Cst2 =
2656 getIConstantVRegValWithLookThrough(VReg: I.getOperand(i: 3).getReg(), MRI);
2657 assert(Cst2 && "Should have gotten a constant for src 2?");
2658 auto LSB = Cst1->Value.getZExtValue();
2659 auto Width = Cst2->Value.getZExtValue();
2660 auto BitfieldInst =
2661 MIB.buildInstr(Opc, DstOps: {I.getOperand(i: 0)}, SrcOps: {I.getOperand(i: 1)})
2662 .addImm(Val: LSB)
2663 .addImm(Val: LSB + Width - 1);
2664 I.eraseFromParent();
2665 constrainSelectedInstRegOperands(I&: *BitfieldInst, TII, TRI, RBI);
2666 return true;
2667 }
2668 case TargetOpcode::G_BRCOND:
2669 return selectCompareBranch(I, MF, MRI);
2670
2671 case TargetOpcode::G_BRJT:
2672 return selectBrJT(I, MRI);
2673
2674 case AArch64::G_ADD_LOW: {
2675 // This op may have been separated from it's ADRP companion by the localizer
2676 // or some other code motion pass. Given that many CPUs will try to
2677 // macro fuse these operations anyway, select this into a MOVaddr pseudo
2678 // which will later be expanded into an ADRP+ADD pair after scheduling.
2679 MachineInstr *BaseMI = MRI.getVRegDef(Reg: I.getOperand(i: 1).getReg());
2680 if (BaseMI->getOpcode() != AArch64::ADRP) {
2681 I.setDesc(TII.get(Opcode: AArch64::ADDXri));
2682 I.addOperand(Op: MachineOperand::CreateImm(Val: 0));
2683 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2684 return true;
2685 }
2686 assert(TM.getCodeModel() == CodeModel::Small &&
2687 "Expected small code model");
2688 auto Op1 = BaseMI->getOperand(i: 1);
2689 auto Op2 = I.getOperand(i: 2);
2690 auto MovAddr = MIB.buildInstr(Opc: AArch64::MOVaddr, DstOps: {I.getOperand(i: 0)}, SrcOps: {})
2691 .addGlobalAddress(GV: Op1.getGlobal(), Offset: Op1.getOffset(),
2692 TargetFlags: Op1.getTargetFlags())
2693 .addGlobalAddress(GV: Op2.getGlobal(), Offset: Op2.getOffset(),
2694 TargetFlags: Op2.getTargetFlags());
2695 I.eraseFromParent();
2696 constrainSelectedInstRegOperands(I&: *MovAddr, TII, TRI, RBI);
2697 return true;
2698 }
2699
2700 case TargetOpcode::G_FCONSTANT: {
2701 const Register DefReg = I.getOperand(i: 0).getReg();
2702 const LLT DefTy = MRI.getType(Reg: DefReg);
2703 const unsigned DefSize = DefTy.getSizeInBits();
2704 const RegisterBank &RB = *RBI.getRegBank(Reg: DefReg, MRI, TRI);
2705
2706 const TargetRegisterClass &FPRRC = *getRegClassForTypeOnBank(Ty: DefTy, RB);
2707 assert(
2708 (DefSize == 16 || DefSize == 32 || DefSize == 64 || DefSize == 128) &&
2709 "Unexpected destination size for G_FCONSTANT?");
2710
2711 // Try to emit as a vector movi.
2712 const ConstantFP *FPImm = I.getOperand(i: 1).getFPImm();
2713 APInt Imm = DefSize >= 64
2714 ? FPImm->getValue().bitcastToAPInt()
2715 : APInt::getSplat(NewLen: 64, V: FPImm->getValue().bitcastToAPInt());
2716 if (auto *Mov = emitConstantMOVVector(
2717 Dst: DefSize >= 64 ? DstOp(DefReg) : DstOp(&AArch64::FPR64RegClass), Imm,
2718 MRI)) {
2719 if (DefSize < 64) {
2720 MIB.buildInstr(Opc: TargetOpcode::COPY, DstOps: {DefReg}, SrcOps: {})
2721 .addReg(RegNo: Mov->getOperand(i: 0).getReg(), Flags: {},
2722 SubReg: DefSize == 16 ? AArch64::hsub : AArch64::ssub);
2723 RBI.constrainGenericRegister(Reg: DefReg, RC: FPRRC, MRI);
2724 }
2725 I.eraseFromParent();
2726 return true;
2727 }
2728
2729 // If TLI says that this fpimm is legal then prefer a copy from GPR movimm,
2730 // otherwise we'll expand to a constant pool load
2731 bool OptForSize = shouldOptForSize(MF: &MF);
2732 const auto &TLI = MF.getSubtarget().getTargetLowering();
2733 if ((DefSize == 32 || DefSize == 64) &&
2734 TLI->isFPImmLegal(I.getOperand(i: 1).getFPImm()->getValueAPF(),
2735 EVT::getFloatingPointVT(BitWidth: DefSize), ForCodeSize: OptForSize)) {
2736 // Emit a copy to emit a normal movimm.
2737 const Register DefGPRReg = MRI.createVirtualRegister(
2738 RegClass: DefSize == 64 ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass);
2739 MachineOperand &RegOp = I.getOperand(i: 0);
2740 RegOp.setReg(DefGPRReg);
2741 MIB.setInsertPt(MBB&: MIB.getMBB(), II: std::next(x: I.getIterator()));
2742 MIB.buildCopy(Res: {DefReg}, Op: {DefGPRReg});
2743
2744 if (!RBI.constrainGenericRegister(Reg: DefReg, RC: FPRRC, MRI)) {
2745 LLVM_DEBUG(dbgs() << "Failed to constrain G_FCONSTANT def operand\n");
2746 return false;
2747 }
2748
2749 MachineOperand &ImmOp = I.getOperand(i: 1);
2750 ImmOp.ChangeToImmediate(
2751 ImmVal: ImmOp.getFPImm()->getValueAPF().bitcastToAPInt().getZExtValue());
2752
2753 const unsigned MovOpc =
2754 DefSize == 64 ? AArch64::MOVi64imm : AArch64::MOVi32imm;
2755 I.setDesc(TII.get(Opcode: MovOpc));
2756 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2757 return true;
2758 }
2759
2760 // Expand to a constant pool.
2761 auto *LoadMI = emitLoadFromConstantPool(CPVal: FPImm, MIRBuilder&: MIB);
2762 if (!LoadMI) {
2763 LLVM_DEBUG(dbgs() << "Failed to load double constant pool entry\n");
2764 return false;
2765 }
2766 MIB.buildCopy(Res: {DefReg}, Op: {LoadMI->getOperand(i: 0).getReg()});
2767 I.eraseFromParent();
2768 return RBI.constrainGenericRegister(Reg: DefReg, RC: FPRRC, MRI);
2769 }
2770 case TargetOpcode::G_EXTRACT: {
2771 Register DstReg = I.getOperand(i: 0).getReg();
2772 Register SrcReg = I.getOperand(i: 1).getReg();
2773 LLT SrcTy = MRI.getType(Reg: SrcReg);
2774 LLT DstTy = MRI.getType(Reg: DstReg);
2775 (void)DstTy;
2776 unsigned SrcSize = SrcTy.getSizeInBits();
2777
2778 if (SrcTy.getSizeInBits() > 64) {
2779 // This should be an extract of an s128, which is like a vector extract.
2780 if (SrcTy.getSizeInBits() != 128)
2781 return false;
2782 // Only support extracting 64 bits from an s128 at the moment.
2783 if (DstTy.getSizeInBits() != 64)
2784 return false;
2785
2786 unsigned Offset = I.getOperand(i: 2).getImm();
2787 if (Offset % 64 != 0)
2788 return false;
2789
2790 // Check we have the right regbank always.
2791 const RegisterBank &SrcRB = *RBI.getRegBank(Reg: SrcReg, MRI, TRI);
2792 const RegisterBank &DstRB = *RBI.getRegBank(Reg: DstReg, MRI, TRI);
2793 assert(SrcRB.getID() == DstRB.getID() && "Wrong extract regbank!");
2794
2795 if (SrcRB.getID() == AArch64::GPRRegBankID) {
2796 auto NewI =
2797 MIB.buildInstr(Opc: TargetOpcode::COPY, DstOps: {DstReg}, SrcOps: {})
2798 .addUse(RegNo: SrcReg, Flags: {},
2799 SubReg: Offset == 0 ? AArch64::sube64 : AArch64::subo64);
2800 constrainOperandRegClass(MF, TRI, MRI, TII, RBI, InsertPt&: *NewI,
2801 RegClass: AArch64::GPR64RegClass, RegMO&: NewI->getOperand(i: 0));
2802 I.eraseFromParent();
2803 return true;
2804 }
2805
2806 // Emit the same code as a vector extract.
2807 // Offset must be a multiple of 64.
2808 unsigned LaneIdx = Offset / 64;
2809 MachineInstr *Extract = emitExtractVectorElt(
2810 DstReg, DstRB, ScalarTy: LLT::scalar(SizeInBits: 64), VecReg: SrcReg, LaneIdx, MIRBuilder&: MIB);
2811 if (!Extract)
2812 return false;
2813 I.eraseFromParent();
2814 return true;
2815 }
2816
2817 I.setDesc(TII.get(Opcode: SrcSize == 64 ? AArch64::UBFMXri : AArch64::UBFMWri));
2818 MachineInstrBuilder(MF, I).addImm(Val: I.getOperand(i: 2).getImm() +
2819 Ty.getSizeInBits() - 1);
2820
2821 if (SrcSize < 64) {
2822 assert(SrcSize == 32 && DstTy.getSizeInBits() == 16 &&
2823 "unexpected G_EXTRACT types");
2824 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2825 return true;
2826 }
2827
2828 DstReg = MRI.createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 64));
2829 MIB.setInsertPt(MBB&: MIB.getMBB(), II: std::next(x: I.getIterator()));
2830 MIB.buildInstr(Opc: TargetOpcode::COPY, DstOps: {I.getOperand(i: 0).getReg()}, SrcOps: {})
2831 .addReg(RegNo: DstReg, Flags: {}, SubReg: AArch64::sub_32);
2832 RBI.constrainGenericRegister(Reg: I.getOperand(i: 0).getReg(),
2833 RC: AArch64::GPR32RegClass, MRI);
2834 I.getOperand(i: 0).setReg(DstReg);
2835
2836 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2837 return true;
2838 }
2839
2840 case TargetOpcode::G_INSERT: {
2841 LLT SrcTy = MRI.getType(Reg: I.getOperand(i: 2).getReg());
2842 LLT DstTy = MRI.getType(Reg: I.getOperand(i: 0).getReg());
2843 unsigned DstSize = DstTy.getSizeInBits();
2844 // Larger inserts are vectors, same-size ones should be something else by
2845 // now (split up or turned into COPYs).
2846 if (Ty.getSizeInBits() > 64 || SrcTy.getSizeInBits() > 32)
2847 return false;
2848
2849 I.setDesc(TII.get(Opcode: DstSize == 64 ? AArch64::BFMXri : AArch64::BFMWri));
2850 unsigned LSB = I.getOperand(i: 3).getImm();
2851 unsigned Width = MRI.getType(Reg: I.getOperand(i: 2).getReg()).getSizeInBits();
2852 I.getOperand(i: 3).setImm((DstSize - LSB) % DstSize);
2853 MachineInstrBuilder(MF, I).addImm(Val: Width - 1);
2854
2855 if (DstSize < 64) {
2856 assert(DstSize == 32 && SrcTy.getSizeInBits() == 16 &&
2857 "unexpected G_INSERT types");
2858 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2859 return true;
2860 }
2861
2862 Register SrcReg = MRI.createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 64));
2863 BuildMI(BB&: MBB, I: I.getIterator(), MIMD: I.getDebugLoc(),
2864 MCID: TII.get(Opcode: AArch64::SUBREG_TO_REG))
2865 .addDef(RegNo: SrcReg)
2866 .addUse(RegNo: I.getOperand(i: 2).getReg())
2867 .addImm(Val: AArch64::sub_32);
2868 RBI.constrainGenericRegister(Reg: I.getOperand(i: 2).getReg(),
2869 RC: AArch64::GPR32RegClass, MRI);
2870 I.getOperand(i: 2).setReg(SrcReg);
2871
2872 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2873 return true;
2874 }
2875 case TargetOpcode::G_FRAME_INDEX: {
2876 // allocas and G_FRAME_INDEX are only supported in addrspace(0).
2877 if (Ty != LLT::pointer(AddressSpace: 0, SizeInBits: 64)) {
2878 LLVM_DEBUG(dbgs() << "G_FRAME_INDEX pointer has type: " << Ty
2879 << ", expected: " << LLT::pointer(0, 64) << '\n');
2880 return false;
2881 }
2882 I.setDesc(TII.get(Opcode: AArch64::ADDXri));
2883
2884 // MOs for a #0 shifted immediate.
2885 I.addOperand(Op: MachineOperand::CreateImm(Val: 0));
2886 I.addOperand(Op: MachineOperand::CreateImm(Val: 0));
2887
2888 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2889 return true;
2890 }
2891
2892 case TargetOpcode::G_GLOBAL_VALUE: {
2893 const GlobalValue *GV = nullptr;
2894 unsigned OpFlags;
2895 if (I.getOperand(i: 1).isSymbol()) {
2896 OpFlags = I.getOperand(i: 1).getTargetFlags();
2897 // Currently only used by "RtLibUseGOT".
2898 assert(OpFlags == AArch64II::MO_GOT);
2899 } else {
2900 GV = I.getOperand(i: 1).getGlobal();
2901 if (GV->isThreadLocal())
2902 return selectTLSGlobalValue(I, MRI);
2903
2904 OpFlags = STI.ClassifyGlobalReference(GV, TM);
2905 }
2906
2907 if (OpFlags & AArch64II::MO_GOT) {
2908 bool IsGOTSigned = MF.getInfo<AArch64FunctionInfo>()->hasELFSignedGOT();
2909 I.setDesc(TII.get(Opcode: IsGOTSigned ? AArch64::LOADgotAUTH : AArch64::LOADgot));
2910 I.getOperand(i: 1).setTargetFlags(OpFlags);
2911 I.addImplicitDefUseOperands(MF);
2912 I.setImplicitPhysRegDefsDead();
2913 } else if (TM.getCodeModel() == CodeModel::Large &&
2914 !TM.isPositionIndependent()) {
2915 // Materialize the global using movz/movk instructions.
2916 materializeLargeCMVal(I, V: GV, OpFlags);
2917 I.eraseFromParent();
2918 return true;
2919 } else if (TM.getCodeModel() == CodeModel::Tiny) {
2920 I.setDesc(TII.get(Opcode: AArch64::ADR));
2921 I.getOperand(i: 1).setTargetFlags(OpFlags);
2922 } else {
2923 I.setDesc(TII.get(Opcode: AArch64::MOVaddr));
2924 I.getOperand(i: 1).setTargetFlags(OpFlags | AArch64II::MO_PAGE);
2925 MachineInstrBuilder MIB(MF, I);
2926 MIB.addGlobalAddress(GV, Offset: I.getOperand(i: 1).getOffset(),
2927 TargetFlags: OpFlags | AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
2928 }
2929 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2930 return true;
2931 }
2932
2933 case TargetOpcode::G_PTRAUTH_GLOBAL_VALUE:
2934 return selectPtrAuthGlobalValue(I, MRI);
2935
2936 case TargetOpcode::G_ZEXTLOAD:
2937 case TargetOpcode::G_LOAD:
2938 case TargetOpcode::G_STORE: {
2939 GLoadStore &LdSt = cast<GLoadStore>(Val&: I);
2940 bool IsZExtLoad = I.getOpcode() == TargetOpcode::G_ZEXTLOAD;
2941 LLT PtrTy = MRI.getType(Reg: LdSt.getPointerReg());
2942
2943 // Can only handle AddressSpace 0, 64-bit pointers.
2944 if (PtrTy != LLT::pointer(AddressSpace: 0, SizeInBits: 64)) {
2945 return false;
2946 }
2947
2948 uint64_t MemSizeInBytes = LdSt.getMemSize().getValue();
2949 unsigned MemSizeInBits = LdSt.getMemSizeInBits().getValue();
2950 AtomicOrdering Order = LdSt.getMMO().getSuccessOrdering();
2951
2952 // Need special instructions for atomics that affect ordering.
2953 if (isStrongerThanMonotonic(AO: Order)) {
2954 assert(!isa<GZExtLoad>(LdSt));
2955 assert(MemSizeInBytes <= 8 &&
2956 "128-bit atomics should already be custom-legalized");
2957
2958 if (isa<GLoad>(Val: LdSt)) {
2959 static constexpr unsigned LDAPROpcodes[] = {
2960 AArch64::LDAPRB, AArch64::LDAPRH, AArch64::LDAPRW, AArch64::LDAPRX};
2961 static constexpr unsigned LDAROpcodes[] = {
2962 AArch64::LDARB, AArch64::LDARH, AArch64::LDARW, AArch64::LDARX};
2963 ArrayRef<unsigned> Opcodes =
2964 STI.hasRCPC() && Order != AtomicOrdering::SequentiallyConsistent
2965 ? LDAPROpcodes
2966 : LDAROpcodes;
2967 I.setDesc(TII.get(Opcode: Opcodes[Log2_32(Value: MemSizeInBytes)]));
2968 } else {
2969 static constexpr unsigned Opcodes[] = {AArch64::STLRB, AArch64::STLRH,
2970 AArch64::STLRW, AArch64::STLRX};
2971 Register ValReg = LdSt.getReg(Idx: 0);
2972 if (MRI.getType(Reg: ValReg).getSizeInBits() == 64 && MemSizeInBits != 64) {
2973 // Emit a subreg copy of 32 bits.
2974 Register NewVal = MRI.createVirtualRegister(RegClass: &AArch64::GPR32RegClass);
2975 MIB.buildInstr(Opc: TargetOpcode::COPY, DstOps: {NewVal}, SrcOps: {})
2976 .addReg(RegNo: I.getOperand(i: 0).getReg(), Flags: {}, SubReg: AArch64::sub_32);
2977 I.getOperand(i: 0).setReg(NewVal);
2978 }
2979 I.setDesc(TII.get(Opcode: Opcodes[Log2_32(Value: MemSizeInBytes)]));
2980 }
2981 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
2982 return true;
2983 }
2984
2985#ifndef NDEBUG
2986 const Register PtrReg = LdSt.getPointerReg();
2987 const RegisterBank &PtrRB = *RBI.getRegBank(PtrReg, MRI, TRI);
2988 // Check that the pointer register is valid.
2989 assert(PtrRB.getID() == AArch64::GPRRegBankID &&
2990 "Load/Store pointer operand isn't a GPR");
2991 assert(MRI.getType(PtrReg).isPointer() &&
2992 "Load/Store pointer operand isn't a pointer");
2993#endif
2994
2995 const Register ValReg = LdSt.getReg(Idx: 0);
2996 const RegisterBank &RB = *RBI.getRegBank(Reg: ValReg, MRI, TRI);
2997 LLT ValTy = MRI.getType(Reg: ValReg);
2998
2999 // The code below doesn't support truncating stores, so we need to split it
3000 // again.
3001 if (isa<GStore>(Val: LdSt) && ValTy.getSizeInBits() > MemSizeInBits &&
3002 RB.getID() == AArch64::FPRRegBankID) {
3003 unsigned SubReg;
3004 LLT MemTy = LdSt.getMMO().getMemoryType();
3005 auto *RC = getRegClassForTypeOnBank(Ty: MemTy, RB);
3006 if (!getSubRegForClass(RC, TRI, SubReg))
3007 return false;
3008
3009 // Generate a subreg copy.
3010 auto Copy = MIB.buildInstr(Opc: TargetOpcode::COPY, DstOps: {MemTy}, SrcOps: {})
3011 .addReg(RegNo: ValReg, Flags: {}, SubReg)
3012 .getReg(Idx: 0);
3013 RBI.constrainGenericRegister(Reg: Copy, RC: *RC, MRI);
3014 LdSt.getOperand(i: 0).setReg(Copy);
3015 } else if (isa<GLoad>(Val: LdSt) && ValTy.getSizeInBits() > MemSizeInBits) {
3016 // If this is an any-extending load from the FPR bank, split it into a regular
3017 // load + extend.
3018 if (RB.getID() == AArch64::FPRRegBankID) {
3019 unsigned SubReg;
3020 LLT MemTy = LdSt.getMMO().getMemoryType();
3021 auto *RC = getRegClassForTypeOnBank(Ty: MemTy, RB);
3022 if (!getSubRegForClass(RC, TRI, SubReg))
3023 return false;
3024 Register OldDst = LdSt.getReg(Idx: 0);
3025 Register NewDst =
3026 MRI.createGenericVirtualRegister(Ty: LdSt.getMMO().getMemoryType());
3027 LdSt.getOperand(i: 0).setReg(NewDst);
3028 MRI.setRegBank(Reg: NewDst, RegBank: RB);
3029 // Generate a SUBREG_TO_REG to extend it.
3030 MIB.setInsertPt(MBB&: MIB.getMBB(), II: std::next(x: LdSt.getIterator()));
3031 MIB.buildInstr(Opc: AArch64::SUBREG_TO_REG, DstOps: {OldDst}, SrcOps: {})
3032 .addUse(RegNo: NewDst)
3033 .addImm(Val: SubReg);
3034 auto SubRegRC = getRegClassForTypeOnBank(Ty: MRI.getType(Reg: OldDst), RB);
3035 RBI.constrainGenericRegister(Reg: OldDst, RC: *SubRegRC, MRI);
3036 MIB.setInstr(LdSt);
3037 ValTy = MemTy; // This is no longer an extending load.
3038 }
3039 }
3040
3041 // Helper lambda for partially selecting I. Either returns the original
3042 // instruction with an updated opcode, or a new instruction.
3043 auto SelectLoadStoreAddressingMode = [&]() -> MachineInstr * {
3044 bool IsStore = isa<GStore>(Val: I);
3045 const unsigned NewOpc =
3046 selectLoadStoreUIOp(GenericOpc: I.getOpcode(), RegBankID: RB.getID(), OpSize: MemSizeInBits);
3047 if (NewOpc == I.getOpcode())
3048 return nullptr;
3049 // Check if we can fold anything into the addressing mode.
3050 auto AddrModeFns =
3051 selectAddrModeIndexed(Root&: I.getOperand(i: 1), Size: MemSizeInBytes);
3052 if (!AddrModeFns) {
3053 // Can't fold anything. Use the original instruction.
3054 I.setDesc(TII.get(Opcode: NewOpc));
3055 I.addOperand(Op: MachineOperand::CreateImm(Val: 0));
3056 return &I;
3057 }
3058
3059 // Folded something. Create a new instruction and return it.
3060 auto NewInst = MIB.buildInstr(Opc: NewOpc, DstOps: {}, SrcOps: {}, Flags: I.getFlags());
3061 Register CurValReg = I.getOperand(i: 0).getReg();
3062 IsStore ? NewInst.addUse(RegNo: CurValReg) : NewInst.addDef(RegNo: CurValReg);
3063 NewInst.cloneMemRefs(OtherMI: I);
3064 for (auto &Fn : *AddrModeFns)
3065 Fn(NewInst);
3066 I.eraseFromParent();
3067 return &*NewInst;
3068 };
3069
3070 MachineInstr *LoadStore = SelectLoadStoreAddressingMode();
3071 if (!LoadStore)
3072 return false;
3073
3074 // If we're storing a 0, use WZR/XZR.
3075 if (Opcode == TargetOpcode::G_STORE) {
3076 auto CVal = getIConstantVRegValWithLookThrough(
3077 VReg: LoadStore->getOperand(i: 0).getReg(), MRI);
3078 if (CVal && CVal->Value == 0) {
3079 switch (LoadStore->getOpcode()) {
3080 case AArch64::STRWui:
3081 case AArch64::STRHHui:
3082 case AArch64::STRBBui:
3083 LoadStore->getOperand(i: 0).setReg(AArch64::WZR);
3084 break;
3085 case AArch64::STRXui:
3086 LoadStore->getOperand(i: 0).setReg(AArch64::XZR);
3087 break;
3088 }
3089 }
3090 }
3091
3092 if (IsZExtLoad || (Opcode == TargetOpcode::G_LOAD &&
3093 ValTy == LLT::scalar(SizeInBits: 64) && MemSizeInBits == 32)) {
3094 // The any/zextload from a smaller type to i32 should be handled by the
3095 // importer.
3096 if (MRI.getType(Reg: LoadStore->getOperand(i: 0).getReg()).getSizeInBits() != 64)
3097 return false;
3098 // If we have an extending load then change the load's type to be a
3099 // narrower reg and zero_extend with SUBREG_TO_REG.
3100 Register LdReg = MRI.createVirtualRegister(RegClass: &AArch64::GPR32RegClass);
3101 Register DstReg = LoadStore->getOperand(i: 0).getReg();
3102 LoadStore->getOperand(i: 0).setReg(LdReg);
3103
3104 MIB.setInsertPt(MBB&: MIB.getMBB(), II: std::next(x: LoadStore->getIterator()));
3105 MIB.buildInstr(Opc: AArch64::SUBREG_TO_REG, DstOps: {DstReg}, SrcOps: {})
3106 .addUse(RegNo: LdReg)
3107 .addImm(Val: AArch64::sub_32);
3108 constrainSelectedInstRegOperands(I&: *LoadStore, TII, TRI, RBI);
3109 return RBI.constrainGenericRegister(Reg: DstReg, RC: AArch64::GPR64allRegClass,
3110 MRI);
3111 }
3112 constrainSelectedInstRegOperands(I&: *LoadStore, TII, TRI, RBI);
3113 return true;
3114 }
3115
3116 case TargetOpcode::G_INDEXED_ZEXTLOAD:
3117 case TargetOpcode::G_INDEXED_SEXTLOAD:
3118 return selectIndexedExtLoad(I, MRI);
3119 case TargetOpcode::G_INDEXED_LOAD:
3120 return selectIndexedLoad(I, MRI);
3121 case TargetOpcode::G_INDEXED_STORE:
3122 return selectIndexedStore(I&: cast<GIndexedStore>(Val&: I), MRI);
3123
3124 case TargetOpcode::G_LSHR:
3125 case TargetOpcode::G_ASHR:
3126 if (MRI.getType(Reg: I.getOperand(i: 0).getReg()).isVector())
3127 return selectVectorAshrLshr(I, MRI);
3128 [[fallthrough]];
3129 case TargetOpcode::G_SHL: {
3130 if (Opcode == TargetOpcode::G_SHL &&
3131 MRI.getType(Reg: I.getOperand(i: 0).getReg()).isVector())
3132 return selectVectorSHL(I, MRI);
3133
3134 // These shifts were legalized to have 64 bit shift amounts because we
3135 // want to take advantage of the selection patterns that assume the
3136 // immediates are s64s, however, selectBinaryOp will assume both operands
3137 // will have the same bit size.
3138 {
3139 Register SrcReg = I.getOperand(i: 1).getReg();
3140 Register ShiftReg = I.getOperand(i: 2).getReg();
3141 const LLT ShiftTy = MRI.getType(Reg: ShiftReg);
3142 const LLT SrcTy = MRI.getType(Reg: SrcReg);
3143 if (!SrcTy.isVector() && SrcTy.getSizeInBits() == 32 &&
3144 ShiftTy.getSizeInBits() == 64) {
3145 assert(!ShiftTy.isVector() && "unexpected vector shift ty");
3146 // Insert a subregister copy to implement a 64->32 trunc
3147 auto Trunc = MIB.buildInstr(Opc: TargetOpcode::COPY, DstOps: {SrcTy}, SrcOps: {})
3148 .addReg(RegNo: ShiftReg, Flags: {}, SubReg: AArch64::sub_32);
3149 MRI.setRegBank(Reg: Trunc.getReg(Idx: 0), RegBank: RBI.getRegBank(ID: AArch64::GPRRegBankID));
3150 I.getOperand(i: 2).setReg(Trunc.getReg(Idx: 0));
3151 }
3152 }
3153
3154 const unsigned OpSize = Ty.getSizeInBits();
3155 const Register DefReg = I.getOperand(i: 0).getReg();
3156 const RegisterBank &RB = *RBI.getRegBank(Reg: DefReg, MRI, TRI);
3157
3158 const unsigned NewOpc = selectBinaryOp(GenericOpc: I.getOpcode(), RegBankID: RB.getID(), OpSize);
3159 if (NewOpc == I.getOpcode())
3160 return false;
3161
3162 I.setDesc(TII.get(Opcode: NewOpc));
3163 // FIXME: Should the type be always reset in setDesc?
3164
3165 // Now that we selected an opcode, we need to constrain the register
3166 // operands to use appropriate classes.
3167 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
3168 return true;
3169 }
3170 case TargetOpcode::G_PTR_ADD: {
3171 emitADD(DefReg: I.getOperand(i: 0).getReg(), LHS&: I.getOperand(i: 1), RHS&: I.getOperand(i: 2), MIRBuilder&: MIB);
3172 I.eraseFromParent();
3173 return true;
3174 }
3175
3176 case TargetOpcode::G_SADDE:
3177 case TargetOpcode::G_UADDE:
3178 case TargetOpcode::G_SSUBE:
3179 case TargetOpcode::G_USUBE:
3180 case TargetOpcode::G_SADDO:
3181 case TargetOpcode::G_UADDO:
3182 case TargetOpcode::G_SSUBO:
3183 case TargetOpcode::G_USUBO:
3184 return selectOverflowOp(I, MRI);
3185
3186 case TargetOpcode::G_PTRMASK: {
3187 Register MaskReg = I.getOperand(i: 2).getReg();
3188 std::optional<int64_t> MaskVal = getIConstantVRegSExtVal(VReg: MaskReg, MRI);
3189 // TODO: Implement arbitrary cases
3190 if (!MaskVal || !isShiftedMask_64(Value: *MaskVal))
3191 return false;
3192
3193 uint64_t Mask = *MaskVal;
3194 I.setDesc(TII.get(Opcode: AArch64::ANDXri));
3195 I.getOperand(i: 2).ChangeToImmediate(
3196 ImmVal: AArch64_AM::encodeLogicalImmediate(imm: Mask, regSize: 64));
3197
3198 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
3199 return true;
3200 }
3201 case TargetOpcode::G_PTRTOINT:
3202 case TargetOpcode::G_TRUNC: {
3203 const LLT DstTy = MRI.getType(Reg: I.getOperand(i: 0).getReg());
3204 const LLT SrcTy = MRI.getType(Reg: I.getOperand(i: 1).getReg());
3205
3206 const Register DstReg = I.getOperand(i: 0).getReg();
3207 const Register SrcReg = I.getOperand(i: 1).getReg();
3208
3209 const RegisterBank &DstRB = *RBI.getRegBank(Reg: DstReg, MRI, TRI);
3210 const RegisterBank &SrcRB = *RBI.getRegBank(Reg: SrcReg, MRI, TRI);
3211
3212 if (DstRB.getID() != SrcRB.getID()) {
3213 LLVM_DEBUG(
3214 dbgs() << "G_TRUNC/G_PTRTOINT input/output on different banks\n");
3215 return false;
3216 }
3217
3218 if (DstRB.getID() == AArch64::GPRRegBankID) {
3219 const TargetRegisterClass *DstRC = getRegClassForTypeOnBank(Ty: DstTy, RB: DstRB);
3220 if (!DstRC)
3221 return false;
3222
3223 const TargetRegisterClass *SrcRC = getRegClassForTypeOnBank(Ty: SrcTy, RB: SrcRB);
3224 if (!SrcRC)
3225 return false;
3226
3227 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI) ||
3228 !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI)) {
3229 LLVM_DEBUG(dbgs() << "Failed to constrain G_TRUNC/G_PTRTOINT\n");
3230 return false;
3231 }
3232
3233 if (DstRC == SrcRC) {
3234 // Nothing to be done
3235 } else if (Opcode == TargetOpcode::G_TRUNC && DstTy == LLT::scalar(SizeInBits: 32) &&
3236 SrcTy == LLT::scalar(SizeInBits: 64)) {
3237 llvm_unreachable("TableGen can import this case");
3238 return false;
3239 } else if (DstRC == &AArch64::GPR32RegClass &&
3240 SrcRC == &AArch64::GPR64RegClass) {
3241 I.getOperand(i: 1).setSubReg(AArch64::sub_32);
3242 } else {
3243 LLVM_DEBUG(
3244 dbgs() << "Unhandled mismatched classes in G_TRUNC/G_PTRTOINT\n");
3245 return false;
3246 }
3247
3248 I.setDesc(TII.get(Opcode: TargetOpcode::COPY));
3249 return true;
3250 } else if (DstRB.getID() == AArch64::FPRRegBankID) {
3251 if (DstTy == LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 16) &&
3252 SrcTy == LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 32)) {
3253 I.setDesc(TII.get(Opcode: AArch64::XTNv4i16));
3254 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
3255 return true;
3256 }
3257
3258 if (!SrcTy.isVector() && SrcTy.getSizeInBits() == 128) {
3259 MachineInstr *Extract = emitExtractVectorElt(
3260 DstReg, DstRB, ScalarTy: LLT::scalar(SizeInBits: DstTy.getSizeInBits()), VecReg: SrcReg, LaneIdx: 0, MIRBuilder&: MIB);
3261 if (!Extract)
3262 return false;
3263 I.eraseFromParent();
3264 return true;
3265 }
3266
3267 // We might have a vector G_PTRTOINT, in which case just emit a COPY.
3268 if (Opcode == TargetOpcode::G_PTRTOINT) {
3269 assert(DstTy.isVector() && "Expected an FPR ptrtoint to be a vector");
3270 I.setDesc(TII.get(Opcode: TargetOpcode::COPY));
3271 return selectCopy(I, TII, MRI, TRI, RBI);
3272 }
3273 }
3274
3275 return false;
3276 }
3277
3278 case TargetOpcode::G_ANYEXT: {
3279 if (selectUSMovFromExtend(I, MRI))
3280 return true;
3281
3282 const Register DstReg = I.getOperand(i: 0).getReg();
3283 const Register SrcReg = I.getOperand(i: 1).getReg();
3284
3285 const RegisterBank &RBDst = *RBI.getRegBank(Reg: DstReg, MRI, TRI);
3286 if (RBDst.getID() != AArch64::GPRRegBankID) {
3287 LLVM_DEBUG(dbgs() << "G_ANYEXT on bank: " << RBDst
3288 << ", expected: GPR\n");
3289 return false;
3290 }
3291
3292 const RegisterBank &RBSrc = *RBI.getRegBank(Reg: SrcReg, MRI, TRI);
3293 if (RBSrc.getID() != AArch64::GPRRegBankID) {
3294 LLVM_DEBUG(dbgs() << "G_ANYEXT on bank: " << RBSrc
3295 << ", expected: GPR\n");
3296 return false;
3297 }
3298
3299 const unsigned DstSize = MRI.getType(Reg: DstReg).getSizeInBits();
3300
3301 if (DstSize == 0) {
3302 LLVM_DEBUG(dbgs() << "G_ANYEXT operand has no size, not a gvreg?\n");
3303 return false;
3304 }
3305
3306 if (DstSize != 64 && DstSize > 32) {
3307 LLVM_DEBUG(dbgs() << "G_ANYEXT to size: " << DstSize
3308 << ", expected: 32 or 64\n");
3309 return false;
3310 }
3311 // At this point G_ANYEXT is just like a plain COPY, but we need
3312 // to explicitly form the 64-bit value if any.
3313 if (DstSize > 32) {
3314 Register ExtSrc = MRI.createVirtualRegister(RegClass: &AArch64::GPR64allRegClass);
3315 BuildMI(BB&: MBB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: AArch64::SUBREG_TO_REG))
3316 .addDef(RegNo: ExtSrc)
3317 .addUse(RegNo: SrcReg)
3318 .addImm(Val: AArch64::sub_32);
3319 I.getOperand(i: 1).setReg(ExtSrc);
3320 }
3321 return selectCopy(I, TII, MRI, TRI, RBI);
3322 }
3323
3324 case TargetOpcode::G_ZEXT:
3325 case TargetOpcode::G_SEXT_INREG:
3326 case TargetOpcode::G_SEXT: {
3327 if (selectUSMovFromExtend(I, MRI))
3328 return true;
3329
3330 unsigned Opcode = I.getOpcode();
3331 const bool IsSigned = Opcode != TargetOpcode::G_ZEXT;
3332 const Register DefReg = I.getOperand(i: 0).getReg();
3333 Register SrcReg = I.getOperand(i: 1).getReg();
3334 const LLT DstTy = MRI.getType(Reg: DefReg);
3335 const LLT SrcTy = MRI.getType(Reg: SrcReg);
3336 unsigned DstSize = DstTy.getSizeInBits();
3337 unsigned SrcSize = SrcTy.getSizeInBits();
3338
3339 // SEXT_INREG has the same src reg size as dst, the size of the value to be
3340 // extended is encoded in the imm.
3341 if (Opcode == TargetOpcode::G_SEXT_INREG)
3342 SrcSize = I.getOperand(i: 2).getImm();
3343
3344 if (DstTy.isVector())
3345 return false; // Should be handled by imported patterns.
3346
3347 assert((*RBI.getRegBank(DefReg, MRI, TRI)).getID() ==
3348 AArch64::GPRRegBankID &&
3349 "Unexpected ext regbank");
3350
3351 MachineInstr *ExtI;
3352
3353 // First check if we're extending the result of a load which has a dest type
3354 // smaller than 32 bits, then this zext is redundant. GPR32 is the smallest
3355 // GPR register on AArch64 and all loads which are smaller automatically
3356 // zero-extend the upper bits. E.g.
3357 // %v(s8) = G_LOAD %p, :: (load 1)
3358 // %v2(s32) = G_ZEXT %v(s8)
3359 if (!IsSigned) {
3360 auto *LoadMI = getOpcodeDef(Opcode: TargetOpcode::G_LOAD, Reg: SrcReg, MRI);
3361 bool IsGPR =
3362 RBI.getRegBank(Reg: SrcReg, MRI, TRI)->getID() == AArch64::GPRRegBankID;
3363 if (LoadMI && IsGPR) {
3364 const MachineMemOperand *MemOp = *LoadMI->memoperands_begin();
3365 unsigned BytesLoaded = MemOp->getSize().getValue();
3366 if (BytesLoaded < 4 && SrcTy.getSizeInBytes() == BytesLoaded)
3367 return selectCopy(I, TII, MRI, TRI, RBI);
3368 }
3369
3370 // For the 32-bit -> 64-bit case, we can emit a mov (ORRWrs)
3371 // + SUBREG_TO_REG.
3372 if (IsGPR && SrcSize == 32 && DstSize == 64) {
3373 Register SubregToRegSrc =
3374 MRI.createVirtualRegister(RegClass: &AArch64::GPR32RegClass);
3375 const Register ZReg = AArch64::WZR;
3376 MIB.buildInstr(Opc: AArch64::ORRWrs, DstOps: {SubregToRegSrc}, SrcOps: {ZReg, SrcReg})
3377 .addImm(Val: 0);
3378
3379 MIB.buildInstr(Opc: AArch64::SUBREG_TO_REG, DstOps: {DefReg}, SrcOps: {})
3380 .addUse(RegNo: SubregToRegSrc)
3381 .addImm(Val: AArch64::sub_32);
3382
3383 if (!RBI.constrainGenericRegister(Reg: DefReg, RC: AArch64::GPR64RegClass,
3384 MRI)) {
3385 LLVM_DEBUG(dbgs() << "Failed to constrain G_ZEXT destination\n");
3386 return false;
3387 }
3388
3389 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: AArch64::GPR32RegClass,
3390 MRI)) {
3391 LLVM_DEBUG(dbgs() << "Failed to constrain G_ZEXT source\n");
3392 return false;
3393 }
3394
3395 I.eraseFromParent();
3396 return true;
3397 }
3398 }
3399
3400 if (DstSize == 64) {
3401 if (Opcode != TargetOpcode::G_SEXT_INREG) {
3402 // FIXME: Can we avoid manually doing this?
3403 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: AArch64::GPR32RegClass,
3404 MRI)) {
3405 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(Opcode)
3406 << " operand\n");
3407 return false;
3408 }
3409 SrcReg = MIB.buildInstr(Opc: AArch64::SUBREG_TO_REG,
3410 DstOps: {&AArch64::GPR64RegClass}, SrcOps: {})
3411 .addUse(RegNo: SrcReg)
3412 .addImm(Val: AArch64::sub_32)
3413 .getReg(Idx: 0);
3414 }
3415
3416 ExtI = MIB.buildInstr(Opc: IsSigned ? AArch64::SBFMXri : AArch64::UBFMXri,
3417 DstOps: {DefReg}, SrcOps: {SrcReg})
3418 .addImm(Val: 0)
3419 .addImm(Val: SrcSize - 1);
3420 } else if (DstSize <= 32) {
3421 ExtI = MIB.buildInstr(Opc: IsSigned ? AArch64::SBFMWri : AArch64::UBFMWri,
3422 DstOps: {DefReg}, SrcOps: {SrcReg})
3423 .addImm(Val: 0)
3424 .addImm(Val: SrcSize - 1);
3425 } else {
3426 return false;
3427 }
3428
3429 constrainSelectedInstRegOperands(I&: *ExtI, TII, TRI, RBI);
3430 I.eraseFromParent();
3431 return true;
3432 }
3433
3434 case TargetOpcode::G_FREEZE:
3435 return selectCopy(I, TII, MRI, TRI, RBI);
3436
3437 case TargetOpcode::G_INTTOPTR:
3438 // The importer is currently unable to import pointer types since they
3439 // didn't exist in SelectionDAG.
3440 return selectCopy(I, TII, MRI, TRI, RBI);
3441
3442 case TargetOpcode::G_BITCAST:
3443 // Imported SelectionDAG rules can handle every bitcast except those that
3444 // bitcast from a type to the same type. Ideally, these shouldn't occur
3445 // but we might not run an optimizer that deletes them. The other exception
3446 // is bitcasts involving pointer types, as SelectionDAG has no knowledge
3447 // of them.
3448 return selectCopy(I, TII, MRI, TRI, RBI);
3449
3450 case TargetOpcode::G_SELECT: {
3451 auto &Sel = cast<GSelect>(Val&: I);
3452 const Register CondReg = Sel.getCondReg();
3453 const Register TReg = Sel.getTrueReg();
3454 const Register FReg = Sel.getFalseReg();
3455
3456 if (tryOptSelect(Sel))
3457 return true;
3458
3459 // Make sure to use an unused vreg instead of wzr, so that the peephole
3460 // optimizations will be able to optimize these.
3461 Register DeadVReg = MRI.createVirtualRegister(RegClass: &AArch64::GPR32RegClass);
3462 auto TstMI = MIB.buildInstr(Opc: AArch64::ANDSWri, DstOps: {DeadVReg}, SrcOps: {CondReg})
3463 .addImm(Val: AArch64_AM::encodeLogicalImmediate(imm: 1, regSize: 32));
3464 constrainSelectedInstRegOperands(I&: *TstMI, TII, TRI, RBI);
3465 if (!emitSelect(Dst: Sel.getReg(Idx: 0), True: TReg, False: FReg, CC: AArch64CC::NE, MIB))
3466 return false;
3467 Sel.eraseFromParent();
3468 return true;
3469 }
3470 case TargetOpcode::G_ICMP: {
3471 if (Ty.isVector())
3472 return false;
3473
3474 if (Ty != LLT::scalar(SizeInBits: 32)) {
3475 LLVM_DEBUG(dbgs() << "G_ICMP result has type: " << Ty
3476 << ", expected: " << LLT::scalar(32) << '\n');
3477 return false;
3478 }
3479
3480 auto &PredOp = I.getOperand(i: 1);
3481 emitIntegerCompare(LHS&: I.getOperand(i: 2), RHS&: I.getOperand(i: 3), Predicate&: PredOp, MIRBuilder&: MIB);
3482 auto Pred = static_cast<CmpInst::Predicate>(PredOp.getPredicate());
3483 const AArch64CC::CondCode InvCC = changeICMPPredToAArch64CC(
3484 P: CmpInst::getInversePredicate(pred: Pred), RHS: I.getOperand(i: 3).getReg(), MRI: &MRI);
3485 emitCSINC(/*Dst=*/I.getOperand(i: 0).getReg(), /*Src1=*/AArch64::WZR,
3486 /*Src2=*/AArch64::WZR, Pred: InvCC, MIRBuilder&: MIB);
3487 I.eraseFromParent();
3488 return true;
3489 }
3490
3491 case TargetOpcode::G_FCMP: {
3492 CmpInst::Predicate Pred =
3493 static_cast<CmpInst::Predicate>(I.getOperand(i: 1).getPredicate());
3494 if (!emitFPCompare(LHS: I.getOperand(i: 2).getReg(), RHS: I.getOperand(i: 3).getReg(), MIRBuilder&: MIB,
3495 Pred) ||
3496 !emitCSetForFCmp(Dst: I.getOperand(i: 0).getReg(), Pred, MIRBuilder&: MIB))
3497 return false;
3498 I.eraseFromParent();
3499 return true;
3500 }
3501 case TargetOpcode::G_VASTART:
3502 return STI.isTargetDarwin() ? selectVaStartDarwin(I, MF, MRI)
3503 : selectVaStartAAPCS(I, MF, MRI);
3504 case TargetOpcode::G_INTRINSIC:
3505 return selectIntrinsic(I, MRI);
3506 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
3507 return selectIntrinsicWithSideEffects(I, MRI);
3508 case TargetOpcode::G_IMPLICIT_DEF: {
3509 I.setDesc(TII.get(Opcode: TargetOpcode::IMPLICIT_DEF));
3510 const LLT DstTy = MRI.getType(Reg: I.getOperand(i: 0).getReg());
3511 const Register DstReg = I.getOperand(i: 0).getReg();
3512 const RegisterBank &DstRB = *RBI.getRegBank(Reg: DstReg, MRI, TRI);
3513 const TargetRegisterClass *DstRC = getRegClassForTypeOnBank(Ty: DstTy, RB: DstRB);
3514 RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI);
3515 return true;
3516 }
3517 case TargetOpcode::G_BLOCK_ADDR: {
3518 Function *BAFn = I.getOperand(i: 1).getBlockAddress()->getFunction();
3519 if (std::optional<uint16_t> BADisc =
3520 STI.getPtrAuthBlockAddressDiscriminatorIfEnabled(ParentFn: *BAFn)) {
3521 MIB.buildInstr(Opcode: AArch64::MOVaddrPAC)
3522 .addBlockAddress(BA: I.getOperand(i: 1).getBlockAddress())
3523 .addImm(Val: AArch64PACKey::IA)
3524 .addReg(/*AddrDisc=*/RegNo: AArch64::XZR)
3525 .addImm(Val: *BADisc)
3526 .constrainAllUses(TII, TRI, RBI);
3527 MIB.buildCopy(Res: I.getOperand(i: 0).getReg(), Op: Register(AArch64::X16));
3528 RBI.constrainGenericRegister(Reg: I.getOperand(i: 0).getReg(),
3529 RC: AArch64::GPR64RegClass, MRI);
3530 I.eraseFromParent();
3531 return true;
3532 }
3533 if (TM.getCodeModel() == CodeModel::Large && !TM.isPositionIndependent()) {
3534 materializeLargeCMVal(I, V: I.getOperand(i: 1).getBlockAddress(), OpFlags: 0);
3535 I.eraseFromParent();
3536 return true;
3537 } else {
3538 I.setDesc(TII.get(Opcode: AArch64::MOVaddrBA));
3539 auto MovMI = BuildMI(BB&: MBB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: AArch64::MOVaddrBA),
3540 DestReg: I.getOperand(i: 0).getReg())
3541 .addBlockAddress(BA: I.getOperand(i: 1).getBlockAddress(),
3542 /* Offset */ 0, TargetFlags: AArch64II::MO_PAGE)
3543 .addBlockAddress(
3544 BA: I.getOperand(i: 1).getBlockAddress(), /* Offset */ 0,
3545 TargetFlags: AArch64II::MO_NC | AArch64II::MO_PAGEOFF);
3546 I.eraseFromParent();
3547 constrainSelectedInstRegOperands(I&: *MovMI, TII, TRI, RBI);
3548 return true;
3549 }
3550 }
3551 case TargetOpcode::G_BUILD_VECTOR:
3552 return selectBuildVector(I, MRI);
3553 case TargetOpcode::G_MERGE_VALUES:
3554 return selectMergeValues(I, MRI);
3555 case TargetOpcode::G_UNMERGE_VALUES:
3556 return selectUnmergeValues(I, MRI);
3557 case TargetOpcode::G_SHUFFLE_VECTOR:
3558 return selectShuffleVector(I, MRI);
3559 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
3560 return selectExtractElt(I, MRI);
3561 case TargetOpcode::G_CONCAT_VECTORS:
3562 return selectConcatVectors(I, MRI);
3563 case TargetOpcode::G_JUMP_TABLE:
3564 return selectJumpTable(I, MRI);
3565 case TargetOpcode::G_MEMCPY:
3566 case TargetOpcode::G_MEMCPY_INLINE:
3567 case TargetOpcode::G_MEMMOVE:
3568 case TargetOpcode::G_MEMSET:
3569 case TargetOpcode::G_MEMSET_INLINE:
3570 assert(STI.hasMOPS() && "Shouldn't get here without +mops feature");
3571 return selectMOPS(I, MRI);
3572 }
3573
3574 return false;
3575}
3576
3577bool AArch64InstructionSelector::selectAndRestoreState(MachineInstr &I) {
3578 MachineIRBuilderState OldMIBState = MIB.getState();
3579 bool Success = select(I);
3580 MIB.setState(OldMIBState);
3581 return Success;
3582}
3583
3584bool AArch64InstructionSelector::selectMOPS(MachineInstr &GI,
3585 MachineRegisterInfo &MRI) {
3586 unsigned Mopcode;
3587 switch (GI.getOpcode()) {
3588 case TargetOpcode::G_MEMCPY:
3589 case TargetOpcode::G_MEMCPY_INLINE:
3590 Mopcode = AArch64::MOPSMemoryCopyPseudo;
3591 break;
3592 case TargetOpcode::G_MEMMOVE:
3593 Mopcode = AArch64::MOPSMemoryMovePseudo;
3594 break;
3595 case TargetOpcode::G_MEMSET:
3596 case TargetOpcode::G_MEMSET_INLINE:
3597 // For tagged memset see llvm.aarch64.mops.memset.tag
3598 Mopcode = AArch64::MOPSMemorySetPseudo;
3599 break;
3600 }
3601
3602 auto &DstPtr = GI.getOperand(i: 0);
3603 auto &SrcOrVal = GI.getOperand(i: 1);
3604 auto &Size = GI.getOperand(i: 2);
3605
3606 // Create copies of the registers that can be clobbered.
3607 const Register DstPtrCopy = MRI.cloneVirtualRegister(VReg: DstPtr.getReg());
3608 const Register SrcValCopy = MRI.cloneVirtualRegister(VReg: SrcOrVal.getReg());
3609 const Register SizeCopy = MRI.cloneVirtualRegister(VReg: Size.getReg());
3610
3611 const bool IsSet = Mopcode == AArch64::MOPSMemorySetPseudo;
3612 const auto &SrcValRegClass =
3613 IsSet ? AArch64::GPR64RegClass : AArch64::GPR64commonRegClass;
3614
3615 // Constrain to specific registers
3616 RBI.constrainGenericRegister(Reg: DstPtrCopy, RC: AArch64::GPR64commonRegClass, MRI);
3617 RBI.constrainGenericRegister(Reg: SrcValCopy, RC: SrcValRegClass, MRI);
3618 RBI.constrainGenericRegister(Reg: SizeCopy, RC: AArch64::GPR64RegClass, MRI);
3619
3620 MIB.buildCopy(Res: DstPtrCopy, Op: DstPtr);
3621 MIB.buildCopy(Res: SrcValCopy, Op: SrcOrVal);
3622 MIB.buildCopy(Res: SizeCopy, Op: Size);
3623
3624 // New instruction uses the copied registers because it must update them.
3625 // The defs are not used since they don't exist in G_MEM*. They are still
3626 // tied.
3627 // Note: order of operands is different from G_MEMSET, G_MEMCPY, G_MEMMOVE
3628 Register DefDstPtr = MRI.createVirtualRegister(RegClass: &AArch64::GPR64commonRegClass);
3629 Register DefSize = MRI.createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
3630 if (IsSet) {
3631 MIB.buildInstr(Opc: Mopcode, DstOps: {DefDstPtr, DefSize},
3632 SrcOps: {DstPtrCopy, SizeCopy, SrcValCopy})
3633 .setOperandDead(5); // implicit-def $nzcv
3634 } else {
3635 Register DefSrcPtr = MRI.createVirtualRegister(RegClass: &SrcValRegClass);
3636 MIB.buildInstr(Opc: Mopcode, DstOps: {DefDstPtr, DefSrcPtr, DefSize},
3637 SrcOps: {DstPtrCopy, SrcValCopy, SizeCopy})
3638 .setOperandDead(6); // implicit-def $nzcv
3639 }
3640
3641 GI.eraseFromParent();
3642 return true;
3643}
3644
3645bool AArch64InstructionSelector::selectBrJT(MachineInstr &I,
3646 MachineRegisterInfo &MRI) {
3647 assert(I.getOpcode() == TargetOpcode::G_BRJT && "Expected G_BRJT");
3648 Register JTAddr = I.getOperand(i: 0).getReg();
3649 unsigned JTI = I.getOperand(i: 1).getIndex();
3650 Register Index = I.getOperand(i: 2).getReg();
3651
3652 MF->getInfo<AArch64FunctionInfo>()->setJumpTableEntryInfo(Idx: JTI, Size: 4, PCRelSym: nullptr);
3653
3654 // With aarch64-jump-table-hardening, we only expand the jump table dispatch
3655 // sequence later, to guarantee the integrity of the intermediate values.
3656 if (MF->getFunction().hasFnAttribute(Kind: "aarch64-jump-table-hardening")) {
3657 CodeModel::Model CM = TM.getCodeModel();
3658 if (STI.isTargetMachO()) {
3659 if (CM != CodeModel::Small && CM != CodeModel::Large)
3660 report_fatal_error(reason: "Unsupported code-model for hardened jump-table");
3661 } else {
3662 // Note that COFF support would likely also need JUMP_TABLE_DEBUG_INFO.
3663 assert(STI.isTargetELF() &&
3664 "jump table hardening only supported on MachO/ELF");
3665 if (CM != CodeModel::Small)
3666 report_fatal_error(reason: "Unsupported code-model for hardened jump-table");
3667 }
3668
3669 MIB.buildCopy(Res: {AArch64::X16}, Op: I.getOperand(i: 2).getReg());
3670 MIB.buildInstr(Opcode: AArch64::BR_JumpTable)
3671 .addJumpTableIndex(Idx: I.getOperand(i: 1).getIndex());
3672 I.eraseFromParent();
3673 return true;
3674 }
3675
3676 Register TargetReg = MRI.createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
3677 Register ScratchReg = MRI.createVirtualRegister(RegClass: &AArch64::GPR64spRegClass);
3678
3679 auto JumpTableInst = MIB.buildInstr(Opc: AArch64::JumpTableDest32,
3680 DstOps: {TargetReg, ScratchReg}, SrcOps: {JTAddr, Index})
3681 .addJumpTableIndex(Idx: JTI);
3682 // Save the jump table info.
3683 MIB.buildInstr(Opc: TargetOpcode::JUMP_TABLE_DEBUG_INFO, DstOps: {},
3684 SrcOps: {static_cast<int64_t>(JTI)});
3685 // Build the indirect branch.
3686 MIB.buildInstr(Opc: AArch64::BR, DstOps: {}, SrcOps: {TargetReg});
3687 I.eraseFromParent();
3688 constrainSelectedInstRegOperands(I&: *JumpTableInst, TII, TRI, RBI);
3689 return true;
3690}
3691
3692bool AArch64InstructionSelector::selectJumpTable(MachineInstr &I,
3693 MachineRegisterInfo &MRI) {
3694 assert(I.getOpcode() == TargetOpcode::G_JUMP_TABLE && "Expected jump table");
3695 assert(I.getOperand(1).isJTI() && "Jump table op should have a JTI!");
3696
3697 Register DstReg = I.getOperand(i: 0).getReg();
3698 unsigned JTI = I.getOperand(i: 1).getIndex();
3699 // We generate a MOVaddrJT which will get expanded to an ADRP + ADD later.
3700 auto MovMI =
3701 MIB.buildInstr(Opc: AArch64::MOVaddrJT, DstOps: {DstReg}, SrcOps: {})
3702 .addJumpTableIndex(Idx: JTI, TargetFlags: AArch64II::MO_PAGE)
3703 .addJumpTableIndex(Idx: JTI, TargetFlags: AArch64II::MO_NC | AArch64II::MO_PAGEOFF);
3704 I.eraseFromParent();
3705 constrainSelectedInstRegOperands(I&: *MovMI, TII, TRI, RBI);
3706 return true;
3707}
3708
3709bool AArch64InstructionSelector::selectTLSLocalExecELF(
3710 const GlobalValue *GV, MachineInstr &I, MachineRegisterInfo &MRI) {
3711 auto ConstrainRegOps = [&](MachineInstrBuilder MIB) {
3712 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
3713 };
3714 Register ThreadBase = MRI.createGenericVirtualRegister(Ty: LLT::pointer(AddressSpace: 0, SizeInBits: 64));
3715 ConstrainRegOps(MIB.buildInstr(Opc: AArch64::MOVbaseTLS, DstOps: {ThreadBase}, SrcOps: {}));
3716
3717 switch (MF->getTarget().Options.TLSSize) {
3718 default:
3719 llvm_unreachable("Unexpected TLS size");
3720 case 12: {
3721 // add x0, x0, :tprel_lo12:a
3722 ConstrainRegOps(
3723 MIB.buildInstr(Opc: AArch64::ADDXri, DstOps: {I.getOperand(i: 0).getReg()},
3724 SrcOps: {ThreadBase})
3725 .addGlobalAddress(GV, Offset: 0, TargetFlags: AArch64II::MO_TLS | AArch64II::MO_PAGEOFF)
3726 .addImm(Val: 0));
3727 break;
3728 }
3729 case 24: {
3730 // add x0, x0, :tprel_hi12:a
3731 // add x0, x0, :tprel_lo12_nc:a
3732 Register Addr = MRI.createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
3733 ConstrainRegOps(
3734 MIB.buildInstr(Opc: AArch64::ADDXri, DstOps: {Addr}, SrcOps: {ThreadBase})
3735 .addGlobalAddress(GV, Offset: 0, TargetFlags: AArch64II::MO_TLS | AArch64II::MO_HI12)
3736 .addImm(Val: 0));
3737 ConstrainRegOps(
3738 MIB.buildInstr(Opc: AArch64::ADDXri, DstOps: {I.getOperand(i: 0).getReg()}, SrcOps: {Addr})
3739 .addGlobalAddress(GV, Offset: 0,
3740 TargetFlags: AArch64II::MO_TLS | AArch64II::MO_PAGEOFF |
3741 AArch64II::MO_NC)
3742 .addImm(Val: 0));
3743 break;
3744 }
3745 case 32: {
3746 // movz x0, #:tprel_g1:a
3747 // movk x0, #:tprel_g0_nc:a
3748 // add x0, x1, x0
3749 Register Addr = MRI.createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
3750 ConstrainRegOps(
3751 MIB.buildInstr(Opc: AArch64::MOVZXi, DstOps: {Addr}, SrcOps: {})
3752 .addGlobalAddress(GV, Offset: 0, TargetFlags: AArch64II::MO_TLS | AArch64II::MO_G1)
3753 .addImm(Val: 16));
3754 Register Addr2 = MRI.createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
3755 ConstrainRegOps(MIB.buildInstr(Opc: AArch64::MOVKXi, DstOps: {Addr2}, SrcOps: {Addr})
3756 .addGlobalAddress(GV, Offset: 0,
3757 TargetFlags: AArch64II::MO_TLS | AArch64II::MO_G0 |
3758 AArch64II::MO_NC)
3759 .addImm(Val: 0));
3760 ConstrainRegOps(MIB.buildInstr(Opc: AArch64::ADDXrr, DstOps: {I.getOperand(i: 0).getReg()},
3761 SrcOps: {ThreadBase, Addr2}));
3762 break;
3763 }
3764 case 48: {
3765 // movz x0, #:tprel_g2:a
3766 // movk x0, #:tprel_g1_nc:a
3767 // movk x0, #:tprel_g0_nc:a
3768 // add x0, x1, x0
3769 Register Addr = MRI.createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
3770 ConstrainRegOps(
3771 MIB.buildInstr(Opc: AArch64::MOVZXi, DstOps: {Addr}, SrcOps: {})
3772 .addGlobalAddress(GV, Offset: 0, TargetFlags: AArch64II::MO_TLS | AArch64II::MO_G2)
3773 .addImm(Val: 32));
3774 Register Addr2 = MRI.createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
3775 ConstrainRegOps(MIB.buildInstr(Opc: AArch64::MOVKXi, DstOps: {Addr2}, SrcOps: {Addr})
3776 .addGlobalAddress(GV, Offset: 0,
3777 TargetFlags: AArch64II::MO_TLS | AArch64II::MO_G1 |
3778 AArch64II::MO_NC)
3779 .addImm(Val: 16));
3780 Register Addr3 = MRI.createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
3781 ConstrainRegOps(MIB.buildInstr(Opc: AArch64::MOVKXi, DstOps: {Addr3}, SrcOps: {Addr2})
3782 .addGlobalAddress(GV, Offset: 0,
3783 TargetFlags: AArch64II::MO_TLS | AArch64II::MO_G0 |
3784 AArch64II::MO_NC)
3785 .addImm(Val: 0));
3786 ConstrainRegOps(MIB.buildInstr(Opc: AArch64::ADDXrr, DstOps: {I.getOperand(i: 0).getReg()},
3787 SrcOps: {ThreadBase, Addr3}));
3788 break;
3789 }
3790 }
3791 I.eraseFromParent();
3792 return true;
3793}
3794
3795// TLS lowering below mirrors the corresponding DAGISel implementation.
3796// See LowerELFTLSModel() for details.
3797bool AArch64InstructionSelector::selectTLSGlobalValueELF(
3798 MachineInstr &I, MachineRegisterInfo &MRI) {
3799 const GlobalValue *GV = I.getOperand(i: 1).getGlobal();
3800 auto *FuncInfo = MF->getInfo<AArch64FunctionInfo>();
3801 TLSModel::Model Model =
3802 AArch64::getELFTLSModel(GV, TM, HasELFSignedGOT: FuncInfo->hasELFSignedGOT());
3803
3804 Register TPOff = MRI.createVirtualRegister(RegClass: &AArch64::GPR64commonRegClass);
3805 switch (Model) {
3806 case TLSModel::LocalExec:
3807 return selectTLSLocalExecELF(GV, I, MRI);
3808 case TLSModel::InitialExec:
3809 MIB.buildInstr(Opc: AArch64::LOADgot, DstOps: {TPOff}, SrcOps: {})
3810 .addGlobalAddress(GV, Offset: 0, TargetFlags: AArch64II::MO_TLS);
3811 break;
3812 case TLSModel::LocalDynamic:
3813 case TLSModel::GeneralDynamic: {
3814#ifndef NDEBUG
3815 SMEAttrs Attrs = MF->getInfo<AArch64FunctionInfo>()->getSMEFnAttrs();
3816 assert(!Attrs.hasZAState() && !Attrs.hasStreamingInterfaceOrBody() &&
3817 !Attrs.hasStreamingCompatibleInterface() &&
3818 "unsupported SME features reached GlobalISel TLS lowering");
3819#endif
3820 unsigned Opcode = FuncInfo->hasELFSignedGOT()
3821 ? AArch64::TLSDESC_AUTH_CALLSEQ
3822 : AArch64::TLSDESC_CALLSEQ;
3823
3824 if (TLSModel::GeneralDynamic == Model) {
3825 MIB.buildInstr(Opc: Opcode, DstOps: {}, SrcOps: {}).addGlobalAddress(GV, Offset: 0, TargetFlags: AArch64II::MO_TLS);
3826 MIB.buildCopy(Res: TPOff, Op: Register(AArch64::X0));
3827 break;
3828 }
3829 assert(TLSModel::LocalDynamic == Model);
3830 // These accesses will need deduplicating if there's more than one.
3831 FuncInfo->incNumLocalDynamicTLSAccesses();
3832
3833 MIB.buildInstr(Opc: Opcode, DstOps: {}, SrcOps: {})
3834 .addExternalSymbol(FnName: "_TLS_MODULE_BASE_", TargetFlags: AArch64II::MO_TLS);
3835 auto Copy = MIB.buildCopy(Res: LLT::scalar(SizeInBits: 64), Op: Register(AArch64::X0));
3836 auto Add1 =
3837 MIB.buildInstr(Opc: AArch64::ADDXri, DstOps: {LLT::scalar(SizeInBits: 64)}, SrcOps: {Copy.getReg(Idx: 0)})
3838 .addGlobalAddress(GV, Offset: 0, TargetFlags: AArch64II::MO_TLS | AArch64II::MO_HI12)
3839 .addImm(Val: 0);
3840 auto Add2 =
3841 MIB.buildInstr(Opc: AArch64::ADDXri, DstOps: {TPOff}, SrcOps: {Add1.getReg(Idx: 0)})
3842 .addGlobalAddress(GV, Offset: 0,
3843 TargetFlags: AArch64II::MO_TLS | AArch64II::MO_PAGEOFF |
3844 AArch64II::MO_NC)
3845 .addImm(Val: 0);
3846 constrainSelectedInstRegOperands(I&: *Add1, TII, TRI, RBI);
3847 constrainSelectedInstRegOperands(I&: *Add2, TII, TRI, RBI);
3848 }
3849 }
3850 Register ThreadBase = MRI.createGenericVirtualRegister(Ty: LLT::pointer(AddressSpace: 0, SizeInBits: 64));
3851 MIB.buildInstr(Opc: AArch64::MOVbaseTLS, DstOps: {ThreadBase}, SrcOps: {});
3852 auto Add = MIB.buildInstr(Opc: AArch64::ADDXrr, DstOps: {I.getOperand(i: 0).getReg()},
3853 SrcOps: {ThreadBase, TPOff});
3854 constrainSelectedInstRegOperands(I&: *Add, TII, TRI, RBI);
3855
3856 I.eraseFromParent();
3857 return true;
3858}
3859
3860bool AArch64InstructionSelector::selectTLSGlobalValueMachO(
3861 MachineInstr &I, MachineRegisterInfo &MRI) {
3862 const auto &GlobalOp = I.getOperand(i: 1);
3863 assert(GlobalOp.getOffset() == 0 &&
3864 "Shouldn't have an offset on TLS globals!");
3865
3866 const GlobalValue &GV = *GlobalOp.getGlobal();
3867 MF->getFrameInfo().setAdjustsStack(true);
3868 auto LoadGOT =
3869 MIB.buildInstr(Opc: AArch64::LOADgot, DstOps: {&AArch64::GPR64commonRegClass}, SrcOps: {})
3870 .addGlobalAddress(GV: &GV, Offset: 0, TargetFlags: AArch64II::MO_TLS);
3871
3872 auto Load = MIB.buildInstr(Opc: AArch64::LDRXui, DstOps: {&AArch64::GPR64commonRegClass},
3873 SrcOps: {LoadGOT.getReg(Idx: 0)})
3874 .addImm(Val: 0);
3875
3876 MIB.buildCopy(Res: Register(AArch64::X0), Op: LoadGOT.getReg(Idx: 0));
3877 // TLS calls preserve all registers except those that absolutely must be
3878 // trashed: X0 (it takes an argument), LR (it's a call) and NZCV (let's not be
3879 // silly).
3880 unsigned Opcode = getBLRCallOpcode(MF: *MF);
3881
3882 // With ptrauth-calls, the tlv access thunk pointer is authenticated (IA, 0).
3883 if (MF->getFunction().hasFnAttribute(Kind: "ptrauth-calls")) {
3884 assert(Opcode == AArch64::BLR);
3885 Opcode = AArch64::BLRAAZ;
3886 }
3887
3888 MIB.buildInstr(Opc: Opcode, DstOps: {}, SrcOps: {Load})
3889 .setOperandDead(1) // implicit-def $lr
3890 .addUse(RegNo: AArch64::X0, Flags: RegState::Implicit)
3891 .addDef(RegNo: AArch64::X0, Flags: RegState::Implicit)
3892 .addRegMask(Mask: TRI.getTLSCallPreservedMask());
3893
3894 MIB.buildCopy(Res: I.getOperand(i: 0).getReg(), Op: Register(AArch64::X0));
3895 RBI.constrainGenericRegister(Reg: I.getOperand(i: 0).getReg(), RC: AArch64::GPR64RegClass,
3896 MRI);
3897 I.eraseFromParent();
3898 return true;
3899}
3900
3901bool AArch64InstructionSelector::selectTLSGlobalValue(
3902 MachineInstr &I, MachineRegisterInfo &MRI) {
3903 // We don't support instructions with emulated TLS variables yet.
3904 if (TM.useEmulatedTLS())
3905 return false;
3906
3907 if (STI.isTargetELF())
3908 return selectTLSGlobalValueELF(I, MRI);
3909
3910 if (STI.isTargetMachO())
3911 return selectTLSGlobalValueMachO(I, MRI);
3912
3913 return false;
3914}
3915
3916MachineInstr *AArch64InstructionSelector::emitScalarToVector(
3917 unsigned EltSize, const TargetRegisterClass *DstRC, Register Scalar,
3918 MachineIRBuilder &MIRBuilder) const {
3919 auto Undef = MIRBuilder.buildInstr(Opc: TargetOpcode::IMPLICIT_DEF, DstOps: {DstRC}, SrcOps: {});
3920
3921 auto BuildFn = [&](unsigned SubregIndex) {
3922 auto Ins =
3923 MIRBuilder
3924 .buildInstr(Opc: TargetOpcode::INSERT_SUBREG, DstOps: {DstRC}, SrcOps: {Undef, Scalar})
3925 .addImm(Val: SubregIndex);
3926 constrainSelectedInstRegOperands(I&: *Undef, TII, TRI, RBI);
3927 constrainSelectedInstRegOperands(I&: *Ins, TII, TRI, RBI);
3928 return &*Ins;
3929 };
3930
3931 switch (EltSize) {
3932 case 8:
3933 return BuildFn(AArch64::bsub);
3934 case 16:
3935 return BuildFn(AArch64::hsub);
3936 case 32:
3937 return BuildFn(AArch64::ssub);
3938 case 64:
3939 return BuildFn(AArch64::dsub);
3940 default:
3941 return nullptr;
3942 }
3943}
3944
3945MachineInstr *
3946AArch64InstructionSelector::emitNarrowVector(Register DstReg, Register SrcReg,
3947 MachineIRBuilder &MIB,
3948 MachineRegisterInfo &MRI) const {
3949 LLT DstTy = MRI.getType(Reg: DstReg);
3950 const TargetRegisterClass *RC =
3951 getRegClassForTypeOnBank(Ty: DstTy, RB: *RBI.getRegBank(Reg: SrcReg, MRI, TRI));
3952 if (RC != &AArch64::FPR32RegClass && RC != &AArch64::FPR64RegClass) {
3953 LLVM_DEBUG(dbgs() << "Unsupported register class!\n");
3954 return nullptr;
3955 }
3956 unsigned SubReg = 0;
3957 if (!getSubRegForClass(RC, TRI, SubReg))
3958 return nullptr;
3959 if (SubReg != AArch64::ssub && SubReg != AArch64::dsub) {
3960 LLVM_DEBUG(dbgs() << "Unsupported destination size! ("
3961 << DstTy.getSizeInBits() << "\n");
3962 return nullptr;
3963 }
3964 auto Copy = MIB.buildInstr(Opc: TargetOpcode::COPY, DstOps: {DstReg}, SrcOps: {})
3965 .addReg(RegNo: SrcReg, Flags: {}, SubReg);
3966 RBI.constrainGenericRegister(Reg: DstReg, RC: *RC, MRI);
3967 return Copy;
3968}
3969
3970bool AArch64InstructionSelector::selectMergeValues(
3971 MachineInstr &I, MachineRegisterInfo &MRI) {
3972 assert(I.getOpcode() == TargetOpcode::G_MERGE_VALUES && "unexpected opcode");
3973 const LLT DstTy = MRI.getType(Reg: I.getOperand(i: 0).getReg());
3974 const LLT SrcTy = MRI.getType(Reg: I.getOperand(i: 1).getReg());
3975 assert(!DstTy.isVector() && !SrcTy.isVector() && "invalid merge operation");
3976 const RegisterBank &RB = *RBI.getRegBank(Reg: I.getOperand(i: 1).getReg(), MRI, TRI);
3977
3978 if (I.getNumOperands() != 3)
3979 return false;
3980
3981 // Merging 2 s64s into an s128.
3982 if (DstTy == LLT::scalar(SizeInBits: 128)) {
3983 if (SrcTy.getSizeInBits() != 64)
3984 return false;
3985 Register DstReg = I.getOperand(i: 0).getReg();
3986 Register Src1Reg = I.getOperand(i: 1).getReg();
3987 Register Src2Reg = I.getOperand(i: 2).getReg();
3988 auto Tmp = MIB.buildInstr(Opc: TargetOpcode::IMPLICIT_DEF, DstOps: {DstTy}, SrcOps: {});
3989 MachineInstr *InsMI = emitLaneInsert(DstReg: std::nullopt, SrcReg: Tmp.getReg(Idx: 0), EltReg: Src1Reg,
3990 /* LaneIdx */ 0, RB, MIRBuilder&: MIB);
3991 if (!InsMI)
3992 return false;
3993 MachineInstr *Ins2MI = emitLaneInsert(DstReg, SrcReg: InsMI->getOperand(i: 0).getReg(),
3994 EltReg: Src2Reg, /* LaneIdx */ 1, RB, MIRBuilder&: MIB);
3995 if (!Ins2MI)
3996 return false;
3997 constrainSelectedInstRegOperands(I&: *InsMI, TII, TRI, RBI);
3998 constrainSelectedInstRegOperands(I&: *Ins2MI, TII, TRI, RBI);
3999 I.eraseFromParent();
4000 return true;
4001 }
4002
4003 if (RB.getID() != AArch64::GPRRegBankID)
4004 return false;
4005
4006 if (DstTy.getSizeInBits() != 64 || SrcTy.getSizeInBits() != 32)
4007 return false;
4008
4009 auto *DstRC = &AArch64::GPR64RegClass;
4010 Register SubToRegDef = MRI.createVirtualRegister(RegClass: DstRC);
4011 MachineInstr &SubRegMI = *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
4012 MCID: TII.get(Opcode: TargetOpcode::SUBREG_TO_REG))
4013 .addDef(RegNo: SubToRegDef)
4014 .addUse(RegNo: I.getOperand(i: 1).getReg())
4015 .addImm(Val: AArch64::sub_32);
4016 Register SubToRegDef2 = MRI.createVirtualRegister(RegClass: DstRC);
4017 // Need to anyext the second scalar before we can use bfm
4018 MachineInstr &SubRegMI2 = *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
4019 MCID: TII.get(Opcode: TargetOpcode::SUBREG_TO_REG))
4020 .addDef(RegNo: SubToRegDef2)
4021 .addUse(RegNo: I.getOperand(i: 2).getReg())
4022 .addImm(Val: AArch64::sub_32);
4023 MachineInstr &BFM =
4024 *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: AArch64::BFMXri))
4025 .addDef(RegNo: I.getOperand(i: 0).getReg())
4026 .addUse(RegNo: SubToRegDef)
4027 .addUse(RegNo: SubToRegDef2)
4028 .addImm(Val: 32)
4029 .addImm(Val: 31);
4030 constrainSelectedInstRegOperands(I&: SubRegMI, TII, TRI, RBI);
4031 constrainSelectedInstRegOperands(I&: SubRegMI2, TII, TRI, RBI);
4032 constrainSelectedInstRegOperands(I&: BFM, TII, TRI, RBI);
4033 I.eraseFromParent();
4034 return true;
4035}
4036
4037static bool getLaneCopyOpcode(unsigned &CopyOpc, unsigned &ExtractSubReg,
4038 const unsigned EltSize) {
4039 // Choose a lane copy opcode and subregister based off of the size of the
4040 // vector's elements.
4041 switch (EltSize) {
4042 case 8:
4043 CopyOpc = AArch64::DUPi8;
4044 ExtractSubReg = AArch64::bsub;
4045 break;
4046 case 16:
4047 CopyOpc = AArch64::DUPi16;
4048 ExtractSubReg = AArch64::hsub;
4049 break;
4050 case 32:
4051 CopyOpc = AArch64::DUPi32;
4052 ExtractSubReg = AArch64::ssub;
4053 break;
4054 case 64:
4055 CopyOpc = AArch64::DUPi64;
4056 ExtractSubReg = AArch64::dsub;
4057 break;
4058 default:
4059 // Unknown size, bail out.
4060 LLVM_DEBUG(dbgs() << "Elt size '" << EltSize << "' unsupported.\n");
4061 return false;
4062 }
4063 return true;
4064}
4065
4066MachineInstr *AArch64InstructionSelector::emitExtractVectorElt(
4067 std::optional<Register> DstReg, const RegisterBank &DstRB, LLT ScalarTy,
4068 Register VecReg, unsigned LaneIdx, MachineIRBuilder &MIRBuilder) const {
4069 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
4070 unsigned CopyOpc = 0;
4071 unsigned ExtractSubReg = 0;
4072 if (!getLaneCopyOpcode(CopyOpc, ExtractSubReg, EltSize: ScalarTy.getSizeInBits())) {
4073 LLVM_DEBUG(
4074 dbgs() << "Couldn't determine lane copy opcode for instruction.\n");
4075 return nullptr;
4076 }
4077
4078 const TargetRegisterClass *DstRC =
4079 getRegClassForTypeOnBank(Ty: ScalarTy, RB: DstRB, GetAllRegSet: true);
4080 if (!DstRC) {
4081 LLVM_DEBUG(dbgs() << "Could not determine destination register class.\n");
4082 return nullptr;
4083 }
4084
4085 const RegisterBank &VecRB = *RBI.getRegBank(Reg: VecReg, MRI, TRI);
4086 const LLT &VecTy = MRI.getType(Reg: VecReg);
4087 const TargetRegisterClass *VecRC =
4088 getRegClassForTypeOnBank(Ty: VecTy, RB: VecRB, GetAllRegSet: true);
4089 if (!VecRC) {
4090 LLVM_DEBUG(dbgs() << "Could not determine source register class.\n");
4091 return nullptr;
4092 }
4093
4094 // The register that we're going to copy into.
4095 Register InsertReg = VecReg;
4096 if (!DstReg)
4097 DstReg = MRI.createVirtualRegister(RegClass: DstRC);
4098 // If the lane index is 0, we just use a subregister COPY.
4099 if (LaneIdx == 0) {
4100 auto Copy = MIRBuilder.buildInstr(Opc: TargetOpcode::COPY, DstOps: {*DstReg}, SrcOps: {})
4101 .addReg(RegNo: VecReg, Flags: {}, SubReg: ExtractSubReg);
4102 RBI.constrainGenericRegister(Reg: *DstReg, RC: *DstRC, MRI);
4103 return &*Copy;
4104 }
4105
4106 // Lane copies require 128-bit wide registers. If we're dealing with an
4107 // unpacked vector, then we need to move up to that width. Insert an implicit
4108 // def and a subregister insert to get us there.
4109 if (VecTy.getSizeInBits() != 128) {
4110 MachineInstr *ScalarToVector = emitScalarToVector(
4111 EltSize: VecTy.getSizeInBits(), DstRC: &AArch64::FPR128RegClass, Scalar: VecReg, MIRBuilder);
4112 if (!ScalarToVector)
4113 return nullptr;
4114 InsertReg = ScalarToVector->getOperand(i: 0).getReg();
4115 }
4116
4117 MachineInstr *LaneCopyMI =
4118 MIRBuilder.buildInstr(Opc: CopyOpc, DstOps: {*DstReg}, SrcOps: {InsertReg}).addImm(Val: LaneIdx);
4119 constrainSelectedInstRegOperands(I&: *LaneCopyMI, TII, TRI, RBI);
4120
4121 // Make sure that we actually constrain the initial copy.
4122 RBI.constrainGenericRegister(Reg: *DstReg, RC: *DstRC, MRI);
4123 return LaneCopyMI;
4124}
4125
4126bool AArch64InstructionSelector::selectExtractElt(
4127 MachineInstr &I, MachineRegisterInfo &MRI) {
4128 assert(I.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT &&
4129 "unexpected opcode!");
4130 Register DstReg = I.getOperand(i: 0).getReg();
4131 const LLT NarrowTy = MRI.getType(Reg: DstReg);
4132 const Register SrcReg = I.getOperand(i: 1).getReg();
4133 const LLT WideTy = MRI.getType(Reg: SrcReg);
4134 assert(WideTy.getSizeInBits() >= NarrowTy.getSizeInBits() &&
4135 "source register size too small!");
4136 assert(!NarrowTy.isVector() && "cannot extract vector into vector!");
4137
4138 // Need the lane index to determine the correct copy opcode.
4139 MachineOperand &LaneIdxOp = I.getOperand(i: 2);
4140 assert(LaneIdxOp.isReg() && "Lane index operand was not a register?");
4141
4142 // Find the index to extract from.
4143 auto VRegAndVal = getIConstantVRegValWithLookThrough(VReg: LaneIdxOp.getReg(), MRI);
4144 if (!VRegAndVal)
4145 return false;
4146 unsigned LaneIdx = VRegAndVal->Value.getSExtValue();
4147
4148 const RegisterBank &DstRB = *RBI.getRegBank(Reg: DstReg, MRI, TRI);
4149 if (DstRB.getID() == AArch64::GPRRegBankID) {
4150 unsigned Opcode;
4151 switch (WideTy.getScalarSizeInBits()) {
4152 case 8:
4153 Opcode = AArch64::UMOVvi8;
4154 break;
4155 case 16:
4156 Opcode = AArch64::UMOVvi16;
4157 break;
4158 case 32:
4159 Opcode = AArch64::UMOVvi32;
4160 break;
4161 default:
4162 return false;
4163 }
4164
4165 if (WideTy.getSizeInBits() != 128) {
4166 MachineInstr *ScalarToVector = emitScalarToVector(
4167 EltSize: WideTy.getSizeInBits(), DstRC: &AArch64::FPR128RegClass, Scalar: SrcReg, MIRBuilder&: MIB);
4168 assert(ScalarToVector && "Didn't expect emitScalarToVector to fail!");
4169 I.getOperand(i: 1).setReg(ScalarToVector->getOperand(i: 0).getReg());
4170 }
4171
4172 I.setDesc(TII.get(Opcode));
4173 I.getOperand(i: 2).ChangeToImmediate(ImmVal: LaneIdx);
4174 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
4175 return true;
4176 }
4177
4178 MachineInstr *Extract = emitExtractVectorElt(DstReg, DstRB, ScalarTy: NarrowTy, VecReg: SrcReg,
4179 LaneIdx, MIRBuilder&: MIB);
4180 if (!Extract)
4181 return false;
4182
4183 I.eraseFromParent();
4184 return true;
4185}
4186
4187bool AArch64InstructionSelector::selectSplitVectorUnmerge(
4188 MachineInstr &I, MachineRegisterInfo &MRI) {
4189 unsigned NumElts = I.getNumOperands() - 1;
4190 Register SrcReg = I.getOperand(i: NumElts).getReg();
4191 const LLT NarrowTy = MRI.getType(Reg: I.getOperand(i: 0).getReg());
4192 const LLT SrcTy = MRI.getType(Reg: SrcReg);
4193
4194 assert(NarrowTy.isVector() && "Expected an unmerge into vectors");
4195 if (SrcTy.getSizeInBits() > 128) {
4196 LLVM_DEBUG(dbgs() << "Unexpected vector type for vec split unmerge");
4197 return false;
4198 }
4199
4200 // We implement a split vector operation by treating the sub-vectors as
4201 // scalars and extracting them.
4202 const RegisterBank &DstRB =
4203 *RBI.getRegBank(Reg: I.getOperand(i: 0).getReg(), MRI, TRI);
4204 for (unsigned OpIdx = 0; OpIdx < NumElts; ++OpIdx) {
4205 Register Dst = I.getOperand(i: OpIdx).getReg();
4206 MachineInstr *Extract =
4207 emitExtractVectorElt(DstReg: Dst, DstRB, ScalarTy: NarrowTy, VecReg: SrcReg, LaneIdx: OpIdx, MIRBuilder&: MIB);
4208 if (!Extract)
4209 return false;
4210 }
4211 I.eraseFromParent();
4212 return true;
4213}
4214
4215bool AArch64InstructionSelector::selectUnmergeValues(MachineInstr &I,
4216 MachineRegisterInfo &MRI) {
4217 assert(I.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
4218 "unexpected opcode");
4219
4220 // The last operand is the vector source register, and every other operand is
4221 // a register to unpack into.
4222 unsigned NumElts = I.getNumOperands() - 1;
4223 Register SrcReg = I.getOperand(i: NumElts).getReg();
4224 Register LoReg = I.getOperand(i: 0).getReg();
4225 Register HiReg = I.getOperand(i: 1).getReg();
4226 const LLT NarrowTy = MRI.getType(Reg: LoReg);
4227 const LLT WideTy = MRI.getType(Reg: SrcReg);
4228 const RegisterBank &LoRB = *RBI.getRegBank(Reg: LoReg, MRI, TRI);
4229 const RegisterBank &HiRB = *RBI.getRegBank(Reg: HiReg, MRI, TRI);
4230 const RegisterBank &SrcRB = *RBI.getRegBank(Reg: SrcReg, MRI, TRI);
4231
4232 // Handle unmerging a 128-bit FPR value into two 64-bit GPR values.
4233 if (NarrowTy == LLT::scalar(SizeInBits: 64) && WideTy == LLT::scalar(SizeInBits: 128) &&
4234 LoRB.getID() == AArch64::GPRRegBankID &&
4235 HiRB.getID() == AArch64::GPRRegBankID &&
4236 SrcRB.getID() == AArch64::FPRRegBankID) {
4237 MachineInstr &Lo = *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
4238 MCID: TII.get(Opcode: AArch64::UMOVvi64), DestReg: LoReg)
4239 .addUse(RegNo: SrcReg)
4240 .addImm(Val: 0);
4241 MachineInstr &Hi = *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
4242 MCID: TII.get(Opcode: AArch64::UMOVvi64), DestReg: HiReg)
4243 .addUse(RegNo: SrcReg)
4244 .addImm(Val: 1);
4245 constrainSelectedInstRegOperands(I&: Lo, TII, TRI, RBI);
4246 constrainSelectedInstRegOperands(I&: Hi, TII, TRI, RBI);
4247 I.eraseFromParent();
4248 return true;
4249 }
4250
4251 // TODO: Handle other unmerges into GPRs and from scalars to scalars.
4252 if (LoRB.getID() != AArch64::FPRRegBankID ||
4253 HiRB.getID() != AArch64::FPRRegBankID) {
4254 LLVM_DEBUG(dbgs() << "Unmerging vector-to-gpr and scalar-to-scalar "
4255 "currently unsupported.\n");
4256 return false;
4257 }
4258
4259 assert(WideTy.getSizeInBits() > NarrowTy.getSizeInBits() &&
4260 "source register size too small!");
4261
4262 if (!NarrowTy.isScalar())
4263 return selectSplitVectorUnmerge(I, MRI);
4264
4265 // Choose a lane copy opcode and subregister based off of the size of the
4266 // vector's elements.
4267 unsigned CopyOpc = 0;
4268 unsigned ExtractSubReg = 0;
4269 if (!getLaneCopyOpcode(CopyOpc, ExtractSubReg, EltSize: NarrowTy.getSizeInBits()))
4270 return false;
4271
4272 // Set up for the lane copies.
4273 MachineBasicBlock &MBB = *I.getParent();
4274
4275 // Stores the registers we'll be copying from.
4276 SmallVector<Register, 4> InsertRegs;
4277
4278 // We'll use the first register twice, so we only need NumElts-1 registers.
4279 unsigned NumInsertRegs = NumElts - 1;
4280
4281 // If our elements fit into exactly 128 bits, then we can copy from the source
4282 // directly. Otherwise, we need to do a bit of setup with some subregister
4283 // inserts.
4284 if (NarrowTy.getSizeInBits() * NumElts == 128) {
4285 InsertRegs.assign(NumElts: NumInsertRegs, Elt: SrcReg);
4286 } else {
4287 // No. We have to perform subregister inserts. For each insert, create an
4288 // implicit def and a subregister insert, and save the register we create.
4289 // For scalar sources, treat as a pseudo-vector of NarrowTy elements.
4290 unsigned EltSize = WideTy.isVector() ? WideTy.getScalarSizeInBits()
4291 : NarrowTy.getSizeInBits();
4292 const TargetRegisterClass *RC = getRegClassForTypeOnBank(
4293 Ty: LLT::fixed_vector(NumElements: NumElts, ScalarSizeInBits: EltSize), RB: *RBI.getRegBank(Reg: SrcReg, MRI, TRI));
4294 unsigned SubReg = 0;
4295 bool Found = getSubRegForClass(RC, TRI, SubReg);
4296 (void)Found;
4297 assert(Found && "expected to find last operand's subeg idx");
4298 for (unsigned Idx = 0; Idx < NumInsertRegs; ++Idx) {
4299 Register ImpDefReg = MRI.createVirtualRegister(RegClass: &AArch64::FPR128RegClass);
4300 MachineInstr &ImpDefMI =
4301 *BuildMI(BB&: MBB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: TargetOpcode::IMPLICIT_DEF),
4302 DestReg: ImpDefReg);
4303
4304 // Now, create the subregister insert from SrcReg.
4305 Register InsertReg = MRI.createVirtualRegister(RegClass: &AArch64::FPR128RegClass);
4306 MachineInstr &InsMI =
4307 *BuildMI(BB&: MBB, I, MIMD: I.getDebugLoc(),
4308 MCID: TII.get(Opcode: TargetOpcode::INSERT_SUBREG), DestReg: InsertReg)
4309 .addUse(RegNo: ImpDefReg)
4310 .addUse(RegNo: SrcReg)
4311 .addImm(Val: SubReg);
4312
4313 constrainSelectedInstRegOperands(I&: ImpDefMI, TII, TRI, RBI);
4314 constrainSelectedInstRegOperands(I&: InsMI, TII, TRI, RBI);
4315
4316 // Save the register so that we can copy from it after.
4317 InsertRegs.push_back(Elt: InsertReg);
4318 }
4319 }
4320
4321 // Now that we've created any necessary subregister inserts, we can
4322 // create the copies.
4323 //
4324 // Perform the first copy separately as a subregister copy.
4325 Register CopyTo = I.getOperand(i: 0).getReg();
4326 auto FirstCopy = MIB.buildInstr(Opc: TargetOpcode::COPY, DstOps: {CopyTo}, SrcOps: {})
4327 .addReg(RegNo: InsertRegs[0], Flags: {}, SubReg: ExtractSubReg);
4328 constrainSelectedInstRegOperands(I&: *FirstCopy, TII, TRI, RBI);
4329
4330 // Now, perform the remaining copies as vector lane copies.
4331 unsigned LaneIdx = 1;
4332 for (Register InsReg : InsertRegs) {
4333 Register CopyTo = I.getOperand(i: LaneIdx).getReg();
4334 MachineInstr &CopyInst =
4335 *BuildMI(BB&: MBB, I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: CopyOpc), DestReg: CopyTo)
4336 .addUse(RegNo: InsReg)
4337 .addImm(Val: LaneIdx);
4338 constrainSelectedInstRegOperands(I&: CopyInst, TII, TRI, RBI);
4339 ++LaneIdx;
4340 }
4341
4342 // Separately constrain the first copy's destination. Because of the
4343 // limitation in constrainOperandRegClass, we can't guarantee that this will
4344 // actually be constrained. So, do it ourselves using the second operand.
4345 const TargetRegisterClass *RC =
4346 MRI.getRegClassOrNull(Reg: I.getOperand(i: 1).getReg());
4347 if (!RC) {
4348 LLVM_DEBUG(dbgs() << "Couldn't constrain copy destination.\n");
4349 return false;
4350 }
4351
4352 RBI.constrainGenericRegister(Reg: CopyTo, RC: *RC, MRI);
4353 I.eraseFromParent();
4354 return true;
4355}
4356
4357bool AArch64InstructionSelector::selectConcatVectors(
4358 MachineInstr &I, MachineRegisterInfo &MRI) {
4359 assert(I.getOpcode() == TargetOpcode::G_CONCAT_VECTORS &&
4360 "Unexpected opcode");
4361 Register Dst = I.getOperand(i: 0).getReg();
4362 Register Op1 = I.getOperand(i: 1).getReg();
4363 Register Op2 = I.getOperand(i: 2).getReg();
4364 MachineInstr *ConcatMI = emitVectorConcat(Dst, Op1, Op2, MIRBuilder&: MIB);
4365 if (!ConcatMI)
4366 return false;
4367 I.eraseFromParent();
4368 return true;
4369}
4370
4371unsigned
4372AArch64InstructionSelector::emitConstantPoolEntry(const Constant *CPVal,
4373 MachineFunction &MF) const {
4374 Type *CPTy = CPVal->getType();
4375 Align Alignment = MF.getDataLayout().getPrefTypeAlign(Ty: CPTy);
4376
4377 MachineConstantPool *MCP = MF.getConstantPool();
4378 return MCP->getConstantPoolIndex(C: CPVal, Alignment);
4379}
4380
4381MachineInstr *AArch64InstructionSelector::emitLoadFromConstantPool(
4382 const Constant *CPVal, MachineIRBuilder &MIRBuilder) const {
4383 const TargetRegisterClass *RC;
4384 unsigned Opc;
4385 bool IsTiny = TM.getCodeModel() == CodeModel::Tiny;
4386 unsigned Size = MIRBuilder.getDataLayout().getTypeStoreSize(Ty: CPVal->getType());
4387 switch (Size) {
4388 case 16:
4389 RC = &AArch64::FPR128RegClass;
4390 Opc = IsTiny ? AArch64::LDRQl : AArch64::LDRQui;
4391 break;
4392 case 8:
4393 RC = &AArch64::FPR64RegClass;
4394 Opc = IsTiny ? AArch64::LDRDl : AArch64::LDRDui;
4395 break;
4396 case 4:
4397 RC = &AArch64::FPR32RegClass;
4398 Opc = IsTiny ? AArch64::LDRSl : AArch64::LDRSui;
4399 break;
4400 case 2:
4401 RC = &AArch64::FPR16RegClass;
4402 Opc = AArch64::LDRHui;
4403 break;
4404 default:
4405 LLVM_DEBUG(dbgs() << "Could not load from constant pool of type "
4406 << *CPVal->getType());
4407 return nullptr;
4408 }
4409
4410 MachineInstr *LoadMI = nullptr;
4411 auto &MF = MIRBuilder.getMF();
4412 unsigned CPIdx = emitConstantPoolEntry(CPVal, MF);
4413 if (IsTiny && (Size == 16 || Size == 8 || Size == 4)) {
4414 // Use load(literal) for tiny code model.
4415 LoadMI = &*MIRBuilder.buildInstr(Opc, DstOps: {RC}, SrcOps: {}).addConstantPoolIndex(Idx: CPIdx);
4416 } else {
4417 auto Adrp =
4418 MIRBuilder.buildInstr(Opc: AArch64::ADRP, DstOps: {&AArch64::GPR64RegClass}, SrcOps: {})
4419 .addConstantPoolIndex(Idx: CPIdx, Offset: 0, TargetFlags: AArch64II::MO_PAGE);
4420
4421 LoadMI = &*MIRBuilder.buildInstr(Opc, DstOps: {RC}, SrcOps: {Adrp})
4422 .addConstantPoolIndex(
4423 Idx: CPIdx, Offset: 0, TargetFlags: AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
4424
4425 constrainSelectedInstRegOperands(I&: *Adrp, TII, TRI, RBI);
4426 }
4427
4428 MachinePointerInfo PtrInfo = MachinePointerInfo::getConstantPool(MF);
4429 LoadMI->addMemOperand(MF, MO: MF.getMachineMemOperand(PtrInfo,
4430 F: MachineMemOperand::MOLoad,
4431 Size, BaseAlignment: Align(Size)));
4432 constrainSelectedInstRegOperands(I&: *LoadMI, TII, TRI, RBI);
4433 return LoadMI;
4434}
4435
4436/// Return an <Opcode, SubregIndex> pair to do an vector elt insert of a given
4437/// size and RB.
4438static std::pair<unsigned, unsigned>
4439getInsertVecEltOpInfo(const RegisterBank &RB, unsigned EltSize) {
4440 unsigned Opc, SubregIdx;
4441 if (RB.getID() == AArch64::GPRRegBankID) {
4442 if (EltSize == 8) {
4443 Opc = AArch64::INSvi8gpr;
4444 SubregIdx = AArch64::bsub;
4445 } else if (EltSize == 16) {
4446 Opc = AArch64::INSvi16gpr;
4447 SubregIdx = AArch64::ssub;
4448 } else if (EltSize == 32) {
4449 Opc = AArch64::INSvi32gpr;
4450 SubregIdx = AArch64::ssub;
4451 } else if (EltSize == 64) {
4452 Opc = AArch64::INSvi64gpr;
4453 SubregIdx = AArch64::dsub;
4454 } else {
4455 llvm_unreachable("invalid elt size!");
4456 }
4457 } else {
4458 if (EltSize == 8) {
4459 Opc = AArch64::INSvi8lane;
4460 SubregIdx = AArch64::bsub;
4461 } else if (EltSize == 16) {
4462 Opc = AArch64::INSvi16lane;
4463 SubregIdx = AArch64::hsub;
4464 } else if (EltSize == 32) {
4465 Opc = AArch64::INSvi32lane;
4466 SubregIdx = AArch64::ssub;
4467 } else if (EltSize == 64) {
4468 Opc = AArch64::INSvi64lane;
4469 SubregIdx = AArch64::dsub;
4470 } else {
4471 llvm_unreachable("invalid elt size!");
4472 }
4473 }
4474 return std::make_pair(x&: Opc, y&: SubregIdx);
4475}
4476
4477MachineInstr *AArch64InstructionSelector::emitInstr(
4478 unsigned Opcode, std::initializer_list<llvm::DstOp> DstOps,
4479 std::initializer_list<llvm::SrcOp> SrcOps, MachineIRBuilder &MIRBuilder,
4480 const ComplexRendererFns &RenderFns) const {
4481 assert(Opcode && "Expected an opcode?");
4482 assert(!isPreISelGenericOpcode(Opcode) &&
4483 "Function should only be used to produce selected instructions!");
4484 auto MI = MIRBuilder.buildInstr(Opc: Opcode, DstOps, SrcOps);
4485 if (RenderFns)
4486 for (auto &Fn : *RenderFns)
4487 Fn(MI);
4488 constrainSelectedInstRegOperands(I&: *MI, TII, TRI, RBI);
4489 return &*MI;
4490}
4491
4492MachineInstr *AArch64InstructionSelector::emitAddSub(
4493 const std::array<std::array<unsigned, 2>, 5> &AddrModeAndSizeToOpcode,
4494 Register Dst, MachineOperand &LHS, MachineOperand &RHS,
4495 MachineIRBuilder &MIRBuilder) const {
4496 MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo();
4497 assert(LHS.isReg() && RHS.isReg() && "Expected register operands?");
4498 auto Ty = MRI.getType(Reg: LHS.getReg());
4499 assert(!Ty.isVector() && "Expected a scalar or pointer?");
4500 unsigned Size = Ty.getSizeInBits();
4501 assert((Size == 32 || Size == 64) && "Expected a 32-bit or 64-bit type only");
4502 bool Is32Bit = Size == 32;
4503
4504 // INSTRri form with positive arithmetic immediate.
4505 if (auto Fns = selectArithImmed(Root&: RHS))
4506 return emitInstr(Opcode: AddrModeAndSizeToOpcode[0][Is32Bit], DstOps: {Dst}, SrcOps: {LHS},
4507 MIRBuilder, RenderFns: Fns);
4508
4509 // INSTRri form with negative arithmetic immediate.
4510 if (auto Fns = selectNegArithImmed(Root&: RHS))
4511 return emitInstr(Opcode: AddrModeAndSizeToOpcode[3][Is32Bit], DstOps: {Dst}, SrcOps: {LHS},
4512 MIRBuilder, RenderFns: Fns);
4513
4514 // INSTRrx form.
4515 if (auto Fns = selectArithExtendedRegister(Root&: RHS))
4516 return emitInstr(Opcode: AddrModeAndSizeToOpcode[4][Is32Bit], DstOps: {Dst}, SrcOps: {LHS},
4517 MIRBuilder, RenderFns: Fns);
4518
4519 // INSTRrs form.
4520 if (auto Fns = selectShiftedRegister(Root&: RHS))
4521 return emitInstr(Opcode: AddrModeAndSizeToOpcode[1][Is32Bit], DstOps: {Dst}, SrcOps: {LHS},
4522 MIRBuilder, RenderFns: Fns);
4523 return emitInstr(Opcode: AddrModeAndSizeToOpcode[2][Is32Bit], DstOps: {Dst}, SrcOps: {LHS, RHS},
4524 MIRBuilder);
4525}
4526
4527MachineInstr *
4528AArch64InstructionSelector::emitADD(Register DefReg, MachineOperand &LHS,
4529 MachineOperand &RHS,
4530 MachineIRBuilder &MIRBuilder) const {
4531 const std::array<std::array<unsigned, 2>, 5> OpcTable{
4532 ._M_elems: {{AArch64::ADDXri, AArch64::ADDWri},
4533 {AArch64::ADDXrs, AArch64::ADDWrs},
4534 {AArch64::ADDXrr, AArch64::ADDWrr},
4535 {AArch64::SUBXri, AArch64::SUBWri},
4536 {AArch64::ADDXrx, AArch64::ADDWrx}}};
4537 return emitAddSub(AddrModeAndSizeToOpcode: OpcTable, Dst: DefReg, LHS, RHS, MIRBuilder);
4538}
4539
4540MachineInstr *
4541AArch64InstructionSelector::emitADDS(Register Dst, MachineOperand &LHS,
4542 MachineOperand &RHS,
4543 MachineIRBuilder &MIRBuilder) const {
4544 const std::array<std::array<unsigned, 2>, 5> OpcTable{
4545 ._M_elems: {{AArch64::ADDSXri, AArch64::ADDSWri},
4546 {AArch64::ADDSXrs, AArch64::ADDSWrs},
4547 {AArch64::ADDSXrr, AArch64::ADDSWrr},
4548 {AArch64::SUBSXri, AArch64::SUBSWri},
4549 {AArch64::ADDSXrx, AArch64::ADDSWrx}}};
4550 return emitAddSub(AddrModeAndSizeToOpcode: OpcTable, Dst, LHS, RHS, MIRBuilder);
4551}
4552
4553MachineInstr *
4554AArch64InstructionSelector::emitSUBS(Register Dst, MachineOperand &LHS,
4555 MachineOperand &RHS,
4556 MachineIRBuilder &MIRBuilder) const {
4557 const std::array<std::array<unsigned, 2>, 5> OpcTable{
4558 ._M_elems: {{AArch64::SUBSXri, AArch64::SUBSWri},
4559 {AArch64::SUBSXrs, AArch64::SUBSWrs},
4560 {AArch64::SUBSXrr, AArch64::SUBSWrr},
4561 {AArch64::ADDSXri, AArch64::ADDSWri},
4562 {AArch64::SUBSXrx, AArch64::SUBSWrx}}};
4563 return emitAddSub(AddrModeAndSizeToOpcode: OpcTable, Dst, LHS, RHS, MIRBuilder);
4564}
4565
4566MachineInstr *
4567AArch64InstructionSelector::emitADCS(Register Dst, MachineOperand &LHS,
4568 MachineOperand &RHS,
4569 MachineIRBuilder &MIRBuilder) const {
4570 assert(LHS.isReg() && RHS.isReg() && "Expected register operands?");
4571 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
4572 bool Is32Bit = (MRI->getType(Reg: LHS.getReg()).getSizeInBits() == 32);
4573 static const unsigned OpcTable[2] = {AArch64::ADCSXr, AArch64::ADCSWr};
4574 return emitInstr(Opcode: OpcTable[Is32Bit], DstOps: {Dst}, SrcOps: {LHS, RHS}, MIRBuilder);
4575}
4576
4577MachineInstr *
4578AArch64InstructionSelector::emitSBCS(Register Dst, MachineOperand &LHS,
4579 MachineOperand &RHS,
4580 MachineIRBuilder &MIRBuilder) const {
4581 assert(LHS.isReg() && RHS.isReg() && "Expected register operands?");
4582 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
4583 bool Is32Bit = (MRI->getType(Reg: LHS.getReg()).getSizeInBits() == 32);
4584 static const unsigned OpcTable[2] = {AArch64::SBCSXr, AArch64::SBCSWr};
4585 return emitInstr(Opcode: OpcTable[Is32Bit], DstOps: {Dst}, SrcOps: {LHS, RHS}, MIRBuilder);
4586}
4587
4588MachineInstr *
4589AArch64InstructionSelector::emitCMP(MachineOperand &LHS, MachineOperand &RHS,
4590 MachineIRBuilder &MIRBuilder) const {
4591 MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo();
4592 bool Is32Bit = MRI.getType(Reg: LHS.getReg()).getSizeInBits() == 32;
4593 auto RC = Is32Bit ? &AArch64::GPR32RegClass : &AArch64::GPR64RegClass;
4594 return emitSUBS(Dst: MRI.createVirtualRegister(RegClass: RC), LHS, RHS, MIRBuilder);
4595}
4596
4597MachineInstr *
4598AArch64InstructionSelector::emitCMN(MachineOperand &LHS, MachineOperand &RHS,
4599 MachineIRBuilder &MIRBuilder) const {
4600 MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo();
4601 bool Is32Bit = (MRI.getType(Reg: LHS.getReg()).getSizeInBits() == 32);
4602 auto RC = Is32Bit ? &AArch64::GPR32RegClass : &AArch64::GPR64RegClass;
4603 return emitADDS(Dst: MRI.createVirtualRegister(RegClass: RC), LHS, RHS, MIRBuilder);
4604}
4605
4606MachineInstr *
4607AArch64InstructionSelector::emitTST(MachineOperand &LHS, MachineOperand &RHS,
4608 MachineIRBuilder &MIRBuilder) const {
4609 assert(LHS.isReg() && RHS.isReg() && "Expected register operands?");
4610 MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo();
4611 LLT Ty = MRI.getType(Reg: LHS.getReg());
4612 unsigned RegSize = Ty.getSizeInBits();
4613 bool Is32Bit = (RegSize == 32);
4614 const unsigned OpcTable[3][2] = {{AArch64::ANDSXri, AArch64::ANDSWri},
4615 {AArch64::ANDSXrs, AArch64::ANDSWrs},
4616 {AArch64::ANDSXrr, AArch64::ANDSWrr}};
4617 // ANDS needs a logical immediate for its immediate form. Check if we can
4618 // fold one in.
4619 if (auto ValAndVReg = getIConstantVRegValWithLookThrough(VReg: RHS.getReg(), MRI)) {
4620 int64_t Imm = ValAndVReg->Value.getSExtValue();
4621
4622 if (AArch64_AM::isLogicalImmediate(imm: Imm, regSize: RegSize)) {
4623 auto TstMI = MIRBuilder.buildInstr(Opc: OpcTable[0][Is32Bit], DstOps: {Ty}, SrcOps: {LHS});
4624 TstMI.addImm(Val: AArch64_AM::encodeLogicalImmediate(imm: Imm, regSize: RegSize));
4625 constrainSelectedInstRegOperands(I&: *TstMI, TII, TRI, RBI);
4626 return &*TstMI;
4627 }
4628 }
4629
4630 if (auto Fns = selectLogicalShiftedRegister(Root&: RHS))
4631 return emitInstr(Opcode: OpcTable[1][Is32Bit], DstOps: {Ty}, SrcOps: {LHS}, MIRBuilder, RenderFns: Fns);
4632 return emitInstr(Opcode: OpcTable[2][Is32Bit], DstOps: {Ty}, SrcOps: {LHS, RHS}, MIRBuilder);
4633}
4634
4635MachineInstr *AArch64InstructionSelector::emitIntegerCompare(
4636 MachineOperand &LHS, MachineOperand &RHS, MachineOperand &Predicate,
4637 MachineIRBuilder &MIRBuilder) const {
4638 assert(LHS.isReg() && RHS.isReg() && "Expected LHS and RHS to be registers!");
4639 assert(Predicate.isPredicate() && "Expected predicate?");
4640 MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo();
4641 LLT CmpTy = MRI.getType(Reg: LHS.getReg());
4642 assert(!CmpTy.isVector() && "Expected scalar or pointer");
4643 unsigned Size = CmpTy.getSizeInBits();
4644 (void)Size;
4645 assert((Size == 32 || Size == 64) && "Expected a 32-bit or 64-bit LHS/RHS?");
4646 // Fold the compare into a cmn or tst if possible.
4647 if (auto FoldCmp = tryFoldIntegerCompare(LHS, RHS, Predicate, MIRBuilder))
4648 return FoldCmp;
4649 return emitCMP(LHS, RHS, MIRBuilder);
4650}
4651
4652MachineInstr *AArch64InstructionSelector::emitCSetForFCmp(
4653 Register Dst, CmpInst::Predicate Pred, MachineIRBuilder &MIRBuilder) const {
4654 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
4655#ifndef NDEBUG
4656 LLT Ty = MRI.getType(Dst);
4657 assert(!Ty.isVector() && Ty.getSizeInBits() == 32 &&
4658 "Expected a 32-bit scalar register?");
4659#endif
4660 const Register ZReg = AArch64::WZR;
4661 AArch64CC::CondCode CC1, CC2;
4662 changeFCMPPredToAArch64CC(P: Pred, CondCode&: CC1, CondCode2&: CC2);
4663 auto InvCC1 = AArch64CC::getInvertedCondCode(Code: CC1);
4664 if (CC2 == AArch64CC::AL)
4665 return emitCSINC(/*Dst=*/Dst, /*Src1=*/ZReg, /*Src2=*/ZReg, Pred: InvCC1,
4666 MIRBuilder);
4667 const TargetRegisterClass *RC = &AArch64::GPR32RegClass;
4668 Register Def1Reg = MRI.createVirtualRegister(RegClass: RC);
4669 Register Def2Reg = MRI.createVirtualRegister(RegClass: RC);
4670 auto InvCC2 = AArch64CC::getInvertedCondCode(Code: CC2);
4671 emitCSINC(/*Dst=*/Def1Reg, /*Src1=*/ZReg, /*Src2=*/ZReg, Pred: InvCC1, MIRBuilder);
4672 emitCSINC(/*Dst=*/Def2Reg, /*Src1=*/ZReg, /*Src2=*/ZReg, Pred: InvCC2, MIRBuilder);
4673 auto OrMI = MIRBuilder.buildInstr(Opc: AArch64::ORRWrr, DstOps: {Dst}, SrcOps: {Def1Reg, Def2Reg});
4674 constrainSelectedInstRegOperands(I&: *OrMI, TII, TRI, RBI);
4675 return &*OrMI;
4676}
4677
4678MachineInstr *AArch64InstructionSelector::emitFPCompare(
4679 Register LHS, Register RHS, MachineIRBuilder &MIRBuilder,
4680 std::optional<CmpInst::Predicate> Pred) const {
4681 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
4682 LLT Ty = MRI.getType(Reg: LHS);
4683 if (Ty.isVector())
4684 return nullptr;
4685 unsigned OpSize = Ty.getSizeInBits();
4686 assert(OpSize == 16 || OpSize == 32 || OpSize == 64);
4687
4688 // If this is a compare against +0.0, then we don't have
4689 // to explicitly materialize a constant.
4690 bool ShouldUseImm = mi_match(R: RHS, MRI, P: m_PosZeroFP());
4691
4692 auto IsEqualityPred = [](CmpInst::Predicate P) {
4693 return P == CmpInst::FCMP_OEQ || P == CmpInst::FCMP_ONE ||
4694 P == CmpInst::FCMP_UEQ || P == CmpInst::FCMP_UNE;
4695 };
4696 if (!ShouldUseImm && Pred && IsEqualityPred(*Pred)) {
4697 // Try commuting the operands.
4698 if (mi_match(R: LHS, MRI, P: m_PosZeroFP())) {
4699 ShouldUseImm = true;
4700 std::swap(a&: LHS, b&: RHS);
4701 }
4702 }
4703 unsigned CmpOpcTbl[2][3] = {
4704 {AArch64::FCMPHrr, AArch64::FCMPSrr, AArch64::FCMPDrr},
4705 {AArch64::FCMPHri, AArch64::FCMPSri, AArch64::FCMPDri}};
4706 unsigned CmpOpc =
4707 CmpOpcTbl[ShouldUseImm][OpSize == 16 ? 0 : (OpSize == 32 ? 1 : 2)];
4708
4709 // Partially build the compare. Decide if we need to add a use for the
4710 // third operand based off whether or not we're comparing against 0.0.
4711 auto CmpMI = MIRBuilder.buildInstr(Opcode: CmpOpc).addUse(RegNo: LHS);
4712 CmpMI.setMIFlags(MachineInstr::NoFPExcept);
4713 if (!ShouldUseImm)
4714 CmpMI.addUse(RegNo: RHS);
4715 constrainSelectedInstRegOperands(I&: *CmpMI, TII, TRI, RBI);
4716 return &*CmpMI;
4717}
4718
4719MachineInstr *AArch64InstructionSelector::emitVectorConcat(
4720 std::optional<Register> Dst, Register Op1, Register Op2,
4721 MachineIRBuilder &MIRBuilder) const {
4722 // We implement a vector concat by:
4723 // 1. Use scalar_to_vector to insert the lower vector into the larger dest
4724 // 2. Insert the upper vector into the destination's upper element
4725 // TODO: some of this code is common with G_BUILD_VECTOR handling.
4726 MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo();
4727
4728 const LLT Op1Ty = MRI.getType(Reg: Op1);
4729 const LLT Op2Ty = MRI.getType(Reg: Op2);
4730
4731 if (Op1Ty != Op2Ty) {
4732 LLVM_DEBUG(dbgs() << "Could not do vector concat of differing vector tys");
4733 return nullptr;
4734 }
4735 assert(Op1Ty.isVector() && "Expected a vector for vector concat");
4736
4737 if (Op1Ty.getSizeInBits() >= 128) {
4738 LLVM_DEBUG(dbgs() << "Vector concat not supported for full size vectors");
4739 return nullptr;
4740 }
4741
4742 // At the moment we just support 64 bit vector concats.
4743 if (Op1Ty.getSizeInBits() != 64) {
4744 LLVM_DEBUG(dbgs() << "Vector concat supported for 64b vectors");
4745 return nullptr;
4746 }
4747
4748 const LLT ScalarTy = LLT::scalar(SizeInBits: Op1Ty.getSizeInBits());
4749 const RegisterBank &FPRBank = *RBI.getRegBank(Reg: Op1, MRI, TRI);
4750 const TargetRegisterClass *DstRC =
4751 getRegClassForTypeOnBank(Ty: Op1Ty.multiplyElements(Factor: 2), RB: FPRBank);
4752
4753 MachineInstr *WidenedOp1 =
4754 emitScalarToVector(EltSize: ScalarTy.getSizeInBits(), DstRC, Scalar: Op1, MIRBuilder);
4755 MachineInstr *WidenedOp2 =
4756 emitScalarToVector(EltSize: ScalarTy.getSizeInBits(), DstRC, Scalar: Op2, MIRBuilder);
4757 if (!WidenedOp1 || !WidenedOp2) {
4758 LLVM_DEBUG(dbgs() << "Could not emit a vector from scalar value");
4759 return nullptr;
4760 }
4761
4762 // Now do the insert of the upper element.
4763 unsigned InsertOpc, InsSubRegIdx;
4764 std::tie(args&: InsertOpc, args&: InsSubRegIdx) =
4765 getInsertVecEltOpInfo(RB: FPRBank, EltSize: ScalarTy.getSizeInBits());
4766
4767 if (!Dst)
4768 Dst = MRI.createVirtualRegister(RegClass: DstRC);
4769 auto InsElt =
4770 MIRBuilder
4771 .buildInstr(Opc: InsertOpc, DstOps: {*Dst}, SrcOps: {WidenedOp1->getOperand(i: 0).getReg()})
4772 .addImm(Val: 1) /* Lane index */
4773 .addUse(RegNo: WidenedOp2->getOperand(i: 0).getReg())
4774 .addImm(Val: 0);
4775 constrainSelectedInstRegOperands(I&: *InsElt, TII, TRI, RBI);
4776 return &*InsElt;
4777}
4778
4779MachineInstr *
4780AArch64InstructionSelector::emitCSINC(Register Dst, Register Src1,
4781 Register Src2, AArch64CC::CondCode Pred,
4782 MachineIRBuilder &MIRBuilder) const {
4783 auto &MRI = *MIRBuilder.getMRI();
4784 const RegClassOrRegBank &RegClassOrBank = MRI.getRegClassOrRegBank(Reg: Dst);
4785 // If we used a register class, then this won't necessarily have an LLT.
4786 // Compute the size based off whether or not we have a class or bank.
4787 unsigned Size;
4788 if (const auto *RC = dyn_cast<const TargetRegisterClass *>(Val: RegClassOrBank))
4789 Size = TRI.getRegSizeInBits(RC: *RC);
4790 else
4791 Size = MRI.getType(Reg: Dst).getSizeInBits();
4792 // Some opcodes use s1.
4793 assert(Size <= 64 && "Expected 64 bits or less only!");
4794 static const unsigned OpcTable[2] = {AArch64::CSINCWr, AArch64::CSINCXr};
4795 unsigned Opc = OpcTable[Size == 64];
4796 auto CSINC = MIRBuilder.buildInstr(Opc, DstOps: {Dst}, SrcOps: {Src1, Src2}).addImm(Val: Pred);
4797 constrainSelectedInstRegOperands(I&: *CSINC, TII, TRI, RBI);
4798 return &*CSINC;
4799}
4800
4801MachineInstr *AArch64InstructionSelector::emitCarryIn(MachineInstr &I,
4802 Register CarryReg) {
4803 MachineRegisterInfo *MRI = MIB.getMRI();
4804 unsigned Opcode = I.getOpcode();
4805
4806 // If the instruction is a SUB, we need to negate the carry,
4807 // because borrowing is indicated by carry-flag == 0.
4808 bool NeedsNegatedCarry =
4809 (Opcode == TargetOpcode::G_USUBE || Opcode == TargetOpcode::G_SSUBE);
4810
4811 // If the previous instruction will already produce the correct carry, do not
4812 // emit a carry generating instruction. E.g. for G_UADDE/G_USUBE sequences
4813 // generated during legalization of wide add/sub. This optimization depends on
4814 // these sequences not being interrupted by other instructions.
4815 // We have to select the previous instruction before the carry-using
4816 // instruction is deleted by the calling function, otherwise the previous
4817 // instruction might become dead and would get deleted.
4818 MachineInstr *SrcMI = MRI->getVRegDef(Reg: CarryReg);
4819
4820 // Debug instructions (e.g., DBG_VALUE) do not modify NZCV, so skip all that
4821 // immediately precede I.
4822 MachineInstr *PrevMI = I.getPrevNode();
4823 while (PrevMI && PrevMI->isDebugInstr())
4824 PrevMI = PrevMI->getPrevNode();
4825
4826 if (SrcMI == PrevMI) {
4827 if (auto *CarrySrcMI = dyn_cast<GAddSubCarryOut>(Val: SrcMI)) {
4828 bool ProducesNegatedCarry = CarrySrcMI->isSub();
4829 if (NeedsNegatedCarry == ProducesNegatedCarry &&
4830 CarrySrcMI->isUnsigned() &&
4831 CarrySrcMI->getCarryOutReg() == CarryReg &&
4832 selectAndRestoreState(I&: *SrcMI))
4833 return nullptr;
4834 }
4835 }
4836
4837 Register DeadReg = MRI->createVirtualRegister(RegClass: &AArch64::GPR32RegClass);
4838
4839 if (NeedsNegatedCarry) {
4840 // (0 - Carry) sets !C in NZCV when Carry == 1
4841 Register ZReg = AArch64::WZR;
4842 return emitInstr(Opcode: AArch64::SUBSWrr, DstOps: {DeadReg}, SrcOps: {ZReg, CarryReg}, MIRBuilder&: MIB);
4843 }
4844
4845 // (Carry - 1) sets !C in NZCV when Carry == 0
4846 auto Fns = select12BitValueWithLeftShift(Immed: 1);
4847 return emitInstr(Opcode: AArch64::SUBSWri, DstOps: {DeadReg}, SrcOps: {CarryReg}, MIRBuilder&: MIB, RenderFns: Fns);
4848}
4849
4850bool AArch64InstructionSelector::selectOverflowOp(MachineInstr &I,
4851 MachineRegisterInfo &MRI) {
4852 auto &CarryMI = cast<GAddSubCarryOut>(Val&: I);
4853
4854 if (auto *CarryInMI = dyn_cast<GAddSubCarryInOut>(Val: &I)) {
4855 // Set NZCV carry according to carry-in VReg
4856 emitCarryIn(I, CarryReg: CarryInMI->getCarryInReg());
4857 }
4858
4859 // Emit the operation and get the correct condition code.
4860 auto OpAndCC = emitOverflowOp(Opcode: I.getOpcode(), Dst: CarryMI.getDstReg(),
4861 LHS&: CarryMI.getLHS(), RHS&: CarryMI.getRHS(), MIRBuilder&: MIB);
4862
4863 Register CarryOutReg = CarryMI.getCarryOutReg();
4864
4865 // Don't convert carry-out to VReg if it is never used
4866 if (MRI.use_nodbg_empty(RegNo: CarryOutReg)) {
4867 OpAndCC.first->addRegisterDead(Reg: AArch64::NZCV, RegInfo: &TRI);
4868 } else {
4869 // Now, put the overflow result in the register given by the first operand
4870 // to the overflow op. CSINC increments the result when the predicate is
4871 // false, so to get the increment when it's true, we need to use the
4872 // inverse. In this case, we want to increment when carry is set.
4873 Register ZReg = AArch64::WZR;
4874 emitCSINC(/*Dst=*/CarryOutReg, /*Src1=*/ZReg, /*Src2=*/ZReg,
4875 Pred: getInvertedCondCode(Code: OpAndCC.second), MIRBuilder&: MIB);
4876 }
4877
4878 I.eraseFromParent();
4879 return true;
4880}
4881
4882std::pair<MachineInstr *, AArch64CC::CondCode>
4883AArch64InstructionSelector::emitOverflowOp(unsigned Opcode, Register Dst,
4884 MachineOperand &LHS,
4885 MachineOperand &RHS,
4886 MachineIRBuilder &MIRBuilder) const {
4887 switch (Opcode) {
4888 default:
4889 llvm_unreachable("Unexpected opcode!");
4890 case TargetOpcode::G_SADDO:
4891 return std::make_pair(x: emitADDS(Dst, LHS, RHS, MIRBuilder), y: AArch64CC::VS);
4892 case TargetOpcode::G_UADDO:
4893 return std::make_pair(x: emitADDS(Dst, LHS, RHS, MIRBuilder), y: AArch64CC::HS);
4894 case TargetOpcode::G_SSUBO:
4895 return std::make_pair(x: emitSUBS(Dst, LHS, RHS, MIRBuilder), y: AArch64CC::VS);
4896 case TargetOpcode::G_USUBO:
4897 return std::make_pair(x: emitSUBS(Dst, LHS, RHS, MIRBuilder), y: AArch64CC::LO);
4898 case TargetOpcode::G_SADDE:
4899 return std::make_pair(x: emitADCS(Dst, LHS, RHS, MIRBuilder), y: AArch64CC::VS);
4900 case TargetOpcode::G_UADDE:
4901 return std::make_pair(x: emitADCS(Dst, LHS, RHS, MIRBuilder), y: AArch64CC::HS);
4902 case TargetOpcode::G_SSUBE:
4903 return std::make_pair(x: emitSBCS(Dst, LHS, RHS, MIRBuilder), y: AArch64CC::VS);
4904 case TargetOpcode::G_USUBE:
4905 return std::make_pair(x: emitSBCS(Dst, LHS, RHS, MIRBuilder), y: AArch64CC::LO);
4906 }
4907}
4908
4909/// Returns true if @p Val is a tree of AND/OR/CMP operations that can be
4910/// expressed as a conjunction.
4911/// \param CanNegate Set to true if we can negate the whole sub-tree just by
4912/// changing the conditions on the CMP tests.
4913/// (this means we can call emitConjunctionRec() with
4914/// Negate==true on this sub-tree)
4915/// \param MustBeFirst Set to true if this subtree needs to be negated and we
4916/// cannot do the negation naturally. We are required to
4917/// emit the subtree first in this case.
4918/// \param WillNegate Is true if are called when the result of this
4919/// subexpression must be negated. This happens when the
4920/// outer expression is an OR. We can use this fact to know
4921/// that we have a double negation (or (or ...) ...) that
4922/// can be implemented for free.
4923static bool canEmitConjunction(Register Val, bool &CanNegate, bool &MustBeFirst,
4924 bool WillNegate, MachineRegisterInfo &MRI,
4925 unsigned Depth = 0) {
4926 if (!MRI.hasOneNonDBGUse(RegNo: Val))
4927 return false;
4928 MachineInstr *ValDef = MRI.getVRegDef(Reg: Val);
4929 unsigned Opcode = ValDef->getOpcode();
4930 if (isa<GAnyCmp>(Val: ValDef)) {
4931 CanNegate = true;
4932 MustBeFirst = false;
4933 return true;
4934 }
4935 // Protect against exponential runtime and stack overflow.
4936 if (Depth > 6)
4937 return false;
4938 if (Opcode == TargetOpcode::G_AND || Opcode == TargetOpcode::G_OR) {
4939 bool IsOR = Opcode == TargetOpcode::G_OR;
4940 Register O0 = ValDef->getOperand(i: 1).getReg();
4941 Register O1 = ValDef->getOperand(i: 2).getReg();
4942 bool CanNegateL;
4943 bool MustBeFirstL;
4944 if (!canEmitConjunction(Val: O0, CanNegate&: CanNegateL, MustBeFirst&: MustBeFirstL, WillNegate: IsOR, MRI, Depth: Depth + 1))
4945 return false;
4946 bool CanNegateR;
4947 bool MustBeFirstR;
4948 if (!canEmitConjunction(Val: O1, CanNegate&: CanNegateR, MustBeFirst&: MustBeFirstR, WillNegate: IsOR, MRI, Depth: Depth + 1))
4949 return false;
4950
4951 if (MustBeFirstL && MustBeFirstR)
4952 return false;
4953
4954 if (IsOR) {
4955 // For an OR expression we need to be able to naturally negate at least
4956 // one side or we cannot do the transformation at all.
4957 if (!CanNegateL && !CanNegateR)
4958 return false;
4959 // If we the result of the OR will be negated and we can naturally negate
4960 // the leaves, then this sub-tree as a whole negates naturally.
4961 CanNegate = WillNegate && CanNegateL && CanNegateR;
4962 // If we cannot naturally negate the whole sub-tree, then this must be
4963 // emitted first.
4964 MustBeFirst = !CanNegate;
4965 } else {
4966 assert(Opcode == TargetOpcode::G_AND && "Must be G_AND");
4967 // We cannot naturally negate an AND operation.
4968 CanNegate = false;
4969 MustBeFirst = MustBeFirstL || MustBeFirstR;
4970 }
4971 return true;
4972 }
4973 return false;
4974}
4975
4976MachineInstr *AArch64InstructionSelector::emitConditionalComparison(
4977 Register LHS, Register RHS, CmpInst::Predicate CC,
4978 AArch64CC::CondCode Predicate, AArch64CC::CondCode OutCC,
4979 MachineIRBuilder &MIB) const {
4980 auto &MRI = *MIB.getMRI();
4981 LLT OpTy = MRI.getType(Reg: LHS);
4982 unsigned CCmpOpc;
4983 std::optional<ValueAndVReg> C;
4984 if (CmpInst::isIntPredicate(P: CC)) {
4985 assert(OpTy.getSizeInBits() == 32 || OpTy.getSizeInBits() == 64);
4986 C = getIConstantVRegValWithLookThrough(VReg: RHS, MRI);
4987 if (!C || C->Value.sgt(RHS: 31) || C->Value.slt(RHS: -31))
4988 CCmpOpc = OpTy.getSizeInBits() == 32 ? AArch64::CCMPWr : AArch64::CCMPXr;
4989 else if (C->Value.ule(RHS: 31))
4990 CCmpOpc = OpTy.getSizeInBits() == 32 ? AArch64::CCMPWi : AArch64::CCMPXi;
4991 else
4992 CCmpOpc = OpTy.getSizeInBits() == 32 ? AArch64::CCMNWi : AArch64::CCMNXi;
4993 } else {
4994 assert(OpTy.getSizeInBits() == 16 || OpTy.getSizeInBits() == 32 ||
4995 OpTy.getSizeInBits() == 64);
4996 switch (OpTy.getSizeInBits()) {
4997 case 16:
4998 assert(STI.hasFullFP16() && "Expected Full FP16 for fp16 comparisons");
4999 CCmpOpc = AArch64::FCCMPHrr;
5000 break;
5001 case 32:
5002 CCmpOpc = AArch64::FCCMPSrr;
5003 break;
5004 case 64:
5005 CCmpOpc = AArch64::FCCMPDrr;
5006 break;
5007 default:
5008 return nullptr;
5009 }
5010 }
5011 AArch64CC::CondCode InvOutCC = AArch64CC::getInvertedCondCode(Code: OutCC);
5012 unsigned NZCV = AArch64CC::getNZCVToSatisfyCondCode(Code: InvOutCC);
5013 auto CCmp =
5014 MIB.buildInstr(Opc: CCmpOpc, DstOps: {}, SrcOps: {LHS});
5015 if (CCmpOpc == AArch64::CCMPWi || CCmpOpc == AArch64::CCMPXi)
5016 CCmp.addImm(Val: C->Value.getZExtValue());
5017 else if (CCmpOpc == AArch64::CCMNWi || CCmpOpc == AArch64::CCMNXi)
5018 CCmp.addImm(Val: C->Value.abs().getZExtValue());
5019 else
5020 CCmp.addReg(RegNo: RHS);
5021 CCmp.addImm(Val: NZCV).addImm(Val: Predicate);
5022 constrainSelectedInstRegOperands(I&: *CCmp, TII, TRI, RBI);
5023 return &*CCmp;
5024}
5025
5026MachineInstr *AArch64InstructionSelector::emitConjunctionRec(
5027 Register Val, AArch64CC::CondCode &OutCC, bool Negate, Register CCOp,
5028 AArch64CC::CondCode Predicate, MachineIRBuilder &MIB) const {
5029 // We're at a tree leaf, produce a conditional comparison operation.
5030 auto &MRI = *MIB.getMRI();
5031 MachineInstr *ValDef = MRI.getVRegDef(Reg: Val);
5032 unsigned Opcode = ValDef->getOpcode();
5033 if (auto *Cmp = dyn_cast<GAnyCmp>(Val: ValDef)) {
5034 Register LHS = Cmp->getLHSReg();
5035 Register RHS = Cmp->getRHSReg();
5036 CmpInst::Predicate CC = Cmp->getCond();
5037 if (Negate)
5038 CC = CmpInst::getInversePredicate(pred: CC);
5039 if (isa<GICmp>(Val: Cmp)) {
5040 OutCC = changeICMPPredToAArch64CC(P: CC, RHS, MRI: MIB.getMRI());
5041 } else {
5042 // Handle special FP cases.
5043 AArch64CC::CondCode ExtraCC;
5044 changeFPCCToANDAArch64CC(CC, CondCode&: OutCC, CondCode2&: ExtraCC);
5045 // Some floating point conditions can't be tested with a single condition
5046 // code. Construct an additional comparison in this case.
5047 if (ExtraCC != AArch64CC::AL) {
5048 MachineInstr *ExtraCmp;
5049 if (!CCOp)
5050 ExtraCmp = emitFPCompare(LHS, RHS, MIRBuilder&: MIB, Pred: CC);
5051 else
5052 ExtraCmp =
5053 emitConditionalComparison(LHS, RHS, CC, Predicate, OutCC: ExtraCC, MIB);
5054 CCOp = ExtraCmp->getOperand(i: 0).getReg();
5055 Predicate = ExtraCC;
5056 }
5057 }
5058
5059 // Produce a normal comparison if we are first in the chain
5060 if (!CCOp) {
5061 if (isa<GICmp>(Val: Cmp))
5062 return emitCMP(LHS&: Cmp->getOperand(i: 2), RHS&: Cmp->getOperand(i: 3), MIRBuilder&: MIB);
5063 return emitFPCompare(LHS: Cmp->getOperand(i: 2).getReg(),
5064 RHS: Cmp->getOperand(i: 3).getReg(), MIRBuilder&: MIB);
5065 }
5066 // Otherwise produce a ccmp.
5067 return emitConditionalComparison(LHS, RHS, CC, Predicate, OutCC, MIB);
5068 }
5069 assert(MRI.hasOneNonDBGUse(Val) && "Valid conjunction/disjunction tree");
5070
5071 bool IsOR = Opcode == TargetOpcode::G_OR;
5072
5073 Register LHS = ValDef->getOperand(i: 1).getReg();
5074 bool CanNegateL;
5075 bool MustBeFirstL;
5076 bool ValidL = canEmitConjunction(Val: LHS, CanNegate&: CanNegateL, MustBeFirst&: MustBeFirstL, WillNegate: IsOR, MRI);
5077 assert(ValidL && "Valid conjunction/disjunction tree");
5078 (void)ValidL;
5079
5080 Register RHS = ValDef->getOperand(i: 2).getReg();
5081 bool CanNegateR;
5082 bool MustBeFirstR;
5083 bool ValidR = canEmitConjunction(Val: RHS, CanNegate&: CanNegateR, MustBeFirst&: MustBeFirstR, WillNegate: IsOR, MRI);
5084 assert(ValidR && "Valid conjunction/disjunction tree");
5085 (void)ValidR;
5086
5087 // Swap sub-tree that must come first to the right side.
5088 if (MustBeFirstL) {
5089 assert(!MustBeFirstR && "Valid conjunction/disjunction tree");
5090 std::swap(a&: LHS, b&: RHS);
5091 std::swap(a&: CanNegateL, b&: CanNegateR);
5092 std::swap(a&: MustBeFirstL, b&: MustBeFirstR);
5093 }
5094
5095 bool NegateR;
5096 bool NegateAfterR;
5097 bool NegateL;
5098 bool NegateAfterAll;
5099 if (Opcode == TargetOpcode::G_OR) {
5100 // Swap the sub-tree that we can negate naturally to the left.
5101 if (!CanNegateL) {
5102 assert(CanNegateR && "at least one side must be negatable");
5103 assert(!MustBeFirstR && "invalid conjunction/disjunction tree");
5104 assert(!Negate);
5105 std::swap(a&: LHS, b&: RHS);
5106 NegateR = false;
5107 NegateAfterR = true;
5108 } else {
5109 // Negate the left sub-tree if possible, otherwise negate the result.
5110 NegateR = CanNegateR;
5111 NegateAfterR = !CanNegateR;
5112 }
5113 NegateL = true;
5114 NegateAfterAll = !Negate;
5115 } else {
5116 assert(Opcode == TargetOpcode::G_AND &&
5117 "Valid conjunction/disjunction tree");
5118 assert(!Negate && "Valid conjunction/disjunction tree");
5119
5120 NegateL = false;
5121 NegateR = false;
5122 NegateAfterR = false;
5123 NegateAfterAll = false;
5124 }
5125
5126 // Emit sub-trees.
5127 AArch64CC::CondCode RHSCC;
5128 MachineInstr *CmpR =
5129 emitConjunctionRec(Val: RHS, OutCC&: RHSCC, Negate: NegateR, CCOp, Predicate, MIB);
5130 if (NegateAfterR)
5131 RHSCC = AArch64CC::getInvertedCondCode(Code: RHSCC);
5132 MachineInstr *CmpL = emitConjunctionRec(
5133 Val: LHS, OutCC, Negate: NegateL, CCOp: CmpR->getOperand(i: 0).getReg(), Predicate: RHSCC, MIB);
5134 if (NegateAfterAll)
5135 OutCC = AArch64CC::getInvertedCondCode(Code: OutCC);
5136 return CmpL;
5137}
5138
5139MachineInstr *AArch64InstructionSelector::emitConjunction(
5140 Register Val, AArch64CC::CondCode &OutCC, MachineIRBuilder &MIB) const {
5141 bool DummyCanNegate;
5142 bool DummyMustBeFirst;
5143 if (!canEmitConjunction(Val, CanNegate&: DummyCanNegate, MustBeFirst&: DummyMustBeFirst, WillNegate: false,
5144 MRI&: *MIB.getMRI()))
5145 return nullptr;
5146 return emitConjunctionRec(Val, OutCC, Negate: false, CCOp: Register(), Predicate: AArch64CC::AL, MIB);
5147}
5148
5149bool AArch64InstructionSelector::tryOptSelectConjunction(GSelect &SelI,
5150 MachineInstr &CondMI) {
5151 AArch64CC::CondCode AArch64CC;
5152 MachineInstr *ConjMI = emitConjunction(Val: SelI.getCondReg(), OutCC&: AArch64CC, MIB);
5153 if (!ConjMI)
5154 return false;
5155
5156 emitSelect(Dst: SelI.getReg(Idx: 0), True: SelI.getTrueReg(), False: SelI.getFalseReg(), CC: AArch64CC, MIB);
5157 SelI.eraseFromParent();
5158 return true;
5159}
5160
5161bool AArch64InstructionSelector::tryOptSelect(GSelect &I) {
5162 MachineRegisterInfo &MRI = *MIB.getMRI();
5163 // We want to recognize this pattern:
5164 //
5165 // $z = G_FCMP pred, $x, $y
5166 // ...
5167 // $w = G_SELECT $z, $a, $b
5168 //
5169 // Where the value of $z is *only* ever used by the G_SELECT (possibly with
5170 // some copies/truncs in between.)
5171 //
5172 // If we see this, then we can emit something like this:
5173 //
5174 // fcmp $x, $y
5175 // fcsel $w, $a, $b, pred
5176 //
5177 // Rather than emitting both of the rather long sequences in the standard
5178 // G_FCMP/G_SELECT select methods.
5179
5180 // First, check if the condition is defined by a compare.
5181 MachineInstr *CondDef = MRI.getVRegDef(Reg: I.getOperand(i: 1).getReg());
5182
5183 // We can only fold if all of the defs have one use.
5184 Register CondDefReg = CondDef->getOperand(i: 0).getReg();
5185 if (!MRI.hasOneNonDBGUse(RegNo: CondDefReg)) {
5186 // Unless it's another select.
5187 for (const MachineInstr &UI : MRI.use_nodbg_instructions(Reg: CondDefReg)) {
5188 if (CondDef == &UI)
5189 continue;
5190 if (UI.getOpcode() != TargetOpcode::G_SELECT)
5191 return false;
5192 }
5193 }
5194
5195 // Is the condition defined by a compare?
5196 unsigned CondOpc = CondDef->getOpcode();
5197 if (CondOpc != TargetOpcode::G_ICMP && CondOpc != TargetOpcode::G_FCMP) {
5198 if (tryOptSelectConjunction(SelI&: I, CondMI&: *CondDef))
5199 return true;
5200 return false;
5201 }
5202
5203 AArch64CC::CondCode CondCode;
5204 if (CondOpc == TargetOpcode::G_ICMP) {
5205 auto &PredOp = CondDef->getOperand(i: 1);
5206 emitIntegerCompare(LHS&: CondDef->getOperand(i: 2), RHS&: CondDef->getOperand(i: 3), Predicate&: PredOp,
5207 MIRBuilder&: MIB);
5208 auto Pred = static_cast<CmpInst::Predicate>(PredOp.getPredicate());
5209 CondCode =
5210 changeICMPPredToAArch64CC(P: Pred, RHS: CondDef->getOperand(i: 3).getReg(), MRI: &MRI);
5211 } else {
5212 // Get the condition code for the select.
5213 auto Pred =
5214 static_cast<CmpInst::Predicate>(CondDef->getOperand(i: 1).getPredicate());
5215 AArch64CC::CondCode CondCode2;
5216 changeFCMPPredToAArch64CC(P: Pred, CondCode, CondCode2);
5217
5218 // changeFCMPPredToAArch64CC sets CondCode2 to AL when we require two
5219 // instructions to emit the comparison.
5220 // TODO: Handle FCMP_UEQ and FCMP_ONE. After that, this check will be
5221 // unnecessary.
5222 if (CondCode2 != AArch64CC::AL)
5223 return false;
5224
5225 if (!emitFPCompare(LHS: CondDef->getOperand(i: 2).getReg(),
5226 RHS: CondDef->getOperand(i: 3).getReg(), MIRBuilder&: MIB)) {
5227 LLVM_DEBUG(dbgs() << "Couldn't emit compare for select!\n");
5228 return false;
5229 }
5230 }
5231
5232 // Emit the select.
5233 emitSelect(Dst: I.getOperand(i: 0).getReg(), True: I.getOperand(i: 2).getReg(),
5234 False: I.getOperand(i: 3).getReg(), CC: CondCode, MIB);
5235 I.eraseFromParent();
5236 return true;
5237}
5238
5239MachineInstr *AArch64InstructionSelector::tryFoldIntegerCompare(
5240 MachineOperand &LHS, MachineOperand &RHS, MachineOperand &Predicate,
5241 MachineIRBuilder &MIRBuilder) const {
5242 assert(LHS.isReg() && RHS.isReg() && Predicate.isPredicate() &&
5243 "Unexpected MachineOperand");
5244 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
5245 // We want to find this sort of thing:
5246 // x = G_SUB 0, y
5247 // G_ICMP z, x
5248 //
5249 // In this case, we can fold the G_SUB into the G_ICMP using a CMN instead.
5250 // e.g:
5251 //
5252 // cmn z, y
5253
5254 // Check if the RHS or LHS of the G_ICMP is defined by a SUB
5255 MachineInstr *LHSDef = getDefIgnoringCopies(Reg: LHS.getReg(), MRI);
5256 MachineInstr *RHSDef = getDefIgnoringCopies(Reg: RHS.getReg(), MRI);
5257 auto P = static_cast<CmpInst::Predicate>(Predicate.getPredicate());
5258
5259 // Given this:
5260 //
5261 // x = G_SUB 0, y
5262 // G_ICMP z, x
5263 //
5264 // Produce this:
5265 //
5266 // cmn z, y
5267 if (isCMN(MaybeSub: RHSDef, Pred: P, MRI))
5268 return emitCMN(LHS, RHS&: RHSDef->getOperand(i: 2), MIRBuilder);
5269
5270 // Same idea here, but with the LHS of the compare instead:
5271 //
5272 // Given this:
5273 //
5274 // x = G_SUB 0, y
5275 // G_ICMP x, z
5276 //
5277 // Produce this:
5278 //
5279 // cmn y, z
5280 //
5281 // But be careful! We need to swap the predicate!
5282 if (isCMN(MaybeSub: LHSDef, Pred: P, MRI)) {
5283 if (!CmpInst::isEquality(pred: P)) {
5284 P = CmpInst::getSwappedPredicate(pred: P);
5285 Predicate = MachineOperand::CreatePredicate(Pred: P);
5286 }
5287 return emitCMN(LHS&: LHSDef->getOperand(i: 2), RHS, MIRBuilder);
5288 }
5289
5290 // Given this:
5291 //
5292 // z = G_AND x, y
5293 // G_ICMP z, 0
5294 //
5295 // Produce this if the compare is signed:
5296 //
5297 // tst x, y
5298 if (!CmpInst::isUnsigned(Pred: P) && LHSDef &&
5299 LHSDef->getOpcode() == TargetOpcode::G_AND) {
5300 // Make sure that the RHS is 0.
5301 auto ValAndVReg = getIConstantVRegValWithLookThrough(VReg: RHS.getReg(), MRI);
5302 if (!ValAndVReg || ValAndVReg->Value != 0)
5303 return nullptr;
5304
5305 return emitTST(LHS&: LHSDef->getOperand(i: 1),
5306 RHS&: LHSDef->getOperand(i: 2), MIRBuilder);
5307 }
5308
5309 return nullptr;
5310}
5311
5312bool AArch64InstructionSelector::selectShuffleVector(
5313 MachineInstr &I, MachineRegisterInfo &MRI) {
5314 const LLT DstTy = MRI.getType(Reg: I.getOperand(i: 0).getReg());
5315 Register Src1Reg = I.getOperand(i: 1).getReg();
5316 Register Src2Reg = I.getOperand(i: 2).getReg();
5317 ArrayRef<int> Mask = I.getOperand(i: 3).getShuffleMask();
5318 assert(DstTy == MRI.getType(Src1Reg) &&
5319 "Expected equal shuffle types during selection");
5320
5321 MachineBasicBlock &MBB = *I.getParent();
5322 MachineFunction &MF = *MBB.getParent();
5323 LLVMContext &Ctx = MF.getFunction().getContext();
5324
5325 unsigned BytesPerElt = DstTy.getElementType().getSizeInBits() / 8;
5326 int NumElts = DstTy.getNumElements();
5327
5328 SmallVector<int> NewMask;
5329 bool FirstUsed = false;
5330 bool SecondUsed = false;
5331 for (int M : Mask) {
5332 // Map any undef or zero lanes to 255.
5333 if (M < 0 || VT->getKnownBits(R: M < NumElts ? Src1Reg : Src2Reg,
5334 DemandedElts: APInt::getOneBitSet(numBits: NumElts, BitNo: M % NumElts))
5335 .isZero()) {
5336 for (unsigned Byte = 0; Byte < BytesPerElt; ++Byte)
5337 NewMask.push_back(Elt: 255);
5338 continue;
5339 }
5340
5341 FirstUsed |= M < NumElts;
5342 SecondUsed |= M >= NumElts;
5343 for (unsigned Byte = 0; Byte < BytesPerElt; ++Byte) {
5344 unsigned Offset = Byte + M * BytesPerElt;
5345 NewMask.push_back(Elt: Offset);
5346 }
5347 }
5348
5349 // If the first is unused or all zeros, use the second src in a tbl1.
5350 if (!FirstUsed) {
5351 int ByteLanes = DstTy.getSizeInBits() == 128 ? 16 : 8;
5352 for (int &M : NewMask) {
5353 if (M != 255) {
5354 assert(M >= ByteLanes && M < 2 * ByteLanes);
5355 M -= ByteLanes;
5356 }
5357 }
5358 std::swap(a&: Src1Reg, b&: Src2Reg);
5359 std::swap(a&: FirstUsed, b&: SecondUsed);
5360 }
5361
5362 // Use a constant pool to load the index vector for TBL.
5363 SmallVector<Constant *> CstIdxs;
5364 transform(Range&: NewMask, d_first: std::back_inserter(x&: CstIdxs), F: [&Ctx](int M) {
5365 return ConstantInt::get(Ty: Type::getInt8Ty(C&: Ctx), V: M);
5366 });
5367 Constant *CPVal = ConstantVector::get(V: CstIdxs);
5368 MachineInstr *IndexLoad = emitLoadFromConstantPool(CPVal, MIRBuilder&: MIB);
5369 if (!IndexLoad) {
5370 LLVM_DEBUG(dbgs() << "Could not load from a constant pool");
5371 return false;
5372 }
5373
5374 if (DstTy.getSizeInBits() != 128) {
5375 assert(DstTy.getSizeInBits() == 64 && "Unexpected shuffle result ty");
5376 // This case can be done with TBL1.
5377 MachineInstr *Concat =
5378 emitVectorConcat(Dst: std::nullopt, Op1: Src1Reg, Op2: Src2Reg, MIRBuilder&: MIB);
5379 if (!Concat) {
5380 LLVM_DEBUG(dbgs() << "Could not do vector concat for tbl1");
5381 return false;
5382 }
5383
5384 // The constant pool load will be 64 bits, so need to convert to FPR128 reg.
5385 IndexLoad = emitScalarToVector(EltSize: 64, DstRC: &AArch64::FPR128RegClass,
5386 Scalar: IndexLoad->getOperand(i: 0).getReg(), MIRBuilder&: MIB);
5387
5388 auto TBL1 = MIB.buildInstr(
5389 Opc: AArch64::TBLv16i8One, DstOps: {&AArch64::FPR128RegClass},
5390 SrcOps: {Concat->getOperand(i: 0).getReg(), IndexLoad->getOperand(i: 0).getReg()});
5391 constrainSelectedInstRegOperands(I&: *TBL1, TII, TRI, RBI);
5392
5393 auto Copy =
5394 MIB.buildInstr(Opc: TargetOpcode::COPY, DstOps: {I.getOperand(i: 0).getReg()}, SrcOps: {})
5395 .addReg(RegNo: TBL1.getReg(Idx: 0), Flags: {}, SubReg: AArch64::dsub);
5396 RBI.constrainGenericRegister(Reg: Copy.getReg(Idx: 0), RC: AArch64::FPR64RegClass, MRI);
5397 I.eraseFromParent();
5398 return true;
5399 }
5400
5401 if (!SecondUsed) {
5402 auto TBL1 = MIB.buildInstr(Opc: AArch64::TBLv16i8One, DstOps: {I.getOperand(i: 0)},
5403 SrcOps: {Src1Reg, IndexLoad->getOperand(i: 0)});
5404 constrainSelectedInstRegOperands(I&: *TBL1, TII, TRI, RBI);
5405 I.eraseFromParent();
5406 return true;
5407 }
5408
5409 // For TBL2 we need to emit a REG_SEQUENCE to tie together two consecutive
5410 // Q registers for regalloc.
5411 SmallVector<Register, 2> Regs = {Src1Reg, Src2Reg};
5412 auto RegSeq = createQTuple(Regs, MIB);
5413 auto TBL2 = MIB.buildInstr(Opc: AArch64::TBLv16i8Two, DstOps: {I.getOperand(i: 0)},
5414 SrcOps: {RegSeq, IndexLoad->getOperand(i: 0)});
5415 constrainSelectedInstRegOperands(I&: *TBL2, TII, TRI, RBI);
5416 I.eraseFromParent();
5417 return true;
5418}
5419
5420MachineInstr *AArch64InstructionSelector::emitLaneInsert(
5421 std::optional<Register> DstReg, Register SrcReg, Register EltReg,
5422 unsigned LaneIdx, const RegisterBank &RB,
5423 MachineIRBuilder &MIRBuilder) const {
5424 MachineInstr *InsElt = nullptr;
5425 const TargetRegisterClass *DstRC = &AArch64::FPR128RegClass;
5426 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
5427
5428 // Create a register to define with the insert if one wasn't passed in.
5429 if (!DstReg)
5430 DstReg = MRI.createVirtualRegister(RegClass: DstRC);
5431
5432 unsigned EltSize = MRI.getType(Reg: EltReg).getSizeInBits();
5433 unsigned Opc = getInsertVecEltOpInfo(RB, EltSize).first;
5434
5435 if (RB.getID() == AArch64::FPRRegBankID) {
5436 auto InsSub = emitScalarToVector(EltSize, DstRC, Scalar: EltReg, MIRBuilder);
5437 InsElt = MIRBuilder.buildInstr(Opc, DstOps: {*DstReg}, SrcOps: {SrcReg})
5438 .addImm(Val: LaneIdx)
5439 .addUse(RegNo: InsSub->getOperand(i: 0).getReg())
5440 .addImm(Val: 0);
5441 } else {
5442 InsElt = MIRBuilder.buildInstr(Opc, DstOps: {*DstReg}, SrcOps: {SrcReg})
5443 .addImm(Val: LaneIdx)
5444 .addUse(RegNo: EltReg);
5445 }
5446
5447 constrainSelectedInstRegOperands(I&: *InsElt, TII, TRI, RBI);
5448 return InsElt;
5449}
5450
5451bool AArch64InstructionSelector::selectUSMovFromExtend(
5452 MachineInstr &MI, MachineRegisterInfo &MRI) {
5453 if (MI.getOpcode() != TargetOpcode::G_SEXT &&
5454 MI.getOpcode() != TargetOpcode::G_ZEXT &&
5455 MI.getOpcode() != TargetOpcode::G_ANYEXT)
5456 return false;
5457 bool IsSigned = MI.getOpcode() == TargetOpcode::G_SEXT;
5458 const Register DefReg = MI.getOperand(i: 0).getReg();
5459 const LLT DstTy = MRI.getType(Reg: DefReg);
5460 unsigned DstSize = DstTy.getSizeInBits();
5461
5462 if (DstSize != 32 && DstSize != 64)
5463 return false;
5464
5465 MachineInstr *Extract = getOpcodeDef(Opcode: TargetOpcode::G_EXTRACT_VECTOR_ELT,
5466 Reg: MI.getOperand(i: 1).getReg(), MRI);
5467 int64_t Lane;
5468 if (!Extract || !mi_match(R: Extract->getOperand(i: 2).getReg(), MRI, P: m_ICst(Cst&: Lane)))
5469 return false;
5470 Register Src0 = Extract->getOperand(i: 1).getReg();
5471
5472 const LLT VecTy = MRI.getType(Reg: Src0);
5473 if (VecTy.isScalableVector())
5474 return false;
5475
5476 if (VecTy.getSizeInBits() != 128) {
5477 const MachineInstr *ScalarToVector = emitScalarToVector(
5478 EltSize: VecTy.getSizeInBits(), DstRC: &AArch64::FPR128RegClass, Scalar: Src0, MIRBuilder&: MIB);
5479 assert(ScalarToVector && "Didn't expect emitScalarToVector to fail!");
5480 Src0 = ScalarToVector->getOperand(i: 0).getReg();
5481 }
5482
5483 unsigned Opcode;
5484 if (DstSize == 64 && VecTy.getScalarSizeInBits() == 32)
5485 Opcode = IsSigned ? AArch64::SMOVvi32to64 : AArch64::UMOVvi32;
5486 else if (DstSize == 64 && VecTy.getScalarSizeInBits() == 16)
5487 Opcode = IsSigned ? AArch64::SMOVvi16to64 : AArch64::UMOVvi16;
5488 else if (DstSize == 64 && VecTy.getScalarSizeInBits() == 8)
5489 Opcode = IsSigned ? AArch64::SMOVvi8to64 : AArch64::UMOVvi8;
5490 else if (DstSize == 32 && VecTy.getScalarSizeInBits() == 16)
5491 Opcode = IsSigned ? AArch64::SMOVvi16to32 : AArch64::UMOVvi16;
5492 else if (DstSize == 32 && VecTy.getScalarSizeInBits() == 8)
5493 Opcode = IsSigned ? AArch64::SMOVvi8to32 : AArch64::UMOVvi8;
5494 else
5495 llvm_unreachable("Unexpected type combo for S/UMov!");
5496
5497 // We may need to generate one of these, depending on the type and sign of the
5498 // input:
5499 // DstReg = SMOV Src0, Lane;
5500 // NewReg = UMOV Src0, Lane; DstReg = SUBREG_TO_REG NewReg, sub_32;
5501 MachineInstr *ExtI = nullptr;
5502 if (DstSize == 64 && !IsSigned) {
5503 Register NewReg = MRI.createVirtualRegister(RegClass: &AArch64::GPR32RegClass);
5504 MIB.buildInstr(Opc: Opcode, DstOps: {NewReg}, SrcOps: {Src0}).addImm(Val: Lane);
5505 ExtI = MIB.buildInstr(Opc: AArch64::SUBREG_TO_REG, DstOps: {DefReg}, SrcOps: {})
5506 .addUse(RegNo: NewReg)
5507 .addImm(Val: AArch64::sub_32);
5508 RBI.constrainGenericRegister(Reg: DefReg, RC: AArch64::GPR64RegClass, MRI);
5509 } else
5510 ExtI = MIB.buildInstr(Opc: Opcode, DstOps: {DefReg}, SrcOps: {Src0}).addImm(Val: Lane);
5511
5512 constrainSelectedInstRegOperands(I&: *ExtI, TII, TRI, RBI);
5513 MI.eraseFromParent();
5514 return true;
5515}
5516
5517bool AArch64InstructionSelector::selectIndexedExtLoad(
5518 MachineInstr &MI, MachineRegisterInfo &MRI) {
5519 auto &ExtLd = cast<GIndexedAnyExtLoad>(Val&: MI);
5520 Register Dst = ExtLd.getDstReg();
5521 Register WriteBack = ExtLd.getWritebackReg();
5522 Register Base = ExtLd.getBaseReg();
5523 Register Offset = ExtLd.getOffsetReg();
5524 LLT Ty = MRI.getType(Reg: Dst);
5525 assert(Ty.getSizeInBits() <= 64); // Only for scalar GPRs.
5526 unsigned MemSizeBits = ExtLd.getMMO().getMemoryType().getSizeInBits();
5527 bool IsPre = ExtLd.isPre();
5528 bool IsSExt = isa<GIndexedSExtLoad>(Val: ExtLd);
5529 unsigned InsertIntoSubReg = 0;
5530 bool IsDst64 = Ty.getSizeInBits() == 64;
5531
5532 // ZExt/SExt should be on gpr but can handle extload and zextload of fpr, so
5533 // long as they are scalar.
5534 bool IsFPR = RBI.getRegBank(Reg: Dst, MRI, TRI)->getID() == AArch64::FPRRegBankID;
5535 if ((IsSExt && IsFPR) || Ty.isVector())
5536 return false;
5537
5538 unsigned Opc = 0;
5539 LLT NewLdDstTy;
5540 LLT s32 = LLT::scalar(SizeInBits: 32);
5541 LLT s64 = LLT::scalar(SizeInBits: 64);
5542
5543 if (MemSizeBits == 8) {
5544 if (IsSExt) {
5545 if (IsDst64)
5546 Opc = IsPre ? AArch64::LDRSBXpre : AArch64::LDRSBXpost;
5547 else
5548 Opc = IsPre ? AArch64::LDRSBWpre : AArch64::LDRSBWpost;
5549 NewLdDstTy = IsDst64 ? s64 : s32;
5550 } else if (IsFPR) {
5551 Opc = IsPre ? AArch64::LDRBpre : AArch64::LDRBpost;
5552 InsertIntoSubReg = AArch64::bsub;
5553 NewLdDstTy = LLT::scalar(SizeInBits: MemSizeBits);
5554 } else {
5555 Opc = IsPre ? AArch64::LDRBBpre : AArch64::LDRBBpost;
5556 InsertIntoSubReg = IsDst64 ? AArch64::sub_32 : 0;
5557 NewLdDstTy = s32;
5558 }
5559 } else if (MemSizeBits == 16) {
5560 if (IsSExt) {
5561 if (IsDst64)
5562 Opc = IsPre ? AArch64::LDRSHXpre : AArch64::LDRSHXpost;
5563 else
5564 Opc = IsPre ? AArch64::LDRSHWpre : AArch64::LDRSHWpost;
5565 NewLdDstTy = IsDst64 ? s64 : s32;
5566 } else if (IsFPR) {
5567 Opc = IsPre ? AArch64::LDRHpre : AArch64::LDRHpost;
5568 InsertIntoSubReg = AArch64::hsub;
5569 NewLdDstTy = LLT::scalar(SizeInBits: MemSizeBits);
5570 } else {
5571 Opc = IsPre ? AArch64::LDRHHpre : AArch64::LDRHHpost;
5572 InsertIntoSubReg = IsDst64 ? AArch64::sub_32 : 0;
5573 NewLdDstTy = s32;
5574 }
5575 } else if (MemSizeBits == 32) {
5576 if (IsSExt) {
5577 Opc = IsPre ? AArch64::LDRSWpre : AArch64::LDRSWpost;
5578 NewLdDstTy = s64;
5579 } else if (IsFPR) {
5580 Opc = IsPre ? AArch64::LDRSpre : AArch64::LDRSpost;
5581 InsertIntoSubReg = AArch64::ssub;
5582 NewLdDstTy = LLT::scalar(SizeInBits: MemSizeBits);
5583 } else {
5584 Opc = IsPre ? AArch64::LDRWpre : AArch64::LDRWpost;
5585 InsertIntoSubReg = IsDst64 ? AArch64::sub_32 : 0;
5586 NewLdDstTy = s32;
5587 }
5588 } else {
5589 llvm_unreachable("Unexpected size for indexed load");
5590 }
5591
5592 auto Cst = getIConstantVRegVal(VReg: Offset, MRI);
5593 if (!Cst)
5594 return false; // Shouldn't happen, but just in case.
5595
5596 auto LdMI = MIB.buildInstr(Opc, DstOps: {WriteBack, NewLdDstTy}, SrcOps: {Base})
5597 .addImm(Val: Cst->getSExtValue());
5598 LdMI.cloneMemRefs(OtherMI: ExtLd);
5599 constrainSelectedInstRegOperands(I&: *LdMI, TII, TRI, RBI);
5600 // Make sure to select the load with the MemTy as the dest type, and then
5601 // insert into a larger reg if needed.
5602 if (InsertIntoSubReg) {
5603 // Generate a SUBREG_TO_REG.
5604 auto SubToReg = MIB.buildInstr(Opc: TargetOpcode::SUBREG_TO_REG, DstOps: {Dst}, SrcOps: {})
5605 .addUse(RegNo: LdMI.getReg(Idx: 1))
5606 .addImm(Val: InsertIntoSubReg);
5607 RBI.constrainGenericRegister(
5608 Reg: SubToReg.getReg(Idx: 0),
5609 RC: *getRegClassForTypeOnBank(Ty: MRI.getType(Reg: Dst),
5610 RB: *RBI.getRegBank(Reg: Dst, MRI, TRI)),
5611 MRI);
5612 } else {
5613 auto Copy = MIB.buildCopy(Res: Dst, Op: LdMI.getReg(Idx: 1));
5614 selectCopy(I&: *Copy, TII, MRI, TRI, RBI);
5615 }
5616 MI.eraseFromParent();
5617
5618 return true;
5619}
5620
5621bool AArch64InstructionSelector::selectIndexedLoad(MachineInstr &MI,
5622 MachineRegisterInfo &MRI) {
5623 auto &Ld = cast<GIndexedLoad>(Val&: MI);
5624 Register Dst = Ld.getDstReg();
5625 Register WriteBack = Ld.getWritebackReg();
5626 Register Base = Ld.getBaseReg();
5627 Register Offset = Ld.getOffsetReg();
5628 assert(MRI.getType(Dst).getSizeInBits() <= 128 &&
5629 "Unexpected type for indexed load");
5630 unsigned MemSize = Ld.getMMO().getMemoryType().getSizeInBytes();
5631
5632 if (MemSize < MRI.getType(Reg: Dst).getSizeInBytes())
5633 return selectIndexedExtLoad(MI, MRI);
5634
5635 unsigned Opc = 0;
5636 if (Ld.isPre()) {
5637 static constexpr unsigned GPROpcodes[] = {
5638 AArch64::LDRBBpre, AArch64::LDRHHpre, AArch64::LDRWpre,
5639 AArch64::LDRXpre};
5640 static constexpr unsigned FPROpcodes[] = {
5641 AArch64::LDRBpre, AArch64::LDRHpre, AArch64::LDRSpre, AArch64::LDRDpre,
5642 AArch64::LDRQpre};
5643 Opc = (RBI.getRegBank(Reg: Dst, MRI, TRI)->getID() == AArch64::FPRRegBankID)
5644 ? FPROpcodes[Log2_32(Value: MemSize)]
5645 : GPROpcodes[Log2_32(Value: MemSize)];
5646 ;
5647 } else {
5648 static constexpr unsigned GPROpcodes[] = {
5649 AArch64::LDRBBpost, AArch64::LDRHHpost, AArch64::LDRWpost,
5650 AArch64::LDRXpost};
5651 static constexpr unsigned FPROpcodes[] = {
5652 AArch64::LDRBpost, AArch64::LDRHpost, AArch64::LDRSpost,
5653 AArch64::LDRDpost, AArch64::LDRQpost};
5654 Opc = (RBI.getRegBank(Reg: Dst, MRI, TRI)->getID() == AArch64::FPRRegBankID)
5655 ? FPROpcodes[Log2_32(Value: MemSize)]
5656 : GPROpcodes[Log2_32(Value: MemSize)];
5657 ;
5658 }
5659 auto Cst = getIConstantVRegVal(VReg: Offset, MRI);
5660 if (!Cst)
5661 return false; // Shouldn't happen, but just in case.
5662 auto LdMI =
5663 MIB.buildInstr(Opc, DstOps: {WriteBack, Dst}, SrcOps: {Base}).addImm(Val: Cst->getSExtValue());
5664 LdMI.cloneMemRefs(OtherMI: Ld);
5665 constrainSelectedInstRegOperands(I&: *LdMI, TII, TRI, RBI);
5666 MI.eraseFromParent();
5667 return true;
5668}
5669
5670bool AArch64InstructionSelector::selectIndexedStore(GIndexedStore &I,
5671 MachineRegisterInfo &MRI) {
5672 Register Dst = I.getWritebackReg();
5673 Register Val = I.getValueReg();
5674 Register Base = I.getBaseReg();
5675 Register Offset = I.getOffsetReg();
5676 assert(MRI.getType(Val).getSizeInBits() <= 128 &&
5677 "Unexpected type for indexed store");
5678
5679 LocationSize MemSize = I.getMMO().getSize();
5680 unsigned MemSizeInBytes = MemSize.getValue();
5681
5682 assert(MemSizeInBytes && MemSizeInBytes <= 16 &&
5683 "Unexpected indexed store size");
5684 unsigned MemSizeLog2 = Log2_32(Value: MemSizeInBytes);
5685
5686 unsigned Opc = 0;
5687 if (I.isPre()) {
5688 static constexpr unsigned GPROpcodes[] = {
5689 AArch64::STRBBpre, AArch64::STRHHpre, AArch64::STRWpre,
5690 AArch64::STRXpre};
5691 static constexpr unsigned FPROpcodes[] = {
5692 AArch64::STRBpre, AArch64::STRHpre, AArch64::STRSpre, AArch64::STRDpre,
5693 AArch64::STRQpre};
5694
5695 if (RBI.getRegBank(Reg: Val, MRI, TRI)->getID() == AArch64::FPRRegBankID)
5696 Opc = FPROpcodes[MemSizeLog2];
5697 else
5698 Opc = GPROpcodes[MemSizeLog2];
5699 } else {
5700 static constexpr unsigned GPROpcodes[] = {
5701 AArch64::STRBBpost, AArch64::STRHHpost, AArch64::STRWpost,
5702 AArch64::STRXpost};
5703 static constexpr unsigned FPROpcodes[] = {
5704 AArch64::STRBpost, AArch64::STRHpost, AArch64::STRSpost,
5705 AArch64::STRDpost, AArch64::STRQpost};
5706
5707 if (RBI.getRegBank(Reg: Val, MRI, TRI)->getID() == AArch64::FPRRegBankID)
5708 Opc = FPROpcodes[MemSizeLog2];
5709 else
5710 Opc = GPROpcodes[MemSizeLog2];
5711 }
5712
5713 auto Cst = getIConstantVRegVal(VReg: Offset, MRI);
5714 if (!Cst)
5715 return false; // Shouldn't happen, but just in case.
5716 auto Str =
5717 MIB.buildInstr(Opc, DstOps: {Dst}, SrcOps: {Val, Base}).addImm(Val: Cst->getSExtValue());
5718 Str.cloneMemRefs(OtherMI: I);
5719 constrainSelectedInstRegOperands(I&: *Str, TII, TRI, RBI);
5720 I.eraseFromParent();
5721 return true;
5722}
5723
5724MachineInstr *
5725AArch64InstructionSelector::emitConstantMOVVector(DstOp Dst, APInt Imm,
5726 MachineRegisterInfo &MRI) {
5727 unsigned DstSize = Imm.getBitWidth();
5728 assert((DstSize == 64 || DstSize == 128) &&
5729 "Unexpected vector constant size");
5730
5731 SmallVector<AArch64_IMM::ImmInsnModel> Insns;
5732 if (AArch64_IMM::expandVectorMOVImm(Imm, Insn&: Insns, ST: STI)) {
5733 for (AArch64_IMM::ImmInsnModel Insn : Insns) {
5734 switch (Insn.Opcode) {
5735 case AArch64::FMOVD0: {
5736 // Currently emit a MOVIv2d_ns in both cases to match SDAG.
5737 auto Mov =
5738 MIB.buildInstr(
5739 Opc: AArch64::MOVIv2d_ns,
5740 DstOps: {DstSize == 128 ? Dst : DstOp(&AArch64::FPR128RegClass)}, SrcOps: {})
5741 .addImm(Val: 0);
5742 constrainSelectedInstRegOperands(I&: *Mov, TII, TRI, RBI);
5743 if (DstSize != 128) {
5744 Mov = MIB.buildInstr(Opc: TargetOpcode::COPY, DstOps: {Dst}, SrcOps: {})
5745 .addReg(RegNo: Mov.getReg(Idx: 0), Flags: {}, SubReg: AArch64::dsub);
5746 RBI.constrainGenericRegister(Reg: Mov.getReg(Idx: 0), RC: AArch64::FPR64RegClass,
5747 MRI);
5748 }
5749 return &*Mov;
5750 }
5751 case AArch64::FMOVDi:
5752 case AArch64::FMOVv2f32_ns:
5753 case AArch64::FMOVv2f64_ns:
5754 case AArch64::FMOVv4f32_ns:
5755 case AArch64::FMOVv4f16_ns:
5756 case AArch64::FMOVv8f16_ns:
5757 case AArch64::MOVID:
5758 case AArch64::MOVIv16b_ns:
5759 case AArch64::MOVIv2d_ns:
5760 case AArch64::MOVIv2i32:
5761 case AArch64::MOVIv2s_msl:
5762 case AArch64::MOVIv4i16:
5763 case AArch64::MOVIv4i32:
5764 case AArch64::MOVIv4s_msl:
5765 case AArch64::MOVIv8b_ns:
5766 case AArch64::MOVIv8i16:
5767 case AArch64::MVNIv2i32:
5768 case AArch64::MVNIv2s_msl:
5769 case AArch64::MVNIv4i16:
5770 case AArch64::MVNIv4i32:
5771 case AArch64::MVNIv4s_msl:
5772 case AArch64::MVNIv8i16: {
5773 auto Mov = MIB.buildInstr(Opc: Insn.Opcode, DstOps: {Dst}, SrcOps: {});
5774 if (Insn.Op1)
5775 Mov.addImm(Val: *Insn.Op1);
5776 if (Insn.Op2)
5777 Mov.addImm(Val: *Insn.Op2);
5778 constrainSelectedInstRegOperands(I&: *Mov, TII, TRI, RBI);
5779 return &*Mov;
5780 }
5781 case AArch64::DUPM_ZI: {
5782 auto Mov = MIB.buildInstr(Opc: Insn.Opcode, DstOps: {&AArch64::ZPRRegClass}, SrcOps: {})
5783 .addImm(Val: *Insn.Op1);
5784 constrainSelectedInstRegOperands(I&: *Mov, TII, TRI, RBI);
5785 Mov = MIB.buildInstr(Opc: TargetOpcode::COPY, DstOps: {Dst}, SrcOps: {})
5786 .addReg(RegNo: Mov.getReg(Idx: 0), Flags: {},
5787 SubReg: DstSize == 64 ? AArch64::dsub : AArch64::zsub);
5788 RBI.constrainGenericRegister(Reg: Mov.getReg(Idx: 0),
5789 RC: DstSize == 64 ? AArch64::FPR64RegClass
5790 : AArch64::FPR128RegClass,
5791 MRI);
5792 return &*Mov;
5793 }
5794 case AArch64::DUP_ZI_D: {
5795 auto Mov = MIB.buildInstr(Opc: Insn.Opcode, DstOps: {&AArch64::ZPRRegClass}, SrcOps: {})
5796 .addImm(Val: *Insn.Op1)
5797 .addImm(Val: *Insn.Op2);
5798 constrainSelectedInstRegOperands(I&: *Mov, TII, TRI, RBI);
5799 Mov = MIB.buildInstr(Opc: TargetOpcode::COPY, DstOps: {Dst}, SrcOps: {})
5800 .addReg(RegNo: Mov.getReg(Idx: 0), Flags: {},
5801 SubReg: DstSize == 64 ? AArch64::dsub : AArch64::zsub);
5802 RBI.constrainGenericRegister(Reg: Mov.getReg(Idx: 0),
5803 RC: DstSize == 64 ? AArch64::FPR64RegClass
5804 : AArch64::FPR128RegClass,
5805 MRI);
5806 return &*Mov;
5807 }
5808 default:
5809 llvm_unreachable("Unexpected node in expandVectorMOVImm\n");
5810 }
5811 }
5812 }
5813 return nullptr;
5814}
5815
5816MachineInstr *AArch64InstructionSelector::emitConstantVector(
5817 Register Dst, Constant *CV, APInt Imm, MachineRegisterInfo &MRI) {
5818 if (auto *MovI = emitConstantMOVVector(Dst, Imm, MRI))
5819 return MovI;
5820
5821 auto *CPLoad = emitLoadFromConstantPool(CPVal: CV, MIRBuilder&: MIB);
5822 if (!CPLoad) {
5823 LLVM_DEBUG(dbgs() << "Could not generate cp load for constant vector!");
5824 return nullptr;
5825 }
5826
5827 auto Copy = MIB.buildCopy(Res: Dst, Op: CPLoad->getOperand(i: 0));
5828 RBI.constrainGenericRegister(
5829 Reg: Dst, RC: *MRI.getRegClass(Reg: CPLoad->getOperand(i: 0).getReg()), MRI);
5830 return &*Copy;
5831}
5832
5833bool AArch64InstructionSelector::tryOptConstantBuildVec(
5834 MachineInstr &I, LLT DstTy, MachineRegisterInfo &MRI) {
5835 assert(I.getOpcode() == TargetOpcode::G_BUILD_VECTOR);
5836 unsigned DstSize = DstTy.getSizeInBits();
5837 assert(DstSize <= 128 && "Unexpected build_vec type!");
5838 if (DstSize < 32)
5839 return false;
5840 // Check if we're building a constant vector, in which case we want to
5841 // generate a constant pool load instead of a vector insert sequence.
5842 SmallVector<Constant *, 16> Csts;
5843 APInt Imm(DstSize, 0);
5844 for (unsigned Idx = 1; Idx < I.getNumOperands(); ++Idx) {
5845 Register OpReg = I.getOperand(i: Idx).getReg();
5846 if (auto AnyConst = getAnyConstantVRegValWithLookThrough(
5847 VReg: OpReg, MRI, /*LookThroughInstrs=*/true,
5848 /*LookThroughAnyExt=*/true)) {
5849 MachineInstr *DefMI = MRI.getVRegDef(Reg: AnyConst->VReg);
5850 Imm |= AnyConst->Value.trunc(width: DstTy.getScalarSizeInBits())
5851 .zext(width: DstSize)
5852 .shl(shiftAmt: (Idx - 1) * DstTy.getScalarSizeInBits());
5853
5854 if (DefMI->getOpcode() == TargetOpcode::G_CONSTANT) {
5855 Csts.emplace_back(
5856 Args: ConstantInt::get(Context&: MIB.getMF().getFunction().getContext(),
5857 V: std::move(AnyConst->Value)));
5858 continue;
5859 }
5860
5861 if (DefMI->getOpcode() == TargetOpcode::G_FCONSTANT) {
5862 Csts.emplace_back(
5863 Args: const_cast<ConstantFP *>(DefMI->getOperand(i: 1).getFPImm()));
5864 continue;
5865 }
5866 }
5867 return false;
5868 }
5869 Constant *CV = ConstantVector::get(V: Csts);
5870 if (!emitConstantVector(Dst: I.getOperand(i: 0).getReg(), CV, Imm, MRI))
5871 return false;
5872 I.eraseFromParent();
5873 return true;
5874}
5875
5876bool AArch64InstructionSelector::tryOptBuildVecToSubregToReg(
5877 MachineInstr &I, MachineRegisterInfo &MRI) {
5878 // Given:
5879 // %vec = G_BUILD_VECTOR %elt, %undef, %undef, ... %undef
5880 //
5881 // Select the G_BUILD_VECTOR as a SUBREG_TO_REG from %elt.
5882 Register Dst = I.getOperand(i: 0).getReg();
5883 Register EltReg = I.getOperand(i: 1).getReg();
5884 LLT EltTy = MRI.getType(Reg: EltReg);
5885 // If the index isn't on the same bank as its elements, then this can't be a
5886 // SUBREG_TO_REG.
5887 const RegisterBank &EltRB = *RBI.getRegBank(Reg: EltReg, MRI, TRI);
5888 const RegisterBank &DstRB = *RBI.getRegBank(Reg: Dst, MRI, TRI);
5889 if (EltRB != DstRB)
5890 return false;
5891 if (any_of(Range: drop_begin(RangeOrContainer: I.operands(), N: 2), P: [&MRI](const MachineOperand &Op) {
5892 return !getOpcodeDef(Opcode: TargetOpcode::G_IMPLICIT_DEF, Reg: Op.getReg(), MRI);
5893 }))
5894 return false;
5895 unsigned SubReg;
5896 const TargetRegisterClass *EltRC = getRegClassForTypeOnBank(Ty: EltTy, RB: EltRB);
5897 if (!EltRC)
5898 return false;
5899 const TargetRegisterClass *DstRC =
5900 getRegClassForTypeOnBank(Ty: MRI.getType(Reg: Dst), RB: DstRB);
5901 if (!DstRC)
5902 return false;
5903 if (!getSubRegForClass(RC: EltRC, TRI, SubReg))
5904 return false;
5905 auto SubregToReg = MIB.buildInstr(Opc: AArch64::SUBREG_TO_REG, DstOps: {Dst}, SrcOps: {})
5906 .addUse(RegNo: EltReg)
5907 .addImm(Val: SubReg);
5908 I.eraseFromParent();
5909 constrainSelectedInstRegOperands(I&: *SubregToReg, TII, TRI, RBI);
5910 return RBI.constrainGenericRegister(Reg: Dst, RC: *DstRC, MRI);
5911}
5912
5913bool AArch64InstructionSelector::selectBuildVector(MachineInstr &I,
5914 MachineRegisterInfo &MRI) {
5915 assert(I.getOpcode() == TargetOpcode::G_BUILD_VECTOR);
5916 // Until we port more of the optimized selections, for now just use a vector
5917 // insert sequence.
5918 const LLT DstTy = MRI.getType(Reg: I.getOperand(i: 0).getReg());
5919 const LLT EltTy = MRI.getType(Reg: I.getOperand(i: 1).getReg());
5920 unsigned EltSize = EltTy.getSizeInBits();
5921
5922 if (tryOptConstantBuildVec(I, DstTy, MRI))
5923 return true;
5924 if (tryOptBuildVecToSubregToReg(I, MRI))
5925 return true;
5926
5927 if (EltSize != 8 && EltSize != 16 && EltSize != 32 && EltSize != 64)
5928 return false; // Don't support all element types yet.
5929 const RegisterBank &RB = *RBI.getRegBank(Reg: I.getOperand(i: 1).getReg(), MRI, TRI);
5930
5931 const TargetRegisterClass *DstRC = &AArch64::FPR128RegClass;
5932 MachineInstr *ScalarToVec =
5933 emitScalarToVector(EltSize: DstTy.getElementType().getSizeInBits(), DstRC,
5934 Scalar: I.getOperand(i: 1).getReg(), MIRBuilder&: MIB);
5935 if (!ScalarToVec)
5936 return false;
5937
5938 Register DstVec = ScalarToVec->getOperand(i: 0).getReg();
5939 unsigned DstSize = DstTy.getSizeInBits();
5940
5941 // Keep track of the last MI we inserted. Later on, we might be able to save
5942 // a copy using it.
5943 MachineInstr *PrevMI = ScalarToVec;
5944 for (unsigned i = 2, e = DstSize / EltSize + 1; i < e; ++i) {
5945 // Note that if we don't do a subregister copy, we can end up making an
5946 // extra register.
5947 Register OpReg = I.getOperand(i).getReg();
5948 // Do not emit inserts for undefs
5949 if (!getOpcodeDef<GImplicitDef>(Reg: OpReg, MRI)) {
5950 PrevMI = &*emitLaneInsert(DstReg: std::nullopt, SrcReg: DstVec, EltReg: OpReg, LaneIdx: i - 1, RB, MIRBuilder&: MIB);
5951 DstVec = PrevMI->getOperand(i: 0).getReg();
5952 }
5953 }
5954
5955 // If DstTy's size in bits is less than 128, then emit a subregister copy
5956 // from DstVec to the last register we've defined.
5957 if (DstSize < 128) {
5958 // Force this to be FPR using the destination vector.
5959 const TargetRegisterClass *RC =
5960 getRegClassForTypeOnBank(Ty: DstTy, RB: *RBI.getRegBank(Reg: DstVec, MRI, TRI));
5961 if (!RC)
5962 return false;
5963 if (RC != &AArch64::FPR32RegClass && RC != &AArch64::FPR64RegClass) {
5964 LLVM_DEBUG(dbgs() << "Unsupported register class!\n");
5965 return false;
5966 }
5967
5968 unsigned SubReg = 0;
5969 if (!getSubRegForClass(RC, TRI, SubReg))
5970 return false;
5971 if (SubReg != AArch64::ssub && SubReg != AArch64::dsub) {
5972 LLVM_DEBUG(dbgs() << "Unsupported destination size! (" << DstSize
5973 << "\n");
5974 return false;
5975 }
5976
5977 Register Reg = MRI.createVirtualRegister(RegClass: RC);
5978 Register DstReg = I.getOperand(i: 0).getReg();
5979
5980 MIB.buildInstr(Opc: TargetOpcode::COPY, DstOps: {DstReg}, SrcOps: {}).addReg(RegNo: DstVec, Flags: {}, SubReg);
5981 MachineOperand &RegOp = I.getOperand(i: 1);
5982 RegOp.setReg(Reg);
5983 RBI.constrainGenericRegister(Reg: DstReg, RC: *RC, MRI);
5984 } else {
5985 // We either have a vector with all elements (except the first one) undef or
5986 // at least one non-undef non-first element. In the first case, we need to
5987 // constrain the output register ourselves as we may have generated an
5988 // INSERT_SUBREG operation which is a generic operation for which the
5989 // output regclass cannot be automatically chosen.
5990 //
5991 // In the second case, there is no need to do this as it may generate an
5992 // instruction like INSvi32gpr where the regclass can be automatically
5993 // chosen.
5994 //
5995 // Also, we save a copy by re-using the destination register on the final
5996 // insert.
5997 PrevMI->getOperand(i: 0).setReg(I.getOperand(i: 0).getReg());
5998 constrainSelectedInstRegOperands(I&: *PrevMI, TII, TRI, RBI);
5999
6000 Register DstReg = PrevMI->getOperand(i: 0).getReg();
6001 if (PrevMI == ScalarToVec && DstReg.isVirtual()) {
6002 const TargetRegisterClass *RC =
6003 getRegClassForTypeOnBank(Ty: DstTy, RB: *RBI.getRegBank(Reg: DstVec, MRI, TRI));
6004 RBI.constrainGenericRegister(Reg: DstReg, RC: *RC, MRI);
6005 }
6006 }
6007
6008 I.eraseFromParent();
6009 return true;
6010}
6011
6012bool AArch64InstructionSelector::selectVectorLoadIntrinsic(unsigned Opc,
6013 unsigned NumVecs,
6014 MachineInstr &I) {
6015 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS);
6016 assert(Opc && "Expected an opcode?");
6017 assert(NumVecs > 1 && NumVecs < 5 && "Only support 2, 3, or 4 vectors");
6018 auto &MRI = *MIB.getMRI();
6019 LLT Ty = MRI.getType(Reg: I.getOperand(i: 0).getReg());
6020 unsigned Size = Ty.getSizeInBits();
6021 assert((Size == 64 || Size == 128) &&
6022 "Destination must be 64 bits or 128 bits?");
6023 unsigned SubReg = Size == 64 ? AArch64::dsub0 : AArch64::qsub0;
6024 auto Ptr = I.getOperand(i: I.getNumOperands() - 1).getReg();
6025 assert(MRI.getType(Ptr).isPointer() && "Expected a pointer type?");
6026 auto Load = MIB.buildInstr(Opc, DstOps: {Ty}, SrcOps: {Ptr});
6027 Load.cloneMemRefs(OtherMI: I);
6028 constrainSelectedInstRegOperands(I&: *Load, TII, TRI, RBI);
6029 Register SelectedLoadDst = Load->getOperand(i: 0).getReg();
6030 for (unsigned Idx = 0; Idx < NumVecs; ++Idx) {
6031 auto Vec = MIB.buildInstr(Opc: TargetOpcode::COPY, DstOps: {I.getOperand(i: Idx)}, SrcOps: {})
6032 .addReg(RegNo: SelectedLoadDst, Flags: {}, SubReg: SubReg + Idx);
6033 // Emit the subreg copies and immediately select them.
6034 // FIXME: We should refactor our copy code into an emitCopy helper and
6035 // clean up uses of this pattern elsewhere in the selector.
6036 selectCopy(I&: *Vec, TII, MRI, TRI, RBI);
6037 }
6038 return true;
6039}
6040
6041bool AArch64InstructionSelector::selectVectorLoadLaneIntrinsic(
6042 unsigned Opc, unsigned NumVecs, MachineInstr &I) {
6043 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS);
6044 assert(Opc && "Expected an opcode?");
6045 assert(NumVecs > 1 && NumVecs < 5 && "Only support 2, 3, or 4 vectors");
6046 auto &MRI = *MIB.getMRI();
6047 LLT Ty = MRI.getType(Reg: I.getOperand(i: 0).getReg());
6048 bool Narrow = Ty.getSizeInBits() == 64;
6049
6050 auto FirstSrcRegIt = I.operands_begin() + NumVecs + 1;
6051 SmallVector<Register, 4> Regs(NumVecs);
6052 std::transform(first: FirstSrcRegIt, last: FirstSrcRegIt + NumVecs, result: Regs.begin(),
6053 unary_op: [](auto MO) { return MO.getReg(); });
6054
6055 if (Narrow) {
6056 transform(Range&: Regs, d_first: Regs.begin(), F: [this](Register Reg) {
6057 return emitScalarToVector(EltSize: 64, DstRC: &AArch64::FPR128RegClass, Scalar: Reg, MIRBuilder&: MIB)
6058 ->getOperand(i: 0)
6059 .getReg();
6060 });
6061 Ty = Ty.multiplyElements(Factor: 2);
6062 }
6063
6064 Register Tuple = createQTuple(Regs, MIB);
6065 auto LaneNo = getIConstantVRegVal(VReg: (FirstSrcRegIt + NumVecs)->getReg(), MRI);
6066 if (!LaneNo)
6067 return false;
6068
6069 Register Ptr = (FirstSrcRegIt + NumVecs + 1)->getReg();
6070 auto Load = MIB.buildInstr(Opc, DstOps: {Ty}, SrcOps: {})
6071 .addReg(RegNo: Tuple)
6072 .addImm(Val: LaneNo->getZExtValue())
6073 .addReg(RegNo: Ptr);
6074 Load.cloneMemRefs(OtherMI: I);
6075 constrainSelectedInstRegOperands(I&: *Load, TII, TRI, RBI);
6076 Register SelectedLoadDst = Load->getOperand(i: 0).getReg();
6077 unsigned SubReg = AArch64::qsub0;
6078 for (unsigned Idx = 0; Idx < NumVecs; ++Idx) {
6079 auto Vec = MIB.buildInstr(Opc: TargetOpcode::COPY,
6080 DstOps: {Narrow ? DstOp(&AArch64::FPR128RegClass)
6081 : DstOp(I.getOperand(i: Idx).getReg())},
6082 SrcOps: {})
6083 .addReg(RegNo: SelectedLoadDst, Flags: {}, SubReg: SubReg + Idx);
6084 Register WideReg = Vec.getReg(Idx: 0);
6085 // Emit the subreg copies and immediately select them.
6086 selectCopy(I&: *Vec, TII, MRI, TRI, RBI);
6087 if (Narrow &&
6088 !emitNarrowVector(DstReg: I.getOperand(i: Idx).getReg(), SrcReg: WideReg, MIB, MRI))
6089 return false;
6090 }
6091 return true;
6092}
6093
6094void AArch64InstructionSelector::selectVectorStoreIntrinsic(MachineInstr &I,
6095 unsigned NumVecs,
6096 unsigned Opc) {
6097 MachineRegisterInfo &MRI = I.getParent()->getParent()->getRegInfo();
6098 LLT Ty = MRI.getType(Reg: I.getOperand(i: 1).getReg());
6099 Register Ptr = I.getOperand(i: 1 + NumVecs).getReg();
6100
6101 SmallVector<Register, 2> Regs(NumVecs);
6102 std::transform(first: I.operands_begin() + 1, last: I.operands_begin() + 1 + NumVecs,
6103 result: Regs.begin(), unary_op: [](auto MO) { return MO.getReg(); });
6104
6105 Register Tuple = Ty.getSizeInBits() == 128 ? createQTuple(Regs, MIB)
6106 : createDTuple(Regs, MIB);
6107 auto Store = MIB.buildInstr(Opc, DstOps: {}, SrcOps: {Tuple, Ptr});
6108 Store.cloneMemRefs(OtherMI: I);
6109 constrainSelectedInstRegOperands(I&: *Store, TII, TRI, RBI);
6110}
6111
6112bool AArch64InstructionSelector::selectVectorStoreLaneIntrinsic(
6113 MachineInstr &I, unsigned NumVecs, unsigned Opc) {
6114 MachineRegisterInfo &MRI = I.getParent()->getParent()->getRegInfo();
6115 LLT Ty = MRI.getType(Reg: I.getOperand(i: 1).getReg());
6116 bool Narrow = Ty.getSizeInBits() == 64;
6117
6118 SmallVector<Register, 2> Regs(NumVecs);
6119 std::transform(first: I.operands_begin() + 1, last: I.operands_begin() + 1 + NumVecs,
6120 result: Regs.begin(), unary_op: [](auto MO) { return MO.getReg(); });
6121
6122 if (Narrow)
6123 transform(Range&: Regs, d_first: Regs.begin(), F: [this](Register Reg) {
6124 return emitScalarToVector(EltSize: 64, DstRC: &AArch64::FPR128RegClass, Scalar: Reg, MIRBuilder&: MIB)
6125 ->getOperand(i: 0)
6126 .getReg();
6127 });
6128
6129 Register Tuple = createQTuple(Regs, MIB);
6130
6131 auto LaneNo = getIConstantVRegVal(VReg: I.getOperand(i: 1 + NumVecs).getReg(), MRI);
6132 if (!LaneNo)
6133 return false;
6134 Register Ptr = I.getOperand(i: 1 + NumVecs + 1).getReg();
6135 auto Store = MIB.buildInstr(Opc, DstOps: {}, SrcOps: {})
6136 .addReg(RegNo: Tuple)
6137 .addImm(Val: LaneNo->getZExtValue())
6138 .addReg(RegNo: Ptr);
6139 Store.cloneMemRefs(OtherMI: I);
6140 constrainSelectedInstRegOperands(I&: *Store, TII, TRI, RBI);
6141 return true;
6142}
6143
6144bool AArch64InstructionSelector::selectIntrinsicWithSideEffects(
6145 MachineInstr &I, MachineRegisterInfo &MRI) {
6146 // Find the intrinsic ID.
6147 unsigned IntrinID = cast<GIntrinsic>(Val&: I).getIntrinsicID();
6148
6149 const LLT S8 = LLT::scalar(SizeInBits: 8);
6150 const LLT S16 = LLT::scalar(SizeInBits: 16);
6151 const LLT S32 = LLT::scalar(SizeInBits: 32);
6152 const LLT S64 = LLT::scalar(SizeInBits: 64);
6153 const LLT P0 = LLT::pointer(AddressSpace: 0, SizeInBits: 64);
6154 // Select the instruction.
6155 switch (IntrinID) {
6156 default:
6157 return false;
6158 case Intrinsic::aarch64_ldxp:
6159 case Intrinsic::aarch64_ldaxp: {
6160 auto NewI = MIB.buildInstr(
6161 Opc: IntrinID == Intrinsic::aarch64_ldxp ? AArch64::LDXPX : AArch64::LDAXPX,
6162 DstOps: {I.getOperand(i: 0).getReg(), I.getOperand(i: 1).getReg()},
6163 SrcOps: {I.getOperand(i: 3)});
6164 NewI.cloneMemRefs(OtherMI: I);
6165 constrainSelectedInstRegOperands(I&: *NewI, TII, TRI, RBI);
6166 break;
6167 }
6168 case Intrinsic::aarch64_neon_ld1x2: {
6169 LLT Ty = MRI.getType(Reg: I.getOperand(i: 0).getReg());
6170 unsigned Opc = 0;
6171 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6172 Opc = AArch64::LD1Twov8b;
6173 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6174 Opc = AArch64::LD1Twov16b;
6175 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6176 Opc = AArch64::LD1Twov4h;
6177 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6178 Opc = AArch64::LD1Twov8h;
6179 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6180 Opc = AArch64::LD1Twov2s;
6181 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6182 Opc = AArch64::LD1Twov4s;
6183 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6184 Opc = AArch64::LD1Twov2d;
6185 else if (Ty == S64 || Ty == P0)
6186 Opc = AArch64::LD1Twov1d;
6187 else
6188 llvm_unreachable("Unexpected type for ld1x2!");
6189 selectVectorLoadIntrinsic(Opc, NumVecs: 2, I);
6190 break;
6191 }
6192 case Intrinsic::aarch64_neon_ld1x3: {
6193 LLT Ty = MRI.getType(Reg: I.getOperand(i: 0).getReg());
6194 unsigned Opc = 0;
6195 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6196 Opc = AArch64::LD1Threev8b;
6197 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6198 Opc = AArch64::LD1Threev16b;
6199 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6200 Opc = AArch64::LD1Threev4h;
6201 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6202 Opc = AArch64::LD1Threev8h;
6203 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6204 Opc = AArch64::LD1Threev2s;
6205 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6206 Opc = AArch64::LD1Threev4s;
6207 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6208 Opc = AArch64::LD1Threev2d;
6209 else if (Ty == S64 || Ty == P0)
6210 Opc = AArch64::LD1Threev1d;
6211 else
6212 llvm_unreachable("Unexpected type for ld1x3!");
6213 selectVectorLoadIntrinsic(Opc, NumVecs: 3, I);
6214 break;
6215 }
6216 case Intrinsic::aarch64_neon_ld1x4: {
6217 LLT Ty = MRI.getType(Reg: I.getOperand(i: 0).getReg());
6218 unsigned Opc = 0;
6219 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6220 Opc = AArch64::LD1Fourv8b;
6221 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6222 Opc = AArch64::LD1Fourv16b;
6223 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6224 Opc = AArch64::LD1Fourv4h;
6225 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6226 Opc = AArch64::LD1Fourv8h;
6227 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6228 Opc = AArch64::LD1Fourv2s;
6229 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6230 Opc = AArch64::LD1Fourv4s;
6231 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6232 Opc = AArch64::LD1Fourv2d;
6233 else if (Ty == S64 || Ty == P0)
6234 Opc = AArch64::LD1Fourv1d;
6235 else
6236 llvm_unreachable("Unexpected type for ld1x4!");
6237 selectVectorLoadIntrinsic(Opc, NumVecs: 4, I);
6238 break;
6239 }
6240 case Intrinsic::aarch64_neon_ld2: {
6241 LLT Ty = MRI.getType(Reg: I.getOperand(i: 0).getReg());
6242 unsigned Opc = 0;
6243 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6244 Opc = AArch64::LD2Twov8b;
6245 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6246 Opc = AArch64::LD2Twov16b;
6247 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6248 Opc = AArch64::LD2Twov4h;
6249 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6250 Opc = AArch64::LD2Twov8h;
6251 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6252 Opc = AArch64::LD2Twov2s;
6253 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6254 Opc = AArch64::LD2Twov4s;
6255 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6256 Opc = AArch64::LD2Twov2d;
6257 else if (Ty == S64 || Ty == P0)
6258 Opc = AArch64::LD1Twov1d;
6259 else
6260 llvm_unreachable("Unexpected type for ld2!");
6261 selectVectorLoadIntrinsic(Opc, NumVecs: 2, I);
6262 break;
6263 }
6264 case Intrinsic::aarch64_neon_ld2lane: {
6265 LLT Ty = MRI.getType(Reg: I.getOperand(i: 0).getReg());
6266 unsigned Opc;
6267 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8) || Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6268 Opc = AArch64::LD2i8;
6269 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16) || Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6270 Opc = AArch64::LD2i16;
6271 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32) || Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6272 Opc = AArch64::LD2i32;
6273 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) ||
6274 Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0) || Ty == S64 || Ty == P0)
6275 Opc = AArch64::LD2i64;
6276 else
6277 llvm_unreachable("Unexpected type for ld2lane!");
6278 if (!selectVectorLoadLaneIntrinsic(Opc, NumVecs: 2, I))
6279 return false;
6280 break;
6281 }
6282 case Intrinsic::aarch64_neon_ld2r: {
6283 LLT Ty = MRI.getType(Reg: I.getOperand(i: 0).getReg());
6284 unsigned Opc = 0;
6285 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6286 Opc = AArch64::LD2Rv8b;
6287 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6288 Opc = AArch64::LD2Rv16b;
6289 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6290 Opc = AArch64::LD2Rv4h;
6291 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6292 Opc = AArch64::LD2Rv8h;
6293 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6294 Opc = AArch64::LD2Rv2s;
6295 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6296 Opc = AArch64::LD2Rv4s;
6297 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6298 Opc = AArch64::LD2Rv2d;
6299 else if (Ty == S64 || Ty == P0)
6300 Opc = AArch64::LD2Rv1d;
6301 else
6302 llvm_unreachable("Unexpected type for ld2r!");
6303 selectVectorLoadIntrinsic(Opc, NumVecs: 2, I);
6304 break;
6305 }
6306 case Intrinsic::aarch64_neon_ld3: {
6307 LLT Ty = MRI.getType(Reg: I.getOperand(i: 0).getReg());
6308 unsigned Opc = 0;
6309 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6310 Opc = AArch64::LD3Threev8b;
6311 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6312 Opc = AArch64::LD3Threev16b;
6313 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6314 Opc = AArch64::LD3Threev4h;
6315 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6316 Opc = AArch64::LD3Threev8h;
6317 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6318 Opc = AArch64::LD3Threev2s;
6319 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6320 Opc = AArch64::LD3Threev4s;
6321 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6322 Opc = AArch64::LD3Threev2d;
6323 else if (Ty == S64 || Ty == P0)
6324 Opc = AArch64::LD1Threev1d;
6325 else
6326 llvm_unreachable("Unexpected type for ld3!");
6327 selectVectorLoadIntrinsic(Opc, NumVecs: 3, I);
6328 break;
6329 }
6330 case Intrinsic::aarch64_neon_ld3lane: {
6331 LLT Ty = MRI.getType(Reg: I.getOperand(i: 0).getReg());
6332 unsigned Opc;
6333 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8) || Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6334 Opc = AArch64::LD3i8;
6335 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16) || Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6336 Opc = AArch64::LD3i16;
6337 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32) || Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6338 Opc = AArch64::LD3i32;
6339 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) ||
6340 Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0) || Ty == S64 || Ty == P0)
6341 Opc = AArch64::LD3i64;
6342 else
6343 llvm_unreachable("Unexpected type for ld3lane!");
6344 if (!selectVectorLoadLaneIntrinsic(Opc, NumVecs: 3, I))
6345 return false;
6346 break;
6347 }
6348 case Intrinsic::aarch64_neon_ld3r: {
6349 LLT Ty = MRI.getType(Reg: I.getOperand(i: 0).getReg());
6350 unsigned Opc = 0;
6351 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6352 Opc = AArch64::LD3Rv8b;
6353 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6354 Opc = AArch64::LD3Rv16b;
6355 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6356 Opc = AArch64::LD3Rv4h;
6357 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6358 Opc = AArch64::LD3Rv8h;
6359 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6360 Opc = AArch64::LD3Rv2s;
6361 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6362 Opc = AArch64::LD3Rv4s;
6363 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6364 Opc = AArch64::LD3Rv2d;
6365 else if (Ty == S64 || Ty == P0)
6366 Opc = AArch64::LD3Rv1d;
6367 else
6368 llvm_unreachable("Unexpected type for ld3r!");
6369 selectVectorLoadIntrinsic(Opc, NumVecs: 3, I);
6370 break;
6371 }
6372 case Intrinsic::aarch64_neon_ld4: {
6373 LLT Ty = MRI.getType(Reg: I.getOperand(i: 0).getReg());
6374 unsigned Opc = 0;
6375 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6376 Opc = AArch64::LD4Fourv8b;
6377 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6378 Opc = AArch64::LD4Fourv16b;
6379 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6380 Opc = AArch64::LD4Fourv4h;
6381 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6382 Opc = AArch64::LD4Fourv8h;
6383 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6384 Opc = AArch64::LD4Fourv2s;
6385 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6386 Opc = AArch64::LD4Fourv4s;
6387 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6388 Opc = AArch64::LD4Fourv2d;
6389 else if (Ty == S64 || Ty == P0)
6390 Opc = AArch64::LD1Fourv1d;
6391 else
6392 llvm_unreachable("Unexpected type for ld4!");
6393 selectVectorLoadIntrinsic(Opc, NumVecs: 4, I);
6394 break;
6395 }
6396 case Intrinsic::aarch64_neon_ld4lane: {
6397 LLT Ty = MRI.getType(Reg: I.getOperand(i: 0).getReg());
6398 unsigned Opc;
6399 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8) || Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6400 Opc = AArch64::LD4i8;
6401 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16) || Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6402 Opc = AArch64::LD4i16;
6403 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32) || Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6404 Opc = AArch64::LD4i32;
6405 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) ||
6406 Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0) || Ty == S64 || Ty == P0)
6407 Opc = AArch64::LD4i64;
6408 else
6409 llvm_unreachable("Unexpected type for ld4lane!");
6410 if (!selectVectorLoadLaneIntrinsic(Opc, NumVecs: 4, I))
6411 return false;
6412 break;
6413 }
6414 case Intrinsic::aarch64_neon_ld4r: {
6415 LLT Ty = MRI.getType(Reg: I.getOperand(i: 0).getReg());
6416 unsigned Opc = 0;
6417 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6418 Opc = AArch64::LD4Rv8b;
6419 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6420 Opc = AArch64::LD4Rv16b;
6421 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6422 Opc = AArch64::LD4Rv4h;
6423 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6424 Opc = AArch64::LD4Rv8h;
6425 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6426 Opc = AArch64::LD4Rv2s;
6427 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6428 Opc = AArch64::LD4Rv4s;
6429 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6430 Opc = AArch64::LD4Rv2d;
6431 else if (Ty == S64 || Ty == P0)
6432 Opc = AArch64::LD4Rv1d;
6433 else
6434 llvm_unreachable("Unexpected type for ld4r!");
6435 selectVectorLoadIntrinsic(Opc, NumVecs: 4, I);
6436 break;
6437 }
6438 case Intrinsic::aarch64_neon_st1x2: {
6439 LLT Ty = MRI.getType(Reg: I.getOperand(i: 1).getReg());
6440 unsigned Opc;
6441 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6442 Opc = AArch64::ST1Twov8b;
6443 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6444 Opc = AArch64::ST1Twov16b;
6445 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6446 Opc = AArch64::ST1Twov4h;
6447 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6448 Opc = AArch64::ST1Twov8h;
6449 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6450 Opc = AArch64::ST1Twov2s;
6451 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6452 Opc = AArch64::ST1Twov4s;
6453 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6454 Opc = AArch64::ST1Twov2d;
6455 else if (Ty == S64 || Ty == P0)
6456 Opc = AArch64::ST1Twov1d;
6457 else
6458 llvm_unreachable("Unexpected type for st1x2!");
6459 selectVectorStoreIntrinsic(I, NumVecs: 2, Opc);
6460 break;
6461 }
6462 case Intrinsic::aarch64_neon_st1x3: {
6463 LLT Ty = MRI.getType(Reg: I.getOperand(i: 1).getReg());
6464 unsigned Opc;
6465 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6466 Opc = AArch64::ST1Threev8b;
6467 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6468 Opc = AArch64::ST1Threev16b;
6469 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6470 Opc = AArch64::ST1Threev4h;
6471 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6472 Opc = AArch64::ST1Threev8h;
6473 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6474 Opc = AArch64::ST1Threev2s;
6475 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6476 Opc = AArch64::ST1Threev4s;
6477 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6478 Opc = AArch64::ST1Threev2d;
6479 else if (Ty == S64 || Ty == P0)
6480 Opc = AArch64::ST1Threev1d;
6481 else
6482 llvm_unreachable("Unexpected type for st1x3!");
6483 selectVectorStoreIntrinsic(I, NumVecs: 3, Opc);
6484 break;
6485 }
6486 case Intrinsic::aarch64_neon_st1x4: {
6487 LLT Ty = MRI.getType(Reg: I.getOperand(i: 1).getReg());
6488 unsigned Opc;
6489 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6490 Opc = AArch64::ST1Fourv8b;
6491 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6492 Opc = AArch64::ST1Fourv16b;
6493 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6494 Opc = AArch64::ST1Fourv4h;
6495 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6496 Opc = AArch64::ST1Fourv8h;
6497 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6498 Opc = AArch64::ST1Fourv2s;
6499 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6500 Opc = AArch64::ST1Fourv4s;
6501 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6502 Opc = AArch64::ST1Fourv2d;
6503 else if (Ty == S64 || Ty == P0)
6504 Opc = AArch64::ST1Fourv1d;
6505 else
6506 llvm_unreachable("Unexpected type for st1x4!");
6507 selectVectorStoreIntrinsic(I, NumVecs: 4, Opc);
6508 break;
6509 }
6510 case Intrinsic::aarch64_neon_st2: {
6511 LLT Ty = MRI.getType(Reg: I.getOperand(i: 1).getReg());
6512 unsigned Opc;
6513 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6514 Opc = AArch64::ST2Twov8b;
6515 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6516 Opc = AArch64::ST2Twov16b;
6517 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6518 Opc = AArch64::ST2Twov4h;
6519 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6520 Opc = AArch64::ST2Twov8h;
6521 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6522 Opc = AArch64::ST2Twov2s;
6523 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6524 Opc = AArch64::ST2Twov4s;
6525 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6526 Opc = AArch64::ST2Twov2d;
6527 else if (Ty == S64 || Ty == P0)
6528 Opc = AArch64::ST1Twov1d;
6529 else
6530 llvm_unreachable("Unexpected type for st2!");
6531 selectVectorStoreIntrinsic(I, NumVecs: 2, Opc);
6532 break;
6533 }
6534 case Intrinsic::aarch64_neon_st3: {
6535 LLT Ty = MRI.getType(Reg: I.getOperand(i: 1).getReg());
6536 unsigned Opc;
6537 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6538 Opc = AArch64::ST3Threev8b;
6539 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6540 Opc = AArch64::ST3Threev16b;
6541 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6542 Opc = AArch64::ST3Threev4h;
6543 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6544 Opc = AArch64::ST3Threev8h;
6545 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6546 Opc = AArch64::ST3Threev2s;
6547 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6548 Opc = AArch64::ST3Threev4s;
6549 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6550 Opc = AArch64::ST3Threev2d;
6551 else if (Ty == S64 || Ty == P0)
6552 Opc = AArch64::ST1Threev1d;
6553 else
6554 llvm_unreachable("Unexpected type for st3!");
6555 selectVectorStoreIntrinsic(I, NumVecs: 3, Opc);
6556 break;
6557 }
6558 case Intrinsic::aarch64_neon_st4: {
6559 LLT Ty = MRI.getType(Reg: I.getOperand(i: 1).getReg());
6560 unsigned Opc;
6561 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8))
6562 Opc = AArch64::ST4Fourv8b;
6563 else if (Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6564 Opc = AArch64::ST4Fourv16b;
6565 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16))
6566 Opc = AArch64::ST4Fourv4h;
6567 else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6568 Opc = AArch64::ST4Fourv8h;
6569 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32))
6570 Opc = AArch64::ST4Fourv2s;
6571 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6572 Opc = AArch64::ST4Fourv4s;
6573 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0))
6574 Opc = AArch64::ST4Fourv2d;
6575 else if (Ty == S64 || Ty == P0)
6576 Opc = AArch64::ST1Fourv1d;
6577 else
6578 llvm_unreachable("Unexpected type for st4!");
6579 selectVectorStoreIntrinsic(I, NumVecs: 4, Opc);
6580 break;
6581 }
6582 case Intrinsic::aarch64_neon_st2lane: {
6583 LLT Ty = MRI.getType(Reg: I.getOperand(i: 1).getReg());
6584 unsigned Opc;
6585 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8) || Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6586 Opc = AArch64::ST2i8;
6587 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16) || Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6588 Opc = AArch64::ST2i16;
6589 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32) || Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6590 Opc = AArch64::ST2i32;
6591 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) ||
6592 Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0) || Ty == S64 || Ty == P0)
6593 Opc = AArch64::ST2i64;
6594 else
6595 llvm_unreachable("Unexpected type for st2lane!");
6596 if (!selectVectorStoreLaneIntrinsic(I, NumVecs: 2, Opc))
6597 return false;
6598 break;
6599 }
6600 case Intrinsic::aarch64_neon_st3lane: {
6601 LLT Ty = MRI.getType(Reg: I.getOperand(i: 1).getReg());
6602 unsigned Opc;
6603 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8) || Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6604 Opc = AArch64::ST3i8;
6605 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16) || Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6606 Opc = AArch64::ST3i16;
6607 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32) || Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6608 Opc = AArch64::ST3i32;
6609 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) ||
6610 Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0) || Ty == S64 || Ty == P0)
6611 Opc = AArch64::ST3i64;
6612 else
6613 llvm_unreachable("Unexpected type for st3lane!");
6614 if (!selectVectorStoreLaneIntrinsic(I, NumVecs: 3, Opc))
6615 return false;
6616 break;
6617 }
6618 case Intrinsic::aarch64_neon_st4lane: {
6619 LLT Ty = MRI.getType(Reg: I.getOperand(i: 1).getReg());
6620 unsigned Opc;
6621 if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S8) || Ty == LLT::fixed_vector(NumElements: 16, ScalarTy: S8))
6622 Opc = AArch64::ST4i8;
6623 else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S16) || Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: S16))
6624 Opc = AArch64::ST4i16;
6625 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S32) || Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: S32))
6626 Opc = AArch64::ST4i32;
6627 else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: S64) ||
6628 Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: P0) || Ty == S64 || Ty == P0)
6629 Opc = AArch64::ST4i64;
6630 else
6631 llvm_unreachable("Unexpected type for st4lane!");
6632 if (!selectVectorStoreLaneIntrinsic(I, NumVecs: 4, Opc))
6633 return false;
6634 break;
6635 }
6636 case Intrinsic::aarch64_mops_memset_tag: {
6637 // Transform
6638 // %dst:gpr(p0) = \
6639 // G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.aarch64.mops.memset.tag),
6640 // \ %dst:gpr(p0), %val:gpr(s64), %n:gpr(s64)
6641 // where %dst is updated, into
6642 // %Rd:GPR64common, %Rn:GPR64) = \
6643 // MOPSMemorySetTaggingPseudo \
6644 // %Rd:GPR64common, %Rn:GPR64, %Rm:GPR64
6645 // where Rd and Rn are tied.
6646 // It is expected that %val has been extended to s64 in legalization.
6647 // Note that the order of the size/value operands are swapped.
6648
6649 Register DstDef = I.getOperand(i: 0).getReg();
6650 // I.getOperand(1) is the intrinsic function
6651 Register DstUse = I.getOperand(i: 2).getReg();
6652 Register ValUse = I.getOperand(i: 3).getReg();
6653 Register SizeUse = I.getOperand(i: 4).getReg();
6654
6655 // MOPSMemorySetTaggingPseudo has two defs; the intrinsic call has only one.
6656 // Therefore an additional virtual register is required for the updated size
6657 // operand. This value is not accessible via the semantics of the intrinsic.
6658 Register SizeDef = MRI.createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 64));
6659
6660 auto Memset = MIB.buildInstr(Opc: AArch64::MOPSMemorySetTaggingPseudo,
6661 DstOps: {DstDef, SizeDef}, SrcOps: {DstUse, SizeUse, ValUse});
6662 Memset.cloneMemRefs(OtherMI: I);
6663 Memset.setOperandDead(5); // implicit-def $nzcv
6664 constrainSelectedInstRegOperands(I&: *Memset, TII, TRI, RBI);
6665 break;
6666 }
6667 case Intrinsic::ptrauth_resign_load_relative: {
6668 Register DstReg = I.getOperand(i: 0).getReg();
6669 Register ValReg = I.getOperand(i: 2).getReg();
6670 uint64_t AUTKey = I.getOperand(i: 3).getImm();
6671 Register AUTDisc = I.getOperand(i: 4).getReg();
6672 uint64_t PACKey = I.getOperand(i: 5).getImm();
6673 Register PACDisc = I.getOperand(i: 6).getReg();
6674 int64_t Addend = I.getOperand(i: 7).getImm();
6675
6676 Register AUTAddrDisc = AUTDisc;
6677 uint16_t AUTConstDiscC = 0;
6678 std::tie(args&: AUTConstDiscC, args&: AUTAddrDisc) =
6679 extractPtrauthBlendDiscriminators(Disc: AUTDisc, MRI);
6680
6681 Register PACAddrDisc = PACDisc;
6682 uint16_t PACConstDiscC = 0;
6683 std::tie(args&: PACConstDiscC, args&: PACAddrDisc) =
6684 extractPtrauthBlendDiscriminators(Disc: PACDisc, MRI);
6685
6686 MIB.buildCopy(Res: {AArch64::X16}, Op: {ValReg});
6687
6688 MIB.buildInstr(Opcode: AArch64::AUTRELLOADPAC)
6689 .addImm(Val: AUTKey)
6690 .addImm(Val: AUTConstDiscC)
6691 .addUse(RegNo: AUTAddrDisc)
6692 .addImm(Val: PACKey)
6693 .addImm(Val: PACConstDiscC)
6694 .addUse(RegNo: PACAddrDisc)
6695 .addImm(Val: Addend)
6696 .setOperandDead(8) // implicit-def $x17
6697 .setOperandDead(9) // implicit-def $nzcv
6698 .constrainAllUses(TII, TRI, RBI);
6699 MIB.buildCopy(Res: {DstReg}, Op: Register(AArch64::X16));
6700
6701 RBI.constrainGenericRegister(Reg: DstReg, RC: AArch64::GPR64RegClass, MRI);
6702 I.eraseFromParent();
6703 return true;
6704 }
6705 }
6706
6707 I.eraseFromParent();
6708 return true;
6709}
6710
6711bool AArch64InstructionSelector::selectIntrinsic(MachineInstr &I,
6712 MachineRegisterInfo &MRI) {
6713 unsigned IntrinID = cast<GIntrinsic>(Val&: I).getIntrinsicID();
6714
6715 switch (IntrinID) {
6716 default:
6717 break;
6718 case Intrinsic::ptrauth_resign: {
6719 Register DstReg = I.getOperand(i: 0).getReg();
6720 Register ValReg = I.getOperand(i: 2).getReg();
6721 uint64_t AUTKey = I.getOperand(i: 3).getImm();
6722 Register AUTDisc = I.getOperand(i: 4).getReg();
6723 uint64_t PACKey = I.getOperand(i: 5).getImm();
6724 Register PACDisc = I.getOperand(i: 6).getReg();
6725
6726 Register AUTAddrDisc = AUTDisc;
6727 uint16_t AUTConstDiscC = 0;
6728 std::tie(args&: AUTConstDiscC, args&: AUTAddrDisc) =
6729 extractPtrauthBlendDiscriminators(Disc: AUTDisc, MRI);
6730
6731 Register PACAddrDisc = PACDisc;
6732 uint16_t PACConstDiscC = 0;
6733 std::tie(args&: PACConstDiscC, args&: PACAddrDisc) =
6734 extractPtrauthBlendDiscriminators(Disc: PACDisc, MRI);
6735
6736 MIB.buildCopy(Res: {AArch64::X16}, Op: {ValReg});
6737 MIB.buildInstr(Opcode: AArch64::AUTPAC)
6738 .addImm(Val: AUTKey)
6739 .addImm(Val: AUTConstDiscC)
6740 .addUse(RegNo: AUTAddrDisc)
6741 .addImm(Val: PACKey)
6742 .addImm(Val: PACConstDiscC)
6743 .addUse(RegNo: PACAddrDisc)
6744 .setOperandDead(7) // implicit-def $x17
6745 .setOperandDead(8) // implicit-def $nzcv
6746 .constrainAllUses(TII, TRI, RBI);
6747 MIB.buildCopy(Res: {DstReg}, Op: Register(AArch64::X16));
6748
6749 RBI.constrainGenericRegister(Reg: DstReg, RC: AArch64::GPR64RegClass, MRI);
6750 I.eraseFromParent();
6751 return true;
6752 }
6753 case Intrinsic::ptrauth_auth_with_pc_and_resign: {
6754 Register DstReg = I.getOperand(i: 0).getReg();
6755 Register ValReg = I.getOperand(i: 2).getReg();
6756 uint64_t AUTKey = I.getOperand(i: 3).getImm();
6757 Register AUTDisc = I.getOperand(i: 4).getReg();
6758 Register AUTPC = I.getOperand(i: 5).getReg();
6759 uint64_t PACKey = I.getOperand(i: 6).getImm();
6760 Register PACDisc = I.getOperand(i: 7).getReg();
6761
6762 assert((AUTKey == AArch64PACKey::IA || AUTKey == AArch64PACKey::IB) &&
6763 "auth_with_pc_and_resign only supports IA and IB keys");
6764
6765 uint16_t PACConstDiscC = 0;
6766 Register PACAddrDisc;
6767 std::tie(args&: PACConstDiscC, args&: PACAddrDisc) =
6768 extractPtrauthBlendDiscriminators(Disc: PACDisc, MRI);
6769
6770 if (!PACAddrDisc.isValid())
6771 PACAddrDisc = AArch64::XZR;
6772
6773 MIB.buildCopy(Res: {AArch64::X17}, Op: {ValReg});
6774 MIB.buildCopy(Res: {AArch64::X16}, Op: {AUTDisc});
6775 MIB.buildCopy(Res: {AArch64::X15}, Op: {AUTPC});
6776
6777 MIB.buildInstr(Opcode: AArch64::AUTPCPAC)
6778 .addImm(Val: AUTKey)
6779 .addImm(Val: PACKey)
6780 .addImm(Val: PACConstDiscC)
6781 .addUse(RegNo: PACAddrDisc)
6782 .setOperandDead(5) // implicit-def $x15
6783 .setOperandDead(6) // implicit-def $x16
6784 .setOperandDead(7) // implicit-def $nzcv
6785 .constrainAllUses(TII, TRI, RBI);
6786
6787 MIB.buildCopy(Res: {DstReg}, Op: Register(AArch64::X17));
6788 RBI.constrainGenericRegister(Reg: DstReg, RC: AArch64::GPR64RegClass, MRI);
6789 I.eraseFromParent();
6790 return true;
6791 }
6792 case Intrinsic::ptrauth_auth: {
6793 Register DstReg = I.getOperand(i: 0).getReg();
6794 Register ValReg = I.getOperand(i: 2).getReg();
6795 uint64_t AUTKey = I.getOperand(i: 3).getImm();
6796 Register AUTDisc = I.getOperand(i: 4).getReg();
6797
6798 Register AUTAddrDisc = AUTDisc;
6799 uint16_t AUTConstDiscC = 0;
6800 std::tie(args&: AUTConstDiscC, args&: AUTAddrDisc) =
6801 extractPtrauthBlendDiscriminators(Disc: AUTDisc, MRI);
6802
6803 if (STI.isX16X17Safer()) {
6804 MIB.buildCopy(Res: {AArch64::X16}, Op: {ValReg});
6805 MIB.buildInstr(Opcode: AArch64::AUTx16x17)
6806 .addImm(Val: AUTKey)
6807 .addImm(Val: AUTConstDiscC)
6808 .addUse(RegNo: AUTAddrDisc)
6809 .setOperandDead(4) // implicit-def $x17
6810 .setOperandDead(5) // implicit-def $nzcv
6811 .constrainAllUses(TII, TRI, RBI);
6812 MIB.buildCopy(Res: {DstReg}, Op: Register(AArch64::X16));
6813 } else {
6814 Register ScratchReg =
6815 MRI.createVirtualRegister(RegClass: &AArch64::GPR64commonRegClass);
6816 auto Auth = MIB.buildInstr(Opcode: AArch64::AUTxMxN)
6817 .addDef(RegNo: DstReg)
6818 .addDef(RegNo: ScratchReg)
6819 .addUse(RegNo: ValReg)
6820 .addImm(Val: AUTKey)
6821 .addImm(Val: AUTConstDiscC)
6822 .addUse(RegNo: AUTAddrDisc);
6823 Auth->setImplicitPhysRegDefsDead();
6824 Auth.constrainAllUses(TII, TRI, RBI);
6825 }
6826
6827 RBI.constrainGenericRegister(Reg: DstReg, RC: AArch64::GPR64RegClass, MRI);
6828 I.eraseFromParent();
6829 return true;
6830 }
6831 case Intrinsic::frameaddress:
6832 case Intrinsic::returnaddress: {
6833 MachineFunction &MF = *I.getParent()->getParent();
6834 MachineFrameInfo &MFI = MF.getFrameInfo();
6835
6836 unsigned Depth = I.getOperand(i: 2).getImm();
6837 Register DstReg = I.getOperand(i: 0).getReg();
6838 RBI.constrainGenericRegister(Reg: DstReg, RC: AArch64::GPR64RegClass, MRI);
6839
6840 if (Depth == 0 && IntrinID == Intrinsic::returnaddress) {
6841 if (!MFReturnAddr) {
6842 // Insert the copy from LR/X30 into the entry block, before it can be
6843 // clobbered by anything.
6844 MFI.setReturnAddressIsTaken(true);
6845 MFReturnAddr = getFunctionLiveInPhysReg(
6846 MF, TII, PhysReg: AArch64::LR, RC: AArch64::GPR64RegClass, DL: I.getDebugLoc());
6847 }
6848
6849 if (STI.hasPAuth()) {
6850 MIB.buildInstr(Opc: AArch64::XPACI, DstOps: {DstReg}, SrcOps: {MFReturnAddr});
6851 } else {
6852 MIB.buildCopy(Res: {Register(AArch64::LR)}, Op: {MFReturnAddr});
6853 MIB.buildInstr(Opcode: AArch64::XPACLRI);
6854 MIB.buildCopy(Res: {DstReg}, Op: {Register(AArch64::LR)});
6855 }
6856
6857 I.eraseFromParent();
6858 return true;
6859 }
6860
6861 MFI.setFrameAddressIsTaken(true);
6862 Register FrameAddr(AArch64::FP);
6863 while (Depth--) {
6864 Register NextFrame = MRI.createVirtualRegister(RegClass: &AArch64::GPR64spRegClass);
6865 auto Ldr =
6866 MIB.buildInstr(Opc: AArch64::LDRXui, DstOps: {NextFrame}, SrcOps: {FrameAddr}).addImm(Val: 0);
6867 constrainSelectedInstRegOperands(I&: *Ldr, TII, TRI, RBI);
6868 FrameAddr = NextFrame;
6869 }
6870
6871 if (IntrinID == Intrinsic::frameaddress)
6872 MIB.buildCopy(Res: {DstReg}, Op: {FrameAddr});
6873 else {
6874 MFI.setReturnAddressIsTaken(true);
6875
6876 if (STI.hasPAuth()) {
6877 Register TmpReg = MRI.createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
6878 MIB.buildInstr(Opc: AArch64::LDRXui, DstOps: {TmpReg}, SrcOps: {FrameAddr}).addImm(Val: 1);
6879 MIB.buildInstr(Opc: AArch64::XPACI, DstOps: {DstReg}, SrcOps: {TmpReg});
6880 } else {
6881 MIB.buildInstr(Opc: AArch64::LDRXui, DstOps: {Register(AArch64::LR)}, SrcOps: {FrameAddr})
6882 .addImm(Val: 1);
6883 MIB.buildInstr(Opcode: AArch64::XPACLRI);
6884 MIB.buildCopy(Res: {DstReg}, Op: {Register(AArch64::LR)});
6885 }
6886 }
6887
6888 I.eraseFromParent();
6889 return true;
6890 }
6891 case Intrinsic::aarch64_neon_tbl2:
6892 SelectTable(I, MRI, NumVecs: 2, Opc1: AArch64::TBLv8i8Two, Opc2: AArch64::TBLv16i8Two, isExt: false);
6893 return true;
6894 case Intrinsic::aarch64_neon_tbl3:
6895 SelectTable(I, MRI, NumVecs: 3, Opc1: AArch64::TBLv8i8Three, Opc2: AArch64::TBLv16i8Three,
6896 isExt: false);
6897 return true;
6898 case Intrinsic::aarch64_neon_tbl4:
6899 SelectTable(I, MRI, NumVecs: 4, Opc1: AArch64::TBLv8i8Four, Opc2: AArch64::TBLv16i8Four, isExt: false);
6900 return true;
6901 case Intrinsic::aarch64_neon_tbx2:
6902 SelectTable(I, MRI, NumVecs: 2, Opc1: AArch64::TBXv8i8Two, Opc2: AArch64::TBXv16i8Two, isExt: true);
6903 return true;
6904 case Intrinsic::aarch64_neon_tbx3:
6905 SelectTable(I, MRI, NumVecs: 3, Opc1: AArch64::TBXv8i8Three, Opc2: AArch64::TBXv16i8Three, isExt: true);
6906 return true;
6907 case Intrinsic::aarch64_neon_tbx4:
6908 SelectTable(I, MRI, NumVecs: 4, Opc1: AArch64::TBXv8i8Four, Opc2: AArch64::TBXv16i8Four, isExt: true);
6909 return true;
6910 case Intrinsic::swift_async_context_addr:
6911 auto Sub = MIB.buildInstr(Opc: AArch64::SUBXri, DstOps: {I.getOperand(i: 0).getReg()},
6912 SrcOps: {Register(AArch64::FP)})
6913 .addImm(Val: 8)
6914 .addImm(Val: 0);
6915 constrainSelectedInstRegOperands(I&: *Sub, TII, TRI, RBI);
6916
6917 MF->getFrameInfo().setFrameAddressIsTaken(true);
6918 MF->getInfo<AArch64FunctionInfo>()->setHasSwiftAsyncContext(true);
6919 I.eraseFromParent();
6920 return true;
6921 }
6922 return false;
6923}
6924
6925// G_PTRAUTH_GLOBAL_VALUE lowering
6926//
6927// We have 3 lowering alternatives to choose from:
6928// - MOVaddrPAC: similar to MOVaddr, with added PAC.
6929// If the GV doesn't need a GOT load (i.e., is locally defined)
6930// materialize the pointer using adrp+add+pac. See LowerMOVaddrPAC.
6931//
6932// - LOADgotPAC: similar to LOADgot, with added PAC.
6933// If the GV needs a GOT load, materialize the pointer using the usual
6934// GOT adrp+ldr, +pac. Pointers in GOT are assumed to be not signed, the GOT
6935// section is assumed to be read-only (for example, via relro mechanism). See
6936// LowerMOVaddrPAC.
6937//
6938// - LOADauthptrstatic: similar to LOADgot, but use a
6939// special stub slot instead of a GOT slot.
6940// Load a signed pointer for symbol 'sym' from a stub slot named
6941// 'sym$auth_ptr$key$disc' filled by dynamic linker during relocation
6942// resolving. This usually lowers to adrp+ldr, but also emits an entry into
6943// .data with an
6944// @AUTH relocation. See LowerLOADauthptrstatic.
6945//
6946// All 3 are pseudos that are expand late to longer sequences: this lets us
6947// provide integrity guarantees on the to-be-signed intermediate values.
6948//
6949// LOADauthptrstatic is undesirable because it requires a large section filled
6950// with often similarly-signed pointers, making it a good harvesting target.
6951// Thus, it's only used for ptrauth references to extern_weak to avoid null
6952// checks.
6953
6954bool AArch64InstructionSelector::selectPtrAuthGlobalValue(
6955 MachineInstr &I, MachineRegisterInfo &MRI) const {
6956 Register DefReg = I.getOperand(i: 0).getReg();
6957 Register Addr = I.getOperand(i: 1).getReg();
6958 uint64_t Key = I.getOperand(i: 2).getImm();
6959 Register AddrDisc = I.getOperand(i: 3).getReg();
6960 uint64_t Disc = I.getOperand(i: 4).getImm();
6961 int64_t Offset = 0;
6962
6963 if (Key > AArch64PACKey::LAST)
6964 report_fatal_error(reason: "key in ptrauth global out of range [0, " +
6965 Twine((int)AArch64PACKey::LAST) + "]");
6966
6967 // Blend only works if the integer discriminator is 16-bit wide.
6968 if (!isUInt<16>(x: Disc))
6969 report_fatal_error(
6970 reason: "constant discriminator in ptrauth global out of range [0, 0xffff]");
6971
6972 // Choosing between 3 lowering alternatives is target-specific.
6973 if (!STI.isTargetELF() && !STI.isTargetMachO())
6974 report_fatal_error(reason: "ptrauth global lowering only supported on MachO/ELF");
6975
6976 if (!MRI.hasOneDef(RegNo: Addr))
6977 return false;
6978
6979 // First match any offset we take from the real global.
6980 const MachineInstr *DefMI = &*MRI.def_instr_begin(RegNo: Addr);
6981 if (DefMI->getOpcode() == TargetOpcode::G_PTR_ADD) {
6982 Register OffsetReg = DefMI->getOperand(i: 2).getReg();
6983 if (!MRI.hasOneDef(RegNo: OffsetReg))
6984 return false;
6985 const MachineInstr &OffsetMI = *MRI.def_instr_begin(RegNo: OffsetReg);
6986 if (OffsetMI.getOpcode() != TargetOpcode::G_CONSTANT)
6987 return false;
6988
6989 Addr = DefMI->getOperand(i: 1).getReg();
6990 if (!MRI.hasOneDef(RegNo: Addr))
6991 return false;
6992
6993 DefMI = &*MRI.def_instr_begin(RegNo: Addr);
6994 Offset = OffsetMI.getOperand(i: 1).getCImm()->getSExtValue();
6995 }
6996
6997 // We should be left with a genuine unauthenticated GlobalValue.
6998 const GlobalValue *GV;
6999 if (DefMI->getOpcode() == TargetOpcode::G_GLOBAL_VALUE) {
7000 GV = DefMI->getOperand(i: 1).getGlobal();
7001 Offset += DefMI->getOperand(i: 1).getOffset();
7002 } else if (DefMI->getOpcode() == AArch64::G_ADD_LOW) {
7003 GV = DefMI->getOperand(i: 2).getGlobal();
7004 Offset += DefMI->getOperand(i: 2).getOffset();
7005 } else {
7006 return false;
7007 }
7008
7009 MachineIRBuilder MIB(I);
7010
7011 // Classify the reference to determine whether it needs a GOT load.
7012 unsigned OpFlags = STI.ClassifyGlobalReference(GV, TM);
7013 const bool NeedsGOTLoad = ((OpFlags & AArch64II::MO_GOT) != 0);
7014 assert(((OpFlags & (~AArch64II::MO_GOT)) == 0) &&
7015 "unsupported non-GOT op flags on ptrauth global reference");
7016 assert((!GV->hasExternalWeakLinkage() || NeedsGOTLoad) &&
7017 "unsupported non-GOT reference to weak ptrauth global");
7018
7019 std::optional<APInt> AddrDiscVal = getIConstantVRegVal(VReg: AddrDisc, MRI);
7020 bool HasAddrDisc = !AddrDiscVal || *AddrDiscVal != 0;
7021
7022 // Non-extern_weak:
7023 // - No GOT load needed -> MOVaddrPAC
7024 // - GOT load for non-extern_weak -> LOADgotPAC
7025 // Note that we disallow extern_weak refs to avoid null checks later.
7026 if (!GV->hasExternalWeakLinkage()) {
7027 MIB.buildInstr(Opcode: NeedsGOTLoad ? AArch64::LOADgotPAC : AArch64::MOVaddrPAC)
7028 .addGlobalAddress(GV, Offset)
7029 .addImm(Val: Key)
7030 .addReg(RegNo: HasAddrDisc ? AddrDisc : AArch64::XZR)
7031 .addImm(Val: Disc)
7032 .constrainAllUses(TII, TRI, RBI);
7033 MIB.buildCopy(Res: DefReg, Op: Register(AArch64::X16));
7034 RBI.constrainGenericRegister(Reg: DefReg, RC: AArch64::GPR64RegClass, MRI);
7035 I.eraseFromParent();
7036 return true;
7037 }
7038
7039 // extern_weak -> LOADauthptrstatic
7040
7041 // Offsets and extern_weak don't mix well: ptrauth aside, you'd get the
7042 // offset alone as a pointer if the symbol wasn't available, which would
7043 // probably break null checks in users. Ptrauth complicates things further:
7044 // error out.
7045 if (Offset != 0)
7046 report_fatal_error(
7047 reason: "unsupported non-zero offset in weak ptrauth global reference");
7048
7049 if (HasAddrDisc)
7050 report_fatal_error(reason: "unsupported weak addr-div ptrauth global");
7051
7052 MIB.buildInstr(Opc: AArch64::LOADauthptrstatic, DstOps: {DefReg}, SrcOps: {})
7053 .addGlobalAddress(GV, Offset)
7054 .addImm(Val: Key)
7055 .addImm(Val: Disc);
7056 RBI.constrainGenericRegister(Reg: DefReg, RC: AArch64::GPR64RegClass, MRI);
7057
7058 I.eraseFromParent();
7059 return true;
7060}
7061
7062void AArch64InstructionSelector::SelectTable(MachineInstr &I,
7063 MachineRegisterInfo &MRI,
7064 unsigned NumVec, unsigned Opc1,
7065 unsigned Opc2, bool isExt) {
7066 Register DstReg = I.getOperand(i: 0).getReg();
7067 unsigned Opc = MRI.getType(Reg: DstReg) == LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 8) ? Opc1 : Opc2;
7068
7069 // Create the REG_SEQUENCE
7070 SmallVector<Register, 4> Regs;
7071 for (unsigned i = 0; i < NumVec; i++)
7072 Regs.push_back(Elt: I.getOperand(i: i + 2 + isExt).getReg());
7073 Register RegSeq = createQTuple(Regs, MIB);
7074
7075 Register IdxReg = I.getOperand(i: 2 + NumVec + isExt).getReg();
7076 MachineInstrBuilder Instr;
7077 if (isExt) {
7078 Register Reg = I.getOperand(i: 2).getReg();
7079 Instr = MIB.buildInstr(Opc, DstOps: {DstReg}, SrcOps: {Reg, RegSeq, IdxReg});
7080 } else
7081 Instr = MIB.buildInstr(Opc, DstOps: {DstReg}, SrcOps: {RegSeq, IdxReg});
7082 constrainSelectedInstRegOperands(I&: *Instr, TII, TRI, RBI);
7083 I.eraseFromParent();
7084}
7085
7086InstructionSelector::ComplexRendererFns
7087AArch64InstructionSelector::selectShiftA_32(const MachineOperand &Root) const {
7088 auto MaybeImmed = getImmedFromMO(Root);
7089 if (MaybeImmed == std::nullopt || *MaybeImmed > 31)
7090 return std::nullopt;
7091 uint64_t Enc = (32 - *MaybeImmed) & 0x1f;
7092 return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Enc); }}};
7093}
7094
7095InstructionSelector::ComplexRendererFns
7096AArch64InstructionSelector::selectShiftB_32(const MachineOperand &Root) const {
7097 auto MaybeImmed = getImmedFromMO(Root);
7098 if (MaybeImmed == std::nullopt || *MaybeImmed > 31)
7099 return std::nullopt;
7100 uint64_t Enc = 31 - *MaybeImmed;
7101 return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Enc); }}};
7102}
7103
7104InstructionSelector::ComplexRendererFns
7105AArch64InstructionSelector::selectShiftA_64(const MachineOperand &Root) const {
7106 auto MaybeImmed = getImmedFromMO(Root);
7107 if (MaybeImmed == std::nullopt || *MaybeImmed > 63)
7108 return std::nullopt;
7109 uint64_t Enc = (64 - *MaybeImmed) & 0x3f;
7110 return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Enc); }}};
7111}
7112
7113InstructionSelector::ComplexRendererFns
7114AArch64InstructionSelector::selectShiftB_64(const MachineOperand &Root) const {
7115 auto MaybeImmed = getImmedFromMO(Root);
7116 if (MaybeImmed == std::nullopt || *MaybeImmed > 63)
7117 return std::nullopt;
7118 uint64_t Enc = 63 - *MaybeImmed;
7119 return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Enc); }}};
7120}
7121
7122template <unsigned ShiftWidth>
7123InstructionSelector::ComplexRendererFns
7124AArch64InstructionSelector::selectShiftMask(MachineOperand &Root) const {
7125 if (!Root.isReg())
7126 return std::nullopt;
7127
7128 MachineRegisterInfo &MRI =
7129 Root.getParent()->getParent()->getParent()->getRegInfo();
7130
7131 Register ShAmtReg = Root.getReg();
7132
7133 // Peek through zext for i32 shifts only. For i64 shifts the zext case
7134 // is already handled by existing patterns in the Shift multiclass.
7135 if (ShiftWidth == 32) {
7136 Register ZExtSrcReg;
7137 if (mi_match(R: ShAmtReg, MRI, P: m_GZExt(Src: m_Reg(R&: ZExtSrcReg))))
7138 ShAmtReg = ZExtSrcReg;
7139 }
7140
7141 // Remove AND if the mask covers at least the low log2(ShiftWidth) bits.
7142 APInt AndMask;
7143 Register AndSrcReg;
7144 if (mi_match(R: ShAmtReg, MRI, P: m_GAnd(L: m_Reg(R&: AndSrcReg), R: m_ICst(Cst&: AndMask))) &&
7145 MRI.getType(Reg: ShAmtReg).getSizeInBits() == ShiftWidth) {
7146 if (AndMask.countr_one() >= Log2_32(Value: ShiftWidth))
7147 ShAmtReg = AndSrcReg;
7148 }
7149
7150 // If shifting by X+/-N where N == 0 mod ShiftWidth, then just shift by X
7151 // to avoid the ADD/SUB. The low log2(ShiftWidth) bits are unchanged, so the
7152 // shift can use X directly; the original ADD/SUB stays for any other users.
7153 Register AddSrcReg;
7154 int64_t AddImm;
7155 if ((mi_match(R: ShAmtReg, MRI,
7156 P: m_GAdd(L: m_Reg(R&: AddSrcReg), R: m_ICstOrSplat(Cst&: AddImm))) ||
7157 mi_match(R: ShAmtReg, MRI,
7158 P: m_GSub(L: m_Reg(R&: AddSrcReg), R: m_ICstOrSplat(Cst&: AddImm)))) &&
7159 (AddImm % ShiftWidth == 0)) {
7160 ShAmtReg = AddSrcReg;
7161 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: ShAmtReg); }}};
7162 }
7163
7164 // If shifting by N-X where N == 0 mod ShiftWidth, then just shift by -X
7165 // to generate a NEG instead of a SUB from a constant.
7166 Register SubSrcReg;
7167 int64_t SubImm;
7168 if (MRI.hasOneUse(RegNo: ShAmtReg) &&
7169 mi_match(R: ShAmtReg, MRI, P: m_GSub(L: m_ICst(Cst&: SubImm), R: m_Reg(R&: SubSrcReg))) &&
7170 SubImm != 0 && (SubImm % ShiftWidth == 0)) {
7171 return {{[=](MachineInstrBuilder &MIB) {
7172 MachineInstr *I = MIB.getInstr();
7173 MachineRegisterInfo &MRI2 = I->getMF()->getRegInfo();
7174 const TargetRegisterClass &RC =
7175 ShiftWidth == 32 ? AArch64::GPR32RegClass : AArch64::GPR64RegClass;
7176 unsigned SubOpc = ShiftWidth == 32 ? AArch64::SUBWrr : AArch64::SUBXrr;
7177 Register ZeroReg = ShiftWidth == 32 ? AArch64::WZR : AArch64::XZR;
7178 Register NegReg = MRI2.createVirtualRegister(RegClass: &RC);
7179 auto NegMI = BuildMI(BB&: *I->getParent(), I&: *I, MIMD: I->getDebugLoc(),
7180 MCID: TII.get(Opcode: SubOpc), DestReg: NegReg)
7181 .addReg(RegNo: ZeroReg)
7182 .addReg(RegNo: SubSrcReg);
7183 constrainSelectedInstRegOperands(I&: *NegMI, TII, TRI, RBI);
7184 MIB.addReg(RegNo: NegReg);
7185 }}};
7186 }
7187
7188 // If shifting by N-X where N == -1 mod ShiftWidth, then just shift by ~X
7189 // to generate a NOT (MVN) instead of a SUB from a constant.
7190 Register NotSrcReg;
7191 int64_t NotImm;
7192 if (MRI.hasOneUse(RegNo: ShAmtReg) &&
7193 mi_match(R: ShAmtReg, MRI, P: m_GSub(L: m_ICst(Cst&: NotImm), R: m_Reg(R&: NotSrcReg))) &&
7194 (NotImm % ShiftWidth == ShiftWidth - 1)) {
7195 return {{[=](MachineInstrBuilder &MIB) {
7196 MachineInstr *I = MIB.getInstr();
7197 MachineRegisterInfo &MRI2 = I->getMF()->getRegInfo();
7198 const TargetRegisterClass &RC =
7199 ShiftWidth == 32 ? AArch64::GPR32RegClass : AArch64::GPR64RegClass;
7200 unsigned NotOpc = ShiftWidth == 32 ? AArch64::ORNWrr : AArch64::ORNXrr;
7201 Register ZeroReg = ShiftWidth == 32 ? AArch64::WZR : AArch64::XZR;
7202 Register NotReg = MRI2.createVirtualRegister(RegClass: &RC);
7203 auto NotMI = BuildMI(BB&: *I->getParent(), I&: *I, MIMD: I->getDebugLoc(),
7204 MCID: TII.get(Opcode: NotOpc), DestReg: NotReg)
7205 .addReg(RegNo: ZeroReg)
7206 .addReg(RegNo: NotSrcReg);
7207 constrainSelectedInstRegOperands(I&: *NotMI, TII, TRI, RBI);
7208 MIB.addReg(RegNo: NotReg);
7209 }}};
7210 }
7211
7212 // Only succeed if we changed the shift amount; otherwise let other
7213 // patterns (e.g. zext GPR32 -> SUBREG_TO_REG) match instead.
7214 if (ShAmtReg == Root.getReg())
7215 return std::nullopt;
7216
7217 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: ShAmtReg); }}};
7218}
7219
7220/// Helper to select an immediate value that can be represented as a 12-bit
7221/// value shifted left by either 0 or 12. If it is possible to do so, return
7222/// the immediate and shift value. If not, return std::nullopt.
7223///
7224/// Used by selectArithImmed and selectNegArithImmed.
7225InstructionSelector::ComplexRendererFns
7226AArch64InstructionSelector::select12BitValueWithLeftShift(
7227 uint64_t Immed) const {
7228 unsigned ShiftAmt;
7229 if (Immed >> 12 == 0) {
7230 ShiftAmt = 0;
7231 } else if ((Immed & 0xfff) == 0 && Immed >> 24 == 0) {
7232 ShiftAmt = 12;
7233 Immed = Immed >> 12;
7234 } else
7235 return std::nullopt;
7236
7237 unsigned ShVal = AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: ShiftAmt);
7238 return {{
7239 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Immed); },
7240 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: ShVal); },
7241 }};
7242}
7243
7244/// SelectArithImmed - Select an immediate value that can be represented as
7245/// a 12-bit value shifted left by either 0 or 12. If so, return true with
7246/// Val set to the 12-bit value and Shift set to the shifter operand.
7247InstructionSelector::ComplexRendererFns
7248AArch64InstructionSelector::selectArithImmed(MachineOperand &Root) const {
7249 // This function is called from the addsub_shifted_imm ComplexPattern,
7250 // which lists [imm] as the list of opcode it's interested in, however
7251 // we still need to check whether the operand is actually an immediate
7252 // here because the ComplexPattern opcode list is only used in
7253 // root-level opcode matching.
7254 auto MaybeImmed = getImmedFromMO(Root);
7255 if (MaybeImmed == std::nullopt)
7256 return std::nullopt;
7257 return select12BitValueWithLeftShift(Immed: *MaybeImmed);
7258}
7259
7260/// SelectNegArithImmed - As above, but negates the value before trying to
7261/// select it.
7262InstructionSelector::ComplexRendererFns
7263AArch64InstructionSelector::selectNegArithImmed(MachineOperand &Root) const {
7264 // We need a register here, because we need to know if we have a 64 or 32
7265 // bit immediate.
7266 if (!Root.isReg())
7267 return std::nullopt;
7268 auto MaybeImmed = getImmedFromMO(Root);
7269 if (MaybeImmed == std::nullopt)
7270 return std::nullopt;
7271 uint64_t Immed = *MaybeImmed;
7272
7273 // This negation is almost always valid, but "cmp wN, #0" and "cmn wN, #0"
7274 // have the opposite effect on the C flag, so this pattern mustn't match under
7275 // those circumstances.
7276 if (Immed == 0)
7277 return std::nullopt;
7278
7279 // Check if we're dealing with a 32-bit type on the root or a 64-bit type on
7280 // the root.
7281 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
7282 if (MRI.getType(Reg: Root.getReg()).getSizeInBits() == 32)
7283 Immed = ~((uint32_t)Immed) + 1;
7284 else
7285 Immed = ~Immed + 1ULL;
7286
7287 if (Immed & 0xFFFFFFFFFF000000ULL)
7288 return std::nullopt;
7289
7290 Immed &= 0xFFFFFFULL;
7291 return select12BitValueWithLeftShift(Immed);
7292}
7293
7294/// Checks if we are sure that folding MI into load/store addressing mode is
7295/// beneficial or not.
7296///
7297/// Returns:
7298/// - true if folding MI would be beneficial.
7299/// - false if folding MI would be bad.
7300/// - std::nullopt if it is not sure whether folding MI is beneficial.
7301///
7302/// \p MI can be the offset operand of G_PTR_ADD, e.g. G_SHL in the example:
7303///
7304/// %13:gpr(s64) = G_CONSTANT i64 1
7305/// %8:gpr(s64) = G_SHL %6, %13(s64)
7306/// %9:gpr(p0) = G_PTR_ADD %0, %8(s64)
7307/// %12:gpr(s32) = G_LOAD %9(p0) :: (load (s16))
7308std::optional<bool> AArch64InstructionSelector::isWorthFoldingIntoAddrMode(
7309 const MachineInstr &MI, const MachineRegisterInfo &MRI) const {
7310 if (MI.getOpcode() == AArch64::G_SHL) {
7311 // Address operands with shifts are free, except for running on subtargets
7312 // with AddrLSLSlow14.
7313 if (const auto ValAndVeg = getIConstantVRegValWithLookThrough(
7314 VReg: MI.getOperand(i: 2).getReg(), MRI)) {
7315 const APInt ShiftVal = ValAndVeg->Value;
7316
7317 // Don't fold if we know this will be slow.
7318 return !(STI.hasAddrLSLSlow14() && (ShiftVal == 1 || ShiftVal == 4));
7319 }
7320 }
7321 return std::nullopt;
7322}
7323
7324/// Return true if it is worth folding MI into an extended register. That is,
7325/// if it's safe to pull it into the addressing mode of a load or store as a
7326/// shift.
7327/// \p IsAddrOperand whether the def of MI is used as an address operand
7328/// (e.g. feeding into an LDR/STR).
7329bool AArch64InstructionSelector::isWorthFoldingIntoExtendedReg(
7330 const MachineInstr &MI, const MachineRegisterInfo &MRI,
7331 bool IsAddrOperand) const {
7332
7333 // Always fold if there is one use, or if we're optimizing for size.
7334 Register DefReg = MI.getOperand(i: 0).getReg();
7335 if (MRI.hasOneNonDBGUse(RegNo: DefReg) ||
7336 MI.getParent()->getParent()->getFunction().hasOptSize())
7337 return true;
7338
7339 if (IsAddrOperand) {
7340 // If we are already sure that folding MI is good or bad, return the result.
7341 if (const auto Worth = isWorthFoldingIntoAddrMode(MI, MRI))
7342 return *Worth;
7343
7344 // Fold G_PTR_ADD if its offset operand can be folded
7345 if (MI.getOpcode() == AArch64::G_PTR_ADD) {
7346 MachineInstr *OffsetInst =
7347 getDefIgnoringCopies(Reg: MI.getOperand(i: 2).getReg(), MRI);
7348
7349 // Note, we already know G_PTR_ADD is used by at least two instructions.
7350 // If we are also sure about whether folding is beneficial or not,
7351 // return the result.
7352 if (const auto Worth = isWorthFoldingIntoAddrMode(MI: *OffsetInst, MRI))
7353 return *Worth;
7354 }
7355 }
7356
7357 // FIXME: Consider checking HasALULSLFast as appropriate.
7358
7359 // We have a fastpath, so folding a shift in and potentially computing it
7360 // many times may be beneficial. Check if this is only used in memory ops.
7361 // If it is, then we should fold.
7362 return all_of(Range: MRI.use_nodbg_instructions(Reg: DefReg),
7363 P: [](MachineInstr &Use) { return Use.mayLoadOrStore(); });
7364}
7365
7366InstructionSelector::ComplexRendererFns
7367AArch64InstructionSelector::selectExtendedSHL(
7368 MachineOperand &Root, MachineOperand &Base, MachineOperand &Offset,
7369 unsigned SizeInBytes, bool WantsExt) const {
7370 assert(Base.isReg() && "Expected base to be a register operand");
7371 assert(Offset.isReg() && "Expected offset to be a register operand");
7372
7373 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
7374 MachineInstr *OffsetInst = MRI.getVRegDef(Reg: Offset.getReg());
7375
7376 unsigned OffsetOpc = OffsetInst->getOpcode();
7377 bool LookedThroughZExt = false;
7378 if (OffsetOpc != TargetOpcode::G_SHL && OffsetOpc != TargetOpcode::G_MUL) {
7379 // Try to look through a ZEXT.
7380 if (OffsetOpc != TargetOpcode::G_ZEXT || !WantsExt)
7381 return std::nullopt;
7382
7383 OffsetInst = MRI.getVRegDef(Reg: OffsetInst->getOperand(i: 1).getReg());
7384 OffsetOpc = OffsetInst->getOpcode();
7385 LookedThroughZExt = true;
7386
7387 if (OffsetOpc != TargetOpcode::G_SHL && OffsetOpc != TargetOpcode::G_MUL)
7388 return std::nullopt;
7389 }
7390 // Make sure that the memory op is a valid size.
7391 int64_t LegalShiftVal = Log2_32(Value: SizeInBytes);
7392 if (LegalShiftVal == 0)
7393 return std::nullopt;
7394 if (!isWorthFoldingIntoExtendedReg(MI: *OffsetInst, MRI, IsAddrOperand: true))
7395 return std::nullopt;
7396
7397 // Now, try to find the specific G_CONSTANT. Start by assuming that the
7398 // register we will offset is the LHS, and the register containing the
7399 // constant is the RHS.
7400 Register OffsetReg = OffsetInst->getOperand(i: 1).getReg();
7401 Register ConstantReg = OffsetInst->getOperand(i: 2).getReg();
7402 auto ValAndVReg = getIConstantVRegValWithLookThrough(VReg: ConstantReg, MRI);
7403 if (!ValAndVReg) {
7404 // We didn't get a constant on the RHS. If the opcode is a shift, then
7405 // we're done.
7406 if (OffsetOpc == TargetOpcode::G_SHL)
7407 return std::nullopt;
7408
7409 // If we have a G_MUL, we can use either register. Try looking at the RHS.
7410 std::swap(a&: OffsetReg, b&: ConstantReg);
7411 ValAndVReg = getIConstantVRegValWithLookThrough(VReg: ConstantReg, MRI);
7412 if (!ValAndVReg)
7413 return std::nullopt;
7414 }
7415
7416 // The value must fit into 3 bits, and must be positive. Make sure that is
7417 // true.
7418 int64_t ImmVal = ValAndVReg->Value.getSExtValue();
7419
7420 // Since we're going to pull this into a shift, the constant value must be
7421 // a power of 2. If we got a multiply, then we need to check this.
7422 if (OffsetOpc == TargetOpcode::G_MUL) {
7423 if (!llvm::has_single_bit<uint32_t>(Value: ImmVal))
7424 return std::nullopt;
7425
7426 // Got a power of 2. So, the amount we'll shift is the log base-2 of that.
7427 ImmVal = Log2_32(Value: ImmVal);
7428 }
7429
7430 if ((ImmVal & 0x7) != ImmVal)
7431 return std::nullopt;
7432
7433 // We are only allowed to shift by LegalShiftVal. This shift value is built
7434 // into the instruction, so we can't just use whatever we want.
7435 if (ImmVal != LegalShiftVal)
7436 return std::nullopt;
7437
7438 unsigned SignExtend = 0;
7439 if (WantsExt) {
7440 // Check if the offset is defined by an extend, unless we looked through a
7441 // G_ZEXT earlier.
7442 if (!LookedThroughZExt) {
7443 MachineInstr *ExtInst = getDefIgnoringCopies(Reg: OffsetReg, MRI);
7444 auto Ext = getExtendTypeForInst(MI&: *ExtInst, MRI, IsLoadStore: true);
7445 if (Ext == AArch64_AM::InvalidShiftExtend)
7446 return std::nullopt;
7447
7448 SignExtend = AArch64_AM::isSignExtendShiftType(Type: Ext) ? 1 : 0;
7449 // We only support SXTW for signed extension here.
7450 if (SignExtend && Ext != AArch64_AM::SXTW)
7451 return std::nullopt;
7452 OffsetReg = ExtInst->getOperand(i: 1).getReg();
7453 }
7454
7455 // Need a 32-bit wide register here.
7456 MachineIRBuilder MIB(*MRI.getVRegDef(Reg: Root.getReg()));
7457 OffsetReg = moveScalarRegClass(Reg: OffsetReg, RC: AArch64::GPR32RegClass, MIB);
7458 }
7459
7460 // We can use the LHS of the GEP as the base, and the LHS of the shift as an
7461 // offset. Signify that we are shifting by setting the shift flag to 1.
7462 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(RegNo: Base.getReg()); },
7463 [=](MachineInstrBuilder &MIB) { MIB.addUse(RegNo: OffsetReg); },
7464 [=](MachineInstrBuilder &MIB) {
7465 // Need to add both immediates here to make sure that they are both
7466 // added to the instruction.
7467 MIB.addImm(Val: SignExtend);
7468 MIB.addImm(Val: 1);
7469 }}};
7470}
7471
7472/// This is used for computing addresses like this:
7473///
7474/// ldr x1, [x2, x3, lsl #3]
7475///
7476/// Where x2 is the base register, and x3 is an offset register. The shift-left
7477/// is a constant value specific to this load instruction. That is, we'll never
7478/// see anything other than a 3 here (which corresponds to the size of the
7479/// element being loaded.)
7480InstructionSelector::ComplexRendererFns
7481AArch64InstructionSelector::selectAddrModeShiftedExtendXReg(
7482 MachineOperand &Root, unsigned SizeInBytes) const {
7483 if (!Root.isReg())
7484 return std::nullopt;
7485 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
7486
7487 // We want to find something like this:
7488 //
7489 // val = G_CONSTANT LegalShiftVal
7490 // shift = G_SHL off_reg val
7491 // ptr = G_PTR_ADD base_reg shift
7492 // x = G_LOAD ptr
7493 //
7494 // And fold it into this addressing mode:
7495 //
7496 // ldr x, [base_reg, off_reg, lsl #LegalShiftVal]
7497
7498 // Check if we can find the G_PTR_ADD.
7499 MachineInstr *PtrAdd =
7500 getOpcodeDef(Opcode: TargetOpcode::G_PTR_ADD, Reg: Root.getReg(), MRI);
7501 if (!PtrAdd || !isWorthFoldingIntoExtendedReg(MI: *PtrAdd, MRI, IsAddrOperand: true))
7502 return std::nullopt;
7503
7504 // Now, try to match an opcode which will match our specific offset.
7505 // We want a G_SHL or a G_MUL.
7506 MachineInstr *OffsetInst =
7507 getDefIgnoringCopies(Reg: PtrAdd->getOperand(i: 2).getReg(), MRI);
7508 return selectExtendedSHL(Root, Base&: PtrAdd->getOperand(i: 1),
7509 Offset&: OffsetInst->getOperand(i: 0), SizeInBytes,
7510 /*WantsExt=*/false);
7511}
7512
7513/// This is used for computing addresses like this:
7514///
7515/// ldr x1, [x2, x3]
7516///
7517/// Where x2 is the base register, and x3 is an offset register.
7518///
7519/// When possible (or profitable) to fold a G_PTR_ADD into the address
7520/// calculation, this will do so. Otherwise, it will return std::nullopt.
7521InstructionSelector::ComplexRendererFns
7522AArch64InstructionSelector::selectAddrModeRegisterOffset(
7523 MachineOperand &Root) const {
7524 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
7525
7526 // We need a GEP.
7527 Register Base, Offset;
7528 if (!mi_match(R: Root.getReg(), MRI, P: m_GPtrAdd(L: m_Reg(R&: Base), R: m_Reg(R&: Offset))))
7529 return std::nullopt;
7530
7531 // If this is used more than once, let's not bother folding.
7532 // TODO: Check if they are memory ops. If they are, then we can still fold
7533 // without having to recompute anything.
7534 if (!MRI.hasOneNonDBGUse(RegNo: Root.getReg()))
7535 return std::nullopt;
7536
7537 // Base is the GEP's LHS, offset is its RHS.
7538 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(RegNo: Base); },
7539 [=](MachineInstrBuilder &MIB) { MIB.addUse(RegNo: Offset); },
7540 [=](MachineInstrBuilder &MIB) {
7541 // Need to add both immediates here to make sure that they are both
7542 // added to the instruction.
7543 MIB.addImm(Val: 0);
7544 MIB.addImm(Val: 0);
7545 }}};
7546}
7547
7548/// This is intended to be equivalent to selectAddrModeXRO in
7549/// AArch64ISelDAGtoDAG. It's used for selecting X register offset loads.
7550InstructionSelector::ComplexRendererFns
7551AArch64InstructionSelector::selectAddrModeXRO(MachineOperand &Root,
7552 unsigned SizeInBytes) const {
7553 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
7554 if (!Root.isReg())
7555 return std::nullopt;
7556 MachineInstr *PtrAdd =
7557 getOpcodeDef(Opcode: TargetOpcode::G_PTR_ADD, Reg: Root.getReg(), MRI);
7558 if (!PtrAdd)
7559 return std::nullopt;
7560
7561 // Check for an immediates which cannot be encoded in the [base + imm]
7562 // addressing mode, and can't be encoded in an add/sub. If this happens, we'll
7563 // end up with code like:
7564 //
7565 // mov x0, wide
7566 // add x1 base, x0
7567 // ldr x2, [x1, x0]
7568 //
7569 // In this situation, we can use the [base, xreg] addressing mode to save an
7570 // add/sub:
7571 //
7572 // mov x0, wide
7573 // ldr x2, [base, x0]
7574 auto ValAndVReg =
7575 getIConstantVRegValWithLookThrough(VReg: PtrAdd->getOperand(i: 2).getReg(), MRI);
7576 if (ValAndVReg) {
7577 unsigned Scale = Log2_32(Value: SizeInBytes);
7578 int64_t ImmOff = ValAndVReg->Value.getSExtValue();
7579
7580 // Skip immediates that can be selected in the load/store addressing
7581 // mode.
7582 if (ImmOff % SizeInBytes == 0 && ImmOff >= 0 &&
7583 ImmOff < (0x1000 << Scale))
7584 return std::nullopt;
7585
7586 // Helper lambda to decide whether or not it is preferable to emit an add.
7587 auto isPreferredADD = [](int64_t ImmOff) {
7588 // Constants in [0x0, 0xfff] can be encoded in an add.
7589 if ((ImmOff & 0xfffffffffffff000LL) == 0x0LL)
7590 return true;
7591
7592 // Can it be encoded in an add lsl #12?
7593 if ((ImmOff & 0xffffffffff000fffLL) != 0x0LL)
7594 return false;
7595
7596 // It can be encoded in an add lsl #12, but we may not want to. If it is
7597 // possible to select this as a single movz, then prefer that. A single
7598 // movz is faster than an add with a shift.
7599 return (ImmOff & 0xffffffffff00ffffLL) != 0x0LL &&
7600 (ImmOff & 0xffffffffffff0fffLL) != 0x0LL;
7601 };
7602
7603 // If the immediate can be encoded in a single add/sub, then bail out.
7604 if (isPreferredADD(ImmOff) || isPreferredADD(-ImmOff))
7605 return std::nullopt;
7606 }
7607
7608 // Try to fold shifts into the addressing mode.
7609 auto AddrModeFns = selectAddrModeShiftedExtendXReg(Root, SizeInBytes);
7610 if (AddrModeFns)
7611 return AddrModeFns;
7612
7613 // If that doesn't work, see if it's possible to fold in registers from
7614 // a GEP.
7615 return selectAddrModeRegisterOffset(Root);
7616}
7617
7618/// This is used for computing addresses like this:
7619///
7620/// ldr x0, [xBase, wOffset, sxtw #LegalShiftVal]
7621///
7622/// Where we have a 64-bit base register, a 32-bit offset register, and an
7623/// extend (which may or may not be signed).
7624InstructionSelector::ComplexRendererFns
7625AArch64InstructionSelector::selectAddrModeWRO(MachineOperand &Root,
7626 unsigned SizeInBytes) const {
7627 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
7628
7629 MachineInstr *PtrAdd =
7630 getOpcodeDef(Opcode: TargetOpcode::G_PTR_ADD, Reg: Root.getReg(), MRI);
7631 if (!PtrAdd || !isWorthFoldingIntoExtendedReg(MI: *PtrAdd, MRI, IsAddrOperand: true))
7632 return std::nullopt;
7633
7634 MachineOperand &LHS = PtrAdd->getOperand(i: 1);
7635 MachineOperand &RHS = PtrAdd->getOperand(i: 2);
7636 MachineInstr *OffsetInst = getDefIgnoringCopies(Reg: RHS.getReg(), MRI);
7637
7638 // The first case is the same as selectAddrModeXRO, except we need an extend.
7639 // In this case, we try to find a shift and extend, and fold them into the
7640 // addressing mode.
7641 //
7642 // E.g.
7643 //
7644 // off_reg = G_Z/S/ANYEXT ext_reg
7645 // val = G_CONSTANT LegalShiftVal
7646 // shift = G_SHL off_reg val
7647 // ptr = G_PTR_ADD base_reg shift
7648 // x = G_LOAD ptr
7649 //
7650 // In this case we can get a load like this:
7651 //
7652 // ldr x0, [base_reg, ext_reg, sxtw #LegalShiftVal]
7653 auto ExtendedShl = selectExtendedSHL(Root, Base&: LHS, Offset&: OffsetInst->getOperand(i: 0),
7654 SizeInBytes, /*WantsExt=*/true);
7655 if (ExtendedShl)
7656 return ExtendedShl;
7657
7658 // There was no shift. We can try and fold a G_Z/S/ANYEXT in alone though.
7659 //
7660 // e.g.
7661 // ldr something, [base_reg, ext_reg, sxtw]
7662 if (!isWorthFoldingIntoExtendedReg(MI: *OffsetInst, MRI, IsAddrOperand: true))
7663 return std::nullopt;
7664
7665 // Check if this is an extend. We'll get an extend type if it is.
7666 AArch64_AM::ShiftExtendType Ext =
7667 getExtendTypeForInst(MI&: *OffsetInst, MRI, /*IsLoadStore=*/true);
7668 if (Ext == AArch64_AM::InvalidShiftExtend)
7669 return std::nullopt;
7670
7671 // Need a 32-bit wide register.
7672 MachineIRBuilder MIB(*PtrAdd);
7673 Register ExtReg = moveScalarRegClass(Reg: OffsetInst->getOperand(i: 1).getReg(),
7674 RC: AArch64::GPR32RegClass, MIB);
7675 unsigned SignExtend = Ext == AArch64_AM::SXTW;
7676
7677 // Base is LHS, offset is ExtReg.
7678 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(RegNo: LHS.getReg()); },
7679 [=](MachineInstrBuilder &MIB) { MIB.addUse(RegNo: ExtReg); },
7680 [=](MachineInstrBuilder &MIB) {
7681 MIB.addImm(Val: SignExtend);
7682 MIB.addImm(Val: 0);
7683 }}};
7684}
7685
7686/// Select a "register plus unscaled signed 9-bit immediate" address. This
7687/// should only match when there is an offset that is not valid for a scaled
7688/// immediate addressing mode. The "Size" argument is the size in bytes of the
7689/// memory reference, which is needed here to know what is valid for a scaled
7690/// immediate.
7691InstructionSelector::ComplexRendererFns
7692AArch64InstructionSelector::selectAddrModeUnscaled(MachineOperand &Root,
7693 unsigned Size) const {
7694 MachineRegisterInfo &MRI =
7695 Root.getParent()->getParent()->getParent()->getRegInfo();
7696
7697 if (!Root.isReg())
7698 return std::nullopt;
7699
7700 if (!isBaseWithConstantOffset(Root, MRI))
7701 return std::nullopt;
7702
7703 MachineInstr *RootDef = MRI.getVRegDef(Reg: Root.getReg());
7704
7705 MachineOperand &OffImm = RootDef->getOperand(i: 2);
7706 if (!OffImm.isReg())
7707 return std::nullopt;
7708 MachineInstr *RHS = MRI.getVRegDef(Reg: OffImm.getReg());
7709 if (RHS->getOpcode() != TargetOpcode::G_CONSTANT)
7710 return std::nullopt;
7711 int64_t RHSC;
7712 MachineOperand &RHSOp1 = RHS->getOperand(i: 1);
7713 if (!RHSOp1.isCImm() || RHSOp1.getCImm()->getBitWidth() > 64)
7714 return std::nullopt;
7715 RHSC = RHSOp1.getCImm()->getSExtValue();
7716
7717 if (RHSC >= -256 && RHSC < 256) {
7718 MachineOperand &Base = RootDef->getOperand(i: 1);
7719 return {{
7720 [=](MachineInstrBuilder &MIB) { MIB.add(MO: Base); },
7721 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: RHSC); },
7722 }};
7723 }
7724 return std::nullopt;
7725}
7726
7727InstructionSelector::ComplexRendererFns
7728AArch64InstructionSelector::tryFoldAddLowIntoImm(MachineInstr &RootDef,
7729 unsigned Size,
7730 MachineRegisterInfo &MRI) const {
7731 if (RootDef.getOpcode() != AArch64::G_ADD_LOW)
7732 return std::nullopt;
7733 MachineInstr &Adrp = *MRI.getVRegDef(Reg: RootDef.getOperand(i: 1).getReg());
7734 if (Adrp.getOpcode() != AArch64::ADRP)
7735 return std::nullopt;
7736
7737 // TODO: add heuristics like isWorthFoldingADDlow() from SelectionDAG.
7738 auto Offset = Adrp.getOperand(i: 1).getOffset();
7739 if (Offset % Size != 0)
7740 return std::nullopt;
7741
7742 auto GV = Adrp.getOperand(i: 1).getGlobal();
7743 if (GV->isThreadLocal())
7744 return std::nullopt;
7745
7746 auto &MF = *RootDef.getParent()->getParent();
7747 if (GV->getPointerAlignment(DL: MF.getDataLayout()) < Size)
7748 return std::nullopt;
7749
7750 unsigned OpFlags = STI.ClassifyGlobalReference(GV, TM: MF.getTarget());
7751 MachineIRBuilder MIRBuilder(RootDef);
7752 Register AdrpReg = Adrp.getOperand(i: 0).getReg();
7753 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(RegNo: AdrpReg); },
7754 [=](MachineInstrBuilder &MIB) {
7755 MIB.addGlobalAddress(GV, Offset,
7756 TargetFlags: OpFlags | AArch64II::MO_PAGEOFF |
7757 AArch64II::MO_NC);
7758 }}};
7759}
7760
7761/// Select a "register plus scaled unsigned 12-bit immediate" address. The
7762/// "Size" argument is the size in bytes of the memory reference, which
7763/// determines the scale.
7764InstructionSelector::ComplexRendererFns
7765AArch64InstructionSelector::selectAddrModeIndexed(MachineOperand &Root,
7766 unsigned Size) const {
7767 MachineFunction &MF = *Root.getParent()->getParent()->getParent();
7768 MachineRegisterInfo &MRI = MF.getRegInfo();
7769
7770 if (!Root.isReg())
7771 return std::nullopt;
7772
7773 MachineInstr *RootDef = MRI.getVRegDef(Reg: Root.getReg());
7774 if (RootDef->getOpcode() == TargetOpcode::G_FRAME_INDEX) {
7775 return {{
7776 [=](MachineInstrBuilder &MIB) { MIB.add(MO: RootDef->getOperand(i: 1)); },
7777 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); },
7778 }};
7779 }
7780
7781 CodeModel::Model CM = MF.getTarget().getCodeModel();
7782 // Check if we can fold in the ADD of small code model ADRP + ADD address.
7783 // HACK: ld64 on Darwin doesn't support relocations on PRFM, so we can't fold
7784 // globals into the offset.
7785 MachineInstr *RootParent = Root.getParent();
7786 if (CM == CodeModel::Small &&
7787 !(RootParent->getOpcode() == AArch64::G_AARCH64_PREFETCH &&
7788 STI.isTargetDarwin())) {
7789 auto OpFns = tryFoldAddLowIntoImm(RootDef&: *RootDef, Size, MRI);
7790 if (OpFns)
7791 return OpFns;
7792 }
7793
7794 if (isBaseWithConstantOffset(Root, MRI)) {
7795 MachineOperand &LHS = RootDef->getOperand(i: 1);
7796 MachineOperand &RHS = RootDef->getOperand(i: 2);
7797 MachineInstr *LHSDef = MRI.getVRegDef(Reg: LHS.getReg());
7798 MachineInstr *RHSDef = MRI.getVRegDef(Reg: RHS.getReg());
7799
7800 int64_t RHSC = (int64_t)RHSDef->getOperand(i: 1).getCImm()->getZExtValue();
7801 unsigned Scale = Log2_32(Value: Size);
7802 if ((RHSC & (Size - 1)) == 0 && RHSC >= 0 && RHSC < (0x1000 << Scale)) {
7803 if (LHSDef->getOpcode() == TargetOpcode::G_FRAME_INDEX)
7804 return {{
7805 [=](MachineInstrBuilder &MIB) { MIB.add(MO: LHSDef->getOperand(i: 1)); },
7806 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: RHSC >> Scale); },
7807 }};
7808
7809 return {{
7810 [=](MachineInstrBuilder &MIB) { MIB.add(MO: LHS); },
7811 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: RHSC >> Scale); },
7812 }};
7813 }
7814 }
7815
7816 // Before falling back to our general case, check if the unscaled
7817 // instructions can handle this. If so, that's preferable.
7818 if (selectAddrModeUnscaled(Root, Size))
7819 return std::nullopt;
7820
7821 return {{
7822 [=](MachineInstrBuilder &MIB) { MIB.add(MO: Root); },
7823 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); },
7824 }};
7825}
7826
7827/// Given a shift instruction, return the correct shift type for that
7828/// instruction.
7829static AArch64_AM::ShiftExtendType getShiftTypeForInst(MachineInstr &MI) {
7830 switch (MI.getOpcode()) {
7831 default:
7832 return AArch64_AM::InvalidShiftExtend;
7833 case TargetOpcode::G_SHL:
7834 return AArch64_AM::LSL;
7835 case TargetOpcode::G_LSHR:
7836 return AArch64_AM::LSR;
7837 case TargetOpcode::G_ASHR:
7838 return AArch64_AM::ASR;
7839 case TargetOpcode::G_ROTR:
7840 return AArch64_AM::ROR;
7841 }
7842}
7843
7844/// Select a "shifted register" operand. If the value is not shifted, set the
7845/// shift operand to a default value of "lsl 0".
7846InstructionSelector::ComplexRendererFns
7847AArch64InstructionSelector::selectShiftedRegister(MachineOperand &Root,
7848 bool AllowROR) const {
7849 if (!Root.isReg())
7850 return std::nullopt;
7851 MachineRegisterInfo &MRI =
7852 Root.getParent()->getParent()->getParent()->getRegInfo();
7853
7854 // Check if the operand is defined by an instruction which corresponds to
7855 // a ShiftExtendType. E.g. a G_SHL, G_LSHR, etc.
7856 MachineInstr *ShiftInst = MRI.getVRegDef(Reg: Root.getReg());
7857 AArch64_AM::ShiftExtendType ShType = getShiftTypeForInst(MI&: *ShiftInst);
7858 if (ShType == AArch64_AM::InvalidShiftExtend)
7859 return std::nullopt;
7860 if (ShType == AArch64_AM::ROR && !AllowROR)
7861 return std::nullopt;
7862 if (!isWorthFoldingIntoExtendedReg(MI: *ShiftInst, MRI, IsAddrOperand: false))
7863 return std::nullopt;
7864
7865 // Need an immediate on the RHS.
7866 MachineOperand &ShiftRHS = ShiftInst->getOperand(i: 2);
7867 auto Immed = getImmedFromMO(Root: ShiftRHS);
7868 if (!Immed)
7869 return std::nullopt;
7870
7871 // We have something that we can fold. Fold in the shift's LHS and RHS into
7872 // the instruction.
7873 MachineOperand &ShiftLHS = ShiftInst->getOperand(i: 1);
7874 Register ShiftReg = ShiftLHS.getReg();
7875
7876 unsigned NumBits = MRI.getType(Reg: ShiftReg).getSizeInBits();
7877 unsigned Val = *Immed & (NumBits - 1);
7878 unsigned ShiftVal = AArch64_AM::getShifterImm(ST: ShType, Imm: Val);
7879
7880 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(RegNo: ShiftReg); },
7881 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: ShiftVal); }}};
7882}
7883
7884AArch64_AM::ShiftExtendType AArch64InstructionSelector::getExtendTypeForInst(
7885 MachineInstr &MI, MachineRegisterInfo &MRI, bool IsLoadStore) const {
7886 unsigned Opc = MI.getOpcode();
7887
7888 // Handle explicit extend instructions first.
7889 if (Opc == TargetOpcode::G_SEXT || Opc == TargetOpcode::G_SEXT_INREG) {
7890 unsigned Size;
7891 if (Opc == TargetOpcode::G_SEXT)
7892 Size = MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getSizeInBits();
7893 else
7894 Size = MI.getOperand(i: 2).getImm();
7895 assert(Size != 64 && "Extend from 64 bits?");
7896 switch (Size) {
7897 case 8:
7898 return IsLoadStore ? AArch64_AM::InvalidShiftExtend : AArch64_AM::SXTB;
7899 case 16:
7900 return IsLoadStore ? AArch64_AM::InvalidShiftExtend : AArch64_AM::SXTH;
7901 case 32:
7902 return AArch64_AM::SXTW;
7903 default:
7904 return AArch64_AM::InvalidShiftExtend;
7905 }
7906 }
7907
7908 if (Opc == TargetOpcode::G_ZEXT || Opc == TargetOpcode::G_ANYEXT) {
7909 unsigned Size = MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getSizeInBits();
7910 assert(Size != 64 && "Extend from 64 bits?");
7911 switch (Size) {
7912 case 8:
7913 return IsLoadStore ? AArch64_AM::InvalidShiftExtend : AArch64_AM::UXTB;
7914 case 16:
7915 return IsLoadStore ? AArch64_AM::InvalidShiftExtend : AArch64_AM::UXTH;
7916 case 32:
7917 return AArch64_AM::UXTW;
7918 default:
7919 return AArch64_AM::InvalidShiftExtend;
7920 }
7921 }
7922
7923 // Don't have an explicit extend. Try to handle a G_AND with a constant mask
7924 // on the RHS.
7925 if (Opc != TargetOpcode::G_AND)
7926 return AArch64_AM::InvalidShiftExtend;
7927
7928 std::optional<uint64_t> MaybeAndMask = getImmedFromMO(Root: MI.getOperand(i: 2));
7929 if (!MaybeAndMask)
7930 return AArch64_AM::InvalidShiftExtend;
7931 uint64_t AndMask = *MaybeAndMask;
7932 switch (AndMask) {
7933 default:
7934 return AArch64_AM::InvalidShiftExtend;
7935 case 0xFF:
7936 return !IsLoadStore ? AArch64_AM::UXTB : AArch64_AM::InvalidShiftExtend;
7937 case 0xFFFF:
7938 return !IsLoadStore ? AArch64_AM::UXTH : AArch64_AM::InvalidShiftExtend;
7939 case 0xFFFFFFFF:
7940 return AArch64_AM::UXTW;
7941 }
7942}
7943
7944Register AArch64InstructionSelector::moveScalarRegClass(
7945 Register Reg, const TargetRegisterClass &RC, MachineIRBuilder &MIB) const {
7946 MachineRegisterInfo &MRI = *MIB.getMRI();
7947 auto Ty = MRI.getType(Reg);
7948 assert(!Ty.isVector() && "Expected scalars only!");
7949 if (Ty.getSizeInBits() == TRI.getRegSizeInBits(RC))
7950 return Reg;
7951
7952 // Create a copy and immediately select it.
7953 // FIXME: We should have an emitCopy function?
7954 auto Copy = MIB.buildCopy(Res: {&RC}, Op: {Reg});
7955 selectCopy(I&: *Copy, TII, MRI, TRI, RBI);
7956 return Copy.getReg(Idx: 0);
7957}
7958
7959/// Select an "extended register" operand. This operand folds in an extend
7960/// followed by an optional left shift.
7961InstructionSelector::ComplexRendererFns
7962AArch64InstructionSelector::selectArithExtendedRegister(
7963 MachineOperand &Root) const {
7964 if (!Root.isReg())
7965 return std::nullopt;
7966 MachineRegisterInfo &MRI =
7967 Root.getParent()->getParent()->getParent()->getRegInfo();
7968
7969 uint64_t ShiftVal = 0;
7970 Register ExtReg;
7971 AArch64_AM::ShiftExtendType Ext;
7972 MachineInstr *RootDef = getDefIgnoringCopies(Reg: Root.getReg(), MRI);
7973 if (!RootDef)
7974 return std::nullopt;
7975
7976 if (!isWorthFoldingIntoExtendedReg(MI: *RootDef, MRI, IsAddrOperand: false))
7977 return std::nullopt;
7978
7979 // Check if we can fold a shift and an extend.
7980 if (RootDef->getOpcode() == TargetOpcode::G_SHL) {
7981 // Look for a constant on the RHS of the shift.
7982 MachineOperand &RHS = RootDef->getOperand(i: 2);
7983 std::optional<uint64_t> MaybeShiftVal = getImmedFromMO(Root: RHS);
7984 if (!MaybeShiftVal)
7985 return std::nullopt;
7986 ShiftVal = *MaybeShiftVal;
7987 if (ShiftVal > 4)
7988 return std::nullopt;
7989 // Look for a valid extend instruction on the LHS of the shift.
7990 MachineOperand &LHS = RootDef->getOperand(i: 1);
7991 MachineInstr *ExtDef = getDefIgnoringCopies(Reg: LHS.getReg(), MRI);
7992 if (!ExtDef)
7993 return std::nullopt;
7994 Ext = getExtendTypeForInst(MI&: *ExtDef, MRI);
7995 if (Ext == AArch64_AM::InvalidShiftExtend)
7996 return std::nullopt;
7997 ExtReg = ExtDef->getOperand(i: 1).getReg();
7998 } else {
7999 // Didn't get a shift. Try just folding an extend.
8000 Ext = getExtendTypeForInst(MI&: *RootDef, MRI);
8001 if (Ext == AArch64_AM::InvalidShiftExtend)
8002 return std::nullopt;
8003 ExtReg = RootDef->getOperand(i: 1).getReg();
8004
8005 // If we have a 32 bit instruction which zeroes out the high half of a
8006 // register, we get an implicit zero extend for free. Check if we have one.
8007 // FIXME: We actually emit the extend right now even though we don't have
8008 // to.
8009 if (Ext == AArch64_AM::UXTW && MRI.getType(Reg: ExtReg).getSizeInBits() == 32) {
8010 MachineInstr *ExtInst = MRI.getVRegDef(Reg: ExtReg);
8011 if (isDef32(MI: *ExtInst))
8012 return std::nullopt;
8013 }
8014 }
8015
8016 // We require a GPR32 here. Narrow the ExtReg if needed using a subregister
8017 // copy.
8018 MachineIRBuilder MIB(*RootDef);
8019 ExtReg = moveScalarRegClass(Reg: ExtReg, RC: AArch64::GPR32RegClass, MIB);
8020
8021 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(RegNo: ExtReg); },
8022 [=](MachineInstrBuilder &MIB) {
8023 MIB.addImm(Val: getArithExtendImm(ET: Ext, Imm: ShiftVal));
8024 }}};
8025}
8026
8027InstructionSelector::ComplexRendererFns
8028AArch64InstructionSelector::selectExtractHigh(MachineOperand &Root) const {
8029 if (!Root.isReg())
8030 return std::nullopt;
8031 MachineRegisterInfo &MRI =
8032 Root.getParent()->getParent()->getParent()->getRegInfo();
8033
8034 auto Extract = getDefSrcRegIgnoringCopies(Reg: Root.getReg(), MRI);
8035 while (Extract && Extract->MI->getOpcode() == TargetOpcode::G_BITCAST &&
8036 STI.isLittleEndian())
8037 Extract =
8038 getDefSrcRegIgnoringCopies(Reg: Extract->MI->getOperand(i: 1).getReg(), MRI);
8039 if (!Extract)
8040 return std::nullopt;
8041
8042 if (auto *Unmerge = dyn_cast<GUnmerge>(Val: Extract->MI)) {
8043 if (Unmerge->getNumDefs() == 2 &&
8044 Extract->Reg == Unmerge->getOperand(i: 1).getReg()) {
8045 Register ExtReg = Unmerge->getSourceReg();
8046 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(RegNo: ExtReg); }}};
8047 }
8048 }
8049 if (auto *ExtElt = dyn_cast<GExtractVectorElement>(Val: Extract->MI)) {
8050 LLT SrcTy = MRI.getType(Reg: ExtElt->getVectorReg());
8051 auto LaneIdx =
8052 getIConstantVRegValWithLookThrough(VReg: ExtElt->getIndexReg(), MRI);
8053 if (LaneIdx && SrcTy == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 64) &&
8054 LaneIdx->Value.getSExtValue() == 1) {
8055 Register ExtReg = ExtElt->getVectorReg();
8056 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(RegNo: ExtReg); }}};
8057 }
8058 }
8059 if (auto *Subvec = dyn_cast<GExtractSubvector>(Val: Extract->MI)) {
8060 LLT SrcTy = MRI.getType(Reg: Subvec->getSrcVec());
8061 auto LaneIdx = Subvec->getIndexImm();
8062 if (LaneIdx == SrcTy.getNumElements() / 2) {
8063 Register ExtReg = Subvec->getSrcVec();
8064 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(RegNo: ExtReg); }}};
8065 }
8066 }
8067
8068 return std::nullopt;
8069}
8070
8071InstructionSelector::ComplexRendererFns
8072AArch64InstructionSelector::selectCVTFixedPointBase(const MachineOperand &Root,
8073 unsigned DstElemWidth,
8074 bool isReciprocal) const {
8075 if (!Root.isReg())
8076 return std::nullopt;
8077 const MachineRegisterInfo &MRI =
8078 Root.getParent()->getParent()->getParent()->getRegInfo();
8079
8080 Register Reg = Root.getReg();
8081 MachineInstr *Dup = getDefIgnoringCopies(Reg, MRI);
8082
8083 if (Dup && Dup->getOpcode() == TargetOpcode::G_SPLAT_VECTOR)
8084 Reg = Dup->getOperand(i: 1).getReg();
8085
8086 std::optional<ValueAndVReg> CstVal =
8087 getAnyConstantVRegValWithLookThrough(VReg: Reg, MRI);
8088
8089 if (!CstVal)
8090 return std::nullopt;
8091
8092 unsigned CstElemWidth = MRI.getType(Reg).getScalarSizeInBits();
8093 APFloat FVal(0.0);
8094 switch (CstElemWidth) {
8095 case 16:
8096 FVal = APFloat(APFloat::IEEEhalf(), CstVal->Value.trunc(width: CstElemWidth));
8097 break;
8098 case 32:
8099 FVal = APFloat(APFloat::IEEEsingle(), CstVal->Value);
8100 break;
8101 case 64:
8102 FVal = APFloat(APFloat::IEEEdouble(), CstVal->Value);
8103 break;
8104 default:
8105 return std::nullopt;
8106 };
8107 if (unsigned FBits =
8108 CheckFixedPointOperandConstant(FVal, RegWidth: DstElemWidth, isReciprocal))
8109 return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Val: FBits); }}};
8110
8111 return std::nullopt;
8112}
8113
8114unsigned AArch64InstructionSelector::getFixedPointWidthFromOperand(
8115 const MachineOperand &Root) const {
8116 return Root.getParent()
8117 ->getMF()
8118 ->getRegInfo()
8119 .getType(Reg: Root.getReg())
8120 .getScalarSizeInBits();
8121}
8122
8123template <unsigned Width>
8124InstructionSelector::ComplexRendererFns
8125AArch64InstructionSelector::selectCVTFixedPoint(MachineOperand &Root) const {
8126 return selectCVTFixedPointBase(Root, DstElemWidth: Width, /*isReciprocal*/ false);
8127}
8128
8129template <unsigned Width>
8130InstructionSelector::ComplexRendererFns
8131AArch64InstructionSelector::selectCVTFixedPosRecipOperand(
8132 MachineOperand &Root) const {
8133 return selectCVTFixedPointBase(Root, DstElemWidth: Width, /*isReciprocal*/ true);
8134}
8135
8136InstructionSelector::ComplexRendererFns
8137AArch64InstructionSelector::selectCVTFixedPointVec(MachineOperand &Root) const {
8138 return selectCVTFixedPointBase(Root, DstElemWidth: getFixedPointWidthFromOperand(Root),
8139 /*isReciprocal*/ false);
8140}
8141
8142InstructionSelector::ComplexRendererFns
8143AArch64InstructionSelector::selectCVTFixedPosRecipOperandVec(
8144 MachineOperand &Root) const {
8145 return selectCVTFixedPointBase(Root, DstElemWidth: getFixedPointWidthFromOperand(Root),
8146 /*isReciprocal*/ true);
8147}
8148
8149void AArch64InstructionSelector::renderFixedPointScalarXForm(
8150 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
8151 assert(OpIdx == 3 && MI.getOperand(OpIdx).isImm() &&
8152 "Expected vecshift immediate operand");
8153 MIB.addImm(Val: MI.getOperand(i: OpIdx).getImm());
8154}
8155
8156void AArch64InstructionSelector::renderFixedPointImm(MachineInstrBuilder &MIB,
8157 const MachineOperand &Root,
8158 unsigned Width,
8159 bool isReciprocal) const {
8160 // FIXME: This is only needed to satisfy the type checking in tablegen, and
8161 // should be able to reuse the Renderers already calculated by
8162 // selectCVTFixedPointBase.
8163 InstructionSelector::ComplexRendererFns Renderer =
8164 selectCVTFixedPointBase(Root, DstElemWidth: Width, isReciprocal);
8165 assert((Renderer && Renderer->size() == 1) &&
8166 "Expected selectCVTFixedPointBase to provide a function\n");
8167 (Renderer->front())(MIB);
8168}
8169
8170void AArch64InstructionSelector::renderFixedPointXForm(MachineInstrBuilder &MIB,
8171 const MachineInstr &MI,
8172 int OpIdx) const {
8173 const MachineOperand &Root = MI.getOperand(i: OpIdx);
8174 renderFixedPointImm(MIB, Root, Width: getFixedPointWidthFromOperand(Root),
8175 /*isReciprocal*/ false);
8176}
8177
8178void AArch64InstructionSelector::renderFixedPointRecipXForm(
8179 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
8180 const MachineOperand &Root = MI.getOperand(i: OpIdx);
8181 renderFixedPointImm(MIB, Root, Width: getFixedPointWidthFromOperand(Root),
8182 /*isReciprocal*/ true);
8183}
8184
8185void AArch64InstructionSelector::renderTruncImm(MachineInstrBuilder &MIB,
8186 const MachineInstr &MI,
8187 int OpIdx) const {
8188 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
8189 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
8190 "Expected G_CONSTANT");
8191 std::optional<int64_t> CstVal =
8192 getIConstantVRegSExtVal(VReg: MI.getOperand(i: 0).getReg(), MRI);
8193 assert(CstVal && "Expected constant value");
8194 MIB.addImm(Val: *CstVal);
8195}
8196
8197void AArch64InstructionSelector::renderLogicalImm32(
8198 MachineInstrBuilder &MIB, const MachineInstr &I, int OpIdx) const {
8199 assert(I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
8200 "Expected G_CONSTANT");
8201 uint64_t CstVal = I.getOperand(i: 1).getCImm()->getZExtValue();
8202 uint64_t Enc = AArch64_AM::encodeLogicalImmediate(imm: CstVal, regSize: 32);
8203 MIB.addImm(Val: Enc);
8204}
8205
8206void AArch64InstructionSelector::renderLogicalImm64(
8207 MachineInstrBuilder &MIB, const MachineInstr &I, int OpIdx) const {
8208 assert(I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
8209 "Expected G_CONSTANT");
8210 uint64_t CstVal = I.getOperand(i: 1).getCImm()->getZExtValue();
8211 uint64_t Enc = AArch64_AM::encodeLogicalImmediate(imm: CstVal, regSize: 64);
8212 MIB.addImm(Val: Enc);
8213}
8214
8215void AArch64InstructionSelector::renderUbsanTrap(MachineInstrBuilder &MIB,
8216 const MachineInstr &MI,
8217 int OpIdx) const {
8218 assert(MI.getOpcode() == TargetOpcode::G_UBSANTRAP && OpIdx == 0 &&
8219 "Expected G_UBSANTRAP");
8220 MIB.addImm(Val: MI.getOperand(i: 0).getImm() | ('U' << 8));
8221}
8222
8223void AArch64InstructionSelector::renderFPImm16(MachineInstrBuilder &MIB,
8224 const MachineInstr &MI,
8225 int OpIdx) const {
8226 assert(MI.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 &&
8227 "Expected G_FCONSTANT");
8228 MIB.addImm(
8229 Val: AArch64_AM::getFP16Imm(FPImm: MI.getOperand(i: 1).getFPImm()->getValueAPF()));
8230}
8231
8232void AArch64InstructionSelector::renderFPImm32(MachineInstrBuilder &MIB,
8233 const MachineInstr &MI,
8234 int OpIdx) const {
8235 assert(MI.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 &&
8236 "Expected G_FCONSTANT");
8237 MIB.addImm(
8238 Val: AArch64_AM::getFP32Imm(FPImm: MI.getOperand(i: 1).getFPImm()->getValueAPF()));
8239}
8240
8241void AArch64InstructionSelector::renderFPImm64(MachineInstrBuilder &MIB,
8242 const MachineInstr &MI,
8243 int OpIdx) const {
8244 assert(MI.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 &&
8245 "Expected G_FCONSTANT");
8246 MIB.addImm(
8247 Val: AArch64_AM::getFP64Imm(FPImm: MI.getOperand(i: 1).getFPImm()->getValueAPF()));
8248}
8249
8250void AArch64InstructionSelector::renderFPImm32SIMDModImmType4(
8251 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
8252 assert(MI.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 &&
8253 "Expected G_FCONSTANT");
8254 MIB.addImm(Val: AArch64_AM::encodeAdvSIMDModImmType4(Imm: MI.getOperand(i: 1)
8255 .getFPImm()
8256 ->getValueAPF()
8257 .bitcastToAPInt()
8258 .getZExtValue()));
8259}
8260
8261bool AArch64InstructionSelector::isLoadStoreOfNumBytes(
8262 const MachineInstr &MI, unsigned NumBytes) const {
8263 if (!MI.mayLoadOrStore())
8264 return false;
8265 assert(MI.hasOneMemOperand() &&
8266 "Expected load/store to have only one mem op!");
8267 return (*MI.memoperands_begin())->getSize() == NumBytes;
8268}
8269
8270bool AArch64InstructionSelector::isDef32(const MachineInstr &MI) const {
8271 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
8272 if (MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits() != 32)
8273 return false;
8274
8275 // Only return true if we know the operation will zero-out the high half of
8276 // the 64-bit register. Truncates can be subregister copies, which don't
8277 // zero out the high bits. Copies and other copy-like instructions can be
8278 // fed by truncates, or could be lowered as subregister copies.
8279 switch (MI.getOpcode()) {
8280 default:
8281 return true;
8282 case TargetOpcode::COPY:
8283 case TargetOpcode::G_BITCAST:
8284 case TargetOpcode::G_TRUNC:
8285 case TargetOpcode::G_PHI:
8286 return false;
8287 }
8288}
8289
8290
8291// Perform fixups on the given PHI instruction's operands to force them all
8292// to be the same as the destination regbank.
8293static void fixupPHIOpBanks(MachineInstr &MI, MachineRegisterInfo &MRI,
8294 const AArch64RegisterBankInfo &RBI) {
8295 assert(MI.getOpcode() == TargetOpcode::G_PHI && "Expected a G_PHI");
8296 Register DstReg = MI.getOperand(i: 0).getReg();
8297 const RegisterBank *DstRB = MRI.getRegBankOrNull(Reg: DstReg);
8298 assert(DstRB && "Expected PHI dst to have regbank assigned");
8299 MachineIRBuilder MIB(MI);
8300
8301 // Go through each operand and ensure it has the same regbank.
8302 for (MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI.operands())) {
8303 if (!MO.isReg())
8304 continue;
8305 Register OpReg = MO.getReg();
8306 const RegisterBank *RB = MRI.getRegBankOrNull(Reg: OpReg);
8307 if (RB != DstRB) {
8308 // Insert a cross-bank copy.
8309 auto *OpDef = MRI.getVRegDef(Reg: OpReg);
8310 const LLT &Ty = MRI.getType(Reg: OpReg);
8311 MachineBasicBlock &OpDefBB = *OpDef->getParent();
8312
8313 // Any instruction we insert must appear after all PHIs in the block
8314 // for the block to be valid MIR.
8315 MachineBasicBlock::iterator InsertPt = std::next(x: OpDef->getIterator());
8316 if (InsertPt != OpDefBB.end() && InsertPt->isPHI())
8317 InsertPt = OpDefBB.getFirstNonPHI();
8318 MIB.setInsertPt(MBB&: *OpDef->getParent(), II: InsertPt);
8319 auto Copy = MIB.buildCopy(Res: Ty, Op: OpReg);
8320 MRI.setRegBank(Reg: Copy.getReg(Idx: 0), RegBank: *DstRB);
8321 MO.setReg(Copy.getReg(Idx: 0));
8322 }
8323 }
8324}
8325
8326void AArch64InstructionSelector::processPHIs(MachineFunction &MF) {
8327 // We're looking for PHIs, build a list so we don't invalidate iterators.
8328 MachineRegisterInfo &MRI = MF.getRegInfo();
8329 SmallVector<MachineInstr *, 32> Phis;
8330 for (auto &BB : MF) {
8331 for (auto &MI : BB) {
8332 if (MI.getOpcode() == TargetOpcode::G_PHI)
8333 Phis.emplace_back(Args: &MI);
8334 }
8335 }
8336
8337 for (auto *MI : Phis) {
8338 // We need to do some work here if the operand types are < 16 bit and they
8339 // are split across fpr/gpr banks. Since all types <32b on gpr
8340 // end up being assigned gpr32 regclasses, we can end up with PHIs here
8341 // which try to select between a gpr32 and an fpr16. Ideally RBS shouldn't
8342 // be selecting heterogenous regbanks for operands if possible, but we
8343 // still need to be able to deal with it here.
8344 //
8345 // To fix this, if we have a gpr-bank operand < 32b in size and at least
8346 // one other operand is on the fpr bank, then we add cross-bank copies
8347 // to homogenize the operand banks. For simplicity the bank that we choose
8348 // to settle on is whatever bank the def operand has. For example:
8349 //
8350 // %endbb:
8351 // %dst:gpr(s16) = G_PHI %in1:gpr(s16), %bb1, %in2:fpr(s16), %bb2
8352 // =>
8353 // %bb2:
8354 // ...
8355 // %in2_copy:gpr(s16) = COPY %in2:fpr(s16)
8356 // ...
8357 // %endbb:
8358 // %dst:gpr(s16) = G_PHI %in1:gpr(s16), %bb1, %in2_copy:gpr(s16), %bb2
8359 bool HasGPROp = false, HasFPROp = false;
8360 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI->operands())) {
8361 if (!MO.isReg())
8362 continue;
8363 const LLT &Ty = MRI.getType(Reg: MO.getReg());
8364 if (!Ty.isValid() || !Ty.isScalar())
8365 break;
8366 if (Ty.getSizeInBits() >= 32)
8367 break;
8368 const RegisterBank *RB = MRI.getRegBankOrNull(Reg: MO.getReg());
8369 // If for some reason we don't have a regbank yet. Don't try anything.
8370 if (!RB)
8371 break;
8372
8373 if (RB->getID() == AArch64::GPRRegBankID)
8374 HasGPROp = true;
8375 else
8376 HasFPROp = true;
8377 }
8378 // We have heterogenous regbanks, need to fixup.
8379 if (HasGPROp && HasFPROp)
8380 fixupPHIOpBanks(MI&: *MI, MRI, RBI);
8381 }
8382}
8383
8384namespace llvm {
8385InstructionSelector *
8386createAArch64InstructionSelector(const AArch64TargetMachine &TM,
8387 const AArch64Subtarget &Subtarget,
8388 const AArch64RegisterBankInfo &RBI) {
8389 return new AArch64InstructionSelector(TM, Subtarget, RBI);
8390}
8391}
8392