1//===- ARMISelLowering.cpp - ARM DAG Lowering Implementation --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the interfaces that ARM uses to lower LLVM code into a
10// selection DAG.
11//
12//===----------------------------------------------------------------------===//
13
14#include "ARMISelLowering.h"
15#include "ARMBaseInstrInfo.h"
16#include "ARMBaseRegisterInfo.h"
17#include "ARMCallingConv.h"
18#include "ARMConstantPoolValue.h"
19#include "ARMMachineFunctionInfo.h"
20#include "ARMPerfectShuffle.h"
21#include "ARMRegisterInfo.h"
22#include "ARMSelectionDAGInfo.h"
23#include "ARMSubtarget.h"
24#include "ARMTargetTransformInfo.h"
25#include "MCTargetDesc/ARMAddressingModes.h"
26#include "MCTargetDesc/ARMBaseInfo.h"
27#include "Utils/ARMBaseInfo.h"
28#include "llvm/ADT/APFloat.h"
29#include "llvm/ADT/APInt.h"
30#include "llvm/ADT/ArrayRef.h"
31#include "llvm/ADT/BitVector.h"
32#include "llvm/ADT/DenseMap.h"
33#include "llvm/ADT/STLExtras.h"
34#include "llvm/ADT/SmallPtrSet.h"
35#include "llvm/ADT/SmallVector.h"
36#include "llvm/ADT/Statistic.h"
37#include "llvm/ADT/StringExtras.h"
38#include "llvm/ADT/StringRef.h"
39#include "llvm/ADT/StringSwitch.h"
40#include "llvm/ADT/Twine.h"
41#include "llvm/Analysis/VectorUtils.h"
42#include "llvm/CodeGen/CallingConvLower.h"
43#include "llvm/CodeGen/ComplexDeinterleavingPass.h"
44#include "llvm/CodeGen/ISDOpcodes.h"
45#include "llvm/CodeGen/MachineBasicBlock.h"
46#include "llvm/CodeGen/MachineConstantPool.h"
47#include "llvm/CodeGen/MachineFrameInfo.h"
48#include "llvm/CodeGen/MachineFunction.h"
49#include "llvm/CodeGen/MachineInstr.h"
50#include "llvm/CodeGen/MachineInstrBuilder.h"
51#include "llvm/CodeGen/MachineJumpTableInfo.h"
52#include "llvm/CodeGen/MachineMemOperand.h"
53#include "llvm/CodeGen/MachineOperand.h"
54#include "llvm/CodeGen/MachineRegisterInfo.h"
55#include "llvm/CodeGen/RuntimeLibcallUtil.h"
56#include "llvm/CodeGen/SelectionDAG.h"
57#include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
58#include "llvm/CodeGen/SelectionDAGNodes.h"
59#include "llvm/CodeGen/TargetInstrInfo.h"
60#include "llvm/CodeGen/TargetLowering.h"
61#include "llvm/CodeGen/TargetOpcodes.h"
62#include "llvm/CodeGen/TargetRegisterInfo.h"
63#include "llvm/CodeGen/TargetSubtargetInfo.h"
64#include "llvm/CodeGen/ValueTypes.h"
65#include "llvm/CodeGenTypes/MachineValueType.h"
66#include "llvm/IR/Attributes.h"
67#include "llvm/IR/CallingConv.h"
68#include "llvm/IR/Constant.h"
69#include "llvm/IR/Constants.h"
70#include "llvm/IR/DataLayout.h"
71#include "llvm/IR/DebugLoc.h"
72#include "llvm/IR/DerivedTypes.h"
73#include "llvm/IR/Function.h"
74#include "llvm/IR/GlobalAlias.h"
75#include "llvm/IR/GlobalValue.h"
76#include "llvm/IR/GlobalVariable.h"
77#include "llvm/IR/IRBuilder.h"
78#include "llvm/IR/InlineAsm.h"
79#include "llvm/IR/Instruction.h"
80#include "llvm/IR/Instructions.h"
81#include "llvm/IR/IntrinsicInst.h"
82#include "llvm/IR/Intrinsics.h"
83#include "llvm/IR/IntrinsicsARM.h"
84#include "llvm/IR/Module.h"
85#include "llvm/IR/Type.h"
86#include "llvm/IR/User.h"
87#include "llvm/IR/Value.h"
88#include "llvm/MC/MCInstrDesc.h"
89#include "llvm/MC/MCInstrItineraries.h"
90#include "llvm/MC/MCSchedule.h"
91#include "llvm/Support/AtomicOrdering.h"
92#include "llvm/Support/BranchProbability.h"
93#include "llvm/Support/Casting.h"
94#include "llvm/Support/CodeGen.h"
95#include "llvm/Support/CommandLine.h"
96#include "llvm/Support/Compiler.h"
97#include "llvm/Support/Debug.h"
98#include "llvm/Support/ErrorHandling.h"
99#include "llvm/Support/KnownBits.h"
100#include "llvm/Support/MathExtras.h"
101#include "llvm/Support/raw_ostream.h"
102#include "llvm/Target/TargetMachine.h"
103#include "llvm/Target/TargetOptions.h"
104#include "llvm/TargetParser/Triple.h"
105#include <algorithm>
106#include <cassert>
107#include <cstdint>
108#include <iterator>
109#include <limits>
110#include <optional>
111#include <tuple>
112#include <utility>
113#include <vector>
114
115using namespace llvm;
116
117#define DEBUG_TYPE "arm-isel"
118
119STATISTIC(NumTailCalls, "Number of tail calls");
120STATISTIC(NumOptimizedImms, "Number of times immediates were optimized");
121STATISTIC(NumMovwMovt, "Number of GAs materialized with movw + movt");
122STATISTIC(NumLoopByVals, "Number of loops generated for byval arguments");
123STATISTIC(NumConstpoolPromoted,
124 "Number of constants with their storage promoted into constant pools");
125
126static cl::opt<bool>
127ARMInterworking("arm-interworking", cl::Hidden,
128 cl::desc("Enable / disable ARM interworking (for debugging only)"),
129 cl::init(Val: true));
130
131static cl::opt<bool> EnableConstpoolPromotion(
132 "arm-promote-constant", cl::Hidden,
133 cl::desc("Enable / disable promotion of unnamed_addr constants into "
134 "constant pools"),
135 cl::init(Val: false)); // FIXME: set to true by default once PR32780 is fixed
136static cl::opt<unsigned> ConstpoolPromotionMaxSize(
137 "arm-promote-constant-max-size", cl::Hidden,
138 cl::desc("Maximum size of constant to promote into a constant pool"),
139 cl::init(Val: 64));
140static cl::opt<unsigned> ConstpoolPromotionMaxTotal(
141 "arm-promote-constant-max-total", cl::Hidden,
142 cl::desc("Maximum size of ALL constants to promote into a constant pool"),
143 cl::init(Val: 128));
144
145cl::opt<unsigned>
146MVEMaxSupportedInterleaveFactor("mve-max-interleave-factor", cl::Hidden,
147 cl::desc("Maximum interleave factor for MVE VLDn to generate."),
148 cl::init(Val: 2));
149
150cl::opt<unsigned> ArmMaxBaseUpdatesToCheck(
151 "arm-max-base-updates-to-check", cl::Hidden,
152 cl::desc("Maximum number of base-updates to check generating postindex."),
153 cl::init(Val: 64));
154
155/// Value type used for "flags" operands / results (either CPSR or FPSCR_NZCV).
156constexpr MVT FlagsVT = MVT::i32;
157
158// The APCS parameter registers.
159static const MCPhysReg GPRArgRegs[] = {
160 ARM::R0, ARM::R1, ARM::R2, ARM::R3
161};
162
163static SDValue handleCMSEValue(const SDValue &Value, const ISD::InputArg &Arg,
164 SelectionDAG &DAG, const SDLoc &DL) {
165 assert(Arg.ArgVT.isScalarInteger());
166 assert(Arg.ArgVT.bitsLT(MVT::i32));
167 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: Arg.ArgVT, Operand: Value);
168 SDValue Ext =
169 DAG.getNode(Opcode: Arg.Flags.isSExt() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL,
170 VT: MVT::i32, Operand: Trunc);
171 return Ext;
172}
173
174void ARMTargetLowering::addTypeForNEON(MVT VT, MVT PromotedLdStVT) {
175 if (VT != PromotedLdStVT) {
176 setOperationAction(Op: ISD::LOAD, VT, Action: Promote);
177 AddPromotedToType (Opc: ISD::LOAD, OrigVT: VT, DestVT: PromotedLdStVT);
178
179 setOperationAction(Op: ISD::STORE, VT, Action: Promote);
180 AddPromotedToType (Opc: ISD::STORE, OrigVT: VT, DestVT: PromotedLdStVT);
181 }
182
183 MVT ElemTy = VT.getVectorElementType();
184 if (ElemTy != MVT::f64)
185 setOperationAction(Op: ISD::SETCC, VT, Action: Custom);
186 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT, Action: Custom);
187 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT, Action: Custom);
188 if (ElemTy == MVT::i32) {
189 setOperationAction(Op: ISD::SINT_TO_FP, VT, Action: Custom);
190 setOperationAction(Op: ISD::UINT_TO_FP, VT, Action: Custom);
191 setOperationAction(Op: ISD::FP_TO_SINT, VT, Action: Custom);
192 setOperationAction(Op: ISD::FP_TO_UINT, VT, Action: Custom);
193 } else {
194 setOperationAction(Op: ISD::SINT_TO_FP, VT, Action: Expand);
195 setOperationAction(Op: ISD::UINT_TO_FP, VT, Action: Expand);
196 setOperationAction(Op: ISD::FP_TO_SINT, VT, Action: Expand);
197 setOperationAction(Op: ISD::FP_TO_UINT, VT, Action: Expand);
198 }
199 setOperationAction(Op: ISD::BUILD_VECTOR, VT, Action: Custom);
200 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT, Action: Custom);
201 setOperationAction(Op: ISD::CONCAT_VECTORS, VT, Action: Legal);
202 setOperationAction(Op: ISD::EXTRACT_SUBVECTOR, VT, Action: Legal);
203 setOperationAction(Op: ISD::SELECT, VT, Action: Expand);
204 setOperationAction(Op: ISD::SELECT_CC, VT, Action: Expand);
205 setOperationAction(Op: ISD::VSELECT, VT, Action: Expand);
206 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT, Action: Expand);
207 if (VT.isInteger()) {
208 setOperationAction(Op: ISD::SHL, VT, Action: Custom);
209 setOperationAction(Op: ISD::SRA, VT, Action: Custom);
210 setOperationAction(Op: ISD::SRL, VT, Action: Custom);
211 }
212
213 // Neon does not support vector divide/remainder operations.
214 setOperationAction(Op: ISD::SDIV, VT, Action: Expand);
215 setOperationAction(Op: ISD::UDIV, VT, Action: Expand);
216 setOperationAction(Op: ISD::FDIV, VT, Action: Expand);
217 setOperationAction(Op: ISD::SREM, VT, Action: Expand);
218 setOperationAction(Op: ISD::UREM, VT, Action: Expand);
219 setOperationAction(Op: ISD::FREM, VT, Action: Expand);
220 setOperationAction(Op: ISD::SDIVREM, VT, Action: Expand);
221 setOperationAction(Op: ISD::UDIVREM, VT, Action: Expand);
222
223 if (!VT.isFloatingPoint() && VT != MVT::v2i64 && VT != MVT::v1i64)
224 for (auto Opcode : {ISD::ABS, ISD::ABDS, ISD::ABDU, ISD::SMIN, ISD::SMAX,
225 ISD::UMIN, ISD::UMAX, ISD::CTLS})
226 setOperationAction(Op: Opcode, VT, Action: Legal);
227 if (!VT.isFloatingPoint())
228 for (auto Opcode : {ISD::SADDSAT, ISD::UADDSAT, ISD::SSUBSAT, ISD::USUBSAT})
229 setOperationAction(Op: Opcode, VT, Action: Legal);
230}
231
232void ARMTargetLowering::addDRTypeForNEON(MVT VT) {
233 addRegisterClass(VT, RC: &ARM::DPRRegClass);
234 addTypeForNEON(VT, PromotedLdStVT: MVT::f64);
235}
236
237void ARMTargetLowering::addQRTypeForNEON(MVT VT) {
238 addRegisterClass(VT, RC: &ARM::DPairRegClass);
239 addTypeForNEON(VT, PromotedLdStVT: MVT::v2f64);
240}
241
242void ARMTargetLowering::setAllExpand(MVT VT) {
243 for (unsigned Opc = 0; Opc < ISD::BUILTIN_OP_END; ++Opc)
244 setOperationAction(Op: Opc, VT, Action: Expand);
245
246 // We support these really simple operations even on types where all
247 // the actual arithmetic has to be broken down into simpler
248 // operations or turned into library calls.
249 setOperationAction(Op: ISD::BITCAST, VT, Action: Legal);
250 setOperationAction(Op: ISD::LOAD, VT, Action: Legal);
251 setOperationAction(Op: ISD::STORE, VT, Action: Legal);
252 setOperationAction(Ops: {ISD::UNDEF, ISD::POISON}, VT, Action: Legal);
253}
254
255void ARMTargetLowering::addAllExtLoads(const MVT From, const MVT To,
256 LegalizeAction Action) {
257 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: From, MemVT: To, Action);
258 setLoadExtAction(ExtType: ISD::ZEXTLOAD, ValVT: From, MemVT: To, Action);
259 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: From, MemVT: To, Action);
260}
261
262void ARMTargetLowering::addMVEVectorTypes(bool HasMVEFP) {
263 const MVT IntTypes[] = { MVT::v16i8, MVT::v8i16, MVT::v4i32 };
264
265 for (auto VT : IntTypes) {
266 addRegisterClass(VT, RC: &ARM::MQPRRegClass);
267 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT, Action: Custom);
268 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT, Action: Custom);
269 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT, Action: Custom);
270 setOperationAction(Op: ISD::BUILD_VECTOR, VT, Action: Custom);
271 setOperationAction(Op: ISD::SHL, VT, Action: Custom);
272 setOperationAction(Op: ISD::SRA, VT, Action: Custom);
273 setOperationAction(Op: ISD::SRL, VT, Action: Custom);
274 setOperationAction(Op: ISD::SMIN, VT, Action: Legal);
275 setOperationAction(Op: ISD::SMAX, VT, Action: Legal);
276 setOperationAction(Op: ISD::UMIN, VT, Action: Legal);
277 setOperationAction(Op: ISD::UMAX, VT, Action: Legal);
278 setOperationAction(Op: ISD::ABS, VT, Action: Legal);
279 setOperationAction(Op: ISD::CTLS, VT, Action: Legal);
280 setOperationAction(Op: ISD::SETCC, VT, Action: Custom);
281 setOperationAction(Op: ISD::MLOAD, VT, Action: Custom);
282 setOperationAction(Op: ISD::MSTORE, VT, Action: Legal);
283 setOperationAction(Op: ISD::CTLZ, VT, Action: Legal);
284 setOperationAction(Op: ISD::CTTZ, VT, Action: Custom);
285 setOperationAction(Op: ISD::BITREVERSE, VT, Action: Legal);
286 setOperationAction(Op: ISD::BSWAP, VT, Action: Legal);
287 setOperationAction(Op: ISD::SADDSAT, VT, Action: Legal);
288 setOperationAction(Op: ISD::UADDSAT, VT, Action: Legal);
289 setOperationAction(Op: ISD::SSUBSAT, VT, Action: Legal);
290 setOperationAction(Op: ISD::USUBSAT, VT, Action: Legal);
291 setOperationAction(Op: ISD::ABDS, VT, Action: Legal);
292 setOperationAction(Op: ISD::ABDU, VT, Action: Legal);
293 setOperationAction(Op: ISD::AVGFLOORS, VT, Action: Legal);
294 setOperationAction(Op: ISD::AVGFLOORU, VT, Action: Legal);
295 setOperationAction(Op: ISD::AVGCEILS, VT, Action: Legal);
296 setOperationAction(Op: ISD::AVGCEILU, VT, Action: Legal);
297
298 // No native support for these.
299 setOperationAction(Op: ISD::UDIV, VT, Action: Expand);
300 setOperationAction(Op: ISD::SDIV, VT, Action: Expand);
301 setOperationAction(Op: ISD::UREM, VT, Action: Expand);
302 setOperationAction(Op: ISD::SREM, VT, Action: Expand);
303 setOperationAction(Op: ISD::UDIVREM, VT, Action: Expand);
304 setOperationAction(Op: ISD::SDIVREM, VT, Action: Expand);
305 setOperationAction(Op: ISD::CTPOP, VT, Action: Expand);
306 setOperationAction(Op: ISD::SELECT, VT, Action: Expand);
307 setOperationAction(Op: ISD::SELECT_CC, VT, Action: Expand);
308
309 // Vector reductions
310 setOperationAction(Op: ISD::VECREDUCE_ADD, VT, Action: Legal);
311 setOperationAction(Op: ISD::VECREDUCE_SMAX, VT, Action: Legal);
312 setOperationAction(Op: ISD::VECREDUCE_UMAX, VT, Action: Legal);
313 setOperationAction(Op: ISD::VECREDUCE_SMIN, VT, Action: Legal);
314 setOperationAction(Op: ISD::VECREDUCE_UMIN, VT, Action: Legal);
315 setOperationAction(Op: ISD::VECREDUCE_MUL, VT, Action: Custom);
316 setOperationAction(Op: ISD::VECREDUCE_AND, VT, Action: Custom);
317 setOperationAction(Op: ISD::VECREDUCE_OR, VT, Action: Custom);
318 setOperationAction(Op: ISD::VECREDUCE_XOR, VT, Action: Custom);
319
320 if (!HasMVEFP) {
321 setOperationAction(Op: ISD::SINT_TO_FP, VT, Action: Expand);
322 setOperationAction(Op: ISD::UINT_TO_FP, VT, Action: Expand);
323 setOperationAction(Op: ISD::FP_TO_SINT, VT, Action: Expand);
324 setOperationAction(Op: ISD::FP_TO_UINT, VT, Action: Expand);
325 } else {
326 setOperationAction(Op: ISD::FP_TO_SINT_SAT, VT, Action: Custom);
327 setOperationAction(Op: ISD::FP_TO_UINT_SAT, VT, Action: Custom);
328 }
329
330 // Pre and Post inc are supported on loads and stores
331 for (unsigned im = (unsigned)ISD::PRE_INC;
332 im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) {
333 setIndexedLoadAction(IdxModes: im, VT, Action: Legal);
334 setIndexedStoreAction(IdxModes: im, VT, Action: Legal);
335 setIndexedMaskedLoadAction(IdxMode: im, VT, Action: Legal);
336 setIndexedMaskedStoreAction(IdxMode: im, VT, Action: Legal);
337 }
338 }
339
340 const MVT FloatTypes[] = { MVT::v8f16, MVT::v4f32 };
341 for (auto VT : FloatTypes) {
342 addRegisterClass(VT, RC: &ARM::MQPRRegClass);
343 if (!HasMVEFP)
344 setAllExpand(VT);
345
346 // These are legal or custom whether we have MVE.fp or not
347 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT, Action: Custom);
348 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT, Action: Custom);
349 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: VT.getVectorElementType(), Action: Custom);
350 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT, Action: Custom);
351 setOperationAction(Op: ISD::BUILD_VECTOR, VT, Action: Custom);
352 setOperationAction(Op: ISD::BUILD_VECTOR, VT: VT.getVectorElementType(), Action: Custom);
353 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT, Action: Legal);
354 setOperationAction(Op: ISD::SETCC, VT, Action: Custom);
355 setOperationAction(Op: ISD::MLOAD, VT, Action: Custom);
356 setOperationAction(Op: ISD::MSTORE, VT, Action: Legal);
357 setOperationAction(Op: ISD::SELECT, VT, Action: Expand);
358 setOperationAction(Op: ISD::SELECT_CC, VT, Action: Expand);
359
360 // Pre and Post inc are supported on loads and stores
361 for (unsigned im = (unsigned)ISD::PRE_INC;
362 im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) {
363 setIndexedLoadAction(IdxModes: im, VT, Action: Legal);
364 setIndexedStoreAction(IdxModes: im, VT, Action: Legal);
365 setIndexedMaskedLoadAction(IdxMode: im, VT, Action: Legal);
366 setIndexedMaskedStoreAction(IdxMode: im, VT, Action: Legal);
367 }
368
369 if (HasMVEFP) {
370 setOperationAction(Op: ISD::FMINNUM, VT, Action: Legal);
371 setOperationAction(Op: ISD::FMAXNUM, VT, Action: Legal);
372 for (auto Op : {ISD::FROUND, ISD::STRICT_FROUND, ISD::FROUNDEVEN,
373 ISD::STRICT_FROUNDEVEN, ISD::FTRUNC, ISD::STRICT_FTRUNC,
374 ISD::FRINT, ISD::STRICT_FRINT, ISD::FFLOOR,
375 ISD::STRICT_FFLOOR, ISD::FCEIL, ISD::STRICT_FCEIL}) {
376 setOperationAction(Op, VT, Action: Legal);
377 }
378 setOperationAction(Op: ISD::VECREDUCE_FADD, VT, Action: Custom);
379 setOperationAction(Op: ISD::VECREDUCE_FMUL, VT, Action: Custom);
380 setOperationAction(Op: ISD::VECREDUCE_FMIN, VT, Action: Custom);
381 setOperationAction(Op: ISD::VECREDUCE_FMAX, VT, Action: Custom);
382
383 // No native support for these.
384 setOperationAction(Op: ISD::FDIV, VT, Action: Expand);
385 setOperationAction(Op: ISD::FREM, VT, Action: Expand);
386 setOperationAction(Op: ISD::FSQRT, VT, Action: Expand);
387 setOperationAction(Op: ISD::FSIN, VT, Action: Expand);
388 setOperationAction(Op: ISD::FCOS, VT, Action: Expand);
389 setOperationAction(Op: ISD::FTAN, VT, Action: Expand);
390 setOperationAction(Op: ISD::FPOW, VT, Action: Expand);
391 setOperationAction(Op: ISD::FLOG, VT, Action: Expand);
392 setOperationAction(Op: ISD::FLOG2, VT, Action: Expand);
393 setOperationAction(Op: ISD::FLOG10, VT, Action: Expand);
394 setOperationAction(Op: ISD::FEXP, VT, Action: Expand);
395 setOperationAction(Op: ISD::FEXP2, VT, Action: Expand);
396 setOperationAction(Op: ISD::FEXP10, VT, Action: Expand);
397 setOperationAction(Op: ISD::FNEARBYINT, VT, Action: Expand);
398 }
399 }
400
401 // Custom Expand smaller than legal vector reductions to prevent false zero
402 // items being added.
403 setOperationAction(Op: ISD::VECREDUCE_FADD, VT: MVT::v4f16, Action: Custom);
404 setOperationAction(Op: ISD::VECREDUCE_FMUL, VT: MVT::v4f16, Action: Custom);
405 setOperationAction(Op: ISD::VECREDUCE_FMIN, VT: MVT::v4f16, Action: Custom);
406 setOperationAction(Op: ISD::VECREDUCE_FMAX, VT: MVT::v4f16, Action: Custom);
407 setOperationAction(Op: ISD::VECREDUCE_FADD, VT: MVT::v2f16, Action: Custom);
408 setOperationAction(Op: ISD::VECREDUCE_FMUL, VT: MVT::v2f16, Action: Custom);
409 setOperationAction(Op: ISD::VECREDUCE_FMIN, VT: MVT::v2f16, Action: Custom);
410 setOperationAction(Op: ISD::VECREDUCE_FMAX, VT: MVT::v2f16, Action: Custom);
411
412 // We 'support' these types up to bitcast/load/store level, regardless of
413 // MVE integer-only / float support. Only doing FP data processing on the FP
414 // vector types is inhibited at integer-only level.
415 const MVT LongTypes[] = { MVT::v2i64, MVT::v2f64 };
416 for (auto VT : LongTypes) {
417 addRegisterClass(VT, RC: &ARM::MQPRRegClass);
418 setAllExpand(VT);
419 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT, Action: Custom);
420 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT, Action: Custom);
421 setOperationAction(Op: ISD::BUILD_VECTOR, VT, Action: Custom);
422 setOperationAction(Op: ISD::VSELECT, VT, Action: Legal);
423 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT, Action: Custom);
424 }
425 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v2f64, Action: Legal);
426
427 // We can do bitwise operations on v2i64 vectors
428 setOperationAction(Op: ISD::AND, VT: MVT::v2i64, Action: Legal);
429 setOperationAction(Op: ISD::OR, VT: MVT::v2i64, Action: Legal);
430 setOperationAction(Op: ISD::XOR, VT: MVT::v2i64, Action: Legal);
431
432 // It is legal to extload from v4i8 to v4i16 or v4i32.
433 addAllExtLoads(From: MVT::v8i16, To: MVT::v8i8, Action: Legal);
434 addAllExtLoads(From: MVT::v4i32, To: MVT::v4i16, Action: Legal);
435 addAllExtLoads(From: MVT::v4i32, To: MVT::v4i8, Action: Legal);
436
437 // It is legal to sign extend from v4i8/v4i16 to v4i32 or v8i8 to v8i16.
438 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::v4i8, Action: Legal);
439 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::v4i16, Action: Legal);
440 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::v4i32, Action: Legal);
441 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::v8i8, Action: Legal);
442 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::v8i16, Action: Legal);
443
444 // Some truncating stores are legal too.
445 setTruncStoreAction(ValVT: MVT::v4i32, MemVT: MVT::v4i16, Action: Legal);
446 setTruncStoreAction(ValVT: MVT::v4i32, MemVT: MVT::v4i8, Action: Legal);
447 setTruncStoreAction(ValVT: MVT::v8i16, MemVT: MVT::v8i8, Action: Legal);
448
449 // Pre and Post inc on these are legal, given the correct extends
450 for (unsigned im = (unsigned)ISD::PRE_INC;
451 im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) {
452 for (auto VT : {MVT::v8i8, MVT::v4i8, MVT::v4i16}) {
453 setIndexedLoadAction(IdxModes: im, VT, Action: Legal);
454 setIndexedStoreAction(IdxModes: im, VT, Action: Legal);
455 setIndexedMaskedLoadAction(IdxMode: im, VT, Action: Legal);
456 setIndexedMaskedStoreAction(IdxMode: im, VT, Action: Legal);
457 }
458 }
459
460 // Predicate types
461 const MVT pTypes[] = {MVT::v16i1, MVT::v8i1, MVT::v4i1, MVT::v2i1};
462 for (auto VT : pTypes) {
463 addRegisterClass(VT, RC: &ARM::VCCRRegClass);
464 setOperationAction(Op: ISD::BUILD_VECTOR, VT, Action: Custom);
465 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT, Action: Custom);
466 setOperationAction(Op: ISD::EXTRACT_SUBVECTOR, VT, Action: Custom);
467 setOperationAction(Op: ISD::CONCAT_VECTORS, VT, Action: Custom);
468 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT, Action: Custom);
469 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT, Action: Custom);
470 setOperationAction(Op: ISD::SETCC, VT, Action: Custom);
471 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT, Action: Expand);
472 setOperationAction(Op: ISD::LOAD, VT, Action: Custom);
473 setOperationAction(Op: ISD::STORE, VT, Action: Custom);
474 setOperationAction(Op: ISD::TRUNCATE, VT, Action: Custom);
475 setOperationAction(Op: ISD::VSELECT, VT, Action: Expand);
476 setOperationAction(Op: ISD::SELECT, VT, Action: Expand);
477 setOperationAction(Op: ISD::SELECT_CC, VT, Action: Expand);
478
479 if (!HasMVEFP) {
480 setOperationAction(Op: ISD::SINT_TO_FP, VT, Action: Expand);
481 setOperationAction(Op: ISD::UINT_TO_FP, VT, Action: Expand);
482 setOperationAction(Op: ISD::FP_TO_SINT, VT, Action: Expand);
483 setOperationAction(Op: ISD::FP_TO_UINT, VT, Action: Expand);
484 }
485 }
486 setOperationAction(Op: ISD::SETCC, VT: MVT::v2i1, Action: Expand);
487 setOperationAction(Op: ISD::TRUNCATE, VT: MVT::v2i1, Action: Expand);
488 setOperationAction(Op: ISD::AND, VT: MVT::v2i1, Action: Expand);
489 setOperationAction(Op: ISD::OR, VT: MVT::v2i1, Action: Expand);
490 setOperationAction(Op: ISD::XOR, VT: MVT::v2i1, Action: Expand);
491 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v2i1, Action: Expand);
492 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v2i1, Action: Expand);
493 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::v2i1, Action: Expand);
494 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::v2i1, Action: Expand);
495
496 setOperationAction(Op: ISD::SIGN_EXTEND, VT: MVT::v8i32, Action: Custom);
497 setOperationAction(Op: ISD::SIGN_EXTEND, VT: MVT::v16i16, Action: Custom);
498 setOperationAction(Op: ISD::SIGN_EXTEND, VT: MVT::v16i32, Action: Custom);
499 setOperationAction(Op: ISD::ZERO_EXTEND, VT: MVT::v8i32, Action: Custom);
500 setOperationAction(Op: ISD::ZERO_EXTEND, VT: MVT::v16i16, Action: Custom);
501 setOperationAction(Op: ISD::ZERO_EXTEND, VT: MVT::v16i32, Action: Custom);
502 setOperationAction(Op: ISD::TRUNCATE, VT: MVT::v8i32, Action: Custom);
503 setOperationAction(Op: ISD::TRUNCATE, VT: MVT::v16i16, Action: Custom);
504}
505
506const ARMBaseTargetMachine &ARMTargetLowering::getTM() const {
507 return static_cast<const ARMBaseTargetMachine &>(getTargetMachine());
508}
509
510ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM_,
511 const ARMSubtarget &STI)
512 : TargetLowering(TM_, STI), Subtarget(&STI),
513 RegInfo(Subtarget->getRegisterInfo()),
514 Itins(Subtarget->getInstrItineraryData()) {
515 const auto &TM = static_cast<const ARMBaseTargetMachine &>(TM_);
516
517 setBooleanContents(ZeroOrOneBooleanContent);
518 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
519
520 const Triple &TT = TM.getTargetTriple();
521
522 if (Subtarget->isThumb1Only())
523 addRegisterClass(VT: MVT::i32, RC: &ARM::tGPRRegClass);
524 else
525 addRegisterClass(VT: MVT::i32, RC: &ARM::GPRRegClass);
526
527 if (!Subtarget->useSoftFloat() && !Subtarget->isThumb1Only() &&
528 Subtarget->hasFPRegs()) {
529 addRegisterClass(VT: MVT::f32, RC: &ARM::SPRRegClass);
530 addRegisterClass(VT: MVT::f64, RC: &ARM::DPRRegClass);
531
532 if (!Subtarget->hasVFP2Base()) {
533 setAllExpand(MVT::f32);
534 } else {
535 setOperationAction(Op: ISD::FP_TO_SINT_SAT, VT: MVT::i32, Action: Custom);
536 setOperationAction(Op: ISD::FP_TO_UINT_SAT, VT: MVT::i32, Action: Custom);
537
538 for (auto Op : {ISD::STRICT_FADD, ISD::STRICT_FSUB, ISD::STRICT_FMUL,
539 ISD::STRICT_FDIV, ISD::STRICT_FMA, ISD::STRICT_FSQRT})
540 setOperationAction(Op, VT: MVT::f32, Action: Legal);
541 }
542 if (!Subtarget->hasFP64()) {
543 setAllExpand(MVT::f64);
544 } else {
545 for (auto Op : {ISD::STRICT_FADD, ISD::STRICT_FSUB, ISD::STRICT_FMUL,
546 ISD::STRICT_FDIV, ISD::STRICT_FMA, ISD::STRICT_FSQRT})
547 setOperationAction(Op, VT: MVT::f64, Action: Legal);
548
549 setOperationAction(Op: ISD::STRICT_FP_ROUND, VT: MVT::f32, Action: Legal);
550 }
551 }
552
553 if (Subtarget->hasFullFP16()) {
554 for (auto Op : {ISD::STRICT_FADD, ISD::STRICT_FSUB, ISD::STRICT_FMUL,
555 ISD::STRICT_FDIV, ISD::STRICT_FMA, ISD::STRICT_FSQRT})
556 setOperationAction(Op, VT: MVT::f16, Action: Legal);
557
558 addRegisterClass(VT: MVT::f16, RC: &ARM::HPRRegClass);
559 setOperationAction(Op: ISD::BITCAST, VT: MVT::i16, Action: Custom);
560 setOperationAction(Op: ISD::BITCAST, VT: MVT::f16, Action: Custom);
561
562 setOperationAction(Op: ISD::FMINNUM, VT: MVT::f16, Action: Legal);
563 setOperationAction(Op: ISD::FMAXNUM, VT: MVT::f16, Action: Legal);
564 setOperationAction(Op: ISD::STRICT_FMINNUM, VT: MVT::f16, Action: Legal);
565 setOperationAction(Op: ISD::STRICT_FMAXNUM, VT: MVT::f16, Action: Legal);
566 }
567
568 if (Subtarget->hasBF16()) {
569 addRegisterClass(VT: MVT::bf16, RC: &ARM::HPRRegClass);
570 setAllExpand(MVT::bf16);
571 if (!Subtarget->hasFullFP16())
572 setOperationAction(Op: ISD::BITCAST, VT: MVT::bf16, Action: Custom);
573 setOperationAction(Op: ISD::FP_ROUND, VT: MVT::bf16, Action: Custom);
574 setOperationAction(Op: ISD::STRICT_FP_ROUND, VT: MVT::bf16, Action: Custom);
575 } else {
576 setOperationAction(Op: ISD::BF16_TO_FP, VT: MVT::f32, Action: Expand);
577 setOperationAction(Op: ISD::BF16_TO_FP, VT: MVT::f64, Action: Expand);
578 setOperationAction(Op: ISD::FP_TO_BF16, VT: MVT::f32, Action: Custom);
579 setOperationAction(Op: ISD::FP_TO_BF16, VT: MVT::f64, Action: Custom);
580 }
581
582 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
583 for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) {
584 setTruncStoreAction(ValVT: VT, MemVT: InnerVT, Action: Expand);
585 addAllExtLoads(From: VT, To: InnerVT, Action: Expand);
586 }
587
588 setOperationAction(Op: ISD::SMUL_LOHI, VT, Action: Expand);
589 setOperationAction(Op: ISD::UMUL_LOHI, VT, Action: Expand);
590
591 setOperationAction(Op: ISD::BSWAP, VT, Action: Expand);
592 }
593
594 if (!Subtarget->isThumb1Only() && !Subtarget->hasV8_1MMainlineOps())
595 setOperationAction(Op: ISD::SCMP, VT: MVT::i32, Action: Custom);
596
597 if (!Subtarget->hasV8_1MMainlineOps())
598 setOperationAction(Op: ISD::UCMP, VT: MVT::i32, Action: Custom);
599
600 if (!Subtarget->isThumb1Only())
601 setOperationAction(Op: ISD::ABS, VT: MVT::i32, Action: Custom);
602
603 setOperationAction(Op: ISD::ConstantFP, VT: MVT::f32, Action: Custom);
604 setOperationAction(Op: ISD::ConstantFP, VT: MVT::f64, Action: Custom);
605
606 setOperationAction(Op: ISD::READ_REGISTER, VT: MVT::i64, Action: Custom);
607 setOperationAction(Op: ISD::WRITE_REGISTER, VT: MVT::i64, Action: Custom);
608
609 if (Subtarget->hasMVEIntegerOps())
610 addMVEVectorTypes(HasMVEFP: Subtarget->hasMVEFloatOps());
611
612 // Combine low-overhead loop intrinsics so that we can lower i1 types.
613 if (Subtarget->hasLOB()) {
614 setTargetDAGCombine({ISD::BRCOND, ISD::BR_CC});
615 }
616
617 if (Subtarget->hasNEON()) {
618 addDRTypeForNEON(VT: MVT::v2f32);
619 addDRTypeForNEON(VT: MVT::v8i8);
620 addDRTypeForNEON(VT: MVT::v4i16);
621 addDRTypeForNEON(VT: MVT::v2i32);
622 addDRTypeForNEON(VT: MVT::v1i64);
623
624 addQRTypeForNEON(VT: MVT::v4f32);
625 addQRTypeForNEON(VT: MVT::v2f64);
626 addQRTypeForNEON(VT: MVT::v16i8);
627 addQRTypeForNEON(VT: MVT::v8i16);
628 addQRTypeForNEON(VT: MVT::v4i32);
629 addQRTypeForNEON(VT: MVT::v2i64);
630
631 if (Subtarget->hasFullFP16()) {
632 addQRTypeForNEON(VT: MVT::v8f16);
633 addDRTypeForNEON(VT: MVT::v4f16);
634 }
635
636 if (Subtarget->hasBF16()) {
637 addQRTypeForNEON(VT: MVT::v8bf16);
638 addDRTypeForNEON(VT: MVT::v4bf16);
639 }
640 }
641
642 if (Subtarget->hasMVEIntegerOps() || Subtarget->hasNEON()) {
643 // v2f64 is legal so that QR subregs can be extracted as f64 elements, but
644 // none of Neon, MVE or VFP supports any arithmetic operations on it.
645 setOperationAction(Op: ISD::FADD, VT: MVT::v2f64, Action: Expand);
646 setOperationAction(Op: ISD::FSUB, VT: MVT::v2f64, Action: Expand);
647 setOperationAction(Op: ISD::FMUL, VT: MVT::v2f64, Action: Expand);
648 // FIXME: Code duplication: FDIV and FREM are expanded always, see
649 // ARMTargetLowering::addTypeForNEON method for details.
650 setOperationAction(Op: ISD::FDIV, VT: MVT::v2f64, Action: Expand);
651 setOperationAction(Op: ISD::FREM, VT: MVT::v2f64, Action: Expand);
652 // FIXME: Create unittest.
653 // In another words, find a way when "copysign" appears in DAG with vector
654 // operands.
655 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::v2f64, Action: Expand);
656 // FIXME: Code duplication: SETCC has custom operation action, see
657 // ARMTargetLowering::addTypeForNEON method for details.
658 setOperationAction(Op: ISD::SETCC, VT: MVT::v2f64, Action: Expand);
659 // FIXME: Create unittest for FNEG and for FABS.
660 setOperationAction(Op: ISD::FNEG, VT: MVT::v2f64, Action: Expand);
661 setOperationAction(Op: ISD::FABS, VT: MVT::v2f64, Action: Expand);
662 setOperationAction(Op: ISD::FSQRT, VT: MVT::v2f64, Action: Expand);
663 setOperationAction(Op: ISD::FSIN, VT: MVT::v2f64, Action: Expand);
664 setOperationAction(Op: ISD::FCOS, VT: MVT::v2f64, Action: Expand);
665 setOperationAction(Op: ISD::FTAN, VT: MVT::v2f64, Action: Expand);
666 setOperationAction(Op: ISD::FPOW, VT: MVT::v2f64, Action: Expand);
667 setOperationAction(Op: ISD::FLOG, VT: MVT::v2f64, Action: Expand);
668 setOperationAction(Op: ISD::FLOG2, VT: MVT::v2f64, Action: Expand);
669 setOperationAction(Op: ISD::FLOG10, VT: MVT::v2f64, Action: Expand);
670 setOperationAction(Op: ISD::FEXP, VT: MVT::v2f64, Action: Expand);
671 setOperationAction(Op: ISD::FEXP2, VT: MVT::v2f64, Action: Expand);
672 setOperationAction(Op: ISD::FEXP10, VT: MVT::v2f64, Action: Expand);
673 setOperationAction(Op: ISD::FCEIL, VT: MVT::v2f64, Action: Expand);
674 setOperationAction(Op: ISD::FTRUNC, VT: MVT::v2f64, Action: Expand);
675 setOperationAction(Op: ISD::FRINT, VT: MVT::v2f64, Action: Expand);
676 setOperationAction(Op: ISD::FROUNDEVEN, VT: MVT::v2f64, Action: Expand);
677 setOperationAction(Op: ISD::FNEARBYINT, VT: MVT::v2f64, Action: Expand);
678 setOperationAction(Op: ISD::FFLOOR, VT: MVT::v2f64, Action: Expand);
679 setOperationAction(Op: ISD::FMA, VT: MVT::v2f64, Action: Expand);
680 }
681
682 if (Subtarget->hasNEON()) {
683 // The same with v4f32. But keep in mind that vadd, vsub, vmul are natively
684 // supported for v4f32.
685 setOperationAction(Op: ISD::FSQRT, VT: MVT::v4f32, Action: Expand);
686 setOperationAction(Op: ISD::FSIN, VT: MVT::v4f32, Action: Expand);
687 setOperationAction(Op: ISD::FCOS, VT: MVT::v4f32, Action: Expand);
688 setOperationAction(Op: ISD::FTAN, VT: MVT::v4f32, Action: Expand);
689 setOperationAction(Op: ISD::FPOW, VT: MVT::v4f32, Action: Expand);
690 setOperationAction(Op: ISD::FLOG, VT: MVT::v4f32, Action: Expand);
691 setOperationAction(Op: ISD::FLOG2, VT: MVT::v4f32, Action: Expand);
692 setOperationAction(Op: ISD::FLOG10, VT: MVT::v4f32, Action: Expand);
693 setOperationAction(Op: ISD::FEXP, VT: MVT::v4f32, Action: Expand);
694 setOperationAction(Op: ISD::FEXP2, VT: MVT::v4f32, Action: Expand);
695 setOperationAction(Op: ISD::FEXP10, VT: MVT::v4f32, Action: Expand);
696 setOperationAction(Op: ISD::FCEIL, VT: MVT::v4f32, Action: Expand);
697 setOperationAction(Op: ISD::FTRUNC, VT: MVT::v4f32, Action: Expand);
698 setOperationAction(Op: ISD::FRINT, VT: MVT::v4f32, Action: Expand);
699 setOperationAction(Op: ISD::FROUNDEVEN, VT: MVT::v4f32, Action: Expand);
700 setOperationAction(Op: ISD::FNEARBYINT, VT: MVT::v4f32, Action: Expand);
701 setOperationAction(Op: ISD::FFLOOR, VT: MVT::v4f32, Action: Expand);
702
703 // Mark v2f32 intrinsics.
704 setOperationAction(Op: ISD::FSQRT, VT: MVT::v2f32, Action: Expand);
705 setOperationAction(Op: ISD::FSIN, VT: MVT::v2f32, Action: Expand);
706 setOperationAction(Op: ISD::FCOS, VT: MVT::v2f32, Action: Expand);
707 setOperationAction(Op: ISD::FTAN, VT: MVT::v2f32, Action: Expand);
708 setOperationAction(Op: ISD::FPOW, VT: MVT::v2f32, Action: Expand);
709 setOperationAction(Op: ISD::FLOG, VT: MVT::v2f32, Action: Expand);
710 setOperationAction(Op: ISD::FLOG2, VT: MVT::v2f32, Action: Expand);
711 setOperationAction(Op: ISD::FLOG10, VT: MVT::v2f32, Action: Expand);
712 setOperationAction(Op: ISD::FEXP, VT: MVT::v2f32, Action: Expand);
713 setOperationAction(Op: ISD::FEXP2, VT: MVT::v2f32, Action: Expand);
714 setOperationAction(Op: ISD::FEXP10, VT: MVT::v2f32, Action: Expand);
715 setOperationAction(Op: ISD::FCEIL, VT: MVT::v2f32, Action: Expand);
716 setOperationAction(Op: ISD::FTRUNC, VT: MVT::v2f32, Action: Expand);
717 setOperationAction(Op: ISD::FRINT, VT: MVT::v2f32, Action: Expand);
718 setOperationAction(Op: ISD::FROUNDEVEN, VT: MVT::v2f32, Action: Expand);
719 setOperationAction(Op: ISD::FNEARBYINT, VT: MVT::v2f32, Action: Expand);
720 setOperationAction(Op: ISD::FFLOOR, VT: MVT::v2f32, Action: Expand);
721
722 for (ISD::NodeType Op : {ISD::FFLOOR, ISD::FNEARBYINT, ISD::FCEIL,
723 ISD::FRINT, ISD::FTRUNC, ISD::FROUNDEVEN}) {
724 setOperationAction(Op, VT: MVT::v4f16, Action: Expand);
725 setOperationAction(Op, VT: MVT::v8f16, Action: Expand);
726 }
727
728 // Neon does not support some operations on v1i64 and v2i64 types.
729 setOperationAction(Op: ISD::MUL, VT: MVT::v1i64, Action: Expand);
730 // Custom handling for some quad-vector types to detect VMULL.
731 setOperationAction(Op: ISD::MUL, VT: MVT::v8i16, Action: Custom);
732 setOperationAction(Op: ISD::MUL, VT: MVT::v4i32, Action: Custom);
733 setOperationAction(Op: ISD::MUL, VT: MVT::v2i64, Action: Custom);
734 // Custom handling for some vector types to avoid expensive expansions
735 setOperationAction(Op: ISD::SDIV, VT: MVT::v4i16, Action: Custom);
736 setOperationAction(Op: ISD::SDIV, VT: MVT::v8i8, Action: Custom);
737 setOperationAction(Op: ISD::UDIV, VT: MVT::v4i16, Action: Custom);
738 setOperationAction(Op: ISD::UDIV, VT: MVT::v8i8, Action: Custom);
739 // Neon does not have single instruction SINT_TO_FP and UINT_TO_FP with
740 // a destination type that is wider than the source, and nor does
741 // it have a FP_TO_[SU]INT instruction with a narrower destination than
742 // source.
743 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v4i16, Action: Custom);
744 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v8i16, Action: Custom);
745 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v4i16, Action: Custom);
746 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v8i16, Action: Custom);
747 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::v4i16, Action: Custom);
748 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::v8i16, Action: Custom);
749 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::v4i16, Action: Custom);
750 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::v8i16, Action: Custom);
751
752 setOperationAction(Op: ISD::FP_ROUND, VT: MVT::v2f32, Action: Expand);
753 setOperationAction(Op: ISD::FP_EXTEND, VT: MVT::v2f64, Action: Expand);
754
755 // NEON does not have single instruction CTPOP for vectors with element
756 // types wider than 8-bits. However, custom lowering can leverage the
757 // v8i8/v16i8 vcnt instruction.
758 setOperationAction(Op: ISD::CTPOP, VT: MVT::v2i32, Action: Custom);
759 setOperationAction(Op: ISD::CTPOP, VT: MVT::v4i32, Action: Custom);
760 setOperationAction(Op: ISD::CTPOP, VT: MVT::v4i16, Action: Custom);
761 setOperationAction(Op: ISD::CTPOP, VT: MVT::v8i16, Action: Custom);
762 setOperationAction(Op: ISD::CTPOP, VT: MVT::v1i64, Action: Custom);
763 setOperationAction(Op: ISD::CTPOP, VT: MVT::v2i64, Action: Custom);
764
765 setOperationAction(Op: ISD::CTLZ, VT: MVT::v1i64, Action: Expand);
766 setOperationAction(Op: ISD::CTLZ, VT: MVT::v2i64, Action: Expand);
767
768 // NEON does not have single instruction CTTZ for vectors.
769 setOperationAction(Op: ISD::CTTZ, VT: MVT::v8i8, Action: Custom);
770 setOperationAction(Op: ISD::CTTZ, VT: MVT::v4i16, Action: Custom);
771 setOperationAction(Op: ISD::CTTZ, VT: MVT::v2i32, Action: Custom);
772 setOperationAction(Op: ISD::CTTZ, VT: MVT::v1i64, Action: Custom);
773
774 setOperationAction(Op: ISD::CTTZ, VT: MVT::v16i8, Action: Custom);
775 setOperationAction(Op: ISD::CTTZ, VT: MVT::v8i16, Action: Custom);
776 setOperationAction(Op: ISD::CTTZ, VT: MVT::v4i32, Action: Custom);
777 setOperationAction(Op: ISD::CTTZ, VT: MVT::v2i64, Action: Custom);
778
779 setOperationAction(Op: ISD::CTTZ_ZERO_POISON, VT: MVT::v8i8, Action: Custom);
780 setOperationAction(Op: ISD::CTTZ_ZERO_POISON, VT: MVT::v4i16, Action: Custom);
781 setOperationAction(Op: ISD::CTTZ_ZERO_POISON, VT: MVT::v2i32, Action: Custom);
782 setOperationAction(Op: ISD::CTTZ_ZERO_POISON, VT: MVT::v1i64, Action: Custom);
783
784 setOperationAction(Op: ISD::CTTZ_ZERO_POISON, VT: MVT::v16i8, Action: Custom);
785 setOperationAction(Op: ISD::CTTZ_ZERO_POISON, VT: MVT::v8i16, Action: Custom);
786 setOperationAction(Op: ISD::CTTZ_ZERO_POISON, VT: MVT::v4i32, Action: Custom);
787 setOperationAction(Op: ISD::CTTZ_ZERO_POISON, VT: MVT::v2i64, Action: Custom);
788
789 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
790 setOperationAction(Op: ISD::MULHS, VT, Action: Expand);
791 setOperationAction(Op: ISD::MULHU, VT, Action: Expand);
792 }
793
794 // NEON only has FMA instructions as of VFP4.
795 if (!Subtarget->hasVFP4Base()) {
796 setOperationAction(Op: ISD::FMA, VT: MVT::v2f32, Action: Expand);
797 setOperationAction(Op: ISD::FMA, VT: MVT::v4f32, Action: Expand);
798 }
799
800 setTargetDAGCombine({ISD::SHL, ISD::SRL, ISD::SRA, ISD::FP_TO_SINT,
801 ISD::FP_TO_UINT, ISD::FMUL, ISD::LOAD});
802
803 // It is legal to extload from v4i8 to v4i16 or v4i32.
804 for (MVT Ty : {MVT::v8i8, MVT::v4i8, MVT::v2i8, MVT::v4i16, MVT::v2i16,
805 MVT::v2i32}) {
806 for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) {
807 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: Ty, Action: Legal);
808 setLoadExtAction(ExtType: ISD::ZEXTLOAD, ValVT: VT, MemVT: Ty, Action: Legal);
809 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: VT, MemVT: Ty, Action: Legal);
810 }
811 }
812
813 for (auto VT : {MVT::v8i8, MVT::v4i16, MVT::v2i32, MVT::v16i8, MVT::v8i16,
814 MVT::v4i32}) {
815 setOperationAction(Op: ISD::VECREDUCE_SMAX, VT, Action: Custom);
816 setOperationAction(Op: ISD::VECREDUCE_UMAX, VT, Action: Custom);
817 setOperationAction(Op: ISD::VECREDUCE_SMIN, VT, Action: Custom);
818 setOperationAction(Op: ISD::VECREDUCE_UMIN, VT, Action: Custom);
819 }
820 }
821
822 if (Subtarget->hasNEON() || Subtarget->hasMVEIntegerOps()) {
823 setTargetDAGCombine(
824 {ISD::BUILD_VECTOR, ISD::VECTOR_SHUFFLE, ISD::INSERT_SUBVECTOR,
825 ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT,
826 ISD::SIGN_EXTEND_INREG, ISD::STORE, ISD::SIGN_EXTEND, ISD::ZERO_EXTEND,
827 ISD::ANY_EXTEND, ISD::INTRINSIC_WO_CHAIN, ISD::INTRINSIC_W_CHAIN,
828 ISD::INTRINSIC_VOID, ISD::VECREDUCE_ADD, ISD::ADD, ISD::BITCAST});
829 }
830 if (Subtarget->hasMVEIntegerOps()) {
831 setTargetDAGCombine({ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX,
832 ISD::FP_EXTEND, ISD::SELECT, ISD::SELECT_CC,
833 ISD::SETCC});
834 }
835 if (Subtarget->hasMVEFloatOps()) {
836 setTargetDAGCombine(ISD::FADD);
837 }
838
839 if (!Subtarget->hasFP64()) {
840 // When targeting a floating-point unit with only single-precision
841 // operations, f64 is legal for the few double-precision instructions which
842 // are present However, no double-precision operations other than moves,
843 // loads and stores are provided by the hardware.
844 setOperationAction(Op: ISD::FADD, VT: MVT::f64, Action: Expand);
845 setOperationAction(Op: ISD::FSUB, VT: MVT::f64, Action: Expand);
846 setOperationAction(Op: ISD::FMUL, VT: MVT::f64, Action: Expand);
847 setOperationAction(Op: ISD::FMA, VT: MVT::f64, Action: Expand);
848 setOperationAction(Op: ISD::FDIV, VT: MVT::f64, Action: Expand);
849 setOperationAction(Op: ISD::FREM, VT: MVT::f64, Action: LibCall);
850 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::f64, Action: Expand);
851 setOperationAction(Op: ISD::FGETSIGN, VT: MVT::f64, Action: Expand);
852 setOperationAction(Op: ISD::FNEG, VT: MVT::f64, Action: Expand);
853 setOperationAction(Op: ISD::FABS, VT: MVT::f64, Action: Expand);
854 setOperationAction(Op: ISD::FSQRT, VT: MVT::f64, Action: Expand);
855 setOperationAction(Op: ISD::FSIN, VT: MVT::f64, Action: Expand);
856 setOperationAction(Op: ISD::FCOS, VT: MVT::f64, Action: Expand);
857 setOperationAction(Op: ISD::FPOW, VT: MVT::f64, Action: Expand);
858 setOperationAction(Op: ISD::FLOG, VT: MVT::f64, Action: Expand);
859 setOperationAction(Op: ISD::FLOG2, VT: MVT::f64, Action: Expand);
860 setOperationAction(Op: ISD::FLOG10, VT: MVT::f64, Action: Expand);
861 setOperationAction(Op: ISD::FEXP, VT: MVT::f64, Action: Expand);
862 setOperationAction(Op: ISD::FEXP2, VT: MVT::f64, Action: Expand);
863 setOperationAction(Op: ISD::FEXP10, VT: MVT::f64, Action: Expand);
864 setOperationAction(Op: ISD::FCEIL, VT: MVT::f64, Action: Expand);
865 setOperationAction(Op: ISD::FTRUNC, VT: MVT::f64, Action: Expand);
866 setOperationAction(Op: ISD::FRINT, VT: MVT::f64, Action: Expand);
867 setOperationAction(Op: ISD::FROUNDEVEN, VT: MVT::f64, Action: Expand);
868 setOperationAction(Op: ISD::FNEARBYINT, VT: MVT::f64, Action: Expand);
869 setOperationAction(Op: ISD::FFLOOR, VT: MVT::f64, Action: Expand);
870 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i32, Action: Custom);
871 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::i32, Action: Custom);
872 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::i32, Action: Custom);
873 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i32, Action: Custom);
874 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::f64, Action: Custom);
875 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::f64, Action: Custom);
876 setOperationAction(Op: ISD::FP_ROUND, VT: MVT::f32, Action: Custom);
877 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::f64, Action: Custom);
878 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::f64, Action: Custom);
879 setOperationAction(Op: ISD::STRICT_FP_ROUND, VT: MVT::f32, Action: Custom);
880 }
881
882 // STRICT_(U/S)INT_TO_FP specifically use the input MVT to register with
883 // setOperationAction() as opposed to other opcodes that use the output MVT
884 // All inputs should be i32 due to type legalization
885 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::i32, Action: Custom);
886 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::i32, Action: Custom);
887
888 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::i32, Action: Custom);
889 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::i32, Action: Custom);
890
891 if (!Subtarget->hasFP64() || !Subtarget->hasFPARMv8Base()) {
892 setOperationAction(Op: ISD::FP_EXTEND, VT: MVT::f64, Action: Custom);
893 setOperationAction(Op: ISD::STRICT_FP_EXTEND, VT: MVT::f64, Action: Custom);
894 if (Subtarget->hasFullFP16()) {
895 setOperationAction(Op: ISD::FP_ROUND, VT: MVT::f16, Action: Custom);
896 setOperationAction(Op: ISD::STRICT_FP_ROUND, VT: MVT::f16, Action: Custom);
897 }
898 } else {
899 setOperationAction(Op: ISD::STRICT_FP_EXTEND, VT: MVT::f64, Action: Legal);
900 }
901
902 if (!Subtarget->hasFP16()) {
903 setOperationAction(Op: ISD::FP_EXTEND, VT: MVT::f32, Action: Custom);
904 setOperationAction(Op: ISD::STRICT_FP_EXTEND, VT: MVT::f32, Action: Custom);
905 } else {
906 setOperationAction(Op: ISD::STRICT_FP_EXTEND, VT: MVT::f32, Action: Legal);
907 setOperationAction(Op: ISD::STRICT_FP_ROUND, VT: MVT::f16, Action: Legal);
908 }
909
910 computeRegisterProperties(TRI: Subtarget->getRegisterInfo());
911
912 // ARM does not have floating-point extending loads.
913 for (MVT VT : MVT::fp_valuetypes()) {
914 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: MVT::f32, Action: Expand);
915 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: MVT::f16, Action: Expand);
916 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: MVT::bf16, Action: Expand);
917 }
918
919 // ... or truncating stores
920 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f32, Action: Expand);
921 setTruncStoreAction(ValVT: MVT::f32, MemVT: MVT::f16, Action: Expand);
922 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f16, Action: Expand);
923 setTruncStoreAction(ValVT: MVT::f32, MemVT: MVT::bf16, Action: Expand);
924 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::bf16, Action: Expand);
925
926 // ARM does not have i1 sign extending load.
927 for (MVT VT : MVT::integer_valuetypes())
928 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: VT, MemVT: MVT::i1, Action: Promote);
929
930 // ARM supports all 4 flavors of integer indexed load / store.
931 if (!Subtarget->isThumb1Only()) {
932 for (unsigned im = (unsigned)ISD::PRE_INC;
933 im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) {
934 setIndexedLoadAction(IdxModes: im, VT: MVT::i1, Action: Legal);
935 setIndexedLoadAction(IdxModes: im, VT: MVT::i8, Action: Legal);
936 setIndexedLoadAction(IdxModes: im, VT: MVT::i16, Action: Legal);
937 setIndexedLoadAction(IdxModes: im, VT: MVT::i32, Action: Legal);
938 setIndexedStoreAction(IdxModes: im, VT: MVT::i1, Action: Legal);
939 setIndexedStoreAction(IdxModes: im, VT: MVT::i8, Action: Legal);
940 setIndexedStoreAction(IdxModes: im, VT: MVT::i16, Action: Legal);
941 setIndexedStoreAction(IdxModes: im, VT: MVT::i32, Action: Legal);
942 }
943 } else {
944 // Thumb-1 has limited post-inc load/store support - LDM r0!, {r1}.
945 setIndexedLoadAction(IdxModes: ISD::POST_INC, VT: MVT::i32, Action: Legal);
946 setIndexedStoreAction(IdxModes: ISD::POST_INC, VT: MVT::i32, Action: Legal);
947 }
948
949 // Custom loads/stores to possible use __aeabi_uread/write*
950 if (TT.isTargetAEABI() && !Subtarget->allowsUnalignedMem()) {
951 setOperationAction(Op: ISD::STORE, VT: MVT::i32, Action: Custom);
952 setOperationAction(Op: ISD::STORE, VT: MVT::i64, Action: Custom);
953 setOperationAction(Op: ISD::LOAD, VT: MVT::i32, Action: Custom);
954 setOperationAction(Op: ISD::LOAD, VT: MVT::i64, Action: Custom);
955 }
956
957 setOperationAction(Op: ISD::SADDO, VT: MVT::i32, Action: Custom);
958 setOperationAction(Op: ISD::UADDO, VT: MVT::i32, Action: Custom);
959 setOperationAction(Op: ISD::SSUBO, VT: MVT::i32, Action: Custom);
960 setOperationAction(Op: ISD::USUBO, VT: MVT::i32, Action: Custom);
961
962 if (!Subtarget->isThumb1Only()) {
963 setOperationAction(Op: ISD::UMULO, VT: MVT::i32, Action: Custom);
964 setOperationAction(Op: ISD::SMULO, VT: MVT::i32, Action: Custom);
965 }
966
967 setOperationAction(Op: ISD::UADDO_CARRY, VT: MVT::i32, Action: Custom);
968 setOperationAction(Op: ISD::USUBO_CARRY, VT: MVT::i32, Action: Custom);
969 setOperationAction(Op: ISD::SADDO_CARRY, VT: MVT::i32, Action: Custom);
970 setOperationAction(Op: ISD::SSUBO_CARRY, VT: MVT::i32, Action: Custom);
971 if (Subtarget->hasDSP()) {
972 setOperationAction(Op: ISD::SADDSAT, VT: MVT::i8, Action: Custom);
973 setOperationAction(Op: ISD::SSUBSAT, VT: MVT::i8, Action: Custom);
974 setOperationAction(Op: ISD::SADDSAT, VT: MVT::i16, Action: Custom);
975 setOperationAction(Op: ISD::SSUBSAT, VT: MVT::i16, Action: Custom);
976 setOperationAction(Op: ISD::UADDSAT, VT: MVT::i8, Action: Custom);
977 setOperationAction(Op: ISD::USUBSAT, VT: MVT::i8, Action: Custom);
978 setOperationAction(Op: ISD::UADDSAT, VT: MVT::i16, Action: Custom);
979 setOperationAction(Op: ISD::USUBSAT, VT: MVT::i16, Action: Custom);
980 }
981 if (Subtarget->hasBaseDSP()) {
982 setOperationAction(Op: ISD::SADDSAT, VT: MVT::i32, Action: Legal);
983 setOperationAction(Op: ISD::SSUBSAT, VT: MVT::i32, Action: Legal);
984 }
985
986 // i64 operation support.
987 setOperationAction(Op: ISD::MUL, VT: MVT::i64, Action: Expand);
988 setOperationAction(Op: ISD::MULHU, VT: MVT::i32, Action: Expand);
989 if (Subtarget->isThumb1Only()) {
990 setOperationAction(Op: ISD::UMUL_LOHI, VT: MVT::i32, Action: Expand);
991 setOperationAction(Op: ISD::SMUL_LOHI, VT: MVT::i32, Action: Expand);
992 }
993 if (Subtarget->isThumb1Only() || !Subtarget->hasV6Ops()
994 || (Subtarget->isThumb2() && !Subtarget->hasDSP()))
995 setOperationAction(Op: ISD::MULHS, VT: MVT::i32, Action: Expand);
996
997 setOperationAction(Op: ISD::SHL_PARTS, VT: MVT::i32, Action: Custom);
998 setOperationAction(Op: ISD::SRA_PARTS, VT: MVT::i32, Action: Custom);
999 setOperationAction(Op: ISD::SRL_PARTS, VT: MVT::i32, Action: Custom);
1000 setOperationAction(Op: ISD::SRL, VT: MVT::i64, Action: Custom);
1001 setOperationAction(Op: ISD::SRA, VT: MVT::i64, Action: Custom);
1002 setOperationAction(Op: ISD::INTRINSIC_VOID, VT: MVT::Other, Action: Custom);
1003 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::i64, Action: Custom);
1004 setOperationAction(Op: ISD::LOAD, VT: MVT::i64, Action: Custom);
1005 setOperationAction(Op: ISD::STORE, VT: MVT::i64, Action: Custom);
1006
1007 // MVE lowers 64 bit shifts to lsll and lsrl
1008 // assuming that ISD::SRL and SRA of i64 are already marked custom
1009 if (Subtarget->hasMVEIntegerOps())
1010 setOperationAction(Op: ISD::SHL, VT: MVT::i64, Action: Custom);
1011
1012 // Expand to __aeabi_l{lsl,lsr,asr} calls for Thumb1.
1013 if (Subtarget->isThumb1Only()) {
1014 setOperationAction(Op: ISD::SHL_PARTS, VT: MVT::i32, Action: Expand);
1015 setOperationAction(Op: ISD::SRA_PARTS, VT: MVT::i32, Action: Expand);
1016 setOperationAction(Op: ISD::SRL_PARTS, VT: MVT::i32, Action: Expand);
1017 }
1018
1019 if (!Subtarget->isThumb1Only() && Subtarget->hasV6T2Ops())
1020 setOperationAction(Op: ISD::BITREVERSE, VT: MVT::i32, Action: Legal);
1021
1022 // ARM does not have ROTL.
1023 setOperationAction(Op: ISD::ROTL, VT: MVT::i32, Action: Expand);
1024 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
1025 setOperationAction(Op: ISD::ROTL, VT, Action: Expand);
1026 setOperationAction(Op: ISD::ROTR, VT, Action: Expand);
1027 }
1028 setOperationAction(Op: ISD::CTTZ, VT: MVT::i32, Action: Custom);
1029 // TODO: These two should be set to LibCall, but this currently breaks
1030 // the Linux kernel build. See #101786.
1031 setOperationAction(Op: ISD::CTPOP, VT: MVT::i32, Action: Expand);
1032 setOperationAction(Op: ISD::CTPOP, VT: MVT::i64, Action: Expand);
1033 if (!Subtarget->hasV5TOps() || Subtarget->isThumb1Only()) {
1034 setOperationAction(Op: ISD::CTLZ, VT: MVT::i32, Action: Expand);
1035 setOperationAction(Op: ISD::CTLZ_ZERO_POISON, VT: MVT::i32, Action: LibCall);
1036 }
1037
1038 // @llvm.readcyclecounter requires the Performance Monitors extension.
1039 // Default to the 0 expansion on unsupported platforms.
1040 // FIXME: Technically there are older ARM CPUs that have
1041 // implementation-specific ways of obtaining this information.
1042 if (Subtarget->hasPerfMon())
1043 setOperationAction(Op: ISD::READCYCLECOUNTER, VT: MVT::i64, Action: Custom);
1044
1045 // Only ARMv6 has BSWAP.
1046 if (!Subtarget->hasV6Ops())
1047 setOperationAction(Op: ISD::BSWAP, VT: MVT::i32, Action: Expand);
1048
1049 bool hasDivide = Subtarget->isThumb() ? Subtarget->hasDivideInThumbMode()
1050 : Subtarget->hasDivideInARMMode();
1051 if (!hasDivide) {
1052 // These are expanded into libcalls if the cpu doesn't have HW divider.
1053 setOperationAction(Op: ISD::SDIV, VT: MVT::i32, Action: LibCall);
1054 setOperationAction(Op: ISD::UDIV, VT: MVT::i32, Action: LibCall);
1055 }
1056
1057 if (TT.isOSWindows() && !Subtarget->hasDivideInThumbMode()) {
1058 setOperationAction(Op: ISD::SDIV, VT: MVT::i32, Action: Custom);
1059 setOperationAction(Op: ISD::UDIV, VT: MVT::i32, Action: Custom);
1060
1061 setOperationAction(Op: ISD::SDIV, VT: MVT::i64, Action: Custom);
1062 setOperationAction(Op: ISD::UDIV, VT: MVT::i64, Action: Custom);
1063 }
1064
1065 setOperationAction(Op: ISD::SREM, VT: MVT::i32, Action: Expand);
1066 setOperationAction(Op: ISD::UREM, VT: MVT::i32, Action: Expand);
1067
1068 // Register based DivRem for AEABI (RTABI 4.2)
1069 if (TT.isTargetAEABI() || TT.isAndroid() || TT.isTargetGNUAEABI() ||
1070 TT.isTargetMuslAEABI() || TT.isOSFuchsia() || TT.isOSWindows()) {
1071 setOperationAction(Op: ISD::SREM, VT: MVT::i64, Action: Custom);
1072 setOperationAction(Op: ISD::UREM, VT: MVT::i64, Action: Custom);
1073 HasStandaloneRem = false;
1074
1075 setOperationAction(Op: ISD::SDIVREM, VT: MVT::i32, Action: Custom);
1076 setOperationAction(Op: ISD::UDIVREM, VT: MVT::i32, Action: Custom);
1077 setOperationAction(Op: ISD::SDIVREM, VT: MVT::i64, Action: Custom);
1078 setOperationAction(Op: ISD::UDIVREM, VT: MVT::i64, Action: Custom);
1079 } else {
1080 setOperationAction(Op: ISD::SDIVREM, VT: MVT::i32, Action: Expand);
1081 setOperationAction(Op: ISD::UDIVREM, VT: MVT::i32, Action: Expand);
1082 }
1083
1084 setOperationAction(Op: ISD::GlobalAddress, VT: MVT::i32, Action: Custom);
1085 setOperationAction(Op: ISD::ConstantPool, VT: MVT::i32, Action: Custom);
1086 setOperationAction(Op: ISD::GlobalTLSAddress, VT: MVT::i32, Action: Custom);
1087 setOperationAction(Op: ISD::BlockAddress, VT: MVT::i32, Action: Custom);
1088
1089 setOperationAction(Op: ISD::TRAP, VT: MVT::Other, Action: Legal);
1090 setOperationAction(Op: ISD::DEBUGTRAP, VT: MVT::Other, Action: Legal);
1091
1092 // Use the default implementation.
1093 setOperationAction(Op: ISD::VASTART, VT: MVT::Other, Action: Custom);
1094 setOperationAction(Op: ISD::VAARG, VT: MVT::Other, Action: Expand);
1095 setOperationAction(Op: ISD::VACOPY, VT: MVT::Other, Action: Expand);
1096 setOperationAction(Op: ISD::VAEND, VT: MVT::Other, Action: Expand);
1097 setOperationAction(Op: ISD::STACKSAVE, VT: MVT::Other, Action: Expand);
1098 setOperationAction(Op: ISD::STACKRESTORE, VT: MVT::Other, Action: Expand);
1099
1100 if (TT.isOSWindows())
1101 setOperationAction(Op: ISD::DYNAMIC_STACKALLOC, VT: MVT::i32, Action: Custom);
1102 else
1103 setOperationAction(Op: ISD::DYNAMIC_STACKALLOC, VT: MVT::i32, Action: Expand);
1104
1105 // ARMv6 Thumb1 (except for CPUs that support dmb / dsb) and earlier use
1106 // the default expansion.
1107 InsertFencesForAtomic = false;
1108 if (Subtarget->hasAnyDataBarrier() &&
1109 (!Subtarget->isThumb() || Subtarget->hasV8MBaselineOps())) {
1110 // ATOMIC_FENCE needs custom lowering; the others should have been expanded
1111 // to ldrex/strex loops already.
1112 setOperationAction(Op: ISD::ATOMIC_FENCE, VT: MVT::Other, Action: Custom);
1113 if (!Subtarget->isThumb() || !Subtarget->isMClass())
1114 setOperationAction(Op: ISD::ATOMIC_CMP_SWAP, VT: MVT::i64, Action: Custom);
1115
1116 // On v8, we have particularly efficient implementations of atomic fences
1117 // if they can be combined with nearby atomic loads and stores.
1118 if (!Subtarget->hasAcquireRelease() ||
1119 getTargetMachine().getOptLevel() == CodeGenOptLevel::None) {
1120 // Automatically insert fences (dmb ish) around ATOMIC_SWAP etc.
1121 InsertFencesForAtomic = true;
1122 }
1123 } else {
1124 // If there's anything we can use as a barrier, go through custom lowering
1125 // for ATOMIC_FENCE.
1126 // If target has DMB in thumb, Fences can be inserted.
1127 if (Subtarget->hasDataBarrier())
1128 InsertFencesForAtomic = true;
1129
1130 setOperationAction(Op: ISD::ATOMIC_FENCE, VT: MVT::Other,
1131 Action: Subtarget->hasAnyDataBarrier() ? Custom : Expand);
1132
1133 // Set them all for libcall, which will force libcalls.
1134 setOperationAction(Op: ISD::ATOMIC_CMP_SWAP, VT: MVT::i32, Action: LibCall);
1135 setOperationAction(Op: ISD::ATOMIC_SWAP, VT: MVT::i32, Action: LibCall);
1136 setOperationAction(Op: ISD::ATOMIC_LOAD_ADD, VT: MVT::i32, Action: LibCall);
1137 setOperationAction(Op: ISD::ATOMIC_LOAD_SUB, VT: MVT::i32, Action: LibCall);
1138 setOperationAction(Op: ISD::ATOMIC_LOAD_AND, VT: MVT::i32, Action: LibCall);
1139 setOperationAction(Op: ISD::ATOMIC_LOAD_OR, VT: MVT::i32, Action: LibCall);
1140 setOperationAction(Op: ISD::ATOMIC_LOAD_XOR, VT: MVT::i32, Action: LibCall);
1141 setOperationAction(Op: ISD::ATOMIC_LOAD_NAND, VT: MVT::i32, Action: LibCall);
1142 setOperationAction(Op: ISD::ATOMIC_LOAD_MIN, VT: MVT::i32, Action: LibCall);
1143 setOperationAction(Op: ISD::ATOMIC_LOAD_MAX, VT: MVT::i32, Action: LibCall);
1144 setOperationAction(Op: ISD::ATOMIC_LOAD_UMIN, VT: MVT::i32, Action: LibCall);
1145 setOperationAction(Op: ISD::ATOMIC_LOAD_UMAX, VT: MVT::i32, Action: LibCall);
1146 // Mark ATOMIC_LOAD and ATOMIC_STORE custom so we can handle the
1147 // Unordered/Monotonic case.
1148 if (!InsertFencesForAtomic) {
1149 setOperationAction(Op: ISD::ATOMIC_LOAD, VT: MVT::i32, Action: Custom);
1150 setOperationAction(Op: ISD::ATOMIC_STORE, VT: MVT::i32, Action: Custom);
1151 }
1152 }
1153
1154 // Compute supported atomic widths.
1155 if (TT.isOSLinux() || (!Subtarget->isMClass() && Subtarget->hasV6Ops())) {
1156 // For targets where __sync_* routines are reliably available, we use them
1157 // if necessary.
1158 //
1159 // ARM Linux always supports 64-bit atomics through kernel-assisted atomic
1160 // routines (kernel 3.1 or later). FIXME: Not with compiler-rt?
1161 //
1162 // ARMv6 targets have native instructions in ARM mode. For Thumb mode,
1163 // such targets should provide __sync_* routines, which use the ARM mode
1164 // instructions. (ARMv6 doesn't have dmb, but it has an equivalent
1165 // encoding; see ARMISD::MEMBARRIER_MCR.)
1166 setMaxAtomicSizeInBitsSupported(64);
1167 } else if ((Subtarget->isMClass() && Subtarget->hasV8MBaselineOps()) ||
1168 Subtarget->hasForced32BitAtomics()) {
1169 // Cortex-M (besides Cortex-M0) have 32-bit atomics.
1170 setMaxAtomicSizeInBitsSupported(32);
1171 } else {
1172 // We can't assume anything about other targets; just use libatomic
1173 // routines.
1174 setMaxAtomicSizeInBitsSupported(0);
1175 }
1176
1177 setMaxDivRemBitWidthSupported(64);
1178
1179 setOperationAction(Op: ISD::PREFETCH, VT: MVT::Other, Action: Custom);
1180
1181 // Requires SXTB/SXTH, available on v6 and up in both ARM and Thumb modes.
1182 if (!Subtarget->hasV6Ops()) {
1183 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i16, Action: Expand);
1184 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i8, Action: Expand);
1185 }
1186 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i1, Action: Expand);
1187
1188 if (!Subtarget->useSoftFloat() && Subtarget->hasFPRegs() &&
1189 !Subtarget->isThumb1Only()) {
1190 // Turn f64->i64 into VMOVRRD, i64 -> f64 to VMOVDRR
1191 // iff target supports vfp2.
1192 setOperationAction(Op: ISD::BITCAST, VT: MVT::i64, Action: Custom);
1193 setOperationAction(Op: ISD::GET_ROUNDING, VT: MVT::i32, Action: Custom);
1194 setOperationAction(Op: ISD::SET_ROUNDING, VT: MVT::Other, Action: Custom);
1195 setOperationAction(Op: ISD::GET_FPENV, VT: MVT::i32, Action: Legal);
1196 setOperationAction(Op: ISD::SET_FPENV, VT: MVT::i32, Action: Legal);
1197 setOperationAction(Op: ISD::RESET_FPENV, VT: MVT::Other, Action: Legal);
1198 setOperationAction(Op: ISD::GET_FPMODE, VT: MVT::i32, Action: Legal);
1199 setOperationAction(Op: ISD::SET_FPMODE, VT: MVT::i32, Action: Custom);
1200 setOperationAction(Op: ISD::RESET_FPMODE, VT: MVT::Other, Action: Custom);
1201 }
1202
1203 // We want to custom lower some of our intrinsics.
1204 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::Other, Action: Custom);
1205 setOperationAction(Op: ISD::EH_SJLJ_SETJMP, VT: MVT::i32, Action: Custom);
1206 setOperationAction(Op: ISD::EH_SJLJ_LONGJMP, VT: MVT::Other, Action: Custom);
1207 setOperationAction(Op: ISD::EH_SJLJ_SETUP_DISPATCH, VT: MVT::Other, Action: Custom);
1208
1209 setOperationAction(Op: ISD::SETCC, VT: MVT::i32, Action: Expand);
1210 setOperationAction(Op: ISD::SETCC, VT: MVT::f32, Action: Expand);
1211 setOperationAction(Op: ISD::SETCC, VT: MVT::f64, Action: Expand);
1212 setOperationAction(Op: ISD::SELECT, VT: MVT::i32, Action: Custom);
1213 setOperationAction(Op: ISD::SELECT, VT: MVT::f32, Action: Custom);
1214 setOperationAction(Op: ISD::SELECT, VT: MVT::f64, Action: Custom);
1215 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::i32, Action: Custom);
1216 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::f32, Action: Custom);
1217 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::f64, Action: Custom);
1218 if (Subtarget->hasFullFP16()) {
1219 setOperationAction(Op: ISD::SETCC, VT: MVT::f16, Action: Expand);
1220 setOperationAction(Op: ISD::SELECT, VT: MVT::f16, Action: Custom);
1221 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::f16, Action: Custom);
1222 }
1223
1224 setOperationAction(Op: ISD::SETCCCARRY, VT: MVT::i32, Action: Custom);
1225
1226 setOperationAction(Op: ISD::BRCOND, VT: MVT::Other, Action: Custom);
1227 setOperationAction(Op: ISD::BR_CC, VT: MVT::i32, Action: Custom);
1228 if (Subtarget->hasFullFP16())
1229 setOperationAction(Op: ISD::BR_CC, VT: MVT::f16, Action: Custom);
1230 setOperationAction(Op: ISD::BR_CC, VT: MVT::f32, Action: Custom);
1231 setOperationAction(Op: ISD::BR_CC, VT: MVT::f64, Action: Custom);
1232 setOperationAction(Op: ISD::BR_JT, VT: MVT::Other, Action: Custom);
1233
1234 // We don't support sin/cos/fmod/copysign/pow
1235 setOperationAction(Op: ISD::FSIN, VT: MVT::f64, Action: Expand);
1236 setOperationAction(Op: ISD::FSIN, VT: MVT::f32, Action: Expand);
1237 setOperationAction(Op: ISD::FCOS, VT: MVT::f32, Action: Expand);
1238 setOperationAction(Op: ISD::FCOS, VT: MVT::f64, Action: Expand);
1239 setOperationAction(Op: ISD::FSINCOS, VT: MVT::f64, Action: Expand);
1240 setOperationAction(Op: ISD::FSINCOS, VT: MVT::f32, Action: Expand);
1241 setOperationAction(Op: ISD::FREM, VT: MVT::f64, Action: LibCall);
1242 setOperationAction(Op: ISD::FREM, VT: MVT::f32, Action: LibCall);
1243 if (!Subtarget->useSoftFloat() && Subtarget->hasVFP2Base() &&
1244 !Subtarget->isThumb1Only()) {
1245 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::f64, Action: Custom);
1246 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::f32, Action: Custom);
1247 }
1248 setOperationAction(Op: ISD::FPOW, VT: MVT::f64, Action: Expand);
1249 setOperationAction(Op: ISD::FPOW, VT: MVT::f32, Action: Expand);
1250
1251 if (!Subtarget->hasVFP4Base()) {
1252 setOperationAction(Op: ISD::FMA, VT: MVT::f64, Action: Expand);
1253 setOperationAction(Op: ISD::FMA, VT: MVT::f32, Action: Expand);
1254 }
1255
1256 // Various VFP goodness
1257 if (!Subtarget->useSoftFloat() && !Subtarget->isThumb1Only()) {
1258 // FP-ARMv8 adds f64 <-> f16 conversion. Before that it should be expanded.
1259 if (!Subtarget->hasFPARMv8Base() || !Subtarget->hasFP64()) {
1260 setOperationAction(Op: ISD::FP16_TO_FP, VT: MVT::f64, Action: Expand);
1261 setOperationAction(Op: ISD::FP_TO_FP16, VT: MVT::f64, Action: Expand);
1262 setOperationAction(Op: ISD::STRICT_FP16_TO_FP, VT: MVT::f64, Action: Expand);
1263 setOperationAction(Op: ISD::STRICT_FP_TO_FP16, VT: MVT::f64, Action: Expand);
1264 }
1265
1266 // fp16 is a special v7 extension that adds f16 <-> f32 conversions.
1267 if (!Subtarget->hasFP16()) {
1268 setOperationAction(Op: ISD::FP16_TO_FP, VT: MVT::f32, Action: Expand);
1269 setOperationAction(Op: ISD::FP_TO_FP16, VT: MVT::f32, Action: Expand);
1270 setOperationAction(Op: ISD::STRICT_FP16_TO_FP, VT: MVT::f32, Action: Expand);
1271 setOperationAction(Op: ISD::STRICT_FP_TO_FP16, VT: MVT::f32, Action: Expand);
1272 }
1273
1274 // Strict floating-point comparisons need custom lowering.
1275 setOperationAction(Op: ISD::STRICT_FSETCC, VT: MVT::f16, Action: Custom);
1276 setOperationAction(Op: ISD::STRICT_FSETCCS, VT: MVT::f16, Action: Custom);
1277 setOperationAction(Op: ISD::STRICT_FSETCC, VT: MVT::f32, Action: Custom);
1278 setOperationAction(Op: ISD::STRICT_FSETCCS, VT: MVT::f32, Action: Custom);
1279 setOperationAction(Op: ISD::STRICT_FSETCC, VT: MVT::f64, Action: Custom);
1280 setOperationAction(Op: ISD::STRICT_FSETCCS, VT: MVT::f64, Action: Custom);
1281 }
1282
1283 // FP-ARMv8 implements a lot of rounding-like FP operations.
1284 if (Subtarget->hasFPARMv8Base()) {
1285 for (auto Op :
1286 {ISD::FFLOOR, ISD::FCEIL, ISD::FROUND,
1287 ISD::FTRUNC, ISD::FNEARBYINT, ISD::FRINT,
1288 ISD::FROUNDEVEN, ISD::FMINNUM, ISD::FMAXNUM,
1289 ISD::STRICT_FFLOOR, ISD::STRICT_FCEIL, ISD::STRICT_FROUND,
1290 ISD::STRICT_FTRUNC, ISD::STRICT_FNEARBYINT, ISD::STRICT_FRINT,
1291 ISD::STRICT_FROUNDEVEN, ISD::STRICT_FMINNUM, ISD::STRICT_FMAXNUM}) {
1292 setOperationAction(Op, VT: MVT::f32, Action: Legal);
1293
1294 if (Subtarget->hasFP64())
1295 setOperationAction(Op, VT: MVT::f64, Action: Legal);
1296 }
1297
1298 if (Subtarget->hasNEON()) {
1299 setOperationAction(Op: ISD::FMINNUM, VT: MVT::v2f32, Action: Legal);
1300 setOperationAction(Op: ISD::FMAXNUM, VT: MVT::v2f32, Action: Legal);
1301 setOperationAction(Op: ISD::FMINNUM, VT: MVT::v4f32, Action: Legal);
1302 setOperationAction(Op: ISD::FMAXNUM, VT: MVT::v4f32, Action: Legal);
1303 }
1304 }
1305
1306 // FP16 often need to be promoted to call lib functions
1307 // clang-format off
1308 if (Subtarget->hasFullFP16()) {
1309 setOperationAction(Op: ISD::LRINT, VT: MVT::f16, Action: Expand);
1310 setOperationAction(Op: ISD::LROUND, VT: MVT::f16, Action: Expand);
1311 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::f16, Action: Expand);
1312
1313 for (auto Op : {ISD::FREM, ISD::FPOW, ISD::FPOWI,
1314 ISD::FCOS, ISD::FSIN, ISD::FSINCOS,
1315 ISD::FSINCOSPI, ISD::FMODF, ISD::FACOS,
1316 ISD::FASIN, ISD::FATAN, ISD::FATAN2,
1317 ISD::FCOSH, ISD::FSINH, ISD::FTANH,
1318 ISD::FTAN, ISD::FEXP, ISD::FEXP2,
1319 ISD::FEXP10, ISD::FLOG, ISD::FLOG2,
1320 ISD::FLOG10, ISD::STRICT_FREM, ISD::STRICT_FPOW,
1321 ISD::STRICT_FPOWI, ISD::STRICT_FCOS, ISD::STRICT_FSIN,
1322 ISD::STRICT_FACOS, ISD::STRICT_FASIN, ISD::STRICT_FATAN,
1323 ISD::STRICT_FATAN2, ISD::STRICT_FCOSH, ISD::STRICT_FSINH,
1324 ISD::STRICT_FTANH, ISD::STRICT_FEXP, ISD::STRICT_FEXP2,
1325 ISD::STRICT_FLOG, ISD::STRICT_FLOG2, ISD::STRICT_FLOG10,
1326 ISD::STRICT_FTAN}) {
1327 setOperationAction(Op, VT: MVT::f16, Action: Promote);
1328 }
1329
1330 // Round-to-integer need custom lowering for fp16, as Promote doesn't work
1331 // because the result type is integer.
1332 for (auto Op : {ISD::STRICT_LROUND, ISD::STRICT_LLROUND, ISD::STRICT_LRINT, ISD::STRICT_LLRINT})
1333 setOperationAction(Op, VT: MVT::f16, Action: Custom);
1334
1335 for (auto Op : {ISD::FROUND, ISD::FROUNDEVEN, ISD::FTRUNC,
1336 ISD::FNEARBYINT, ISD::FRINT, ISD::FFLOOR,
1337 ISD::FCEIL, ISD::STRICT_FROUND, ISD::STRICT_FROUNDEVEN,
1338 ISD::STRICT_FTRUNC, ISD::STRICT_FNEARBYINT, ISD::STRICT_FRINT,
1339 ISD::STRICT_FFLOOR, ISD::STRICT_FCEIL}) {
1340 setOperationAction(Op, VT: MVT::f16, Action: Legal);
1341 }
1342 // clang-format on
1343 }
1344
1345 if (Subtarget->hasNEON()) {
1346 // vmin and vmax aren't available in a scalar form, so we can use
1347 // a NEON instruction with an undef lane instead.
1348 setOperationAction(Op: ISD::FMINIMUM, VT: MVT::f32, Action: Legal);
1349 setOperationAction(Op: ISD::FMAXIMUM, VT: MVT::f32, Action: Legal);
1350 setOperationAction(Op: ISD::FMINIMUM, VT: MVT::f16, Action: Legal);
1351 setOperationAction(Op: ISD::FMAXIMUM, VT: MVT::f16, Action: Legal);
1352 setOperationAction(Op: ISD::FMINIMUM, VT: MVT::v2f32, Action: Legal);
1353 setOperationAction(Op: ISD::FMAXIMUM, VT: MVT::v2f32, Action: Legal);
1354 setOperationAction(Op: ISD::FMINIMUM, VT: MVT::v4f32, Action: Legal);
1355 setOperationAction(Op: ISD::FMAXIMUM, VT: MVT::v4f32, Action: Legal);
1356
1357 if (Subtarget->hasV8Ops()) {
1358 for (auto Op : {ISD::FROUND, ISD::STRICT_FROUND, ISD::FROUNDEVEN,
1359 ISD::STRICT_FROUNDEVEN, ISD::FTRUNC, ISD::STRICT_FTRUNC,
1360 ISD::FRINT, ISD::STRICT_FRINT, ISD::FFLOOR,
1361 ISD::STRICT_FFLOOR, ISD::FCEIL, ISD::STRICT_FCEIL}) {
1362 setOperationAction(Op, VT: MVT::v2f32, Action: Legal);
1363 setOperationAction(Op, VT: MVT::v4f32, Action: Legal);
1364 }
1365 }
1366
1367 if (Subtarget->hasFullFP16()) {
1368 setOperationAction(Op: ISD::FMINNUM, VT: MVT::v4f16, Action: Legal);
1369 setOperationAction(Op: ISD::FMAXNUM, VT: MVT::v4f16, Action: Legal);
1370 setOperationAction(Op: ISD::FMINNUM, VT: MVT::v8f16, Action: Legal);
1371 setOperationAction(Op: ISD::FMAXNUM, VT: MVT::v8f16, Action: Legal);
1372
1373 setOperationAction(Op: ISD::FMINIMUM, VT: MVT::v4f16, Action: Legal);
1374 setOperationAction(Op: ISD::FMAXIMUM, VT: MVT::v4f16, Action: Legal);
1375 setOperationAction(Op: ISD::FMINIMUM, VT: MVT::v8f16, Action: Legal);
1376 setOperationAction(Op: ISD::FMAXIMUM, VT: MVT::v8f16, Action: Legal);
1377
1378 for (auto Op : {ISD::FROUND, ISD::STRICT_FROUND, ISD::FROUNDEVEN,
1379 ISD::STRICT_FROUNDEVEN, ISD::FTRUNC, ISD::STRICT_FTRUNC,
1380 ISD::FRINT, ISD::STRICT_FRINT, ISD::FFLOOR,
1381 ISD::STRICT_FFLOOR, ISD::FCEIL, ISD::STRICT_FCEIL}) {
1382 setOperationAction(Op, VT: MVT::v4f16, Action: Legal);
1383 setOperationAction(Op, VT: MVT::v8f16, Action: Legal);
1384 }
1385 }
1386 }
1387
1388 // On MSVC, both 32-bit and 64-bit, ldexpf(f32) is not defined. MinGW has
1389 // it, but it's just a wrapper around ldexp.
1390 if (TT.isOSWindows()) {
1391 for (ISD::NodeType Op : {ISD::FLDEXP, ISD::STRICT_FLDEXP, ISD::FFREXP})
1392 if (isOperationExpand(Op, VT: MVT::f32))
1393 setOperationAction(Op, VT: MVT::f32, Action: Promote);
1394 }
1395
1396 // LegalizeDAG currently can't expand fp16 LDEXP/FREXP on targets where i16
1397 // isn't legal.
1398 for (ISD::NodeType Op : {ISD::FLDEXP, ISD::STRICT_FLDEXP, ISD::FFREXP})
1399 if (isOperationExpand(Op, VT: MVT::f16))
1400 setOperationAction(Op, VT: MVT::f16, Action: Promote);
1401
1402 // We have target-specific dag combine patterns for the following nodes:
1403 // ARMISD::VMOVRRD - No need to call setTargetDAGCombine
1404 setTargetDAGCombine(
1405 {ISD::ADD, ISD::SUB, ISD::MUL, ISD::AND, ISD::OR, ISD::XOR});
1406
1407 if (Subtarget->hasMVEIntegerOps())
1408 setTargetDAGCombine(ISD::VSELECT);
1409
1410 if (Subtarget->hasV6Ops())
1411 setTargetDAGCombine(ISD::SRL);
1412 if (Subtarget->isThumb1Only())
1413 setTargetDAGCombine(ISD::SHL);
1414 // Attempt to lower smin/smax to ssat/usat
1415 if ((!Subtarget->isThumb() && Subtarget->hasV6Ops()) ||
1416 Subtarget->isThumb2()) {
1417 setTargetDAGCombine({ISD::SMIN, ISD::SMAX});
1418 }
1419
1420 setStackPointerRegisterToSaveRestore(ARM::SP);
1421
1422 if (Subtarget->useSoftFloat() || Subtarget->isThumb1Only() ||
1423 !Subtarget->hasVFP2Base() || Subtarget->hasMinSize())
1424 setSchedulingPreference(Sched::RegPressure);
1425 else
1426 setSchedulingPreference(Sched::Hybrid);
1427
1428 //// temporary - rewrite interface to use type
1429 MaxStoresPerMemset = 8;
1430 MaxStoresPerMemsetOptSize = 4;
1431 MaxStoresPerMemcpy = 4; // For @llvm.memcpy -> sequence of stores
1432 MaxStoresPerMemcpyOptSize = 2;
1433 MaxStoresPerMemmove = 4; // For @llvm.memmove -> sequence of stores
1434 MaxStoresPerMemmoveOptSize = 2;
1435
1436 // On ARM arguments smaller than 4 bytes are extended, so all arguments
1437 // are at least 4 bytes aligned.
1438 setMinStackArgumentAlignment(Align(4));
1439
1440 // Prefer likely predicted branches to selects on out-of-order cores.
1441 PredictableSelectIsExpensive = Subtarget->getSchedModel().isOutOfOrder();
1442
1443 setPrefLoopAlignment(
1444 Align::fromLog2(Log2Value: Subtarget->getPreferBranchLogAlignment()));
1445 setPrefFunctionAlignment(
1446 Align::fromLog2(Log2Value: Subtarget->getPreferBranchLogAlignment()));
1447
1448 setMinFunctionAlignment(Subtarget->isThumb() ? Align(2) : Align(4));
1449
1450 IsStrictFPEnabled = true;
1451}
1452
1453bool ARMTargetLowering::useSoftFloat() const {
1454 return Subtarget->useSoftFloat();
1455}
1456
1457bool ARMTargetLowering::preferSelectsOverBooleanArithmetic(EVT VT) const {
1458 return !Subtarget->isThumb1Only() && VT.getSizeInBits() <= 32;
1459}
1460
1461// FIXME: It might make sense to define the representative register class as the
1462// nearest super-register that has a non-null superset. For example, DPR_VFP2 is
1463// a super-register of SPR, and DPR is a superset if DPR_VFP2. Consequently,
1464// SPR's representative would be DPR_VFP2. This should work well if register
1465// pressure tracking were modified such that a register use would increment the
1466// pressure of the register class's representative and all of it's super
1467// classes' representatives transitively. We have not implemented this because
1468// of the difficulty prior to coalescing of modeling operand register classes
1469// due to the common occurrence of cross class copies and subregister insertions
1470// and extractions.
1471std::pair<const TargetRegisterClass *, uint8_t>
1472ARMTargetLowering::findRepresentativeClass(const TargetRegisterInfo *TRI,
1473 MVT VT) const {
1474 const TargetRegisterClass *RRC = nullptr;
1475 uint8_t Cost = 1;
1476 switch (VT.SimpleTy) {
1477 default:
1478 return TargetLowering::findRepresentativeClass(TRI, VT);
1479 // Use DPR as representative register class for all floating point
1480 // and vector types. Since there are 32 SPR registers and 32 DPR registers so
1481 // the cost is 1 for both f32 and f64.
1482 case MVT::f32: case MVT::f64: case MVT::v8i8: case MVT::v4i16:
1483 case MVT::v2i32: case MVT::v1i64: case MVT::v2f32:
1484 RRC = &ARM::DPRRegClass;
1485 // When NEON is used for SP, only half of the register file is available
1486 // because operations that define both SP and DP results will be constrained
1487 // to the VFP2 class (D0-D15). We currently model this constraint prior to
1488 // coalescing by double-counting the SP regs. See the FIXME above.
1489 if (Subtarget->useNEONForSinglePrecisionFP())
1490 Cost = 2;
1491 break;
1492 case MVT::v16i8: case MVT::v8i16: case MVT::v4i32: case MVT::v2i64:
1493 case MVT::v4f32: case MVT::v2f64:
1494 RRC = &ARM::DPRRegClass;
1495 Cost = 2;
1496 break;
1497 case MVT::v4i64:
1498 RRC = &ARM::DPRRegClass;
1499 Cost = 4;
1500 break;
1501 case MVT::v8i64:
1502 RRC = &ARM::DPRRegClass;
1503 Cost = 8;
1504 break;
1505 }
1506 return std::make_pair(x&: RRC, y&: Cost);
1507}
1508
1509EVT ARMTargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &C,
1510 EVT VT) const {
1511 if (!VT.isVector())
1512 return getPointerTy(DL);
1513
1514 // MVE has a predicate register.
1515 if (Subtarget->hasMVEIntegerOps())
1516 return EVT::getVectorVT(Context&: C, VT: MVT::i1, EC: VT.getVectorElementCount());
1517
1518 return VT.changeVectorElementTypeToInteger();
1519}
1520
1521/// getRegClassFor - Return the register class that should be used for the
1522/// specified value type.
1523const TargetRegisterClass *
1524ARMTargetLowering::getRegClassFor(MVT VT, bool isDivergent) const {
1525 (void)isDivergent;
1526 // Map v4i64 to QQ registers but do not make the type legal. Similarly map
1527 // v8i64 to QQQQ registers. v4i64 and v8i64 are only used for REG_SEQUENCE to
1528 // load / store 4 to 8 consecutive NEON D registers, or 2 to 4 consecutive
1529 // MVE Q registers.
1530 if (Subtarget->hasNEON()) {
1531 if (VT == MVT::v4i64)
1532 return &ARM::QQPRRegClass;
1533 if (VT == MVT::v8i64)
1534 return &ARM::QQQQPRRegClass;
1535 }
1536 if (Subtarget->hasMVEIntegerOps()) {
1537 if (VT == MVT::v4i64)
1538 return &ARM::MQQPRRegClass;
1539 if (VT == MVT::v8i64)
1540 return &ARM::MQQQQPRRegClass;
1541 }
1542 return TargetLowering::getRegClassFor(VT);
1543}
1544
1545// memcpy, and other memory intrinsics, typically tries to use LDM/STM if the
1546// source/dest is aligned and the copy size is large enough. We therefore want
1547// to align such objects passed to memory intrinsics.
1548bool ARMTargetLowering::shouldAlignPointerArgs(CallInst *CI, unsigned &MinSize,
1549 Align &PrefAlign) const {
1550 if (!isa<MemIntrinsic>(Val: CI))
1551 return false;
1552 MinSize = 8;
1553 // On ARM11 onwards (excluding M class) 8-byte aligned LDM is typically 1
1554 // cycle faster than 4-byte aligned LDM.
1555 PrefAlign =
1556 (Subtarget->hasV6Ops() && !Subtarget->isMClass() ? Align(8) : Align(4));
1557 return true;
1558}
1559
1560// Create a fast isel object.
1561FastISel *ARMTargetLowering::createFastISel(
1562 FunctionLoweringInfo &funcInfo, const TargetLibraryInfo *libInfo,
1563 const LibcallLoweringInfo *libcallLowering) const {
1564 return ARM::createFastISel(funcInfo, libInfo, libcallLowering);
1565}
1566
1567Sched::Preference ARMTargetLowering::getSchedulingPreference(SDNode *N) const {
1568 unsigned NumVals = N->getNumValues();
1569 if (!NumVals)
1570 return Sched::RegPressure;
1571
1572 for (unsigned i = 0; i != NumVals; ++i) {
1573 EVT VT = N->getValueType(ResNo: i);
1574 if (VT == MVT::Glue || VT == MVT::Other)
1575 continue;
1576 if (VT.isFloatingPoint() || VT.isVector())
1577 return Sched::ILP;
1578 }
1579
1580 if (!N->isMachineOpcode())
1581 return Sched::RegPressure;
1582
1583 // Load are scheduled for latency even if there instruction itinerary
1584 // is not available.
1585 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
1586 const MCInstrDesc &MCID = TII->get(Opcode: N->getMachineOpcode());
1587
1588 if (MCID.getNumDefs() == 0)
1589 return Sched::RegPressure;
1590 if (!Itins->isEmpty() &&
1591 Itins->getOperandCycle(ItinClassIndx: MCID.getSchedClass(), OperandIdx: 0) > 2U)
1592 return Sched::ILP;
1593
1594 return Sched::RegPressure;
1595}
1596
1597//===----------------------------------------------------------------------===//
1598// Lowering Code
1599//===----------------------------------------------------------------------===//
1600
1601static bool isSRL16(const SDValue &Op) {
1602 if (Op.getOpcode() != ISD::SRL)
1603 return false;
1604 if (auto Const = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1)))
1605 return Const->getZExtValue() == 16;
1606 return false;
1607}
1608
1609static bool isSRA16(const SDValue &Op) {
1610 if (Op.getOpcode() != ISD::SRA)
1611 return false;
1612 if (auto Const = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1)))
1613 return Const->getZExtValue() == 16;
1614 return false;
1615}
1616
1617static bool isSHL16(const SDValue &Op) {
1618 if (Op.getOpcode() != ISD::SHL)
1619 return false;
1620 if (auto Const = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1)))
1621 return Const->getZExtValue() == 16;
1622 return false;
1623}
1624
1625// Check for a signed 16-bit value. We special case SRA because it makes it
1626// more simple when also looking for SRAs that aren't sign extending a
1627// smaller value. Without the check, we'd need to take extra care with
1628// checking order for some operations.
1629static bool isS16(const SDValue &Op, SelectionDAG &DAG) {
1630 if (isSRA16(Op))
1631 return isSHL16(Op: Op.getOperand(i: 0));
1632 return DAG.ComputeNumSignBits(Op) == 17;
1633}
1634
1635/// IntCCToARMCC - Convert a DAG integer condition code to an ARM CC
1636static ARMCC::CondCodes IntCCToARMCC(ISD::CondCode CC) {
1637 switch (CC) {
1638 default: llvm_unreachable("Unknown condition code!");
1639 case ISD::SETNE: return ARMCC::NE;
1640 case ISD::SETEQ: return ARMCC::EQ;
1641 case ISD::SETGT: return ARMCC::GT;
1642 case ISD::SETGE: return ARMCC::GE;
1643 case ISD::SETLT: return ARMCC::LT;
1644 case ISD::SETLE: return ARMCC::LE;
1645 case ISD::SETUGT: return ARMCC::HI;
1646 case ISD::SETUGE: return ARMCC::HS;
1647 case ISD::SETULT: return ARMCC::LO;
1648 case ISD::SETULE: return ARMCC::LS;
1649 }
1650}
1651
1652/// FPCCToARMCC - Convert a DAG fp condition code to an ARM CC.
1653static void FPCCToARMCC(ISD::CondCode CC, ARMCC::CondCodes &CondCode,
1654 ARMCC::CondCodes &CondCode2) {
1655 CondCode2 = ARMCC::AL;
1656 switch (CC) {
1657 default: llvm_unreachable("Unknown FP condition!");
1658 case ISD::SETEQ:
1659 case ISD::SETOEQ: CondCode = ARMCC::EQ; break;
1660 case ISD::SETGT:
1661 case ISD::SETOGT: CondCode = ARMCC::GT; break;
1662 case ISD::SETGE:
1663 case ISD::SETOGE: CondCode = ARMCC::GE; break;
1664 case ISD::SETOLT: CondCode = ARMCC::MI; break;
1665 case ISD::SETOLE: CondCode = ARMCC::LS; break;
1666 case ISD::SETONE: CondCode = ARMCC::MI; CondCode2 = ARMCC::GT; break;
1667 case ISD::SETO: CondCode = ARMCC::VC; break;
1668 case ISD::SETUO: CondCode = ARMCC::VS; break;
1669 case ISD::SETUEQ: CondCode = ARMCC::EQ; CondCode2 = ARMCC::VS; break;
1670 case ISD::SETUGT: CondCode = ARMCC::HI; break;
1671 case ISD::SETUGE: CondCode = ARMCC::PL; break;
1672 case ISD::SETLT:
1673 case ISD::SETULT: CondCode = ARMCC::LT; break;
1674 case ISD::SETLE:
1675 case ISD::SETULE: CondCode = ARMCC::LE; break;
1676 case ISD::SETNE:
1677 case ISD::SETUNE: CondCode = ARMCC::NE; break;
1678 }
1679}
1680
1681//===----------------------------------------------------------------------===//
1682// Calling Convention Implementation
1683//===----------------------------------------------------------------------===//
1684
1685/// getEffectiveCallingConv - Get the effective calling convention, taking into
1686/// account presence of floating point hardware and calling convention
1687/// limitations, such as support for variadic functions.
1688CallingConv::ID
1689ARMTargetLowering::getEffectiveCallingConv(CallingConv::ID CC,
1690 bool isVarArg) const {
1691 switch (CC) {
1692 default:
1693 // Unknown CCs are rejected when calling convention lowering is required.
1694 case CallingConv::ARM_AAPCS:
1695 case CallingConv::ARM_APCS:
1696 case CallingConv::GHC:
1697 case CallingConv::CFGuard_Check:
1698 return CC;
1699 case CallingConv::PreserveMost:
1700 return CallingConv::PreserveMost;
1701 case CallingConv::PreserveAll:
1702 return CallingConv::PreserveAll;
1703 case CallingConv::ARM_AAPCS_VFP:
1704 case CallingConv::Swift:
1705 case CallingConv::SwiftTail:
1706 return isVarArg ? CallingConv::ARM_AAPCS : CallingConv::ARM_AAPCS_VFP;
1707 case CallingConv::C:
1708 case CallingConv::Tail:
1709 if (!Subtarget->isAAPCS_ABI())
1710 return CallingConv::ARM_APCS;
1711 else if (Subtarget->isTargetHardFloat() && !isVarArg)
1712 return CallingConv::ARM_AAPCS_VFP;
1713 else
1714 return CallingConv::ARM_AAPCS;
1715 case CallingConv::Fast:
1716 case CallingConv::CXX_FAST_TLS:
1717 if (!Subtarget->isAAPCS_ABI()) {
1718 if (Subtarget->hasFPRegs() && !Subtarget->isThumb1Only() && !isVarArg)
1719 return CallingConv::Fast;
1720 return CallingConv::ARM_APCS;
1721 } else if (Subtarget->hasFPRegs() && !Subtarget->isThumb1Only() &&
1722 !isVarArg)
1723 return CallingConv::ARM_AAPCS_VFP;
1724 else
1725 return CallingConv::ARM_AAPCS;
1726 }
1727}
1728
1729CCAssignFn *ARMTargetLowering::CCAssignFnForCall(CallingConv::ID CC,
1730 bool isVarArg) const {
1731 return CCAssignFnForNode(CC, Return: false, isVarArg);
1732}
1733
1734CCAssignFn *ARMTargetLowering::CCAssignFnForReturn(CallingConv::ID CC,
1735 bool isVarArg) const {
1736 return CCAssignFnForNode(CC, Return: true, isVarArg);
1737}
1738
1739/// CCAssignFnForNode - Selects the correct CCAssignFn for the given
1740/// CallingConvention.
1741CCAssignFn *ARMTargetLowering::CCAssignFnForNode(CallingConv::ID CC,
1742 bool Return,
1743 bool isVarArg) const {
1744 switch (getEffectiveCallingConv(CC, isVarArg)) {
1745 default:
1746 report_fatal_error(reason: "Unsupported calling convention");
1747 case CallingConv::ARM_APCS:
1748 return (Return ? RetCC_ARM_APCS : CC_ARM_APCS);
1749 case CallingConv::ARM_AAPCS:
1750 return (Return ? RetCC_ARM_AAPCS : CC_ARM_AAPCS);
1751 case CallingConv::ARM_AAPCS_VFP:
1752 return (Return ? RetCC_ARM_AAPCS_VFP : CC_ARM_AAPCS_VFP);
1753 case CallingConv::Fast:
1754 return (Return ? RetFastCC_ARM_APCS : FastCC_ARM_APCS);
1755 case CallingConv::GHC:
1756 return (Return ? RetCC_ARM_APCS : CC_ARM_APCS_GHC);
1757 case CallingConv::PreserveMost:
1758 return (Return ? RetCC_ARM_AAPCS : CC_ARM_AAPCS);
1759 case CallingConv::PreserveAll:
1760 return (Return ? RetCC_ARM_AAPCS : CC_ARM_AAPCS);
1761 case CallingConv::CFGuard_Check:
1762 return (Return ? RetCC_ARM_AAPCS : CC_ARM_Win32_CFGuard_Check);
1763 }
1764}
1765
1766SDValue ARMTargetLowering::MoveToHPR(const SDLoc &dl, SelectionDAG &DAG,
1767 MVT LocVT, MVT ValVT, SDValue Val) const {
1768 Val = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::getIntegerVT(BitWidth: LocVT.getSizeInBits()),
1769 Operand: Val);
1770 if (Subtarget->hasFullFP16()) {
1771 Val = DAG.getNode(Opcode: ARMISD::VMOVhr, DL: dl, VT: ValVT, Operand: Val);
1772 } else {
1773 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl,
1774 VT: MVT::getIntegerVT(BitWidth: ValVT.getSizeInBits()), Operand: Val);
1775 Val = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: ValVT, Operand: Val);
1776 }
1777 return Val;
1778}
1779
1780SDValue ARMTargetLowering::MoveFromHPR(const SDLoc &dl, SelectionDAG &DAG,
1781 MVT LocVT, MVT ValVT,
1782 SDValue Val) const {
1783 if (Subtarget->hasFullFP16()) {
1784 Val = DAG.getNode(Opcode: ARMISD::VMOVrh, DL: dl,
1785 VT: MVT::getIntegerVT(BitWidth: LocVT.getSizeInBits()), Operand: Val);
1786 } else {
1787 Val = DAG.getNode(Opcode: ISD::BITCAST, DL: dl,
1788 VT: MVT::getIntegerVT(BitWidth: ValVT.getSizeInBits()), Operand: Val);
1789 Val = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl,
1790 VT: MVT::getIntegerVT(BitWidth: LocVT.getSizeInBits()), Operand: Val);
1791 }
1792 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LocVT, Operand: Val);
1793}
1794
1795/// LowerCallResult - Lower the result values of a call into the
1796/// appropriate copies out of appropriate physical registers.
1797SDValue ARMTargetLowering::LowerCallResult(
1798 SDValue Chain, SDValue InGlue, CallingConv::ID CallConv, bool isVarArg,
1799 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
1800 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals, bool isThisReturn,
1801 SDValue ThisVal, bool isCmseNSCall) const {
1802 // Assign locations to each value returned by this call.
1803 SmallVector<CCValAssign, 16> RVLocs;
1804 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
1805 *DAG.getContext());
1806 CCInfo.AnalyzeCallResult(Ins, Fn: CCAssignFnForReturn(CC: CallConv, isVarArg));
1807
1808 // Copy all of the result registers out of their specified physreg.
1809 for (unsigned i = 0; i != RVLocs.size(); ++i) {
1810 CCValAssign VA = RVLocs[i];
1811
1812 // Pass 'this' value directly from the argument to return value, to avoid
1813 // reg unit interference
1814 if (i == 0 && isThisReturn) {
1815 assert(!VA.needsCustom() && VA.getLocVT() == MVT::i32 &&
1816 "unexpected return calling convention register assignment");
1817 InVals.push_back(Elt: ThisVal);
1818 continue;
1819 }
1820
1821 SDValue Val;
1822 if (VA.needsCustom() &&
1823 (VA.getLocVT() == MVT::f64 || VA.getLocVT() == MVT::v2f64)) {
1824 // Handle f64 or half of a v2f64.
1825 SDValue Lo = DAG.getCopyFromReg(Chain, dl, Reg: VA.getLocReg(), VT: MVT::i32,
1826 Glue: InGlue);
1827 Chain = Lo.getValue(R: 1);
1828 InGlue = Lo.getValue(R: 2);
1829 VA = RVLocs[++i]; // skip ahead to next loc
1830 SDValue Hi = DAG.getCopyFromReg(Chain, dl, Reg: VA.getLocReg(), VT: MVT::i32,
1831 Glue: InGlue);
1832 Chain = Hi.getValue(R: 1);
1833 InGlue = Hi.getValue(R: 2);
1834 if (!Subtarget->isLittle())
1835 std::swap (a&: Lo, b&: Hi);
1836 Val = DAG.getNode(Opcode: ARMISD::VMOVDRR, DL: dl, VT: MVT::f64, N1: Lo, N2: Hi);
1837
1838 if (VA.getLocVT() == MVT::v2f64) {
1839 SDValue Vec = DAG.getNode(Opcode: ISD::UNDEF, DL: dl, VT: MVT::v2f64);
1840 Vec = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: MVT::v2f64, N1: Vec, N2: Val,
1841 N3: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
1842
1843 VA = RVLocs[++i]; // skip ahead to next loc
1844 Lo = DAG.getCopyFromReg(Chain, dl, Reg: VA.getLocReg(), VT: MVT::i32, Glue: InGlue);
1845 Chain = Lo.getValue(R: 1);
1846 InGlue = Lo.getValue(R: 2);
1847 VA = RVLocs[++i]; // skip ahead to next loc
1848 Hi = DAG.getCopyFromReg(Chain, dl, Reg: VA.getLocReg(), VT: MVT::i32, Glue: InGlue);
1849 Chain = Hi.getValue(R: 1);
1850 InGlue = Hi.getValue(R: 2);
1851 if (!Subtarget->isLittle())
1852 std::swap (a&: Lo, b&: Hi);
1853 Val = DAG.getNode(Opcode: ARMISD::VMOVDRR, DL: dl, VT: MVT::f64, N1: Lo, N2: Hi);
1854 Val = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: MVT::v2f64, N1: Vec, N2: Val,
1855 N3: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
1856 }
1857 } else {
1858 Val = DAG.getCopyFromReg(Chain, dl, Reg: VA.getLocReg(), VT: VA.getLocVT(),
1859 Glue: InGlue);
1860 Chain = Val.getValue(R: 1);
1861 InGlue = Val.getValue(R: 2);
1862 }
1863
1864 switch (VA.getLocInfo()) {
1865 default: llvm_unreachable("Unknown loc info!");
1866 case CCValAssign::Full: break;
1867 case CCValAssign::BCvt:
1868 Val = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: VA.getValVT(), Operand: Val);
1869 break;
1870 }
1871
1872 // f16 arguments have their size extended to 4 bytes and passed as if they
1873 // had been copied to the LSBs of a 32-bit register.
1874 // For that, it's passed extended to i32 (soft ABI) or to f32 (hard ABI)
1875 if (VA.needsCustom() &&
1876 (VA.getValVT() == MVT::f16 || VA.getValVT() == MVT::bf16))
1877 Val = MoveToHPR(dl, DAG, LocVT: VA.getLocVT(), ValVT: VA.getValVT(), Val);
1878
1879 // On CMSE Non-secure Calls, call results (returned values) whose bitwidth
1880 // is less than 32 bits must be sign- or zero-extended after the call for
1881 // security reasons. Although the ABI mandates an extension done by the
1882 // callee, the latter cannot be trusted to follow the rules of the ABI.
1883 const ISD::InputArg &Arg = Ins[VA.getValNo()];
1884 if (isCmseNSCall && Arg.ArgVT.isScalarInteger() &&
1885 VA.getLocVT().isScalarInteger() && Arg.ArgVT.bitsLT(VT: MVT::i32))
1886 Val = handleCMSEValue(Value: Val, Arg, DAG, DL: dl);
1887
1888 InVals.push_back(Elt: Val);
1889 }
1890
1891 return Chain;
1892}
1893
1894std::pair<SDValue, MachinePointerInfo> ARMTargetLowering::computeAddrForCallArg(
1895 const SDLoc &dl, SelectionDAG &DAG, const CCValAssign &VA, SDValue StackPtr,
1896 bool IsTailCall, int SPDiff) const {
1897 SDValue DstAddr;
1898 MachinePointerInfo DstInfo;
1899 int32_t Offset = VA.getLocMemOffset();
1900 MachineFunction &MF = DAG.getMachineFunction();
1901
1902 if (IsTailCall) {
1903 Offset += SPDiff;
1904 auto PtrVT = getPointerTy(DL: DAG.getDataLayout());
1905 int Size = VA.getLocVT().getFixedSizeInBits() / 8;
1906 int FI = MF.getFrameInfo().CreateFixedObject(Size, SPOffset: Offset, IsImmutable: true);
1907 DstAddr = DAG.getFrameIndex(FI, VT: PtrVT);
1908 DstInfo =
1909 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI);
1910 } else {
1911 SDValue PtrOff = DAG.getIntPtrConstant(Val: Offset, DL: dl);
1912 DstAddr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: getPointerTy(DL: DAG.getDataLayout()),
1913 N1: StackPtr, N2: PtrOff);
1914 DstInfo =
1915 MachinePointerInfo::getStack(MF&: DAG.getMachineFunction(), Offset);
1916 }
1917
1918 return std::make_pair(x&: DstAddr, y&: DstInfo);
1919}
1920
1921// Returns the type of copying which is required to set up a byval argument to
1922// a tail-called function. This isn't needed for non-tail calls, because they
1923// always need the equivalent of CopyOnce, but tail-calls sometimes need two to
1924// avoid clobbering another argument (CopyViaTemp), and sometimes can be
1925// optimised to zero copies when forwarding an argument from the caller's
1926// caller (NoCopy).
1927ARMTargetLowering::ByValCopyKind ARMTargetLowering::ByValNeedsCopyForTailCall(
1928 SelectionDAG &DAG, SDValue Src, SDValue Dst, ISD::ArgFlagsTy Flags) const {
1929 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
1930 ARMFunctionInfo *AFI = DAG.getMachineFunction().getInfo<ARMFunctionInfo>();
1931
1932 // Globals are always safe to copy from.
1933 if (isa<GlobalAddressSDNode>(Val: Src) || isa<ExternalSymbolSDNode>(Val: Src))
1934 return CopyOnce;
1935
1936 // Can only analyse frame index nodes, conservatively assume we need a
1937 // temporary.
1938 auto *SrcFrameIdxNode = dyn_cast<FrameIndexSDNode>(Val&: Src);
1939 auto *DstFrameIdxNode = dyn_cast<FrameIndexSDNode>(Val&: Dst);
1940 if (!SrcFrameIdxNode || !DstFrameIdxNode)
1941 return CopyViaTemp;
1942
1943 int SrcFI = SrcFrameIdxNode->getIndex();
1944 int DstFI = DstFrameIdxNode->getIndex();
1945 assert(MFI.isFixedObjectIndex(DstFI) &&
1946 "byval passed in non-fixed stack slot");
1947
1948 int64_t SrcOffset = MFI.getObjectOffset(ObjectIdx: SrcFI);
1949 int64_t DstOffset = MFI.getObjectOffset(ObjectIdx: DstFI);
1950
1951 // If the source is in the local frame, then the copy to the argument memory
1952 // is always valid.
1953 bool FixedSrc = MFI.isFixedObjectIndex(ObjectIdx: SrcFI);
1954 if (!FixedSrc ||
1955 (FixedSrc && SrcOffset < -(int64_t)AFI->getArgRegsSaveSize()))
1956 return CopyOnce;
1957
1958 // In the case of byval arguments split between registers and the stack,
1959 // computeAddrForCallArg returns a FrameIndex which corresponds only to the
1960 // stack portion, but the Src SDValue will refer to the full value, including
1961 // the local stack memory that the register portion gets stored into. We only
1962 // need to compare them for equality, so normalise on the full value version.
1963 uint64_t RegSize = Flags.getByValSize() - MFI.getObjectSize(ObjectIdx: DstFI);
1964 DstOffset -= RegSize;
1965
1966 // If the value is already in the correct location, then no copying is
1967 // needed. If not, then we need to copy via a temporary.
1968 if (SrcOffset == DstOffset)
1969 return NoCopy;
1970 else
1971 return CopyViaTemp;
1972}
1973
1974void ARMTargetLowering::PassF64ArgInRegs(const SDLoc &dl, SelectionDAG &DAG,
1975 SDValue Chain, SDValue &Arg,
1976 RegsToPassVector &RegsToPass,
1977 CCValAssign &VA, CCValAssign &NextVA,
1978 SDValue &StackPtr,
1979 SmallVectorImpl<SDValue> &MemOpChains,
1980 bool IsTailCall,
1981 int SPDiff) const {
1982 SDValue fmrrd = DAG.getNode(Opcode: ARMISD::VMOVRRD, DL: dl,
1983 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N: Arg);
1984 unsigned id = Subtarget->isLittle() ? 0 : 1;
1985 RegsToPass.push_back(Elt: std::make_pair(x: VA.getLocReg(), y: fmrrd.getValue(R: id)));
1986
1987 if (NextVA.isRegLoc())
1988 RegsToPass.push_back(Elt: std::make_pair(x: NextVA.getLocReg(), y: fmrrd.getValue(R: 1-id)));
1989 else {
1990 assert(NextVA.isMemLoc());
1991 if (!StackPtr.getNode())
1992 StackPtr = DAG.getCopyFromReg(Chain, dl, Reg: ARM::SP,
1993 VT: getPointerTy(DL: DAG.getDataLayout()));
1994
1995 SDValue DstAddr;
1996 MachinePointerInfo DstInfo;
1997 std::tie(args&: DstAddr, args&: DstInfo) =
1998 computeAddrForCallArg(dl, DAG, VA: NextVA, StackPtr, IsTailCall, SPDiff);
1999 MemOpChains.push_back(
2000 Elt: DAG.getStore(Chain, dl, Val: fmrrd.getValue(R: 1 - id), Ptr: DstAddr, PtrInfo: DstInfo));
2001 }
2002}
2003
2004static bool canGuaranteeTCO(CallingConv::ID CC, bool GuaranteeTailCalls) {
2005 return (CC == CallingConv::Fast && GuaranteeTailCalls) ||
2006 CC == CallingConv::Tail || CC == CallingConv::SwiftTail;
2007}
2008
2009/// LowerCall - Lowering a call into a callseq_start <-
2010/// ARMISD:CALL <- callseq_end chain. Also add input and output parameter
2011/// nodes.
2012SDValue
2013ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
2014 SmallVectorImpl<SDValue> &InVals) const {
2015 SelectionDAG &DAG = CLI.DAG;
2016 SDLoc &dl = CLI.DL;
2017 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
2018 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
2019 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
2020 SDValue Chain = CLI.Chain;
2021 SDValue Callee = CLI.Callee;
2022 bool &isTailCall = CLI.IsTailCall;
2023 CallingConv::ID CallConv = CLI.CallConv;
2024 bool doesNotRet = CLI.DoesNotReturn;
2025 bool isVarArg = CLI.IsVarArg;
2026 const CallBase *CB = CLI.CB;
2027
2028 MachineFunction &MF = DAG.getMachineFunction();
2029 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
2030 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
2031 MachineFunction::CallSiteInfo CSInfo;
2032 bool isStructRet = (Outs.empty()) ? false : Outs[0].Flags.isSRet();
2033 bool isThisReturn = false;
2034 bool isCmseNSCall = false;
2035 bool isSibCall = false;
2036 bool PreferIndirect = false;
2037 bool GuardWithBTI = false;
2038
2039 // Analyze operands of the call, assigning locations to each operand.
2040 SmallVector<CCValAssign, 16> ArgLocs;
2041 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs,
2042 *DAG.getContext());
2043 CCInfo.AnalyzeCallOperands(Outs, Fn: CCAssignFnForCall(CC: CallConv, isVarArg));
2044
2045 // Lower 'returns_twice' calls to a pseudo-instruction.
2046 if (CLI.CB && CLI.CB->getAttributes().hasFnAttr(Kind: Attribute::ReturnsTwice) &&
2047 !Subtarget->noBTIAtReturnTwice())
2048 GuardWithBTI = AFI->branchTargetEnforcement();
2049
2050 // Set type id for call site info.
2051 setTypeIdForCallsiteInfo(CB, MF, CSInfo);
2052
2053 // Determine whether this is a non-secure function call.
2054 if (CLI.CB && CLI.CB->getAttributes().hasFnAttr(Kind: "cmse_nonsecure_call"))
2055 isCmseNSCall = true;
2056
2057 // Disable tail calls if they're not supported.
2058 if (!Subtarget->supportsTailCall())
2059 isTailCall = false;
2060
2061 // For both the non-secure calls and the returns from a CMSE entry function,
2062 // the function needs to do some extra work after the call, or before the
2063 // return, respectively, thus it cannot end with a tail call
2064 if (isCmseNSCall || AFI->isCmseNSEntryFunction())
2065 isTailCall = false;
2066
2067 if (isa<GlobalAddressSDNode>(Val: Callee)) {
2068 // If we're optimizing for minimum size and the function is called three or
2069 // more times in this block, we can improve codesize by calling indirectly
2070 // as BLXr has a 16-bit encoding.
2071 auto *GV = cast<GlobalAddressSDNode>(Val&: Callee)->getGlobal();
2072 if (CLI.CB) {
2073 auto *BB = CLI.CB->getParent();
2074 PreferIndirect = Subtarget->isThumb() && Subtarget->hasMinSize() &&
2075 count_if(Range: GV->users(), P: [&BB](const User *U) {
2076 return isa<Instruction>(Val: U) &&
2077 cast<Instruction>(Val: U)->getParent() == BB;
2078 }) > 2;
2079 }
2080 }
2081 if (isTailCall) {
2082 // Check if it's really possible to do a tail call.
2083 isTailCall =
2084 IsEligibleForTailCallOptimization(CLI, CCInfo, ArgLocs, isIndirect: PreferIndirect);
2085
2086 if (isTailCall && !getTargetMachine().Options.GuaranteedTailCallOpt &&
2087 CallConv != CallingConv::Tail && CallConv != CallingConv::SwiftTail)
2088 isSibCall = true;
2089
2090 // We don't support GuaranteedTailCallOpt for ARM, only automatically
2091 // detected sibcalls.
2092 if (isTailCall)
2093 ++NumTailCalls;
2094 }
2095
2096 if (!isTailCall && CLI.CB && CLI.CB->isMustTailCall())
2097 report_fatal_error(reason: "failed to perform tail call elimination on a call "
2098 "site marked musttail");
2099
2100 // Get a count of how many bytes are to be pushed on the stack.
2101 unsigned NumBytes = CCInfo.getStackSize();
2102
2103 // SPDiff is the byte offset of the call's argument area from the callee's.
2104 // Stores to callee stack arguments will be placed in FixedStackSlots offset
2105 // by this amount for a tail call. In a sibling call it must be 0 because the
2106 // caller will deallocate the entire stack and the callee still expects its
2107 // arguments to begin at SP+0. Completely unused for non-tail calls.
2108 int SPDiff = 0;
2109
2110 if (isTailCall && !isSibCall) {
2111 auto FuncInfo = MF.getInfo<ARMFunctionInfo>();
2112 unsigned NumReusableBytes = FuncInfo->getArgumentStackSize();
2113
2114 // Since callee will pop argument stack as a tail call, we must keep the
2115 // popped size 16-byte aligned.
2116 MaybeAlign StackAlign = DAG.getDataLayout().getStackAlignment();
2117 assert(StackAlign && "data layout string is missing stack alignment");
2118 NumBytes = alignTo(Size: NumBytes, A: *StackAlign);
2119
2120 // SPDiff will be negative if this tail call requires more space than we
2121 // would automatically have in our incoming argument space. Positive if we
2122 // can actually shrink the stack.
2123 SPDiff = NumReusableBytes - NumBytes;
2124
2125 // If this call requires more stack than we have available from
2126 // LowerFormalArguments, tell FrameLowering to reserve space for it.
2127 if (SPDiff < 0 && AFI->getArgRegsSaveSize() < (unsigned)-SPDiff)
2128 AFI->setArgRegsSaveSize(-SPDiff);
2129 }
2130
2131 if (isSibCall) {
2132 // For sibling tail calls, memory operands are available in our caller's stack.
2133 NumBytes = 0;
2134 } else {
2135 // Adjust the stack pointer for the new arguments...
2136 // These operations are automatically eliminated by the prolog/epilog pass
2137 Chain = DAG.getCALLSEQ_START(Chain, InSize: isTailCall ? 0 : NumBytes, OutSize: 0, DL: dl);
2138 }
2139
2140 SDValue StackPtr =
2141 DAG.getCopyFromReg(Chain, dl, Reg: ARM::SP, VT: getPointerTy(DL: DAG.getDataLayout()));
2142
2143 RegsToPassVector RegsToPass;
2144 SmallVector<SDValue, 8> MemOpChains;
2145
2146 // If we are doing a tail-call, any byval arguments will be written to stack
2147 // space which was used for incoming arguments. If any the values being used
2148 // are incoming byval arguments to this function, then they might be
2149 // overwritten by the stores of the outgoing arguments. To avoid this, we
2150 // need to make a temporary copy of them in local stack space, then copy back
2151 // to the argument area.
2152 DenseMap<unsigned, SDValue> ByValTemporaries;
2153 SDValue ByValTempChain;
2154 if (isTailCall) {
2155 SmallVector<SDValue, 8> ByValCopyChains;
2156 for (const CCValAssign &VA : ArgLocs) {
2157 unsigned ArgIdx = VA.getValNo();
2158 SDValue Src = OutVals[ArgIdx];
2159 ISD::ArgFlagsTy Flags = Outs[ArgIdx].Flags;
2160
2161 if (!Flags.isByVal())
2162 continue;
2163
2164 SDValue Dst;
2165 MachinePointerInfo DstInfo;
2166 std::tie(args&: Dst, args&: DstInfo) =
2167 computeAddrForCallArg(dl, DAG, VA, StackPtr: SDValue(), IsTailCall: true, SPDiff);
2168 ByValCopyKind Copy = ByValNeedsCopyForTailCall(DAG, Src, Dst, Flags);
2169
2170 if (Copy == NoCopy) {
2171 // If the argument is already at the correct offset on the stack
2172 // (because we are forwarding a byval argument from our caller), we
2173 // don't need any copying.
2174 continue;
2175 } else if (Copy == CopyOnce) {
2176 // If the argument is in our local stack frame, no other argument
2177 // preparation can clobber it, so we can copy it to the final location
2178 // later.
2179 ByValTemporaries[ArgIdx] = Src;
2180 } else {
2181 assert(Copy == CopyViaTemp && "unexpected enum value");
2182 // If we might be copying this argument from the outgoing argument
2183 // stack area, we need to copy via a temporary in the local stack
2184 // frame.
2185 int TempFrameIdx = MFI.CreateStackObject(
2186 Size: Flags.getByValSize(), Alignment: Flags.getNonZeroByValAlign(), isSpillSlot: false);
2187 SDValue Temp =
2188 DAG.getFrameIndex(FI: TempFrameIdx, VT: getPointerTy(DL: DAG.getDataLayout()));
2189
2190 SDValue SizeNode = DAG.getConstant(Val: Flags.getByValSize(), DL: dl, VT: MVT::i32);
2191 SDValue AlignNode =
2192 DAG.getConstant(Val: Flags.getNonZeroByValAlign().value(), DL: dl, VT: MVT::i32);
2193
2194 SDVTList VTs = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
2195 SDValue Ops[] = {Chain, Temp, Src, SizeNode, AlignNode};
2196 ByValCopyChains.push_back(
2197 Elt: DAG.getNode(Opcode: ARMISD::COPY_STRUCT_BYVAL, DL: dl, VTList: VTs, Ops));
2198 ByValTemporaries[ArgIdx] = Temp;
2199 }
2200 }
2201 if (!ByValCopyChains.empty())
2202 ByValTempChain =
2203 DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: ByValCopyChains);
2204 }
2205
2206 // During a tail call, stores to the argument area must happen after all of
2207 // the function's incoming arguments have been loaded because they may alias.
2208 // This is done by folding in a TokenFactor from LowerFormalArguments, but
2209 // there's no point in doing so repeatedly so this tracks whether that's
2210 // happened yet.
2211 bool AfterFormalArgLoads = false;
2212
2213 // Walk the register/memloc assignments, inserting copies/loads. In the case
2214 // of tail call optimization, arguments are handled later.
2215 for (unsigned i = 0, realArgIdx = 0, e = ArgLocs.size();
2216 i != e;
2217 ++i, ++realArgIdx) {
2218 CCValAssign &VA = ArgLocs[i];
2219 SDValue Arg = OutVals[realArgIdx];
2220 ISD::ArgFlagsTy Flags = Outs[realArgIdx].Flags;
2221 bool isByVal = Flags.isByVal();
2222
2223 // Promote the value if needed.
2224 switch (VA.getLocInfo()) {
2225 default: llvm_unreachable("Unknown loc info!");
2226 case CCValAssign::Full: break;
2227 case CCValAssign::SExt:
2228 Arg = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: Arg);
2229 break;
2230 case CCValAssign::ZExt:
2231 Arg = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: Arg);
2232 break;
2233 case CCValAssign::AExt:
2234 Arg = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: Arg);
2235 break;
2236 case CCValAssign::BCvt:
2237 Arg = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: VA.getLocVT(), Operand: Arg);
2238 break;
2239 }
2240
2241 if (isTailCall && VA.isMemLoc() && !AfterFormalArgLoads) {
2242 Chain = DAG.getStackArgumentTokenFactor(Chain);
2243 if (ByValTempChain) {
2244 // In case of large byval copies, re-using the stackframe for tail-calls
2245 // can lead to overwriting incoming arguments on the stack. Force
2246 // loading these stack arguments before the copy to avoid that.
2247 SmallVector<SDValue, 8> IncomingLoad;
2248 for (unsigned I = 0; I < OutVals.size(); ++I) {
2249 if (Outs[I].Flags.isByVal())
2250 continue;
2251
2252 SDValue OutVal = OutVals[I];
2253 LoadSDNode *OutLN = dyn_cast_or_null<LoadSDNode>(Val&: OutVal);
2254 if (!OutLN)
2255 continue;
2256
2257 FrameIndexSDNode *FIN =
2258 dyn_cast_or_null<FrameIndexSDNode>(Val: OutLN->getBasePtr());
2259 if (!FIN)
2260 continue;
2261
2262 if (!MFI.isFixedObjectIndex(ObjectIdx: FIN->getIndex()))
2263 continue;
2264
2265 for (const CCValAssign &VA : ArgLocs) {
2266 if (VA.isMemLoc())
2267 IncomingLoad.push_back(Elt: OutVal.getValue(R: 1));
2268 }
2269 }
2270
2271 // Update the chain to force loads for potentially clobbered argument
2272 // loads to happen before the byval copy.
2273 if (!IncomingLoad.empty()) {
2274 IncomingLoad.push_back(Elt: Chain);
2275 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: IncomingLoad);
2276 }
2277
2278 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Chain,
2279 N2: ByValTempChain);
2280 }
2281 AfterFormalArgLoads = true;
2282 }
2283
2284 // f16 arguments have their size extended to 4 bytes and passed as if they
2285 // had been copied to the LSBs of a 32-bit register.
2286 // For that, it's passed extended to i32 (soft ABI) or to f32 (hard ABI)
2287 if (VA.needsCustom() &&
2288 (VA.getValVT() == MVT::f16 || VA.getValVT() == MVT::bf16)) {
2289 Arg = MoveFromHPR(dl, DAG, LocVT: VA.getLocVT(), ValVT: VA.getValVT(), Val: Arg);
2290 } else {
2291 // f16 arguments could have been extended prior to argument lowering.
2292 // Mask them arguments if this is a CMSE nonsecure call.
2293 auto ArgVT = Outs[realArgIdx].ArgVT;
2294 if (isCmseNSCall && (ArgVT == MVT::f16)) {
2295 auto LocBits = VA.getLocVT().getSizeInBits();
2296 auto MaskValue = APInt::getLowBitsSet(numBits: LocBits, loBitsSet: ArgVT.getSizeInBits());
2297 SDValue Mask =
2298 DAG.getConstant(Val: MaskValue, DL: dl, VT: MVT::getIntegerVT(BitWidth: LocBits));
2299 Arg = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::getIntegerVT(BitWidth: LocBits), Operand: Arg);
2300 Arg = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::getIntegerVT(BitWidth: LocBits), N1: Arg, N2: Mask);
2301 Arg = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: VA.getLocVT(), Operand: Arg);
2302 }
2303 }
2304
2305 // f64 and v2f64 might be passed in i32 pairs and must be split into pieces
2306 if (VA.needsCustom() && VA.getLocVT() == MVT::v2f64) {
2307 SDValue Op0 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: MVT::f64, N1: Arg,
2308 N2: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
2309 SDValue Op1 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: MVT::f64, N1: Arg,
2310 N2: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
2311
2312 PassF64ArgInRegs(dl, DAG, Chain, Arg&: Op0, RegsToPass, VA, NextVA&: ArgLocs[++i],
2313 StackPtr, MemOpChains, IsTailCall: isTailCall, SPDiff);
2314
2315 VA = ArgLocs[++i]; // skip ahead to next loc
2316 if (VA.isRegLoc()) {
2317 PassF64ArgInRegs(dl, DAG, Chain, Arg&: Op1, RegsToPass, VA, NextVA&: ArgLocs[++i],
2318 StackPtr, MemOpChains, IsTailCall: isTailCall, SPDiff);
2319 } else {
2320 assert(VA.isMemLoc());
2321 SDValue DstAddr;
2322 MachinePointerInfo DstInfo;
2323 std::tie(args&: DstAddr, args&: DstInfo) =
2324 computeAddrForCallArg(dl, DAG, VA, StackPtr, IsTailCall: isTailCall, SPDiff);
2325 MemOpChains.push_back(Elt: DAG.getStore(Chain, dl, Val: Op1, Ptr: DstAddr, PtrInfo: DstInfo));
2326 }
2327 } else if (VA.needsCustom() && VA.getLocVT() == MVT::f64) {
2328 PassF64ArgInRegs(dl, DAG, Chain, Arg, RegsToPass, VA, NextVA&: ArgLocs[++i],
2329 StackPtr, MemOpChains, IsTailCall: isTailCall, SPDiff);
2330 } else if (VA.isRegLoc()) {
2331 if (realArgIdx == 0 && Flags.isReturned() && !Flags.isSwiftSelf() &&
2332 Outs[0].VT == MVT::i32) {
2333 assert(VA.getLocVT() == MVT::i32 &&
2334 "unexpected calling convention register assignment");
2335 assert(!Ins.empty() && Ins[0].VT == MVT::i32 &&
2336 "unexpected use of 'returned'");
2337 isThisReturn = true;
2338 }
2339 const TargetOptions &Options = DAG.getTarget().Options;
2340 if (Options.EmitCallSiteInfo)
2341 CSInfo.ArgRegPairs.emplace_back(Args: VA.getLocReg(), Args&: i);
2342 RegsToPass.push_back(Elt: std::make_pair(x: VA.getLocReg(), y&: Arg));
2343 } else if (isByVal) {
2344 assert(VA.isMemLoc());
2345 unsigned offset = 0;
2346
2347 // True if this byval aggregate will be split between registers
2348 // and memory.
2349 unsigned ByValArgsCount = CCInfo.getInRegsParamsCount();
2350 unsigned CurByValIdx = CCInfo.getInRegsParamsProcessed();
2351
2352 SDValue ByValSrc;
2353 bool NeedsStackCopy;
2354 if (auto It = ByValTemporaries.find(Val: realArgIdx);
2355 It != ByValTemporaries.end()) {
2356 ByValSrc = It->second;
2357 NeedsStackCopy = true;
2358 } else {
2359 ByValSrc = Arg;
2360 NeedsStackCopy = !isTailCall;
2361 }
2362
2363 // If part of the argument is in registers, load them.
2364 if (CurByValIdx < ByValArgsCount) {
2365 unsigned RegBegin, RegEnd;
2366 CCInfo.getInRegsParamInfo(InRegsParamRecordIndex: CurByValIdx, BeginReg&: RegBegin, EndReg&: RegEnd);
2367
2368 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
2369 unsigned int i, j;
2370 for (i = 0, j = RegBegin; j < RegEnd; i++, j++) {
2371 SDValue Const = DAG.getConstant(Val: 4*i, DL: dl, VT: MVT::i32);
2372 SDValue AddArg = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: ByValSrc, N2: Const);
2373 SDValue Load =
2374 DAG.getLoad(VT: PtrVT, dl, Chain, Ptr: AddArg, PtrInfo: MachinePointerInfo(),
2375 Alignment: DAG.InferPtrAlign(Ptr: AddArg));
2376 MemOpChains.push_back(Elt: Load.getValue(R: 1));
2377 RegsToPass.push_back(Elt: std::make_pair(x&: j, y&: Load));
2378 }
2379
2380 // If parameter size outsides register area, "offset" value
2381 // helps us to calculate stack slot for remained part properly.
2382 offset = RegEnd - RegBegin;
2383
2384 CCInfo.nextInRegsParam();
2385 }
2386
2387 // If the memory part of the argument isn't already in the correct place
2388 // (which can happen with tail calls), copy it into the argument area.
2389 if (NeedsStackCopy && Flags.getByValSize() > 4 * offset) {
2390 auto PtrVT = getPointerTy(DL: DAG.getDataLayout());
2391 SDValue Dst;
2392 MachinePointerInfo DstInfo;
2393 std::tie(args&: Dst, args&: DstInfo) =
2394 computeAddrForCallArg(dl, DAG, VA, StackPtr, IsTailCall: isTailCall, SPDiff);
2395 SDValue SrcOffset = DAG.getIntPtrConstant(Val: 4*offset, DL: dl);
2396 SDValue Src = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: ByValSrc, N2: SrcOffset);
2397 SDValue SizeNode = DAG.getConstant(Val: Flags.getByValSize() - 4*offset, DL: dl,
2398 VT: MVT::i32);
2399 SDValue AlignNode =
2400 DAG.getConstant(Val: Flags.getNonZeroByValAlign().value(), DL: dl, VT: MVT::i32);
2401
2402 SDVTList VTs = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
2403 SDValue Ops[] = { Chain, Dst, Src, SizeNode, AlignNode};
2404 MemOpChains.push_back(Elt: DAG.getNode(Opcode: ARMISD::COPY_STRUCT_BYVAL, DL: dl, VTList: VTs,
2405 Ops));
2406 }
2407 } else {
2408 assert(VA.isMemLoc());
2409 SDValue DstAddr;
2410 MachinePointerInfo DstInfo;
2411 std::tie(args&: DstAddr, args&: DstInfo) =
2412 computeAddrForCallArg(dl, DAG, VA, StackPtr, IsTailCall: isTailCall, SPDiff);
2413
2414 SDValue Store = DAG.getStore(Chain, dl, Val: Arg, Ptr: DstAddr, PtrInfo: DstInfo);
2415 MemOpChains.push_back(Elt: Store);
2416 }
2417 }
2418
2419 if (!MemOpChains.empty())
2420 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: MemOpChains);
2421
2422 // Build a sequence of copy-to-reg nodes chained together with token chain
2423 // and flag operands which copy the outgoing args into the appropriate regs.
2424 SDValue InGlue;
2425 for (const auto &[Reg, N] : RegsToPass) {
2426 Chain = DAG.getCopyToReg(Chain, dl, Reg, N, Glue: InGlue);
2427 InGlue = Chain.getValue(R: 1);
2428 }
2429
2430 // If the callee is a GlobalAddress/ExternalSymbol node (quite common, every
2431 // direct call is) turn it into a TargetGlobalAddress/TargetExternalSymbol
2432 // node so that legalize doesn't hack it.
2433 bool isDirect = false;
2434
2435 const TargetMachine &TM = getTargetMachine();
2436 const Triple &TT = TM.getTargetTriple();
2437 const GlobalValue *GVal = nullptr;
2438 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Val&: Callee))
2439 GVal = G->getGlobal();
2440 bool isStub = !TM.shouldAssumeDSOLocal(GV: GVal) && TT.isOSBinFormatMachO();
2441
2442 bool isARMFunc = !Subtarget->isThumb() || (isStub && !Subtarget->isMClass());
2443 bool isLocalARMFunc = false;
2444 auto PtrVt = getPointerTy(DL: DAG.getDataLayout());
2445
2446 if (Subtarget->genLongCalls()) {
2447 bool isPIC = isPositionIndependent() && !TT.isOSWindows();
2448 if (isPIC && Subtarget->genExecuteOnly())
2449 reportFatalUsageError(reason: "long-calls with execute-only and "
2450 "position-independent code is not supported");
2451 if (Subtarget->isROPI())
2452 reportFatalUsageError(reason: "long-calls with ROPI is not currently supported");
2453
2454 // Handle a global address or an external symbol. If it's not one of
2455 // those, the target's already in a register, so we don't need to do
2456 // anything extra.
2457 if (isa<GlobalAddressSDNode>(Val: Callee)) {
2458 if (Subtarget->genExecuteOnly()) {
2459 // Execute-only forbids constant pools in .text, so use movw/movt.
2460 // fPIC is not supported with execute-only.
2461 if (Subtarget->useMovt())
2462 ++NumMovwMovt;
2463 Callee = DAG.getNode(Opcode: ARMISD::Wrapper, DL: dl, VT: PtrVt,
2464 Operand: DAG.getTargetGlobalAddress(GV: GVal, DL: dl, VT: PtrVt));
2465 } else if (isPIC) {
2466 // PIC without execute-only: use GOT-based addressing.
2467 // DSO-local symbols use a plain PC-relative WrapperPIC;
2468 // non-DSO-local symbols additionally load the address from the GOT.
2469 SDValue G = DAG.getTargetGlobalAddress(
2470 GV: GVal, DL: dl, VT: PtrVt, offset: 0, TargetFlags: GVal->isDSOLocal() ? 0 : ARMII::MO_GOT);
2471 Callee = DAG.getNode(Opcode: ARMISD::WrapperPIC, DL: dl, VT: PtrVt, Operand: G);
2472 if (!GVal->isDSOLocal())
2473 Callee =
2474 DAG.getLoad(VT: PtrVt, dl, Chain: DAG.getEntryNode(), Ptr: Callee,
2475 PtrInfo: MachinePointerInfo::getGOT(MF&: DAG.getMachineFunction()));
2476 } else {
2477 // Neither execute-only nor PIC: load the address from a constant pool.
2478 unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
2479 ARMConstantPoolValue *CPV = ARMConstantPoolConstant::Create(
2480 C: GVal, ID: ARMPCLabelIndex, Kind: ARMCP::CPValue, PCAdj: 0);
2481
2482 // Get the address of the callee into a register
2483 SDValue Addr = DAG.getTargetConstantPool(C: CPV, VT: PtrVt, Align: Align(4));
2484 Addr = DAG.getNode(Opcode: ARMISD::Wrapper, DL: dl, VT: MVT::i32, Operand: Addr);
2485 Callee = DAG.getLoad(
2486 VT: PtrVt, dl, Chain: DAG.getEntryNode(), Ptr: Addr,
2487 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
2488 }
2489 } else if (ExternalSymbolSDNode *S=dyn_cast<ExternalSymbolSDNode>(Val&: Callee)) {
2490 const char *Sym = S->getSymbol();
2491
2492 if (Subtarget->genExecuteOnly()) {
2493 // Execute-only forbids constant pools in .text, so use movw/movt.
2494 // fPIC is not supported with execute-only.
2495 if (Subtarget->useMovt())
2496 ++NumMovwMovt;
2497 Callee = DAG.getNode(Opcode: ARMISD::Wrapper, DL: dl, VT: PtrVt,
2498 Operand: DAG.getTargetExternalSymbol(Sym, VT: PtrVt, TargetFlags: 0));
2499 } else if (isPIC) {
2500 // PIC without execute-only: load the symbol's address from the GOT via
2501 // a GOT_PREL constant pool entry consumed by a PICLDR.
2502 unsigned PCAdj = Subtarget->isThumb() ? 4 : 8;
2503 unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
2504 ARMConstantPoolValue *CPV = ARMConstantPoolSymbol::Create(
2505 C&: *DAG.getContext(), s: Sym, ID: ARMPCLabelIndex, PCAdj, Modifier: ARMCP::GOT_PREL,
2506 /*AddCurrentAddress=*/true);
2507 SDValue CPAddr = DAG.getTargetConstantPool(C: CPV, VT: PtrVt, Align: Align(4));
2508 CPAddr = DAG.getNode(Opcode: ARMISD::Wrapper, DL: dl, VT: MVT::i32, Operand: CPAddr);
2509 SDValue GOTOffset = DAG.getLoad(
2510 VT: PtrVt, dl, Chain: DAG.getEntryNode(), Ptr: CPAddr,
2511 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
2512 SDValue PICLabel = DAG.getConstant(Val: ARMPCLabelIndex, DL: dl, VT: MVT::i32);
2513 Callee = DAG.getNode(Opcode: ARMISD::PIC_ADD, DL: dl, VT: PtrVt, N1: GOTOffset, N2: PICLabel);
2514 Callee =
2515 DAG.getLoad(VT: PtrVt, dl, Chain: DAG.getEntryNode(), Ptr: Callee,
2516 PtrInfo: MachinePointerInfo::getGOT(MF&: DAG.getMachineFunction()));
2517 } else {
2518 // Neither execute-only nor PIC: load the address from a constant pool.
2519 unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
2520 ARMConstantPoolValue *CPV = ARMConstantPoolSymbol::Create(
2521 C&: *DAG.getContext(), s: Sym, ID: ARMPCLabelIndex, PCAdj: 0);
2522
2523 // Get the address of the callee into a register
2524 SDValue Addr = DAG.getTargetConstantPool(C: CPV, VT: PtrVt, Align: Align(4));
2525 Addr = DAG.getNode(Opcode: ARMISD::Wrapper, DL: dl, VT: MVT::i32, Operand: Addr);
2526 Callee = DAG.getLoad(
2527 VT: PtrVt, dl, Chain: DAG.getEntryNode(), Ptr: Addr,
2528 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
2529 }
2530 }
2531 } else if (isa<GlobalAddressSDNode>(Val: Callee)) {
2532 if (!PreferIndirect) {
2533 isDirect = true;
2534 bool isDef = GVal->isStrongDefinitionForLinker();
2535
2536 // ARM call to a local ARM function is predicable.
2537 isLocalARMFunc = !Subtarget->isThumb() && (isDef || !ARMInterworking);
2538 // tBX takes a register source operand.
2539 if (isStub && Subtarget->isThumb1Only() && !Subtarget->hasV5TOps()) {
2540 assert(TT.isOSBinFormatMachO() && "WrapperPIC use on non-MachO?");
2541 Callee = DAG.getNode(
2542 Opcode: ARMISD::WrapperPIC, DL: dl, VT: PtrVt,
2543 Operand: DAG.getTargetGlobalAddress(GV: GVal, DL: dl, VT: PtrVt, offset: 0, TargetFlags: ARMII::MO_NONLAZY));
2544 Callee = DAG.getLoad(
2545 VT: PtrVt, dl, Chain: DAG.getEntryNode(), Ptr: Callee,
2546 PtrInfo: MachinePointerInfo::getGOT(MF&: DAG.getMachineFunction()), Alignment: MaybeAlign(),
2547 MMOFlags: MachineMemOperand::MODereferenceable |
2548 MachineMemOperand::MOInvariant);
2549 } else if (Subtarget->isTargetCOFF()) {
2550 assert(Subtarget->isTargetWindows() &&
2551 "Windows is the only supported COFF target");
2552 unsigned TargetFlags = ARMII::MO_NO_FLAG;
2553 if (GVal->hasDLLImportStorageClass())
2554 TargetFlags = ARMII::MO_DLLIMPORT;
2555 else if (!TM.shouldAssumeDSOLocal(GV: GVal))
2556 TargetFlags = ARMII::MO_COFFSTUB;
2557 Callee = DAG.getTargetGlobalAddress(GV: GVal, DL: dl, VT: PtrVt, /*offset=*/0,
2558 TargetFlags);
2559 if (TargetFlags & (ARMII::MO_DLLIMPORT | ARMII::MO_COFFSTUB))
2560 Callee =
2561 DAG.getLoad(VT: PtrVt, dl, Chain: DAG.getEntryNode(),
2562 Ptr: DAG.getNode(Opcode: ARMISD::Wrapper, DL: dl, VT: PtrVt, Operand: Callee),
2563 PtrInfo: MachinePointerInfo::getGOT(MF&: DAG.getMachineFunction()));
2564 } else {
2565 Callee = DAG.getTargetGlobalAddress(GV: GVal, DL: dl, VT: PtrVt, offset: 0, TargetFlags: 0);
2566 }
2567 }
2568 } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Val&: Callee)) {
2569 isDirect = true;
2570 // tBX takes a register source operand.
2571 const char *Sym = S->getSymbol();
2572 if (isARMFunc && Subtarget->isThumb1Only() && !Subtarget->hasV5TOps()) {
2573 unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
2574 ARMConstantPoolValue *CPV =
2575 ARMConstantPoolSymbol::Create(C&: *DAG.getContext(), s: Sym,
2576 ID: ARMPCLabelIndex, PCAdj: 4);
2577 SDValue CPAddr = DAG.getTargetConstantPool(C: CPV, VT: PtrVt, Align: Align(4));
2578 CPAddr = DAG.getNode(Opcode: ARMISD::Wrapper, DL: dl, VT: MVT::i32, Operand: CPAddr);
2579 Callee = DAG.getLoad(
2580 VT: PtrVt, dl, Chain: DAG.getEntryNode(), Ptr: CPAddr,
2581 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
2582 SDValue PICLabel = DAG.getConstant(Val: ARMPCLabelIndex, DL: dl, VT: MVT::i32);
2583 Callee = DAG.getNode(Opcode: ARMISD::PIC_ADD, DL: dl, VT: PtrVt, N1: Callee, N2: PICLabel);
2584 } else {
2585 Callee = DAG.getTargetExternalSymbol(Sym, VT: PtrVt, TargetFlags: 0);
2586 }
2587 }
2588
2589 if (isCmseNSCall) {
2590 assert(!isARMFunc && !isDirect &&
2591 "Cannot handle call to ARM function or direct call");
2592 if (NumBytes > 0) {
2593 DAG.getContext()->diagnose(
2594 DI: DiagnosticInfoUnsupported(DAG.getMachineFunction().getFunction(),
2595 "call to non-secure function would require "
2596 "passing arguments on stack",
2597 dl.getDebugLoc()));
2598 }
2599 if (isStructRet) {
2600 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
2601 DAG.getMachineFunction().getFunction(),
2602 "call to non-secure function would return value through pointer",
2603 dl.getDebugLoc()));
2604 }
2605 }
2606
2607 // FIXME: handle tail calls differently.
2608 unsigned CallOpc;
2609 if (Subtarget->isThumb()) {
2610 if (GuardWithBTI)
2611 CallOpc = ARMISD::t2CALL_BTI;
2612 else if (isCmseNSCall)
2613 CallOpc = ARMISD::tSECALL;
2614 else if ((!isDirect || isARMFunc) && !Subtarget->hasV5TOps())
2615 CallOpc = ARMISD::CALL_NOLINK;
2616 else
2617 CallOpc = ARMISD::CALL;
2618 } else {
2619 if (!isDirect && !Subtarget->hasV5TOps())
2620 CallOpc = ARMISD::CALL_NOLINK;
2621 else if (doesNotRet && isDirect && Subtarget->hasRetAddrStack() &&
2622 // Emit regular call when code size is the priority
2623 !Subtarget->hasMinSize())
2624 // "mov lr, pc; b _foo" to avoid confusing the RSP
2625 CallOpc = ARMISD::CALL_NOLINK;
2626 else
2627 CallOpc = isLocalARMFunc ? ARMISD::CALL_PRED : ARMISD::CALL;
2628 }
2629
2630 // We don't usually want to end the call-sequence here because we would tidy
2631 // the frame up *after* the call, however in the ABI-changing tail-call case
2632 // we've carefully laid out the parameters so that when sp is reset they'll be
2633 // in the correct location.
2634 if (isTailCall && !isSibCall) {
2635 Chain = DAG.getCALLSEQ_END(Chain, Size1: 0, Size2: 0, Glue: InGlue, DL: dl);
2636 InGlue = Chain.getValue(R: 1);
2637 }
2638
2639 std::vector<SDValue> Ops;
2640 Ops.push_back(x: Chain);
2641 Ops.push_back(x: Callee);
2642
2643 if (isTailCall) {
2644 Ops.push_back(x: DAG.getSignedTargetConstant(Val: SPDiff, DL: dl, VT: MVT::i32));
2645 }
2646
2647 // Add argument registers to the end of the list so that they are known live
2648 // into the call.
2649 for (const auto &[Reg, N] : RegsToPass)
2650 Ops.push_back(x: DAG.getRegister(Reg, VT: N.getValueType()));
2651
2652 // Add a register mask operand representing the call-preserved registers.
2653 const uint32_t *Mask;
2654 const ARMBaseRegisterInfo *ARI = Subtarget->getRegisterInfo();
2655 if (isThisReturn) {
2656 // For 'this' returns, use the R0-preserving mask if applicable
2657 Mask = ARI->getThisReturnPreservedMask(MF, CallConv);
2658 if (!Mask) {
2659 // Set isThisReturn to false if the calling convention is not one that
2660 // allows 'returned' to be modeled in this way, so LowerCallResult does
2661 // not try to pass 'this' straight through
2662 isThisReturn = false;
2663 Mask = ARI->getCallPreservedMask(MF, CallConv);
2664 }
2665 } else
2666 Mask = ARI->getCallPreservedMask(MF, CallConv);
2667
2668 assert(Mask && "Missing call preserved mask for calling convention");
2669 Ops.push_back(x: DAG.getRegisterMask(RegMask: Mask));
2670
2671 if (InGlue.getNode())
2672 Ops.push_back(x: InGlue);
2673
2674 if (isTailCall) {
2675 MF.getFrameInfo().setHasTailCall();
2676 SDValue Ret = DAG.getNode(Opcode: ARMISD::TC_RETURN, DL: dl, VT: MVT::Other, Ops);
2677 if (CLI.CFIType)
2678 Ret.getNode()->setCFIType(CLI.CFIType->getZExtValue());
2679 DAG.addNoMergeSiteInfo(Node: Ret.getNode(), NoMerge: CLI.NoMerge);
2680 DAG.addCallSiteInfo(Node: Ret.getNode(), CallInfo: std::move(CSInfo));
2681 return Ret;
2682 }
2683
2684 // Returns a chain and a flag for retval copy to use.
2685 Chain = DAG.getNode(Opcode: CallOpc, DL: dl, ResultTys: {MVT::Other, MVT::Glue}, Ops);
2686 if (CLI.CFIType)
2687 Chain.getNode()->setCFIType(CLI.CFIType->getZExtValue());
2688 DAG.addNoMergeSiteInfo(Node: Chain.getNode(), NoMerge: CLI.NoMerge);
2689 InGlue = Chain.getValue(R: 1);
2690 DAG.addCallSiteInfo(Node: Chain.getNode(), CallInfo: std::move(CSInfo));
2691
2692 // If we're guaranteeing tail-calls will be honoured, the callee must
2693 // pop its own argument stack on return. But this call is *not* a tail call so
2694 // we need to undo that after it returns to restore the status-quo.
2695 bool TailCallOpt = getTargetMachine().Options.GuaranteedTailCallOpt;
2696 uint64_t CalleePopBytes =
2697 canGuaranteeTCO(CC: CallConv, GuaranteeTailCalls: TailCallOpt) ? alignTo(Value: NumBytes, Align: 16) : -1U;
2698
2699 Chain = DAG.getCALLSEQ_END(Chain, Size1: NumBytes, Size2: CalleePopBytes, Glue: InGlue, DL: dl);
2700 if (!Ins.empty())
2701 InGlue = Chain.getValue(R: 1);
2702
2703 // Handle result values, copying them out of physregs into vregs that we
2704 // return.
2705 return LowerCallResult(Chain, InGlue, CallConv, isVarArg, Ins, dl, DAG,
2706 InVals, isThisReturn,
2707 ThisVal: isThisReturn ? OutVals[0] : SDValue(), isCmseNSCall);
2708}
2709
2710/// HandleByVal - Every parameter *after* a byval parameter is passed
2711/// on the stack. Remember the next parameter register to allocate,
2712/// and then confiscate the rest of the parameter registers to insure
2713/// this.
2714void ARMTargetLowering::HandleByVal(CCState *State, unsigned &Size,
2715 Align Alignment) const {
2716 // Byval (as with any stack) slots are always at least 4 byte aligned.
2717 Alignment = std::max(a: Alignment, b: Align(4));
2718
2719 MCRegister Reg = State->AllocateReg(Regs: GPRArgRegs);
2720 if (!Reg)
2721 return;
2722
2723 unsigned AlignInRegs = Alignment.value() / 4;
2724 unsigned Waste = (ARM::R4 - Reg) % AlignInRegs;
2725 for (unsigned i = 0; i < Waste; ++i)
2726 Reg = State->AllocateReg(Regs: GPRArgRegs);
2727
2728 if (!Reg)
2729 return;
2730
2731 unsigned Excess = 4 * (ARM::R4 - Reg);
2732
2733 // Special case when NSAA != SP and parameter size greater than size of
2734 // all remained GPR regs. In that case we can't split parameter, we must
2735 // send it to stack. We also must set NCRN to R4, so waste all
2736 // remained registers.
2737 const unsigned NSAAOffset = State->getStackSize();
2738 if (NSAAOffset != 0 && Size > Excess) {
2739 while (State->AllocateReg(Regs: GPRArgRegs))
2740 ;
2741 return;
2742 }
2743
2744 // First register for byval parameter is the first register that wasn't
2745 // allocated before this method call, so it would be "reg".
2746 // If parameter is small enough to be saved in range [reg, r4), then
2747 // the end (first after last) register would be reg + param-size-in-regs,
2748 // else parameter would be splitted between registers and stack,
2749 // end register would be r4 in this case.
2750 unsigned ByValRegBegin = Reg;
2751 unsigned ByValRegEnd = std::min<unsigned>(a: Reg + Size / 4, b: ARM::R4);
2752 State->addInRegsParamInfo(RegBegin: ByValRegBegin, RegEnd: ByValRegEnd);
2753 // Note, first register is allocated in the beginning of function already,
2754 // allocate remained amount of registers we need.
2755 for (unsigned i = Reg + 1; i != ByValRegEnd; ++i)
2756 State->AllocateReg(Regs: GPRArgRegs);
2757 // A byval parameter that is split between registers and memory needs its
2758 // size truncated here.
2759 // In the case where the entire structure fits in registers, we set the
2760 // size in memory to zero.
2761 Size = std::max<int>(a: Size - Excess, b: 0);
2762}
2763
2764/// IsEligibleForTailCallOptimization - Check whether the call is eligible
2765/// for tail call optimization. Targets which want to do tail call
2766/// optimization should implement this function. Note that this function also
2767/// processes musttail calls, so when this function returns false on a valid
2768/// musttail call, a fatal backend error occurs.
2769bool ARMTargetLowering::IsEligibleForTailCallOptimization(
2770 TargetLowering::CallLoweringInfo &CLI, CCState &CCInfo,
2771 SmallVectorImpl<CCValAssign> &ArgLocs, const bool isIndirect) const {
2772 CallingConv::ID CalleeCC = CLI.CallConv;
2773 SDValue Callee = CLI.Callee;
2774 bool isVarArg = CLI.IsVarArg;
2775 const SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
2776 const SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
2777 const SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
2778 const SelectionDAG &DAG = CLI.DAG;
2779 MachineFunction &MF = DAG.getMachineFunction();
2780 const Function &CallerF = MF.getFunction();
2781 CallingConv::ID CallerCC = CallerF.getCallingConv();
2782
2783 assert(Subtarget->supportsTailCall());
2784
2785 // Indirect tail-calls require a register to hold the target address. That
2786 // register must be:
2787 // * Allocatable (i.e. r0-r7 if the target is Thumb1).
2788 // * Not callee-saved, so must be one of r0-r3 or r12.
2789 // * Not used to hold an argument to the tail-called function, which might be
2790 // in r0-r3.
2791 // * Not used to hold the return address authentication code, which is in r12
2792 // if enabled.
2793 // Sometimes, no register matches all of these conditions, so we can't do a
2794 // tail-call.
2795 if (!isa<GlobalAddressSDNode>(Val: Callee.getNode()) || isIndirect) {
2796 SmallSet<MCPhysReg, 5> AddressRegisters = {ARM::R0, ARM::R1, ARM::R2,
2797 ARM::R3};
2798 if (!(Subtarget->isThumb1Only() ||
2799 MF.getInfo<ARMFunctionInfo>()->shouldSignReturnAddress(SpillsLR: true)))
2800 AddressRegisters.insert(V: ARM::R12);
2801 for (const CCValAssign &AL : ArgLocs)
2802 if (AL.isRegLoc())
2803 AddressRegisters.erase(V: AL.getLocReg());
2804 if (AddressRegisters.empty()) {
2805 LLVM_DEBUG(dbgs() << "false (no reg to hold function pointer)\n");
2806 return false;
2807 }
2808 }
2809
2810 // Look for obvious safe cases to perform tail call optimization that do not
2811 // require ABI changes. This is what gcc calls sibcall.
2812
2813 // Exception-handling functions need a special set of instructions to indicate
2814 // a return to the hardware. Tail-calling another function would probably
2815 // break this.
2816 if (CallerF.hasFnAttribute(Kind: "interrupt")) {
2817 LLVM_DEBUG(dbgs() << "false (interrupt attribute)\n");
2818 return false;
2819 }
2820
2821 if (canGuaranteeTCO(CC: CalleeCC,
2822 GuaranteeTailCalls: getTargetMachine().Options.GuaranteedTailCallOpt)) {
2823 LLVM_DEBUG(dbgs() << (CalleeCC == CallerCC ? "true" : "false")
2824 << " (guaranteed tail-call CC)\n");
2825 return CalleeCC == CallerCC;
2826 }
2827
2828 // Also avoid sibcall optimization if either caller or callee uses struct
2829 // return semantics.
2830 bool isCalleeStructRet = Outs.empty() ? false : Outs[0].Flags.isSRet();
2831 bool isCallerStructRet = MF.getFunction().hasStructRetAttr();
2832 if (isCalleeStructRet != isCallerStructRet) {
2833 LLVM_DEBUG(dbgs() << "false (struct-ret)\n");
2834 return false;
2835 }
2836
2837 // Externally-defined functions with weak linkage should not be
2838 // tail-called on ARM when the OS does not support dynamic
2839 // pre-emption of symbols, as the AAELF spec requires normal calls
2840 // to undefined weak functions to be replaced with a NOP or jump to the
2841 // next instruction. The behaviour of branch instructions in this
2842 // situation (as used for tail calls) is implementation-defined, so we
2843 // cannot rely on the linker replacing the tail call with a return.
2844 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Val&: Callee)) {
2845 const GlobalValue *GV = G->getGlobal();
2846 const Triple &TT = GV->getParent()->getTargetTriple();
2847 if (GV->hasExternalWeakLinkage() &&
2848 (!TT.isOSWindows() || TT.isOSBinFormatELF() ||
2849 TT.isOSBinFormatMachO())) {
2850 LLVM_DEBUG(dbgs() << "false (external weak linkage)\n");
2851 return false;
2852 }
2853 }
2854
2855 // Check that the call results are passed in the same way.
2856 LLVMContext &C = *DAG.getContext();
2857 if (!CCState::resultsCompatible(
2858 CalleeCC: getEffectiveCallingConv(CC: CalleeCC, isVarArg),
2859 CallerCC: getEffectiveCallingConv(CC: CallerCC, isVarArg: CallerF.isVarArg()), MF, C, Ins,
2860 CalleeFn: CCAssignFnForReturn(CC: CalleeCC, isVarArg),
2861 CallerFn: CCAssignFnForReturn(CC: CallerCC, isVarArg: CallerF.isVarArg()))) {
2862 LLVM_DEBUG(dbgs() << "false (incompatible results)\n");
2863 return false;
2864 }
2865 // The callee has to preserve all registers the caller needs to preserve.
2866 const ARMBaseRegisterInfo *TRI = Subtarget->getRegisterInfo();
2867 const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
2868 if (CalleeCC != CallerCC) {
2869 const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
2870 if (!TRI->regmaskSubsetEqual(mask0: CallerPreserved, mask1: CalleePreserved)) {
2871 LLVM_DEBUG(dbgs() << "false (not all registers preserved)\n");
2872 return false;
2873 }
2874 }
2875
2876 // If Caller's vararg argument has been split between registers and stack, do
2877 // not perform tail call, since part of the argument is in caller's local
2878 // frame.
2879 const ARMFunctionInfo *AFI_Caller = MF.getInfo<ARMFunctionInfo>();
2880 if (CLI.IsVarArg && AFI_Caller->getArgRegsSaveSize()) {
2881 LLVM_DEBUG(dbgs() << "false (arg reg save area)\n");
2882 return false;
2883 }
2884
2885 // If the callee takes no arguments then go on to check the results of the
2886 // call.
2887 const MachineRegisterInfo &MRI = MF.getRegInfo();
2888 if (!parametersInCSRMatch(MRI, CallerPreservedMask: CallerPreserved, ArgLocs, OutVals)) {
2889 LLVM_DEBUG(dbgs() << "false (parameters in CSRs do not match)\n");
2890 return false;
2891 }
2892
2893 // If the stack arguments for this call do not fit into our own save area then
2894 // the call cannot be made tail.
2895 if (CCInfo.getStackSize() > AFI_Caller->getArgumentStackSize())
2896 return false;
2897
2898 LLVM_DEBUG(dbgs() << "true\n");
2899 return true;
2900}
2901
2902bool
2903ARMTargetLowering::CanLowerReturn(CallingConv::ID CallConv,
2904 MachineFunction &MF, bool isVarArg,
2905 const SmallVectorImpl<ISD::OutputArg> &Outs,
2906 LLVMContext &Context, const Type *RetTy) const {
2907 SmallVector<CCValAssign, 16> RVLocs;
2908 CCState CCInfo(CallConv, isVarArg, MF, RVLocs, Context);
2909 return CCInfo.CheckReturn(Outs, Fn: CCAssignFnForReturn(CC: CallConv, isVarArg));
2910}
2911
2912static SDValue LowerInterruptReturn(SmallVectorImpl<SDValue> &RetOps,
2913 const SDLoc &DL, SelectionDAG &DAG) {
2914 const MachineFunction &MF = DAG.getMachineFunction();
2915 const Function &F = MF.getFunction();
2916
2917 StringRef IntKind = F.getFnAttribute(Kind: "interrupt").getValueAsString();
2918
2919 // See ARM ARM v7 B1.8.3. On exception entry LR is set to a possibly offset
2920 // version of the "preferred return address". These offsets affect the return
2921 // instruction if this is a return from PL1 without hypervisor extensions.
2922 // IRQ/FIQ: +4 "subs pc, lr, #4"
2923 // SWI: 0 "subs pc, lr, #0"
2924 // ABORT: +4 "subs pc, lr, #4"
2925 // UNDEF: +4/+2 "subs pc, lr, #0"
2926 // UNDEF varies depending on where the exception came from ARM or Thumb
2927 // mode. Alongside GCC, we throw our hands up in disgust and pretend it's 0.
2928
2929 int64_t LROffset;
2930 if (IntKind == "" || IntKind == "IRQ" || IntKind == "FIQ" ||
2931 IntKind == "ABORT")
2932 LROffset = 4;
2933 else if (IntKind == "SWI" || IntKind == "UNDEF")
2934 LROffset = 0;
2935 else
2936 report_fatal_error(reason: "Unsupported interrupt attribute. If present, value "
2937 "must be one of: IRQ, FIQ, SWI, ABORT or UNDEF");
2938
2939 RetOps.insert(I: RetOps.begin() + 1,
2940 Elt: DAG.getConstant(Val: LROffset, DL, VT: MVT::i32, isTarget: false));
2941
2942 return DAG.getNode(Opcode: ARMISD::INTRET_GLUE, DL, VT: MVT::Other, Ops: RetOps);
2943}
2944
2945SDValue
2946ARMTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
2947 bool isVarArg,
2948 const SmallVectorImpl<ISD::OutputArg> &Outs,
2949 const SmallVectorImpl<SDValue> &OutVals,
2950 const SDLoc &dl, SelectionDAG &DAG) const {
2951 // CCValAssign - represent the assignment of the return value to a location.
2952 SmallVector<CCValAssign, 16> RVLocs;
2953
2954 // CCState - Info about the registers and stack slots.
2955 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
2956 *DAG.getContext());
2957
2958 // Analyze outgoing return values.
2959 CCInfo.AnalyzeReturn(Outs, Fn: CCAssignFnForReturn(CC: CallConv, isVarArg));
2960
2961 SDValue Glue;
2962 SmallVector<SDValue, 4> RetOps;
2963 RetOps.push_back(Elt: Chain); // Operand #0 = Chain (updated below)
2964 bool isLittleEndian = Subtarget->isLittle();
2965
2966 MachineFunction &MF = DAG.getMachineFunction();
2967 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
2968 AFI->setReturnRegsCount(RVLocs.size());
2969
2970 // Report error if cmse entry function returns structure through first ptr arg.
2971 if (AFI->isCmseNSEntryFunction() && MF.getFunction().hasStructRetAttr()) {
2972 // Note: using an empty SDLoc(), as the first line of the function is a
2973 // better place to report than the last line.
2974 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
2975 DAG.getMachineFunction().getFunction(),
2976 "secure entry function would return value through pointer",
2977 SDLoc().getDebugLoc()));
2978 }
2979
2980 // Copy the result values into the output registers.
2981 for (unsigned i = 0, realRVLocIdx = 0;
2982 i != RVLocs.size();
2983 ++i, ++realRVLocIdx) {
2984 CCValAssign &VA = RVLocs[i];
2985 assert(VA.isRegLoc() && "Can only return in registers!");
2986
2987 SDValue Arg = OutVals[realRVLocIdx];
2988 bool ReturnF16 = false;
2989
2990 if (Subtarget->hasFullFP16() && Subtarget->isTargetHardFloat()) {
2991 // Half-precision return values can be returned like this:
2992 //
2993 // t11 f16 = fadd ...
2994 // t12: i16 = bitcast t11
2995 // t13: i32 = zero_extend t12
2996 // t14: f32 = bitcast t13 <~~~~~~~ Arg
2997 //
2998 // to avoid code generation for bitcasts, we simply set Arg to the node
2999 // that produces the f16 value, t11 in this case.
3000 //
3001 if (Arg.getValueType() == MVT::f32 && Arg.getOpcode() == ISD::BITCAST) {
3002 SDValue ZE = Arg.getOperand(i: 0);
3003 if (ZE.getOpcode() == ISD::ZERO_EXTEND && ZE.getValueType() == MVT::i32) {
3004 SDValue BC = ZE.getOperand(i: 0);
3005 if (BC.getOpcode() == ISD::BITCAST && BC.getValueType() == MVT::i16) {
3006 Arg = BC.getOperand(i: 0);
3007 ReturnF16 = true;
3008 }
3009 }
3010 }
3011 }
3012
3013 switch (VA.getLocInfo()) {
3014 default: llvm_unreachable("Unknown loc info!");
3015 case CCValAssign::Full: break;
3016 case CCValAssign::BCvt:
3017 if (!ReturnF16)
3018 Arg = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: VA.getLocVT(), Operand: Arg);
3019 break;
3020 }
3021
3022 // Mask f16 arguments if this is a CMSE nonsecure entry.
3023 auto RetVT = Outs[realRVLocIdx].ArgVT;
3024 if (AFI->isCmseNSEntryFunction() && (RetVT == MVT::f16)) {
3025 if (VA.needsCustom() && VA.getValVT() == MVT::f16) {
3026 Arg = MoveFromHPR(dl, DAG, LocVT: VA.getLocVT(), ValVT: VA.getValVT(), Val: Arg);
3027 } else {
3028 auto LocBits = VA.getLocVT().getSizeInBits();
3029 auto MaskValue = APInt::getLowBitsSet(numBits: LocBits, loBitsSet: RetVT.getSizeInBits());
3030 SDValue Mask =
3031 DAG.getConstant(Val: MaskValue, DL: dl, VT: MVT::getIntegerVT(BitWidth: LocBits));
3032 Arg = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::getIntegerVT(BitWidth: LocBits), Operand: Arg);
3033 Arg = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::getIntegerVT(BitWidth: LocBits), N1: Arg, N2: Mask);
3034 Arg = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: VA.getLocVT(), Operand: Arg);
3035 }
3036 }
3037
3038 if (VA.needsCustom() &&
3039 (VA.getLocVT() == MVT::v2f64 || VA.getLocVT() == MVT::f64)) {
3040 if (VA.getLocVT() == MVT::v2f64) {
3041 // Extract the first half and return it in two registers.
3042 SDValue Half = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: MVT::f64, N1: Arg,
3043 N2: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
3044 SDValue HalfGPRs = DAG.getNode(Opcode: ARMISD::VMOVRRD, DL: dl,
3045 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N: Half);
3046
3047 Chain =
3048 DAG.getCopyToReg(Chain, dl, Reg: VA.getLocReg(),
3049 N: HalfGPRs.getValue(R: isLittleEndian ? 0 : 1), Glue);
3050 Glue = Chain.getValue(R: 1);
3051 RetOps.push_back(Elt: DAG.getRegister(Reg: VA.getLocReg(), VT: VA.getLocVT()));
3052 VA = RVLocs[++i]; // skip ahead to next loc
3053 Chain =
3054 DAG.getCopyToReg(Chain, dl, Reg: VA.getLocReg(),
3055 N: HalfGPRs.getValue(R: isLittleEndian ? 1 : 0), Glue);
3056 Glue = Chain.getValue(R: 1);
3057 RetOps.push_back(Elt: DAG.getRegister(Reg: VA.getLocReg(), VT: VA.getLocVT()));
3058 VA = RVLocs[++i]; // skip ahead to next loc
3059
3060 // Extract the 2nd half and fall through to handle it as an f64 value.
3061 Arg = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: MVT::f64, N1: Arg,
3062 N2: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
3063 }
3064 // Legalize ret f64 -> ret 2 x i32. We always have fmrrd if f64 is
3065 // available.
3066 SDValue fmrrd = DAG.getNode(Opcode: ARMISD::VMOVRRD, DL: dl,
3067 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N: Arg);
3068 Chain = DAG.getCopyToReg(Chain, dl, Reg: VA.getLocReg(),
3069 N: fmrrd.getValue(R: isLittleEndian ? 0 : 1), Glue);
3070 Glue = Chain.getValue(R: 1);
3071 RetOps.push_back(Elt: DAG.getRegister(Reg: VA.getLocReg(), VT: VA.getLocVT()));
3072 VA = RVLocs[++i]; // skip ahead to next loc
3073 Chain = DAG.getCopyToReg(Chain, dl, Reg: VA.getLocReg(),
3074 N: fmrrd.getValue(R: isLittleEndian ? 1 : 0), Glue);
3075 } else
3076 Chain = DAG.getCopyToReg(Chain, dl, Reg: VA.getLocReg(), N: Arg, Glue);
3077
3078 // Guarantee that all emitted copies are
3079 // stuck together, avoiding something bad.
3080 Glue = Chain.getValue(R: 1);
3081 RetOps.push_back(Elt: DAG.getRegister(
3082 Reg: VA.getLocReg(), VT: ReturnF16 ? Arg.getValueType() : VA.getLocVT()));
3083 }
3084 const ARMBaseRegisterInfo *TRI = Subtarget->getRegisterInfo();
3085 const MCPhysReg *I =
3086 TRI->getCalleeSavedRegsViaCopy(MF: &DAG.getMachineFunction());
3087 if (I) {
3088 for (; *I; ++I) {
3089 if (ARM::GPRRegClass.contains(Reg: *I))
3090 RetOps.push_back(Elt: DAG.getRegister(Reg: *I, VT: MVT::i32));
3091 else if (ARM::DPRRegClass.contains(Reg: *I))
3092 RetOps.push_back(Elt: DAG.getRegister(Reg: *I, VT: MVT::getFloatingPointVT(BitWidth: 64)));
3093 else
3094 llvm_unreachable("Unexpected register class in CSRsViaCopy!");
3095 }
3096 }
3097
3098 // Update chain and glue.
3099 RetOps[0] = Chain;
3100 if (Glue.getNode())
3101 RetOps.push_back(Elt: Glue);
3102
3103 // CPUs which aren't M-class use a special sequence to return from
3104 // exceptions (roughly, any instruction setting pc and cpsr simultaneously,
3105 // though we use "subs pc, lr, #N").
3106 //
3107 // M-class CPUs actually use a normal return sequence with a special
3108 // (hardware-provided) value in LR, so the normal code path works.
3109 if (DAG.getMachineFunction().getFunction().hasFnAttribute(Kind: "interrupt") &&
3110 !Subtarget->isMClass()) {
3111 if (Subtarget->isThumb1Only())
3112 report_fatal_error(reason: "interrupt attribute is not supported in Thumb1");
3113 return LowerInterruptReturn(RetOps, DL: dl, DAG);
3114 }
3115
3116 unsigned RetNode =
3117 AFI->isCmseNSEntryFunction() ? ARMISD::SERET_GLUE : ARMISD::RET_GLUE;
3118 return DAG.getNode(Opcode: RetNode, DL: dl, VT: MVT::Other, Ops: RetOps);
3119}
3120
3121bool ARMTargetLowering::isUsedByReturnOnly(SDNode *N, SDValue &Chain) const {
3122 if (N->getNumValues() != 1)
3123 return false;
3124 if (!N->hasNUsesOfValue(NUses: 1, Value: 0))
3125 return false;
3126
3127 SDValue TCChain = Chain;
3128 SDNode *Copy = *N->user_begin();
3129 if (Copy->getOpcode() == ISD::CopyToReg) {
3130 // If the copy has a glue operand, we conservatively assume it isn't safe to
3131 // perform a tail call.
3132 if (Copy->getOperand(Num: Copy->getNumOperands()-1).getValueType() == MVT::Glue)
3133 return false;
3134 TCChain = Copy->getOperand(Num: 0);
3135 } else if (Copy->getOpcode() == ARMISD::VMOVRRD) {
3136 SDNode *VMov = Copy;
3137 // f64 returned in a pair of GPRs.
3138 SmallPtrSet<SDNode*, 2> Copies;
3139 for (SDNode *U : VMov->users()) {
3140 if (U->getOpcode() != ISD::CopyToReg)
3141 return false;
3142 Copies.insert(Ptr: U);
3143 }
3144 if (Copies.size() > 2)
3145 return false;
3146
3147 for (SDNode *U : VMov->users()) {
3148 SDValue UseChain = U->getOperand(Num: 0);
3149 if (Copies.count(Ptr: UseChain.getNode()))
3150 // Second CopyToReg
3151 Copy = U;
3152 else {
3153 // We are at the top of this chain.
3154 // If the copy has a glue operand, we conservatively assume it
3155 // isn't safe to perform a tail call.
3156 if (U->getOperand(Num: U->getNumOperands() - 1).getValueType() == MVT::Glue)
3157 return false;
3158 // First CopyToReg
3159 TCChain = UseChain;
3160 }
3161 }
3162 } else if (Copy->getOpcode() == ISD::BITCAST) {
3163 // f32 returned in a single GPR.
3164 if (!Copy->hasOneUse())
3165 return false;
3166 Copy = *Copy->user_begin();
3167 if (Copy->getOpcode() != ISD::CopyToReg || !Copy->hasNUsesOfValue(NUses: 1, Value: 0))
3168 return false;
3169 // If the copy has a glue operand, we conservatively assume it isn't safe to
3170 // perform a tail call.
3171 if (Copy->getOperand(Num: Copy->getNumOperands()-1).getValueType() == MVT::Glue)
3172 return false;
3173 TCChain = Copy->getOperand(Num: 0);
3174 } else {
3175 return false;
3176 }
3177
3178 bool HasRet = false;
3179 for (const SDNode *U : Copy->users()) {
3180 if (U->getOpcode() != ARMISD::RET_GLUE &&
3181 U->getOpcode() != ARMISD::INTRET_GLUE)
3182 return false;
3183 HasRet = true;
3184 }
3185
3186 if (!HasRet)
3187 return false;
3188
3189 Chain = TCChain;
3190 return true;
3191}
3192
3193bool ARMTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
3194 if (!Subtarget->supportsTailCall())
3195 return false;
3196
3197 if (!CI->isTailCall())
3198 return false;
3199
3200 return true;
3201}
3202
3203// Trying to write a 64 bit value so need to split into two 32 bit values first,
3204// and pass the lower and high parts through.
3205static SDValue LowerWRITE_REGISTER(SDValue Op, SelectionDAG &DAG) {
3206 SDLoc DL(Op);
3207 SDValue WriteValue = Op->getOperand(Num: 2);
3208
3209 // This function is only supposed to be called for i64 type argument.
3210 assert(WriteValue.getValueType() == MVT::i64
3211 && "LowerWRITE_REGISTER called for non-i64 type argument.");
3212
3213 SDValue Lo, Hi;
3214 std::tie(args&: Lo, args&: Hi) = DAG.SplitScalar(N: WriteValue, DL, LoVT: MVT::i32, HiVT: MVT::i32);
3215 SDValue Ops[] = { Op->getOperand(Num: 0), Op->getOperand(Num: 1), Lo, Hi };
3216 return DAG.getNode(Opcode: ISD::WRITE_REGISTER, DL, VT: MVT::Other, Ops);
3217}
3218
3219// ConstantPool, JumpTable, GlobalAddress, and ExternalSymbol are lowered as
3220// their target counterpart wrapped in the ARMISD::Wrapper node. Suppose N is
3221// one of the above mentioned nodes. It has to be wrapped because otherwise
3222// Select(N) returns N. So the raw TargetGlobalAddress nodes, etc. can only
3223// be used to form addressing mode. These wrapped nodes will be selected
3224// into MOVi.
3225SDValue ARMTargetLowering::LowerConstantPool(SDValue Op,
3226 SelectionDAG &DAG) const {
3227 EVT PtrVT = Op.getValueType();
3228 // FIXME there is no actual debug info here
3229 SDLoc dl(Op);
3230 ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Val&: Op);
3231 SDValue Res;
3232
3233 // When generating execute-only code Constant Pools must be promoted to the
3234 // global data section. It's a bit ugly that we can't share them across basic
3235 // blocks, but this way we guarantee that execute-only behaves correct with
3236 // position-independent addressing modes.
3237 if (Subtarget->genExecuteOnly()) {
3238 auto AFI = DAG.getMachineFunction().getInfo<ARMFunctionInfo>();
3239 auto *T = CP->getType();
3240 auto C = const_cast<Constant*>(CP->getConstVal());
3241 auto M = DAG.getMachineFunction().getFunction().getParent();
3242 auto GV = new GlobalVariable(
3243 *M, T, /*isConstant=*/true, GlobalVariable::InternalLinkage, C,
3244 Twine(DAG.getDataLayout().getInternalSymbolPrefix()) + "CP" +
3245 Twine(DAG.getMachineFunction().getFunctionNumber()) + "_" +
3246 Twine(AFI->createPICLabelUId()));
3247 SDValue GA = DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT);
3248 return LowerGlobalAddress(Op: GA, DAG);
3249 }
3250
3251 // The 16-bit ADR instruction can only encode offsets that are multiples of 4,
3252 // so we need to align to at least 4 bytes when we don't have 32-bit ADR.
3253 Align CPAlign = CP->getAlign();
3254 if (Subtarget->isThumb1Only())
3255 CPAlign = std::max(a: CPAlign, b: Align(4));
3256 if (CP->isMachineConstantPoolEntry())
3257 Res =
3258 DAG.getTargetConstantPool(C: CP->getMachineCPVal(), VT: PtrVT, Align: CPAlign);
3259 else
3260 Res = DAG.getTargetConstantPool(C: CP->getConstVal(), VT: PtrVT, Align: CPAlign);
3261 return DAG.getNode(Opcode: ARMISD::Wrapper, DL: dl, VT: MVT::i32, Operand: Res);
3262}
3263
3264unsigned ARMTargetLowering::getJumpTableEncoding() const {
3265 // If we don't have a 32-bit pc-relative branch instruction then the jump
3266 // table consists of block addresses. Usually this is inline, but for
3267 // execute-only it must be placed out-of-line.
3268 if (Subtarget->genExecuteOnly() && !Subtarget->hasV8MBaselineOps())
3269 return MachineJumpTableInfo::EK_BlockAddress;
3270 return MachineJumpTableInfo::EK_Inline;
3271}
3272
3273SDValue ARMTargetLowering::LowerBlockAddress(SDValue Op,
3274 SelectionDAG &DAG) const {
3275 MachineFunction &MF = DAG.getMachineFunction();
3276 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
3277 unsigned ARMPCLabelIndex = 0;
3278 SDLoc DL(Op);
3279 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
3280 const BlockAddress *BA = cast<BlockAddressSDNode>(Val&: Op)->getBlockAddress();
3281 SDValue CPAddr;
3282 bool IsPositionIndependent = isPositionIndependent() || Subtarget->isROPI();
3283 if (!IsPositionIndependent) {
3284 CPAddr = DAG.getTargetConstantPool(C: BA, VT: PtrVT, Align: Align(4));
3285 } else {
3286 unsigned PCAdj = Subtarget->isThumb() ? 4 : 8;
3287 ARMPCLabelIndex = AFI->createPICLabelUId();
3288 ARMConstantPoolValue *CPV =
3289 ARMConstantPoolConstant::Create(C: BA, ID: ARMPCLabelIndex,
3290 Kind: ARMCP::CPBlockAddress, PCAdj);
3291 CPAddr = DAG.getTargetConstantPool(C: CPV, VT: PtrVT, Align: Align(4));
3292 }
3293 CPAddr = DAG.getNode(Opcode: ARMISD::Wrapper, DL, VT: PtrVT, Operand: CPAddr);
3294 SDValue Result = DAG.getLoad(
3295 VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: CPAddr,
3296 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
3297 if (!IsPositionIndependent)
3298 return Result;
3299 SDValue PICLabel = DAG.getConstant(Val: ARMPCLabelIndex, DL, VT: MVT::i32);
3300 return DAG.getNode(Opcode: ARMISD::PIC_ADD, DL, VT: PtrVT, N1: Result, N2: PICLabel);
3301}
3302
3303/// Convert a TLS address reference into the correct sequence of loads
3304/// and calls to compute the variable's address for Darwin, and return an
3305/// SDValue containing the final node.
3306
3307/// Darwin only has one TLS scheme which must be capable of dealing with the
3308/// fully general situation, in the worst case. This means:
3309/// + "extern __thread" declaration.
3310/// + Defined in a possibly unknown dynamic library.
3311///
3312/// The general system is that each __thread variable has a [3 x i32] descriptor
3313/// which contains information used by the runtime to calculate the address. The
3314/// only part of this the compiler needs to know about is the first word, which
3315/// contains a function pointer that must be called with the address of the
3316/// entire descriptor in "r0".
3317///
3318/// Since this descriptor may be in a different unit, in general access must
3319/// proceed along the usual ARM rules. A common sequence to produce is:
3320///
3321/// movw rT1, :lower16:_var$non_lazy_ptr
3322/// movt rT1, :upper16:_var$non_lazy_ptr
3323/// ldr r0, [rT1]
3324/// ldr rT2, [r0]
3325/// blx rT2
3326/// [...address now in r0...]
3327SDValue
3328ARMTargetLowering::LowerGlobalTLSAddressDarwin(SDValue Op,
3329 SelectionDAG &DAG) const {
3330 assert(getTargetMachine().getTargetTriple().isOSDarwin() &&
3331 "This function expects a Darwin target");
3332 SDLoc DL(Op);
3333
3334 // First step is to get the address of the actua global symbol. This is where
3335 // the TLS descriptor lives.
3336 SDValue DescAddr = LowerGlobalAddressDarwin(Op, DAG);
3337
3338 // The first entry in the descriptor is a function pointer that we must call
3339 // to obtain the address of the variable.
3340 SDValue Chain = DAG.getEntryNode();
3341 SDValue FuncTLVGet = DAG.getLoad(
3342 VT: MVT::i32, dl: DL, Chain, Ptr: DescAddr,
3343 PtrInfo: MachinePointerInfo::getGOT(MF&: DAG.getMachineFunction()), Alignment: Align(4),
3344 MMOFlags: MachineMemOperand::MONonTemporal | MachineMemOperand::MODereferenceable |
3345 MachineMemOperand::MOInvariant);
3346 Chain = FuncTLVGet.getValue(R: 1);
3347
3348 MachineFunction &F = DAG.getMachineFunction();
3349 MachineFrameInfo &MFI = F.getFrameInfo();
3350 MFI.setAdjustsStack(true);
3351
3352 // TLS calls preserve all registers except those that absolutely must be
3353 // trashed: R0 (it takes an argument), LR (it's a call) and CPSR (let's not be
3354 // silly).
3355 auto TRI =
3356 getTargetMachine().getSubtargetImpl(F.getFunction())->getRegisterInfo();
3357 auto ARI = static_cast<const ARMRegisterInfo *>(TRI);
3358 const uint32_t *Mask = ARI->getTLSCallPreservedMask(MF: DAG.getMachineFunction());
3359
3360 // Finally, we can make the call. This is just a degenerate version of a
3361 // normal AArch64 call node: r0 takes the address of the descriptor, and
3362 // returns the address of the variable in this thread.
3363 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: ARM::R0, N: DescAddr, Glue: SDValue());
3364 Chain =
3365 DAG.getNode(Opcode: ARMISD::CALL, DL, VTList: DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue),
3366 N1: Chain, N2: FuncTLVGet, N3: DAG.getRegister(Reg: ARM::R0, VT: MVT::i32),
3367 N4: DAG.getRegisterMask(RegMask: Mask), N5: Chain.getValue(R: 1));
3368 return DAG.getCopyFromReg(Chain, dl: DL, Reg: ARM::R0, VT: MVT::i32, Glue: Chain.getValue(R: 1));
3369}
3370
3371SDValue
3372ARMTargetLowering::LowerGlobalTLSAddressWindows(SDValue Op,
3373 SelectionDAG &DAG) const {
3374 assert(getTargetMachine().getTargetTriple().isOSWindows() &&
3375 "Windows specific TLS lowering");
3376
3377 SDValue Chain = DAG.getEntryNode();
3378 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
3379 SDLoc DL(Op);
3380
3381 // Load the current TEB (thread environment block)
3382 SDValue Ops[] = {Chain,
3383 DAG.getTargetConstant(Val: Intrinsic::arm_mrc, DL, VT: MVT::i32),
3384 DAG.getTargetConstant(Val: 15, DL, VT: MVT::i32),
3385 DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32),
3386 DAG.getTargetConstant(Val: 13, DL, VT: MVT::i32),
3387 DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32),
3388 DAG.getTargetConstant(Val: 2, DL, VT: MVT::i32)};
3389 SDValue CurrentTEB = DAG.getNode(Opcode: ISD::INTRINSIC_W_CHAIN, DL,
3390 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other), Ops);
3391
3392 SDValue TEB = CurrentTEB.getValue(R: 0);
3393 Chain = CurrentTEB.getValue(R: 1);
3394
3395 // Load the ThreadLocalStoragePointer from the TEB
3396 // A pointer to the TLS array is located at offset 0x2c from the TEB.
3397 SDValue TLSArray =
3398 DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: TEB, N2: DAG.getIntPtrConstant(Val: 0x2c, DL));
3399 TLSArray = DAG.getLoad(VT: PtrVT, dl: DL, Chain, Ptr: TLSArray, PtrInfo: MachinePointerInfo());
3400
3401 // The pointer to the thread's TLS data area is at the TLS Index scaled by 4
3402 // offset into the TLSArray.
3403
3404 // Load the TLS index from the C runtime
3405 SDValue TLSIndex =
3406 DAG.getTargetExternalSymbol(Sym: "_tls_index", VT: PtrVT, TargetFlags: ARMII::MO_NO_FLAG);
3407 TLSIndex = DAG.getNode(Opcode: ARMISD::Wrapper, DL, VT: PtrVT, Operand: TLSIndex);
3408 TLSIndex = DAG.getLoad(VT: PtrVT, dl: DL, Chain, Ptr: TLSIndex, PtrInfo: MachinePointerInfo());
3409
3410 SDValue Slot = DAG.getNode(Opcode: ISD::SHL, DL, VT: PtrVT, N1: TLSIndex,
3411 N2: DAG.getConstant(Val: 2, DL, VT: MVT::i32));
3412 SDValue TLS = DAG.getLoad(VT: PtrVT, dl: DL, Chain,
3413 Ptr: DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: TLSArray, N2: Slot),
3414 PtrInfo: MachinePointerInfo());
3415
3416 // Get the offset of the start of the .tls section (section base)
3417 const auto *GA = cast<GlobalAddressSDNode>(Val&: Op);
3418 auto *CPV = ARMConstantPoolConstant::Create(GV: GA->getGlobal(), Modifier: ARMCP::SECREL);
3419 SDValue Offset = DAG.getLoad(
3420 VT: PtrVT, dl: DL, Chain,
3421 Ptr: DAG.getNode(Opcode: ARMISD::Wrapper, DL, VT: MVT::i32,
3422 Operand: DAG.getTargetConstantPool(C: CPV, VT: PtrVT, Align: Align(4))),
3423 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
3424
3425 return DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: TLS, N2: Offset);
3426}
3427
3428// Lower ISD::GlobalTLSAddress using the "general dynamic" model
3429SDValue
3430ARMTargetLowering::LowerToTLSGeneralDynamicModel(GlobalAddressSDNode *GA,
3431 SelectionDAG &DAG) const {
3432 SDLoc dl(GA);
3433 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
3434 unsigned char PCAdj = Subtarget->isThumb() ? 4 : 8;
3435 MachineFunction &MF = DAG.getMachineFunction();
3436 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
3437 unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
3438 ARMConstantPoolValue *CPV =
3439 ARMConstantPoolConstant::Create(C: GA->getGlobal(), ID: ARMPCLabelIndex,
3440 Kind: ARMCP::CPValue, PCAdj, Modifier: ARMCP::TLSGD, AddCurrentAddress: true);
3441 SDValue Argument = DAG.getTargetConstantPool(C: CPV, VT: PtrVT, Align: Align(4));
3442 Argument = DAG.getNode(Opcode: ARMISD::Wrapper, DL: dl, VT: MVT::i32, Operand: Argument);
3443 Argument = DAG.getLoad(
3444 VT: PtrVT, dl, Chain: DAG.getEntryNode(), Ptr: Argument,
3445 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
3446 SDValue Chain = Argument.getValue(R: 1);
3447
3448 SDValue PICLabel = DAG.getConstant(Val: ARMPCLabelIndex, DL: dl, VT: MVT::i32);
3449 Argument = DAG.getNode(Opcode: ARMISD::PIC_ADD, DL: dl, VT: PtrVT, N1: Argument, N2: PICLabel);
3450
3451 // call __tls_get_addr.
3452 ArgListTy Args;
3453 Args.emplace_back(args&: Argument, args: Type::getInt32Ty(C&: *DAG.getContext()));
3454
3455 // FIXME: is there useful debug info available here?
3456 TargetLowering::CallLoweringInfo CLI(DAG);
3457 CLI.setDebugLoc(dl).setChain(Chain).setLibCallee(
3458 CC: CallingConv::C, ResultType: Type::getInt32Ty(C&: *DAG.getContext()),
3459 Target: DAG.getExternalSymbol(Sym: "__tls_get_addr", VT: PtrVT), ArgsList: std::move(Args));
3460
3461 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
3462 return CallResult.first;
3463}
3464
3465// Lower ISD::GlobalTLSAddress using the "initial exec" or
3466// "local exec" model.
3467SDValue
3468ARMTargetLowering::LowerToTLSExecModels(GlobalAddressSDNode *GA,
3469 SelectionDAG &DAG,
3470 TLSModel::Model model) const {
3471 const GlobalValue *GV = GA->getGlobal();
3472 SDLoc dl(GA);
3473 SDValue Offset;
3474 SDValue Chain = DAG.getEntryNode();
3475 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
3476 // Get the Thread Pointer
3477 SDValue ThreadPointer = DAG.getNode(Opcode: ARMISD::THREAD_POINTER, DL: dl, VT: PtrVT);
3478
3479 if (model == TLSModel::InitialExec) {
3480 MachineFunction &MF = DAG.getMachineFunction();
3481 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
3482 unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
3483 // Initial exec model.
3484 unsigned char PCAdj = Subtarget->isThumb() ? 4 : 8;
3485 ARMConstantPoolValue *CPV =
3486 ARMConstantPoolConstant::Create(C: GA->getGlobal(), ID: ARMPCLabelIndex,
3487 Kind: ARMCP::CPValue, PCAdj, Modifier: ARMCP::GOTTPOFF,
3488 AddCurrentAddress: true);
3489 Offset = DAG.getTargetConstantPool(C: CPV, VT: PtrVT, Align: Align(4));
3490 Offset = DAG.getNode(Opcode: ARMISD::Wrapper, DL: dl, VT: MVT::i32, Operand: Offset);
3491 Offset = DAG.getLoad(
3492 VT: PtrVT, dl, Chain, Ptr: Offset,
3493 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
3494 Chain = Offset.getValue(R: 1);
3495
3496 SDValue PICLabel = DAG.getConstant(Val: ARMPCLabelIndex, DL: dl, VT: MVT::i32);
3497 Offset = DAG.getNode(Opcode: ARMISD::PIC_ADD, DL: dl, VT: PtrVT, N1: Offset, N2: PICLabel);
3498
3499 Offset = DAG.getLoad(
3500 VT: PtrVT, dl, Chain, Ptr: Offset,
3501 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
3502 } else {
3503 // local exec model
3504 assert(model == TLSModel::LocalExec);
3505 ARMConstantPoolValue *CPV =
3506 ARMConstantPoolConstant::Create(GV, Modifier: ARMCP::TPOFF);
3507 Offset = DAG.getTargetConstantPool(C: CPV, VT: PtrVT, Align: Align(4));
3508 Offset = DAG.getNode(Opcode: ARMISD::Wrapper, DL: dl, VT: MVT::i32, Operand: Offset);
3509 Offset = DAG.getLoad(
3510 VT: PtrVT, dl, Chain, Ptr: Offset,
3511 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
3512 }
3513
3514 // The address of the thread local variable is the add of the thread
3515 // pointer with the offset of the variable.
3516 return DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: ThreadPointer, N2: Offset);
3517}
3518
3519SDValue
3520ARMTargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const {
3521 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Val&: Op);
3522 if (DAG.getTarget().useEmulatedTLS())
3523 return LowerToTLSEmulatedModel(GA, DAG);
3524
3525 const Triple &TT = getTargetMachine().getTargetTriple();
3526 if (TT.isOSDarwin())
3527 return LowerGlobalTLSAddressDarwin(Op, DAG);
3528
3529 if (TT.isOSWindows())
3530 return LowerGlobalTLSAddressWindows(Op, DAG);
3531
3532 // TODO: implement the "local dynamic" model
3533 assert(TT.isOSBinFormatELF() && "Only ELF implemented here");
3534 TLSModel::Model model = getTargetMachine().getTLSModel(GV: GA->getGlobal());
3535
3536 switch (model) {
3537 case TLSModel::GeneralDynamic:
3538 case TLSModel::LocalDynamic:
3539 return LowerToTLSGeneralDynamicModel(GA, DAG);
3540 case TLSModel::InitialExec:
3541 case TLSModel::LocalExec:
3542 return LowerToTLSExecModels(GA, DAG, model);
3543 }
3544 llvm_unreachable("bogus TLS model");
3545}
3546
3547/// Return true if all users of V are within function F, looking through
3548/// ConstantExprs.
3549static bool allUsersAreInFunction(const Value *V, const Function *F) {
3550 SmallVector<const User*,4> Worklist(V->users());
3551 while (!Worklist.empty()) {
3552 auto *U = Worklist.pop_back_val();
3553 if (isa<ConstantExpr>(Val: U)) {
3554 append_range(C&: Worklist, R: U->users());
3555 continue;
3556 }
3557
3558 auto *I = dyn_cast<Instruction>(Val: U);
3559 if (!I || I->getParent()->getParent() != F)
3560 return false;
3561 }
3562 return true;
3563}
3564
3565static SDValue promoteToConstantPool(const ARMTargetLowering *TLI,
3566 const GlobalValue *GV, SelectionDAG &DAG,
3567 EVT PtrVT, const SDLoc &dl) {
3568 // If we're creating a pool entry for a constant global with unnamed address,
3569 // and the global is small enough, we can emit it inline into the constant pool
3570 // to save ourselves an indirection.
3571 //
3572 // This is a win if the constant is only used in one function (so it doesn't
3573 // need to be duplicated) or duplicating the constant wouldn't increase code
3574 // size (implying the constant is no larger than 4 bytes).
3575 const Function &F = DAG.getMachineFunction().getFunction();
3576
3577 // We rely on this decision to inline being idempotent and unrelated to the
3578 // use-site. We know that if we inline a variable at one use site, we'll
3579 // inline it elsewhere too (and reuse the constant pool entry). Fast-isel
3580 // doesn't know about this optimization, so bail out if it's enabled else
3581 // we could decide to inline here (and thus never emit the GV) but require
3582 // the GV from fast-isel generated code.
3583 if (!EnableConstpoolPromotion ||
3584 DAG.getMachineFunction().getTarget().Options.EnableFastISel)
3585 return SDValue();
3586
3587 auto *GVar = dyn_cast<GlobalVariable>(Val: GV);
3588 if (!GVar || !GVar->hasInitializer() ||
3589 !GVar->isConstant() || !GVar->hasGlobalUnnamedAddr() ||
3590 !GVar->hasLocalLinkage())
3591 return SDValue();
3592
3593 // If we inline a value that contains relocations, we move the relocations
3594 // from .data to .text. This is not allowed in position-independent code.
3595 auto *Init = GVar->getInitializer();
3596 if ((TLI->isPositionIndependent() || TLI->getSubtarget()->isROPI()) &&
3597 Init->needsDynamicRelocation())
3598 return SDValue();
3599
3600 // The constant islands pass can only really deal with alignment requests
3601 // <= 4 bytes and cannot pad constants itself. Therefore we cannot promote
3602 // any type wanting greater alignment requirements than 4 bytes. We also
3603 // can only promote constants that are multiples of 4 bytes in size or
3604 // are paddable to a multiple of 4. Currently we only try and pad constants
3605 // that are strings for simplicity.
3606 auto *CDAInit = dyn_cast<ConstantDataArray>(Val: Init);
3607 unsigned Size = DAG.getDataLayout().getTypeAllocSize(Ty: Init->getType());
3608 Align PrefAlign = DAG.getDataLayout().getPreferredAlign(GV: GVar);
3609 unsigned RequiredPadding = 4 - (Size % 4);
3610 bool PaddingPossible =
3611 RequiredPadding == 4 || (CDAInit && CDAInit->isString());
3612 if (!PaddingPossible || PrefAlign > 4 || Size > ConstpoolPromotionMaxSize ||
3613 Size == 0)
3614 return SDValue();
3615
3616 unsigned PaddedSize = Size + ((RequiredPadding == 4) ? 0 : RequiredPadding);
3617 MachineFunction &MF = DAG.getMachineFunction();
3618 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
3619
3620 // We can't bloat the constant pool too much, else the ConstantIslands pass
3621 // may fail to converge. If we haven't promoted this global yet (it may have
3622 // multiple uses), and promoting it would increase the constant pool size (Sz
3623 // > 4), ensure we have space to do so up to MaxTotal.
3624 if (!AFI->getGlobalsPromotedToConstantPool().count(Ptr: GVar) && Size > 4)
3625 if (AFI->getPromotedConstpoolIncrease() + PaddedSize - 4 >=
3626 ConstpoolPromotionMaxTotal)
3627 return SDValue();
3628
3629 // This is only valid if all users are in a single function; we can't clone
3630 // the constant in general. The LLVM IR unnamed_addr allows merging
3631 // constants, but not cloning them.
3632 //
3633 // We could potentially allow cloning if we could prove all uses of the
3634 // constant in the current function don't care about the address, like
3635 // printf format strings. But that isn't implemented for now.
3636 if (!allUsersAreInFunction(V: GVar, F: &F))
3637 return SDValue();
3638
3639 // We're going to inline this global. Pad it out if needed.
3640 if (RequiredPadding != 4) {
3641 StringRef S = CDAInit->getAsString();
3642
3643 SmallVector<uint8_t,16> V(S.size());
3644 std::copy(first: S.bytes_begin(), last: S.bytes_end(), result: V.begin());
3645 while (RequiredPadding--)
3646 V.push_back(Elt: 0);
3647 Init = ConstantDataArray::get(Context&: *DAG.getContext(), Elts&: V);
3648 }
3649
3650 auto CPVal = ARMConstantPoolConstant::Create(GV: GVar, Initializer: Init);
3651 SDValue CPAddr = DAG.getTargetConstantPool(C: CPVal, VT: PtrVT, Align: Align(4));
3652 if (!AFI->getGlobalsPromotedToConstantPool().count(Ptr: GVar)) {
3653 AFI->markGlobalAsPromotedToConstantPool(GV: GVar);
3654 AFI->setPromotedConstpoolIncrease(AFI->getPromotedConstpoolIncrease() +
3655 PaddedSize - 4);
3656 }
3657 ++NumConstpoolPromoted;
3658 return DAG.getNode(Opcode: ARMISD::Wrapper, DL: dl, VT: MVT::i32, Operand: CPAddr);
3659}
3660
3661bool ARMTargetLowering::isReadOnly(const GlobalValue *GV) const {
3662 if (const GlobalAlias *GA = dyn_cast<GlobalAlias>(Val: GV))
3663 if (!(GV = GA->getAliaseeObject()))
3664 return false;
3665 if (const auto *V = dyn_cast<GlobalVariable>(Val: GV))
3666 return V->isConstant();
3667 return isa<Function>(Val: GV);
3668}
3669
3670SDValue ARMTargetLowering::LowerGlobalAddress(SDValue Op,
3671 SelectionDAG &DAG) const {
3672 switch (Subtarget->getTargetTriple().getObjectFormat()) {
3673 default: llvm_unreachable("unknown object format");
3674 case Triple::COFF:
3675 return LowerGlobalAddressWindows(Op, DAG);
3676 case Triple::ELF:
3677 return LowerGlobalAddressELF(Op, DAG);
3678 case Triple::MachO:
3679 return LowerGlobalAddressDarwin(Op, DAG);
3680 }
3681}
3682
3683SDValue ARMTargetLowering::LowerGlobalAddressELF(SDValue Op,
3684 SelectionDAG &DAG) const {
3685 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
3686 SDLoc dl(Op);
3687 const GlobalValue *GV = cast<GlobalAddressSDNode>(Val&: Op)->getGlobal();
3688 bool IsRO = isReadOnly(GV);
3689
3690 // promoteToConstantPool only if not generating XO text section
3691 if (GV->isDSOLocal() && !Subtarget->genExecuteOnly())
3692 if (SDValue V = promoteToConstantPool(TLI: this, GV, DAG, PtrVT, dl))
3693 return V;
3694
3695 if (isPositionIndependent()) {
3696 SDValue G = DAG.getTargetGlobalAddress(
3697 GV, DL: dl, VT: PtrVT, offset: 0, TargetFlags: GV->isDSOLocal() ? 0 : ARMII::MO_GOT);
3698 SDValue Result = DAG.getNode(Opcode: ARMISD::WrapperPIC, DL: dl, VT: PtrVT, Operand: G);
3699 if (!GV->isDSOLocal())
3700 Result =
3701 DAG.getLoad(VT: PtrVT, dl, Chain: DAG.getEntryNode(), Ptr: Result,
3702 PtrInfo: MachinePointerInfo::getGOT(MF&: DAG.getMachineFunction()));
3703 return Result;
3704 } else if (Subtarget->isROPI() && IsRO) {
3705 // PC-relative.
3706 SDValue G = DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT);
3707 SDValue Result = DAG.getNode(Opcode: ARMISD::WrapperPIC, DL: dl, VT: PtrVT, Operand: G);
3708 return Result;
3709 } else if (Subtarget->isRWPI() && !IsRO) {
3710 // SB-relative.
3711 SDValue RelAddr;
3712 if (Subtarget->useMovt()) {
3713 ++NumMovwMovt;
3714 SDValue G = DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: 0, TargetFlags: ARMII::MO_SBREL);
3715 RelAddr = DAG.getNode(Opcode: ARMISD::Wrapper, DL: dl, VT: PtrVT, Operand: G);
3716 } else { // use literal pool for address constant
3717 ARMConstantPoolValue *CPV =
3718 ARMConstantPoolConstant::Create(GV, Modifier: ARMCP::SBREL);
3719 SDValue CPAddr = DAG.getTargetConstantPool(C: CPV, VT: PtrVT, Align: Align(4));
3720 CPAddr = DAG.getNode(Opcode: ARMISD::Wrapper, DL: dl, VT: MVT::i32, Operand: CPAddr);
3721 RelAddr = DAG.getLoad(
3722 VT: PtrVT, dl, Chain: DAG.getEntryNode(), Ptr: CPAddr,
3723 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
3724 }
3725 SDValue SB = DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl, Reg: ARM::R9, VT: PtrVT);
3726 SDValue Result = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: SB, N2: RelAddr);
3727 return Result;
3728 }
3729
3730 // If we have T2 ops, we can materialize the address directly via movt/movw
3731 // pair. This is always cheaper. If need to generate Execute Only code, and we
3732 // only have Thumb1 available, we can't use a constant pool and are forced to
3733 // use immediate relocations.
3734 if (Subtarget->useMovt() || Subtarget->genExecuteOnly()) {
3735 if (Subtarget->useMovt())
3736 ++NumMovwMovt;
3737 // FIXME: Once remat is capable of dealing with instructions with register
3738 // operands, expand this into two nodes.
3739 return DAG.getNode(Opcode: ARMISD::Wrapper, DL: dl, VT: PtrVT,
3740 Operand: DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT));
3741 } else {
3742 SDValue CPAddr = DAG.getTargetConstantPool(C: GV, VT: PtrVT, Align: Align(4));
3743 CPAddr = DAG.getNode(Opcode: ARMISD::Wrapper, DL: dl, VT: MVT::i32, Operand: CPAddr);
3744 return DAG.getLoad(
3745 VT: PtrVT, dl, Chain: DAG.getEntryNode(), Ptr: CPAddr,
3746 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
3747 }
3748}
3749
3750SDValue ARMTargetLowering::LowerGlobalAddressDarwin(SDValue Op,
3751 SelectionDAG &DAG) const {
3752 assert(!Subtarget->isROPI() && !Subtarget->isRWPI() &&
3753 "ROPI/RWPI not currently supported for Darwin");
3754 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
3755 SDLoc dl(Op);
3756 const GlobalValue *GV = cast<GlobalAddressSDNode>(Val&: Op)->getGlobal();
3757
3758 if (Subtarget->useMovt())
3759 ++NumMovwMovt;
3760
3761 // FIXME: Once remat is capable of dealing with instructions with register
3762 // operands, expand this into multiple nodes
3763 unsigned Wrapper =
3764 isPositionIndependent() ? ARMISD::WrapperPIC : ARMISD::Wrapper;
3765
3766 SDValue G = DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: 0, TargetFlags: ARMII::MO_NONLAZY);
3767 SDValue Result = DAG.getNode(Opcode: Wrapper, DL: dl, VT: PtrVT, Operand: G);
3768
3769 if (Subtarget->isGVIndirectSymbol(GV))
3770 Result = DAG.getLoad(VT: PtrVT, dl, Chain: DAG.getEntryNode(), Ptr: Result,
3771 PtrInfo: MachinePointerInfo::getGOT(MF&: DAG.getMachineFunction()));
3772 return Result;
3773}
3774
3775SDValue ARMTargetLowering::LowerGlobalAddressWindows(SDValue Op,
3776 SelectionDAG &DAG) const {
3777 assert(getTargetMachine().getTargetTriple().isOSWindows() &&
3778 "non-Windows COFF is not supported");
3779 assert(Subtarget->useMovt() &&
3780 "Windows on ARM expects to use movw/movt");
3781 assert(!Subtarget->isROPI() && !Subtarget->isRWPI() &&
3782 "ROPI/RWPI not currently supported for Windows");
3783
3784 const TargetMachine &TM = getTargetMachine();
3785 const GlobalValue *GV = cast<GlobalAddressSDNode>(Val&: Op)->getGlobal();
3786 ARMII::TOF TargetFlags = ARMII::MO_NO_FLAG;
3787 if (GV->hasDLLImportStorageClass())
3788 TargetFlags = ARMII::MO_DLLIMPORT;
3789 else if (!TM.shouldAssumeDSOLocal(GV))
3790 TargetFlags = ARMII::MO_COFFSTUB;
3791 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
3792 SDValue Result;
3793 SDLoc DL(Op);
3794
3795 ++NumMovwMovt;
3796
3797 // FIXME: Once remat is capable of dealing with instructions with register
3798 // operands, expand this into two nodes.
3799 Result = DAG.getNode(Opcode: ARMISD::Wrapper, DL, VT: PtrVT,
3800 Operand: DAG.getTargetGlobalAddress(GV, DL, VT: PtrVT, /*offset=*/0,
3801 TargetFlags));
3802 if (TargetFlags & (ARMII::MO_DLLIMPORT | ARMII::MO_COFFSTUB))
3803 Result = DAG.getLoad(VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: Result,
3804 PtrInfo: MachinePointerInfo::getGOT(MF&: DAG.getMachineFunction()));
3805 return Result;
3806}
3807
3808SDValue
3809ARMTargetLowering::LowerEH_SJLJ_SETJMP(SDValue Op, SelectionDAG &DAG) const {
3810 SDLoc dl(Op);
3811 SDValue Val = DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32);
3812 return DAG.getNode(Opcode: ARMISD::EH_SJLJ_SETJMP, DL: dl,
3813 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other), N1: Op.getOperand(i: 0),
3814 N2: Op.getOperand(i: 1), N3: Val);
3815}
3816
3817SDValue
3818ARMTargetLowering::LowerEH_SJLJ_LONGJMP(SDValue Op, SelectionDAG &DAG) const {
3819 SDLoc dl(Op);
3820 return DAG.getNode(Opcode: ARMISD::EH_SJLJ_LONGJMP, DL: dl, VT: MVT::Other, N1: Op.getOperand(i: 0),
3821 N2: Op.getOperand(i: 1), N3: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
3822}
3823
3824SDValue ARMTargetLowering::LowerEH_SJLJ_SETUP_DISPATCH(SDValue Op,
3825 SelectionDAG &DAG) const {
3826 SDLoc dl(Op);
3827 return DAG.getNode(Opcode: ARMISD::EH_SJLJ_SETUP_DISPATCH, DL: dl, VT: MVT::Other,
3828 Operand: Op.getOperand(i: 0));
3829}
3830
3831SDValue ARMTargetLowering::LowerINTRINSIC_VOID(
3832 SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget) const {
3833 unsigned IntNo =
3834 Op.getConstantOperandVal(i: Op.getOperand(i: 0).getValueType() == MVT::Other);
3835 switch (IntNo) {
3836 default:
3837 return SDValue(); // Don't custom lower most intrinsics.
3838 case Intrinsic::arm_gnu_eabi_mcount: {
3839 MachineFunction &MF = DAG.getMachineFunction();
3840 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
3841 SDLoc dl(Op);
3842 SDValue Chain = Op.getOperand(i: 0);
3843 // call "\01__gnu_mcount_nc"
3844 const ARMBaseRegisterInfo *ARI = Subtarget->getRegisterInfo();
3845 const uint32_t *Mask =
3846 ARI->getCallPreservedMask(MF: DAG.getMachineFunction(), CallingConv::C);
3847 assert(Mask && "Missing call preserved mask for calling convention");
3848 // Mark LR an implicit live-in.
3849 Register Reg = MF.addLiveIn(PReg: ARM::LR, RC: getRegClassFor(VT: MVT::i32));
3850 SDValue ReturnAddress =
3851 DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl, Reg, VT: PtrVT);
3852 constexpr EVT ResultTys[] = {MVT::Other, MVT::Glue};
3853 SDValue Callee =
3854 DAG.getTargetExternalSymbol(Sym: "\01__gnu_mcount_nc", VT: PtrVT, TargetFlags: 0);
3855 SDValue RegisterMask = DAG.getRegisterMask(RegMask: Mask);
3856 if (Subtarget->isThumb())
3857 return SDValue(
3858 DAG.getMachineNode(
3859 Opcode: ARM::tBL_PUSHLR, dl, ResultTys,
3860 Ops: {ReturnAddress, DAG.getTargetConstant(Val: ARMCC::AL, DL: dl, VT: PtrVT),
3861 DAG.getRegister(Reg: 0, VT: PtrVT), Callee, RegisterMask, Chain}),
3862 0);
3863 return SDValue(
3864 DAG.getMachineNode(Opcode: ARM::BL_PUSHLR, dl, ResultTys,
3865 Ops: {ReturnAddress, Callee, RegisterMask, Chain}),
3866 0);
3867 }
3868 }
3869}
3870
3871SDValue
3872ARMTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG,
3873 const ARMSubtarget *Subtarget) const {
3874 unsigned IntNo = Op.getConstantOperandVal(i: 0);
3875 SDLoc dl(Op);
3876 switch (IntNo) {
3877 default: return SDValue(); // Don't custom lower most intrinsics.
3878 case Intrinsic::localaddress: {
3879 const MachineFunction &MF = DAG.getMachineFunction();
3880 const auto *RegInfo = Subtarget->getRegisterInfo();
3881 unsigned Reg = RegInfo->getLocalAddressRegister(MF);
3882 return DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl, Reg,
3883 VT: Op.getSimpleValueType());
3884 }
3885 case Intrinsic::eh_recoverfp: {
3886 SDValue FnOp = Op.getOperand(i: 1);
3887 GlobalAddressSDNode *GSD = dyn_cast<GlobalAddressSDNode>(Val&: FnOp);
3888 auto *Fn = dyn_cast_or_null<Function>(Val: GSD ? GSD->getGlobal() : nullptr);
3889 if (!Fn)
3890 report_fatal_error(
3891 reason: "llvm.eh.recoverfp must take a function as the first argument");
3892 const auto *RegInfo = Subtarget->getRegisterInfo();
3893 Register BaseReg = RegInfo->getBaseRegister();
3894 MachineFunction &MF = DAG.getMachineFunction();
3895 MachineBasicBlock &MBB = *MF.begin();
3896 if (!MBB.isLiveIn(Reg: BaseReg))
3897 MBB.addLiveIn(PhysReg: BaseReg);
3898 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
3899 return DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl, Reg: BaseReg, VT: PtrVT);
3900 }
3901 case Intrinsic::thread_pointer: {
3902 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
3903 return DAG.getNode(Opcode: ARMISD::THREAD_POINTER, DL: dl, VT: PtrVT);
3904 }
3905 case Intrinsic::arm_cls: {
3906 // Note: arm_cls and arm_cls64 intrinsics are expanded directly here
3907 // in LowerINTRINSIC_WO_CHAIN since there's no native scalar CLS
3908 // instruction.
3909 const SDValue &Operand = Op.getOperand(i: 1);
3910 const EVT VTy = Op.getValueType();
3911 return DAG.getNode(Opcode: ISD::CTLS, DL: dl, VT: VTy, Operand);
3912 }
3913 case Intrinsic::arm_cls64: {
3914 // arm_cls64 returns i32 but takes i64 input.
3915 // Use ISD::CTLS for i64 and truncate the result.
3916 SDValue CTLS64 = DAG.getNode(Opcode: ISD::CTLS, DL: dl, VT: MVT::i64, Operand: Op.getOperand(i: 1));
3917 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i32, Operand: CTLS64);
3918 }
3919 case Intrinsic::arm_neon_vcls:
3920 case Intrinsic::arm_mve_vcls: {
3921 // Lower vector CLS intrinsics to ISD::CTLS.
3922 // Vector CTLS is Legal when NEON/MVE is available (set elsewhere).
3923 const EVT VTy = Op.getValueType();
3924 return DAG.getNode(Opcode: ISD::CTLS, DL: dl, VT: VTy, Operand: Op.getOperand(i: 1));
3925 }
3926 case Intrinsic::eh_sjlj_lsda: {
3927 MachineFunction &MF = DAG.getMachineFunction();
3928 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
3929 unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
3930 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
3931 SDValue CPAddr;
3932 bool IsPositionIndependent = isPositionIndependent();
3933 unsigned PCAdj = IsPositionIndependent ? (Subtarget->isThumb() ? 4 : 8) : 0;
3934 ARMConstantPoolValue *CPV =
3935 ARMConstantPoolConstant::Create(C: &MF.getFunction(), ID: ARMPCLabelIndex,
3936 Kind: ARMCP::CPLSDA, PCAdj);
3937 CPAddr = DAG.getTargetConstantPool(C: CPV, VT: PtrVT, Align: Align(4));
3938 CPAddr = DAG.getNode(Opcode: ARMISD::Wrapper, DL: dl, VT: MVT::i32, Operand: CPAddr);
3939 SDValue Result = DAG.getLoad(
3940 VT: PtrVT, dl, Chain: DAG.getEntryNode(), Ptr: CPAddr,
3941 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
3942
3943 if (IsPositionIndependent) {
3944 SDValue PICLabel = DAG.getConstant(Val: ARMPCLabelIndex, DL: dl, VT: MVT::i32);
3945 Result = DAG.getNode(Opcode: ARMISD::PIC_ADD, DL: dl, VT: PtrVT, N1: Result, N2: PICLabel);
3946 }
3947 return Result;
3948 }
3949 case Intrinsic::arm_neon_vabs:
3950 return DAG.getNode(Opcode: ISD::ABS, DL: SDLoc(Op), VT: Op.getValueType(),
3951 Operand: Op.getOperand(i: 1));
3952 case Intrinsic::arm_neon_vabds:
3953 if (Op.getValueType().isInteger())
3954 return DAG.getNode(Opcode: ISD::ABDS, DL: SDLoc(Op), VT: Op.getValueType(),
3955 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
3956 return SDValue();
3957 case Intrinsic::arm_neon_vabdu:
3958 return DAG.getNode(Opcode: ISD::ABDU, DL: SDLoc(Op), VT: Op.getValueType(),
3959 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
3960 case Intrinsic::arm_neon_vmulls:
3961 case Intrinsic::arm_neon_vmullu: {
3962 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmulls)
3963 ? ARMISD::VMULLs : ARMISD::VMULLu;
3964 return DAG.getNode(Opcode: NewOpc, DL: SDLoc(Op), VT: Op.getValueType(),
3965 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
3966 }
3967 case Intrinsic::arm_neon_vminnm:
3968 case Intrinsic::arm_neon_vmaxnm: {
3969 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vminnm)
3970 ? ISD::FMINNUM : ISD::FMAXNUM;
3971 return DAG.getNode(Opcode: NewOpc, DL: SDLoc(Op), VT: Op.getValueType(),
3972 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
3973 }
3974 case Intrinsic::arm_neon_vminu:
3975 case Intrinsic::arm_neon_vmaxu: {
3976 if (Op.getValueType().isFloatingPoint())
3977 return SDValue();
3978 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vminu)
3979 ? ISD::UMIN : ISD::UMAX;
3980 return DAG.getNode(Opcode: NewOpc, DL: SDLoc(Op), VT: Op.getValueType(),
3981 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
3982 }
3983 case Intrinsic::arm_neon_vmins:
3984 case Intrinsic::arm_neon_vmaxs: {
3985 // v{min,max}s is overloaded between signed integers and floats.
3986 if (!Op.getValueType().isFloatingPoint()) {
3987 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmins)
3988 ? ISD::SMIN : ISD::SMAX;
3989 return DAG.getNode(Opcode: NewOpc, DL: SDLoc(Op), VT: Op.getValueType(),
3990 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
3991 }
3992 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmins)
3993 ? ISD::FMINIMUM : ISD::FMAXIMUM;
3994 return DAG.getNode(Opcode: NewOpc, DL: SDLoc(Op), VT: Op.getValueType(),
3995 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
3996 }
3997 case Intrinsic::arm_neon_vtbl1:
3998 return DAG.getNode(Opcode: ARMISD::VTBL1, DL: SDLoc(Op), VT: Op.getValueType(),
3999 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
4000 case Intrinsic::arm_neon_vtbl2:
4001 return DAG.getNode(Opcode: ARMISD::VTBL2, DL: SDLoc(Op), VT: Op.getValueType(),
4002 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
4003 case Intrinsic::arm_mve_pred_i2v:
4004 case Intrinsic::arm_mve_pred_v2i:
4005 return DAG.getNode(Opcode: ARMISD::PREDICATE_CAST, DL: SDLoc(Op), VT: Op.getValueType(),
4006 Operand: Op.getOperand(i: 1));
4007 case Intrinsic::arm_mve_vreinterpretq:
4008 return DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: SDLoc(Op), VT: Op.getValueType(),
4009 Operand: Op.getOperand(i: 1));
4010 case Intrinsic::arm_mve_lsll:
4011 return DAG.getNode(Opcode: ARMISD::LSLL, DL: SDLoc(Op), VTList: Op->getVTList(),
4012 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
4013 case Intrinsic::arm_mve_asrl:
4014 return DAG.getNode(Opcode: ARMISD::ASRL, DL: SDLoc(Op), VTList: Op->getVTList(),
4015 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
4016 case Intrinsic::arm_mve_vsli:
4017 return DAG.getNode(Opcode: ARMISD::VSLIIMM, DL: SDLoc(Op), VTList: Op->getVTList(),
4018 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
4019 case Intrinsic::arm_mve_vsri:
4020 return DAG.getNode(Opcode: ARMISD::VSRIIMM, DL: SDLoc(Op), VTList: Op->getVTList(),
4021 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
4022 }
4023}
4024
4025static SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG,
4026 const ARMSubtarget *Subtarget) {
4027 SDLoc dl(Op);
4028 auto SSID = static_cast<SyncScope::ID>(Op.getConstantOperandVal(i: 2));
4029 if (SSID == SyncScope::SingleThread)
4030 return Op;
4031
4032 if (!Subtarget->hasDataBarrier()) {
4033 // Some ARMv6 cpus can support data barriers with an mcr instruction.
4034 // Thumb1 and pre-v6 ARM mode use a libcall instead and should never get
4035 // here.
4036 assert(Subtarget->hasV6Ops() && !Subtarget->isThumb() &&
4037 "Unexpected ISD::ATOMIC_FENCE encountered. Should be libcall!");
4038 return DAG.getNode(Opcode: ARMISD::MEMBARRIER_MCR, DL: dl, VT: MVT::Other, N1: Op.getOperand(i: 0),
4039 N2: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
4040 }
4041
4042 AtomicOrdering Ord =
4043 static_cast<AtomicOrdering>(Op.getConstantOperandVal(i: 1));
4044 ARM_MB::MemBOpt Domain = ARM_MB::ISH;
4045 if (Subtarget->isMClass()) {
4046 // Only a full system barrier exists in the M-class architectures.
4047 Domain = ARM_MB::SY;
4048 } else if (Subtarget->preferISHSTBarriers() &&
4049 Ord == AtomicOrdering::Release) {
4050 // Swift happens to implement ISHST barriers in a way that's compatible with
4051 // Release semantics but weaker than ISH so we'd be fools not to use
4052 // it. Beware: other processors probably don't!
4053 Domain = ARM_MB::ISHST;
4054 }
4055
4056 return DAG.getNode(Opcode: ISD::INTRINSIC_VOID, DL: dl, VT: MVT::Other, N1: Op.getOperand(i: 0),
4057 N2: DAG.getConstant(Val: Intrinsic::arm_dmb, DL: dl, VT: MVT::i32),
4058 N3: DAG.getConstant(Val: Domain, DL: dl, VT: MVT::i32));
4059}
4060
4061static SDValue LowerPREFETCH(SDValue Op, SelectionDAG &DAG,
4062 const ARMSubtarget *Subtarget) {
4063 // ARM pre v5TE and Thumb1 does not have preload instructions.
4064 if (!(Subtarget->isThumb2() ||
4065 (!Subtarget->isThumb1Only() && Subtarget->hasV5TEOps())))
4066 // Just preserve the chain.
4067 return Op.getOperand(i: 0);
4068
4069 SDLoc dl(Op);
4070 unsigned isRead = ~Op.getConstantOperandVal(i: 2) & 1;
4071 if (!isRead &&
4072 (!Subtarget->hasV7Ops() || !Subtarget->hasMPExtension()))
4073 // ARMv7 with MP extension has PLDW.
4074 return Op.getOperand(i: 0);
4075
4076 unsigned isData = Op.getConstantOperandVal(i: 4);
4077 if (Subtarget->isThumb()) {
4078 // Invert the bits.
4079 isRead = ~isRead & 1;
4080 isData = ~isData & 1;
4081 }
4082
4083 return DAG.getNode(Opcode: ARMISD::PRELOAD, DL: dl, VT: MVT::Other, N1: Op.getOperand(i: 0),
4084 N2: Op.getOperand(i: 1), N3: DAG.getConstant(Val: isRead, DL: dl, VT: MVT::i32),
4085 N4: DAG.getConstant(Val: isData, DL: dl, VT: MVT::i32));
4086}
4087
4088static SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) {
4089 MachineFunction &MF = DAG.getMachineFunction();
4090 ARMFunctionInfo *FuncInfo = MF.getInfo<ARMFunctionInfo>();
4091
4092 // vastart just stores the address of the VarArgsFrameIndex slot into the
4093 // memory location argument.
4094 SDLoc dl(Op);
4095 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DL: DAG.getDataLayout());
4096 SDValue FR = DAG.getFrameIndex(FI: FuncInfo->getVarArgsFrameIndex(), VT: PtrVT);
4097 const Value *SV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 2))->getValue();
4098 return DAG.getStore(Chain: Op.getOperand(i: 0), dl, Val: FR, Ptr: Op.getOperand(i: 1),
4099 PtrInfo: MachinePointerInfo(SV));
4100}
4101
4102SDValue ARMTargetLowering::GetF64FormalArgument(CCValAssign &VA,
4103 CCValAssign &NextVA,
4104 SDValue &Root,
4105 SelectionDAG &DAG,
4106 const SDLoc &dl) const {
4107 MachineFunction &MF = DAG.getMachineFunction();
4108 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
4109
4110 const TargetRegisterClass *RC;
4111 if (AFI->isThumb1OnlyFunction())
4112 RC = &ARM::tGPRRegClass;
4113 else
4114 RC = &ARM::GPRRegClass;
4115
4116 // Transform the arguments stored in physical registers into virtual ones.
4117 Register Reg = MF.addLiveIn(PReg: VA.getLocReg(), RC);
4118 SDValue ArgValue = DAG.getCopyFromReg(Chain: Root, dl, Reg, VT: MVT::i32);
4119
4120 SDValue ArgValue2;
4121 if (NextVA.isMemLoc()) {
4122 MachineFrameInfo &MFI = MF.getFrameInfo();
4123 int FI = MFI.CreateFixedObject(Size: 4, SPOffset: NextVA.getLocMemOffset(), IsImmutable: true);
4124
4125 // Create load node to retrieve arguments from the stack.
4126 SDValue FIN = DAG.getFrameIndex(FI, VT: getPointerTy(DL: DAG.getDataLayout()));
4127 ArgValue2 = DAG.getLoad(
4128 VT: MVT::i32, dl, Chain: Root, Ptr: FIN,
4129 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI));
4130 } else {
4131 Reg = MF.addLiveIn(PReg: NextVA.getLocReg(), RC);
4132 ArgValue2 = DAG.getCopyFromReg(Chain: Root, dl, Reg, VT: MVT::i32);
4133 }
4134 if (!Subtarget->isLittle())
4135 std::swap (a&: ArgValue, b&: ArgValue2);
4136 return DAG.getNode(Opcode: ARMISD::VMOVDRR, DL: dl, VT: MVT::f64, N1: ArgValue, N2: ArgValue2);
4137}
4138
4139// The remaining GPRs hold either the beginning of variable-argument
4140// data, or the beginning of an aggregate passed by value (usually
4141// byval). Either way, we allocate stack slots adjacent to the data
4142// provided by our caller, and store the unallocated registers there.
4143// If this is a variadic function, the va_list pointer will begin with
4144// these values; otherwise, this reassembles a (byval) structure that
4145// was split between registers and memory.
4146// Return: The frame index registers were stored into.
4147int ARMTargetLowering::StoreByValRegs(CCState &CCInfo, SelectionDAG &DAG,
4148 const SDLoc &dl, SDValue &Chain,
4149 const Value *OrigArg,
4150 unsigned InRegsParamRecordIdx,
4151 int ArgOffset, unsigned ArgSize) const {
4152 // Currently, two use-cases possible:
4153 // Case #1. Non-var-args function, and we meet first byval parameter.
4154 // Setup first unallocated register as first byval register;
4155 // eat all remained registers
4156 // (these two actions are performed by HandleByVal method).
4157 // Then, here, we initialize stack frame with
4158 // "store-reg" instructions.
4159 // Case #2. Var-args function, that doesn't contain byval parameters.
4160 // The same: eat all remained unallocated registers,
4161 // initialize stack frame.
4162
4163 MachineFunction &MF = DAG.getMachineFunction();
4164 MachineFrameInfo &MFI = MF.getFrameInfo();
4165 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
4166 unsigned RBegin, REnd;
4167 if (InRegsParamRecordIdx < CCInfo.getInRegsParamsCount()) {
4168 CCInfo.getInRegsParamInfo(InRegsParamRecordIndex: InRegsParamRecordIdx, BeginReg&: RBegin, EndReg&: REnd);
4169 } else {
4170 unsigned RBeginIdx = CCInfo.getFirstUnallocated(Regs: GPRArgRegs);
4171 RBegin = RBeginIdx == 4 ? (unsigned)ARM::R4 : GPRArgRegs[RBeginIdx];
4172 REnd = ARM::R4;
4173 }
4174
4175 if (REnd != RBegin)
4176 ArgOffset = -4 * (ARM::R4 - RBegin);
4177
4178 auto PtrVT = getPointerTy(DL: DAG.getDataLayout());
4179 int FrameIndex = MFI.CreateFixedObject(Size: ArgSize, SPOffset: ArgOffset, IsImmutable: false);
4180 SDValue FIN = DAG.getFrameIndex(FI: FrameIndex, VT: PtrVT);
4181
4182 SmallVector<SDValue, 4> MemOps;
4183 const TargetRegisterClass *RC =
4184 AFI->isThumb1OnlyFunction() ? &ARM::tGPRRegClass : &ARM::GPRRegClass;
4185
4186 for (unsigned Reg = RBegin, i = 0; Reg < REnd; ++Reg, ++i) {
4187 Register VReg = MF.addLiveIn(PReg: Reg, RC);
4188 SDValue Val = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: MVT::i32);
4189 SDValue Store = DAG.getStore(Chain: Val.getValue(R: 1), dl, Val, Ptr: FIN,
4190 PtrInfo: MachinePointerInfo(OrigArg, 4 * i));
4191 MemOps.push_back(Elt: Store);
4192 FIN = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: FIN, N2: DAG.getConstant(Val: 4, DL: dl, VT: PtrVT));
4193 }
4194
4195 if (!MemOps.empty())
4196 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: MemOps);
4197 return FrameIndex;
4198}
4199
4200// Setup stack frame, the va_list pointer will start from.
4201void ARMTargetLowering::VarArgStyleRegisters(CCState &CCInfo, SelectionDAG &DAG,
4202 const SDLoc &dl, SDValue &Chain,
4203 unsigned ArgOffset,
4204 unsigned TotalArgRegsSaveSize,
4205 bool ForceMutable) const {
4206 MachineFunction &MF = DAG.getMachineFunction();
4207 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
4208
4209 // Try to store any remaining integer argument regs
4210 // to their spots on the stack so that they may be loaded by dereferencing
4211 // the result of va_next.
4212 // If there is no regs to be stored, just point address after last
4213 // argument passed via stack.
4214 int FrameIndex = StoreByValRegs(
4215 CCInfo, DAG, dl, Chain, OrigArg: nullptr, InRegsParamRecordIdx: CCInfo.getInRegsParamsCount(),
4216 ArgOffset: CCInfo.getStackSize(), ArgSize: std::max(a: 4U, b: TotalArgRegsSaveSize));
4217 AFI->setVarArgsFrameIndex(FrameIndex);
4218}
4219
4220bool ARMTargetLowering::splitValueIntoRegisterParts(
4221 SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
4222 unsigned NumParts, MVT PartVT, std::optional<CallingConv::ID> CC) const {
4223 EVT ValueVT = Val.getValueType();
4224 if ((ValueVT == MVT::f16 || ValueVT == MVT::bf16) && PartVT == MVT::f32) {
4225 unsigned ValueBits = ValueVT.getSizeInBits();
4226 unsigned PartBits = PartVT.getSizeInBits();
4227 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::getIntegerVT(BitWidth: ValueBits), Operand: Val);
4228 Val = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::getIntegerVT(BitWidth: PartBits), Operand: Val);
4229 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: PartVT, Operand: Val);
4230 Parts[0] = Val;
4231 return true;
4232 }
4233 return false;
4234}
4235
4236SDValue ARMTargetLowering::joinRegisterPartsIntoValue(
4237 SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts,
4238 MVT PartVT, EVT ValueVT, std::optional<CallingConv::ID> CC) const {
4239 if ((ValueVT == MVT::f16 || ValueVT == MVT::bf16) && PartVT == MVT::f32) {
4240 unsigned ValueBits = ValueVT.getSizeInBits();
4241 unsigned PartBits = PartVT.getSizeInBits();
4242 SDValue Val = Parts[0];
4243
4244 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::getIntegerVT(BitWidth: PartBits), Operand: Val);
4245 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::getIntegerVT(BitWidth: ValueBits), Operand: Val);
4246 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ValueVT, Operand: Val);
4247 return Val;
4248 }
4249 return SDValue();
4250}
4251
4252SDValue ARMTargetLowering::LowerFormalArguments(
4253 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
4254 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
4255 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
4256 MachineFunction &MF = DAG.getMachineFunction();
4257 MachineFrameInfo &MFI = MF.getFrameInfo();
4258
4259 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
4260
4261 // Assign locations to all of the incoming arguments.
4262 SmallVector<CCValAssign, 16> ArgLocs;
4263 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs,
4264 *DAG.getContext());
4265 CCInfo.AnalyzeFormalArguments(Ins, Fn: CCAssignFnForCall(CC: CallConv, isVarArg));
4266
4267 Function::const_arg_iterator CurOrigArg = MF.getFunction().arg_begin();
4268 unsigned CurArgIdx = 0;
4269
4270 // Initially ArgRegsSaveSize is zero.
4271 // Then we increase this value each time we meet byval parameter.
4272 // We also increase this value in case of varargs function.
4273 AFI->setArgRegsSaveSize(0);
4274
4275 // Calculate the amount of stack space that we need to allocate to store
4276 // byval and variadic arguments that are passed in registers.
4277 // We need to know this before we allocate the first byval or variadic
4278 // argument, as they will be allocated a stack slot below the CFA (Canonical
4279 // Frame Address, the stack pointer at entry to the function).
4280 unsigned ArgRegBegin = ARM::R4;
4281 for (const CCValAssign &VA : ArgLocs) {
4282 if (CCInfo.getInRegsParamsProcessed() >= CCInfo.getInRegsParamsCount())
4283 break;
4284
4285 unsigned Index = VA.getValNo();
4286 ISD::ArgFlagsTy Flags = Ins[Index].Flags;
4287 if (!Flags.isByVal())
4288 continue;
4289
4290 assert(VA.isMemLoc() && "unexpected byval pointer in reg");
4291 unsigned RBegin, REnd;
4292 CCInfo.getInRegsParamInfo(InRegsParamRecordIndex: CCInfo.getInRegsParamsProcessed(), BeginReg&: RBegin, EndReg&: REnd);
4293 ArgRegBegin = std::min(a: ArgRegBegin, b: RBegin);
4294
4295 CCInfo.nextInRegsParam();
4296 }
4297 CCInfo.rewindByValRegsInfo();
4298
4299 int lastInsIndex = -1;
4300 if (isVarArg && MFI.hasVAStart()) {
4301 unsigned RegIdx = CCInfo.getFirstUnallocated(Regs: GPRArgRegs);
4302 if (RegIdx != std::size(GPRArgRegs))
4303 ArgRegBegin = std::min(a: ArgRegBegin, b: (unsigned)GPRArgRegs[RegIdx]);
4304 }
4305
4306 unsigned TotalArgRegsSaveSize = 4 * (ARM::R4 - ArgRegBegin);
4307 AFI->setArgRegsSaveSize(TotalArgRegsSaveSize);
4308 auto PtrVT = getPointerTy(DL: DAG.getDataLayout());
4309
4310 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
4311 CCValAssign &VA = ArgLocs[i];
4312 if (Ins[VA.getValNo()].isOrigArg()) {
4313 std::advance(i&: CurOrigArg,
4314 n: Ins[VA.getValNo()].getOrigArgIndex() - CurArgIdx);
4315 CurArgIdx = Ins[VA.getValNo()].getOrigArgIndex();
4316 }
4317 // Arguments stored in registers.
4318 if (VA.isRegLoc()) {
4319 EVT RegVT = VA.getLocVT();
4320 SDValue ArgValue;
4321
4322 if (VA.needsCustom() && VA.getLocVT() == MVT::v2f64) {
4323 // f64 and vector types are split up into multiple registers or
4324 // combinations of registers and stack slots.
4325 SDValue ArgValue1 =
4326 GetF64FormalArgument(VA, NextVA&: ArgLocs[++i], Root&: Chain, DAG, dl);
4327 VA = ArgLocs[++i]; // skip ahead to next loc
4328 SDValue ArgValue2;
4329 if (VA.isMemLoc()) {
4330 int FI = MFI.CreateFixedObject(Size: 8, SPOffset: VA.getLocMemOffset(), IsImmutable: true);
4331 SDValue FIN = DAG.getFrameIndex(FI, VT: PtrVT);
4332 ArgValue2 = DAG.getLoad(
4333 VT: MVT::f64, dl, Chain, Ptr: FIN,
4334 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI));
4335 } else {
4336 ArgValue2 = GetF64FormalArgument(VA, NextVA&: ArgLocs[++i], Root&: Chain, DAG, dl);
4337 }
4338 ArgValue = DAG.getNode(Opcode: ISD::UNDEF, DL: dl, VT: MVT::v2f64);
4339 ArgValue = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: MVT::v2f64, N1: ArgValue,
4340 N2: ArgValue1, N3: DAG.getIntPtrConstant(Val: 0, DL: dl));
4341 ArgValue = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: MVT::v2f64, N1: ArgValue,
4342 N2: ArgValue2, N3: DAG.getIntPtrConstant(Val: 1, DL: dl));
4343 } else if (VA.needsCustom() && VA.getLocVT() == MVT::f64) {
4344 ArgValue = GetF64FormalArgument(VA, NextVA&: ArgLocs[++i], Root&: Chain, DAG, dl);
4345 } else {
4346 const TargetRegisterClass *RC;
4347
4348 if (RegVT == MVT::f16 || RegVT == MVT::bf16)
4349 RC = &ARM::HPRRegClass;
4350 else if (RegVT == MVT::f32)
4351 RC = &ARM::SPRRegClass;
4352 else if (RegVT == MVT::f64 || RegVT == MVT::v4f16 ||
4353 RegVT == MVT::v4bf16)
4354 RC = &ARM::DPRRegClass;
4355 else if (RegVT == MVT::v2f64 || RegVT == MVT::v8f16 ||
4356 RegVT == MVT::v8bf16)
4357 RC = &ARM::QPRRegClass;
4358 else if (RegVT == MVT::i32)
4359 RC = AFI->isThumb1OnlyFunction() ? &ARM::tGPRRegClass
4360 : &ARM::GPRRegClass;
4361 else
4362 llvm_unreachable("RegVT not supported by FORMAL_ARGUMENTS Lowering");
4363
4364 // Transform the arguments in physical registers into virtual ones.
4365 Register Reg = MF.addLiveIn(PReg: VA.getLocReg(), RC);
4366 ArgValue = DAG.getCopyFromReg(Chain, dl, Reg, VT: RegVT);
4367
4368 // If this value is passed in r0 and has the returned attribute (e.g.
4369 // C++ 'structors), record this fact for later use.
4370 if (VA.getLocReg() == ARM::R0 && Ins[VA.getValNo()].Flags.isReturned()) {
4371 AFI->setPreservesR0();
4372 }
4373 }
4374
4375 // If this is an 8 or 16-bit value, it is really passed promoted
4376 // to 32 bits. Insert an assert[sz]ext to capture this, then
4377 // truncate to the right size.
4378 switch (VA.getLocInfo()) {
4379 default: llvm_unreachable("Unknown loc info!");
4380 case CCValAssign::Full: break;
4381 case CCValAssign::BCvt:
4382 ArgValue = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: VA.getValVT(), Operand: ArgValue);
4383 break;
4384 }
4385
4386 // f16 arguments have their size extended to 4 bytes and passed as if they
4387 // had been copied to the LSBs of a 32-bit register.
4388 // For that, it's passed extended to i32 (soft ABI) or to f32 (hard ABI)
4389 if (VA.needsCustom() &&
4390 (VA.getValVT() == MVT::f16 || VA.getValVT() == MVT::bf16))
4391 ArgValue = MoveToHPR(dl, DAG, LocVT: VA.getLocVT(), ValVT: VA.getValVT(), Val: ArgValue);
4392
4393 // On CMSE Entry Functions, formal integer arguments whose bitwidth is
4394 // less than 32 bits must be sign- or zero-extended in the callee for
4395 // security reasons. Although the ABI mandates an extension done by the
4396 // caller, the latter cannot be trusted to follow the rules of the ABI.
4397 const ISD::InputArg &Arg = Ins[VA.getValNo()];
4398 if (AFI->isCmseNSEntryFunction() && Arg.ArgVT.isScalarInteger() &&
4399 RegVT.isScalarInteger() && Arg.ArgVT.bitsLT(VT: MVT::i32))
4400 ArgValue = handleCMSEValue(Value: ArgValue, Arg, DAG, DL: dl);
4401
4402 InVals.push_back(Elt: ArgValue);
4403 } else { // VA.isRegLoc()
4404 // Only arguments passed on the stack should make it here.
4405 assert(VA.isMemLoc());
4406 assert(VA.getValVT() != MVT::i64 && "i64 should already be lowered");
4407
4408 int index = VA.getValNo();
4409
4410 // Some Ins[] entries become multiple ArgLoc[] entries.
4411 // Process them only once.
4412 if (index != lastInsIndex)
4413 {
4414 ISD::ArgFlagsTy Flags = Ins[index].Flags;
4415 // FIXME: For now, all byval parameter objects are marked mutable.
4416 // This can be changed with more analysis.
4417 // In case of tail call optimization mark all arguments mutable.
4418 // Since they could be overwritten by lowering of arguments in case of
4419 // a tail call.
4420 if (Flags.isByVal()) {
4421 assert(Ins[index].isOrigArg() &&
4422 "Byval arguments cannot be implicit");
4423 unsigned CurByValIndex = CCInfo.getInRegsParamsProcessed();
4424
4425 int FrameIndex = StoreByValRegs(
4426 CCInfo, DAG, dl, Chain, OrigArg: &*CurOrigArg, InRegsParamRecordIdx: CurByValIndex,
4427 ArgOffset: VA.getLocMemOffset(), ArgSize: Flags.getByValSize());
4428 InVals.push_back(Elt: DAG.getFrameIndex(FI: FrameIndex, VT: PtrVT));
4429 CCInfo.nextInRegsParam();
4430 } else if (VA.needsCustom() && (VA.getValVT() == MVT::f16 ||
4431 VA.getValVT() == MVT::bf16)) {
4432 // f16 and bf16 values are passed in the least-significant half of
4433 // a 4 byte stack slot. This is done as-if the extension was done
4434 // in a 32-bit register, so the actual bytes used for the value
4435 // differ between little and big endian.
4436 assert(VA.getLocVT().getSizeInBits() == 32);
4437 unsigned FIOffset = VA.getLocMemOffset();
4438 int FI = MFI.CreateFixedObject(Size: VA.getLocVT().getSizeInBits() / 8,
4439 SPOffset: FIOffset, IsImmutable: true);
4440
4441 SDValue Addr = DAG.getFrameIndex(FI, VT: PtrVT);
4442 if (DAG.getDataLayout().isBigEndian())
4443 Addr = DAG.getObjectPtrOffset(SL: dl, Ptr: Addr, Offset: TypeSize::getFixed(ExactSize: 2));
4444
4445 InVals.push_back(Elt: DAG.getLoad(VT: VA.getValVT(), dl, Chain, Ptr: Addr,
4446 PtrInfo: MachinePointerInfo::getFixedStack(
4447 MF&: DAG.getMachineFunction(), FI)));
4448
4449 } else {
4450 unsigned FIOffset = VA.getLocMemOffset();
4451 int FI = MFI.CreateFixedObject(Size: VA.getLocVT().getSizeInBits()/8,
4452 SPOffset: FIOffset, IsImmutable: true);
4453
4454 // Create load nodes to retrieve arguments from the stack.
4455 SDValue FIN = DAG.getFrameIndex(FI, VT: PtrVT);
4456 InVals.push_back(Elt: DAG.getLoad(VT: VA.getValVT(), dl, Chain, Ptr: FIN,
4457 PtrInfo: MachinePointerInfo::getFixedStack(
4458 MF&: DAG.getMachineFunction(), FI)));
4459 }
4460 lastInsIndex = index;
4461 }
4462 }
4463 }
4464
4465 // varargs
4466 if (isVarArg && MFI.hasVAStart()) {
4467 VarArgStyleRegisters(CCInfo, DAG, dl, Chain, ArgOffset: CCInfo.getStackSize(),
4468 TotalArgRegsSaveSize);
4469 if (AFI->isCmseNSEntryFunction()) {
4470 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
4471 DAG.getMachineFunction().getFunction(),
4472 "secure entry function must not be variadic", dl.getDebugLoc()));
4473 }
4474 }
4475
4476 unsigned StackArgSize = CCInfo.getStackSize();
4477 bool TailCallOpt = MF.getTarget().Options.GuaranteedTailCallOpt;
4478 if (canGuaranteeTCO(CC: CallConv, GuaranteeTailCalls: TailCallOpt)) {
4479 // The only way to guarantee a tail call is if the callee restores its
4480 // argument area, but it must also keep the stack aligned when doing so.
4481 MaybeAlign StackAlign = DAG.getDataLayout().getStackAlignment();
4482 assert(StackAlign && "data layout string is missing stack alignment");
4483 StackArgSize = alignTo(Size: StackArgSize, A: *StackAlign);
4484
4485 AFI->setArgumentStackToRestore(StackArgSize);
4486 }
4487 AFI->setArgumentStackSize(StackArgSize);
4488
4489 if (CCInfo.getStackSize() > 0 && AFI->isCmseNSEntryFunction()) {
4490 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
4491 DAG.getMachineFunction().getFunction(),
4492 "secure entry function requires arguments on stack", dl.getDebugLoc()));
4493 }
4494
4495 return Chain;
4496}
4497
4498/// isFloatingPointZero - Return true if this is +0.0.
4499static bool isFloatingPointZero(SDValue Op) {
4500 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Val&: Op))
4501 return CFP->getValueAPF().isPosZero();
4502 else if (ISD::isEXTLoad(N: Op.getNode()) || ISD::isNON_EXTLoad(N: Op.getNode())) {
4503 // Maybe this has already been legalized into the constant pool?
4504 if (Op.getOperand(i: 1).getOpcode() == ARMISD::Wrapper) {
4505 SDValue WrapperOp = Op.getOperand(i: 1).getOperand(i: 0);
4506 if (ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(Val&: WrapperOp))
4507 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(Val: CP->getConstVal()))
4508 return CFP->getValueAPF().isPosZero();
4509 }
4510 } else if (Op->getOpcode() == ISD::BITCAST &&
4511 Op->getValueType(ResNo: 0) == MVT::f64) {
4512 // Handle (ISD::BITCAST (ARMISD::VMOVIMM (ISD::TargetConstant 0)) MVT::f64)
4513 // created by LowerConstantFP().
4514 SDValue BitcastOp = Op->getOperand(Num: 0);
4515 if (BitcastOp->getOpcode() == ARMISD::VMOVIMM &&
4516 isNullConstant(V: BitcastOp->getOperand(Num: 0)))
4517 return true;
4518 }
4519 return false;
4520}
4521
4522static bool isSafeSignedCMN(SDValue Op, SelectionDAG &DAG) {
4523 // 0 - INT_MIN sign wraps, so no signed wrap means cmn is safe.
4524 if (Op->getFlags().hasNoSignedWrap())
4525 return true;
4526
4527 // We can still figure out if the second operand is safe to use
4528 // in a CMN instruction by checking if it is known to be not the minimum
4529 // signed value. If it is not, then we can safely use CMN.
4530 // Note: We can eventually remove this check and simply rely on
4531 // Op->getFlags().hasNoSignedWrap() once SelectionDAG/ISelLowering
4532 // consistently sets them appropriately when making said nodes.
4533
4534 KnownBits KnownSrc = DAG.computeKnownBits(Op: Op.getOperand(i: 1));
4535 return !KnownSrc.getSignedMinValue().isMinSignedValue();
4536}
4537
4538static bool isCMN(SDValue Op, ISD::CondCode CC, SelectionDAG &DAG) {
4539 return Op.getOpcode() == ISD::SUB && isNullConstant(V: Op.getOperand(i: 0)) &&
4540 (isIntEqualitySetCC(Code: CC) ||
4541 (isUnsignedIntSetCC(Code: CC) && DAG.isKnownNeverZero(Op: Op.getOperand(i: 1))) ||
4542 (isSignedIntSetCC(Code: CC) && isSafeSignedCMN(Op, DAG)));
4543}
4544
4545/// Returns how profitable it is to fold a comparison's operand's shift and/or
4546/// extension operations into the comparison instruction's second operand
4547/// (so_reg_imm / so_reg_reg for ARM, t2_so_reg for Thumb-2).
4548static unsigned getCmpOperandFoldingProfit(SDValue Op, const ARMSubtarget &ST) {
4549 // Thumb-1 CMP does not support shifted second operands.
4550 if (ST.isThumb1Only() || !Op.hasOneUse())
4551 return 0;
4552
4553 unsigned Opc = Op.getOpcode();
4554 if (Opc == ISD::SHL || Opc == ISD::SRL || Opc == ISD::SRA) {
4555 if (auto *ShiftAmt = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1)))
4556 return ShiftAmt->getZExtValue() <= 31 ? 1 : 0;
4557 // Register-controlled shift: only ARM-mode CMP/CMN (so_reg_reg) supports
4558 // this; Thumb-2 t2_so_reg requires an immediate shift amount.
4559 return ST.isThumb() ? 0 : 1;
4560 }
4561
4562 if (Opc == ISD::ROTR) {
4563 // Rotr constants will be normalized via mod 32, or & 31,
4564 // so we do not have to bounds check.
4565 if (isa<ConstantSDNode>(Val: Op.getOperand(i: 1)))
4566 return 1;
4567 return ST.isThumb() ? 0 : 1;
4568 }
4569
4570 return 0;
4571}
4572
4573/// Returns appropriate ARM CMP (cmp) and corresponding condition code for
4574/// the given operands.
4575SDValue ARMTargetLowering::getARMCmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
4576 SDValue &ARMcc, SelectionDAG &DAG,
4577 const SDLoc &dl) const {
4578 if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(Val: RHS.getNode())) {
4579 unsigned C = RHSC->getZExtValue();
4580 if (!isLegalICmpImmediate(Imm: (int32_t)C)) {
4581 // Constant does not fit, try adjusting it by one.
4582 switch (CC) {
4583 default: break;
4584 case ISD::SETLT:
4585 case ISD::SETGE:
4586 if (C != 0x80000000 && isLegalICmpImmediate(Imm: C-1)) {
4587 CC = (CC == ISD::SETLT) ? ISD::SETLE : ISD::SETGT;
4588 RHS = DAG.getConstant(Val: C - 1, DL: dl, VT: MVT::i32);
4589 }
4590 break;
4591 case ISD::SETULT:
4592 case ISD::SETUGE:
4593 if (C != 0 && isLegalICmpImmediate(Imm: C-1)) {
4594 CC = (CC == ISD::SETULT) ? ISD::SETULE : ISD::SETUGT;
4595 RHS = DAG.getConstant(Val: C - 1, DL: dl, VT: MVT::i32);
4596 }
4597 break;
4598 case ISD::SETLE:
4599 case ISD::SETGT:
4600 if (C != 0x7fffffff && isLegalICmpImmediate(Imm: C+1)) {
4601 CC = (CC == ISD::SETLE) ? ISD::SETLT : ISD::SETGE;
4602 RHS = DAG.getConstant(Val: C + 1, DL: dl, VT: MVT::i32);
4603 }
4604 break;
4605 case ISD::SETULE:
4606 case ISD::SETUGT:
4607 if (C != 0xffffffff && isLegalICmpImmediate(Imm: C+1)) {
4608 CC = (CC == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE;
4609 RHS = DAG.getConstant(Val: C + 1, DL: dl, VT: MVT::i32);
4610 }
4611 break;
4612 }
4613 }
4614 }
4615
4616 // Thumb1 has very limited immediate modes, so turning an "and" into a
4617 // shift can save multiple instructions.
4618 //
4619 // If we have (x & C1), and C1 is an appropriate mask, we can transform it
4620 // into "((x << n) >> n)". But that isn't necessarily profitable on its
4621 // own. If it's the operand to an unsigned comparison with an immediate,
4622 // we can eliminate one of the shifts: we transform
4623 // "((x << n) >> n) == C2" to "(x << n) == (C2 << n)".
4624 //
4625 // We avoid transforming cases which aren't profitable due to encoding
4626 // details:
4627 //
4628 // 1. C2 fits into the immediate field of a cmp, and the transformed version
4629 // would not; in that case, we're essentially trading one immediate load for
4630 // another.
4631 // 2. C1 is 255 or 65535, so we can use uxtb or uxth.
4632 // 3. C2 is zero; we have other code for this special case.
4633 //
4634 // FIXME: Figure out profitability for Thumb2; we usually can't save an
4635 // instruction, since the AND is always one instruction anyway, but we could
4636 // use narrow instructions in some cases.
4637 if (Subtarget->isThumb1Only() && LHS->getOpcode() == ISD::AND &&
4638 LHS->hasOneUse() && isa<ConstantSDNode>(Val: LHS.getOperand(i: 1)) &&
4639 LHS.getValueType() == MVT::i32 && isa<ConstantSDNode>(Val: RHS) &&
4640 !isSignedIntSetCC(Code: CC)) {
4641 unsigned Mask = LHS.getConstantOperandVal(i: 1);
4642 auto *RHSC = cast<ConstantSDNode>(Val: RHS.getNode());
4643 uint64_t RHSV = RHSC->getZExtValue();
4644 if (isMask_32(Value: Mask) && (RHSV & ~Mask) == 0 && Mask != 255 && Mask != 65535) {
4645 unsigned ShiftBits = llvm::countl_zero(Val: Mask);
4646 if (RHSV && (RHSV > 255 || (RHSV << ShiftBits) <= 255)) {
4647 SDValue ShiftAmt = DAG.getConstant(Val: ShiftBits, DL: dl, VT: MVT::i32);
4648 LHS = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: MVT::i32, N1: LHS.getOperand(i: 0), N2: ShiftAmt);
4649 RHS = DAG.getConstant(Val: RHSV << ShiftBits, DL: dl, VT: MVT::i32);
4650 }
4651 }
4652 }
4653
4654 // The specific comparison "(x<<c) > 0x80000000U" can be optimized to a
4655 // single "lsls x, c+1". The shift sets the "C" and "Z" flags the same
4656 // way a cmp would.
4657 // FIXME: Add support for ARM/Thumb2; this would need isel patterns, and
4658 // some tweaks to the heuristics for the previous and->shift transform.
4659 // FIXME: Optimize cases where the LHS isn't a shift.
4660 if (Subtarget->isThumb1Only() && LHS->getOpcode() == ISD::SHL &&
4661 isa<ConstantSDNode>(Val: RHS) && RHS->getAsZExtVal() == 0x80000000U &&
4662 CC == ISD::SETUGT && isa<ConstantSDNode>(Val: LHS.getOperand(i: 1)) &&
4663 LHS.getConstantOperandVal(i: 1) < 31) {
4664 unsigned ShiftAmt = LHS.getConstantOperandVal(i: 1) + 1;
4665 SDValue Shift =
4666 DAG.getNode(Opcode: ARMISD::LSLS, DL: dl, VTList: DAG.getVTList(VT1: MVT::i32, VT2: FlagsVT),
4667 N1: LHS.getOperand(i: 0), N2: DAG.getConstant(Val: ShiftAmt, DL: dl, VT: MVT::i32));
4668 ARMcc = DAG.getConstant(Val: ARMCC::HI, DL: dl, VT: MVT::i32);
4669 return Shift.getValue(R: 1);
4670 }
4671
4672 ARMCC::CondCodes CondCode = IntCCToARMCC(CC);
4673
4674 unsigned CompareType;
4675 switch (CondCode) {
4676 default:
4677 CompareType = ARMISD::CMP;
4678 break;
4679 case ARMCC::EQ:
4680 case ARMCC::NE:
4681 // Uses only Z Flag
4682 CompareType = ARMISD::CMPZ;
4683 break;
4684 }
4685
4686 // TODO: Remove CMPZ check once we generalize and remove the CMPZ enum from
4687 // the codebase.
4688
4689 // TODO: When we have a solution to the vselect predicate not allowing pl/mi
4690 // all the time, allow those cases to be cmn too no matter what.
4691 if (CompareType != ARMISD::CMPZ && isCMN(Op: RHS, CC, DAG)) {
4692 CompareType = ARMISD::CMN;
4693 RHS = RHS.getOperand(i: 1);
4694 } else if (CompareType != ARMISD::CMPZ && isCMN(Op: LHS, CC, DAG)) {
4695 CompareType = ARMISD::CMN;
4696 LHS = LHS.getOperand(i: 1);
4697 CondCode = IntCCToARMCC(CC: ISD::getSetCCSwappedOperands(Operation: CC));
4698 }
4699
4700 // Prefer folding shifts / CMN into the cmp/cmn second operand (so_reg /
4701 // t2_so_reg). When both sides compete, pick the higher
4702 // getCmpOperandFoldingProfit. Only when RHS is not a legal icmp
4703 // immediate: otherwise keep the canonical (reg, imm) form.
4704 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val: RHS.getNode());
4705 if (!C || !isLegalICmpImmediate(Imm: C->getSExtValue())) {
4706 if (getCmpOperandFoldingProfit(Op: LHS, ST: *Subtarget) >
4707 getCmpOperandFoldingProfit(Op: RHS, ST: *Subtarget)) {
4708 std::swap(a&: LHS, b&: RHS);
4709 if (CompareType == ARMISD::CMP)
4710 CondCode = ARMCC::getSwappedCondition(CC: CondCode);
4711 }
4712 }
4713
4714 // If the RHS is a constant zero then the V (overflow) flag will never be
4715 // set. This can allow us to simplify GE to PL or LT to MI, which can be
4716 // simpler for other passes (like the peephole optimiser) to deal with.
4717 if (isNullConstant(V: RHS)) {
4718 switch (CondCode) {
4719 default:
4720 break;
4721 case ARMCC::GE:
4722 CondCode = ARMCC::PL;
4723 break;
4724 case ARMCC::LT:
4725 CondCode = ARMCC::MI;
4726 break;
4727 }
4728 }
4729
4730 ARMcc = DAG.getConstant(Val: CondCode, DL: dl, VT: MVT::i32);
4731 return DAG.getNode(Opcode: CompareType, DL: dl, VT: FlagsVT, N1: LHS, N2: RHS);
4732}
4733
4734/// Returns a appropriate VFP CMP (fcmp{s|d}+fmstat) for the given operands.
4735SDValue ARMTargetLowering::getVFPCmp(SDValue LHS, SDValue RHS,
4736 SelectionDAG &DAG, const SDLoc &dl,
4737 bool Signaling) const {
4738 assert(Subtarget->hasFP64() || RHS.getValueType() != MVT::f64);
4739 SDValue Flags;
4740 if (!isFloatingPointZero(Op: RHS))
4741 Flags = DAG.getNode(Opcode: Signaling ? ARMISD::CMPFPE : ARMISD::CMPFP, DL: dl, VT: FlagsVT,
4742 N1: LHS, N2: RHS);
4743 else
4744 Flags = DAG.getNode(Opcode: Signaling ? ARMISD::CMPFPEw0 : ARMISD::CMPFPw0, DL: dl,
4745 VT: FlagsVT, Operand: LHS);
4746 return DAG.getNode(Opcode: ARMISD::FMSTAT, DL: dl, VT: FlagsVT, Operand: Flags);
4747}
4748
4749// This function returns three things: the arithmetic computation itself
4750// (Value), a comparison (OverflowCmp), and a condition code (ARMcc). The
4751// comparison and the condition code define the case in which the arithmetic
4752// computation *does not* overflow.
4753std::pair<SDValue, SDValue>
4754ARMTargetLowering::getARMXALUOOp(SDValue Op, SelectionDAG &DAG,
4755 SDValue &ARMcc) const {
4756 assert(Op.getValueType() == MVT::i32 && "Unsupported value type");
4757
4758 SDValue Value, OverflowCmp;
4759 SDValue LHS = Op.getOperand(i: 0);
4760 SDValue RHS = Op.getOperand(i: 1);
4761 SDLoc dl(Op);
4762
4763 // FIXME: We are currently always generating CMPs because we don't support
4764 // generating CMN through the backend. This is not as good as the natural
4765 // CMP case because it causes a register dependency and cannot be folded
4766 // later.
4767
4768 switch (Op.getOpcode()) {
4769 default:
4770 llvm_unreachable("Unknown overflow instruction!");
4771 case ISD::SADDO:
4772 ARMcc = DAG.getConstant(Val: ARMCC::VC, DL: dl, VT: MVT::i32);
4773 Value = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: Op.getValueType(), N1: LHS, N2: RHS);
4774 OverflowCmp = DAG.getNode(Opcode: ARMISD::CMP, DL: dl, VT: FlagsVT, N1: Value, N2: LHS);
4775 break;
4776 case ISD::UADDO:
4777 ARMcc = DAG.getConstant(Val: ARMCC::HS, DL: dl, VT: MVT::i32);
4778 // We use ADDC here to correspond to its use in LowerALUO.
4779 // We do not use it in the USUBO case as Value may not be used.
4780 Value = DAG.getNode(Opcode: ARMISD::ADDC, DL: dl,
4781 VTList: DAG.getVTList(VT1: Op.getValueType(), VT2: MVT::i32), N1: LHS, N2: RHS)
4782 .getValue(R: 0);
4783 OverflowCmp = DAG.getNode(Opcode: ARMISD::CMP, DL: dl, VT: FlagsVT, N1: Value, N2: LHS);
4784 break;
4785 case ISD::SSUBO:
4786 ARMcc = DAG.getConstant(Val: ARMCC::VC, DL: dl, VT: MVT::i32);
4787 Value = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: Op.getValueType(), N1: LHS, N2: RHS);
4788 OverflowCmp = DAG.getNode(Opcode: ARMISD::CMP, DL: dl, VT: FlagsVT, N1: LHS, N2: RHS);
4789 break;
4790 case ISD::USUBO:
4791 ARMcc = DAG.getConstant(Val: ARMCC::HS, DL: dl, VT: MVT::i32);
4792 Value = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: Op.getValueType(), N1: LHS, N2: RHS);
4793 OverflowCmp = DAG.getNode(Opcode: ARMISD::CMP, DL: dl, VT: FlagsVT, N1: LHS, N2: RHS);
4794 break;
4795 case ISD::UMULO:
4796 // We generate a UMUL_LOHI and then check if the high word is 0.
4797 ARMcc = DAG.getConstant(Val: ARMCC::EQ, DL: dl, VT: MVT::i32);
4798 Value = DAG.getNode(Opcode: ISD::UMUL_LOHI, DL: dl,
4799 VTList: DAG.getVTList(VT1: Op.getValueType(), VT2: Op.getValueType()),
4800 N1: LHS, N2: RHS);
4801 OverflowCmp = DAG.getNode(Opcode: ARMISD::CMPZ, DL: dl, VT: FlagsVT, N1: Value.getValue(R: 1),
4802 N2: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
4803 Value = Value.getValue(R: 0); // We only want the low 32 bits for the result.
4804 break;
4805 case ISD::SMULO:
4806 // We generate a SMUL_LOHI and then check if all the bits of the high word
4807 // are the same as the sign bit of the low word.
4808 ARMcc = DAG.getConstant(Val: ARMCC::EQ, DL: dl, VT: MVT::i32);
4809 Value = DAG.getNode(Opcode: ISD::SMUL_LOHI, DL: dl,
4810 VTList: DAG.getVTList(VT1: Op.getValueType(), VT2: Op.getValueType()),
4811 N1: LHS, N2: RHS);
4812 OverflowCmp = DAG.getNode(Opcode: ARMISD::CMPZ, DL: dl, VT: FlagsVT, N1: Value.getValue(R: 1),
4813 N2: DAG.getNode(Opcode: ISD::SRA, DL: dl, VT: Op.getValueType(),
4814 N1: Value.getValue(R: 0),
4815 N2: DAG.getConstant(Val: 31, DL: dl, VT: MVT::i32)));
4816 Value = Value.getValue(R: 0); // We only want the low 32 bits for the result.
4817 break;
4818 } // switch (...)
4819
4820 return std::make_pair(x&: Value, y&: OverflowCmp);
4821}
4822
4823static SDValue valueToCarryFlag(SDValue Value, SelectionDAG &DAG, bool Invert) {
4824 SDLoc DL(Value);
4825 EVT VT = Value.getValueType();
4826
4827 if (Invert)
4828 Value = DAG.getNode(Opcode: ISD::SUB, DL, VT: MVT::i32,
4829 N1: DAG.getConstant(Val: 1, DL, VT: MVT::i32), N2: Value);
4830
4831 SDValue Cmp = DAG.getNode(Opcode: ARMISD::SUBC, DL, VTList: DAG.getVTList(VT1: VT, VT2: MVT::i32),
4832 N1: Value, N2: DAG.getConstant(Val: 1, DL, VT));
4833 return Cmp.getValue(R: 1);
4834}
4835
4836static SDValue carryFlagToValue(SDValue Flags, EVT VT, SelectionDAG &DAG,
4837 bool Invert) {
4838 SDLoc DL(Flags);
4839
4840 if (Invert) {
4841 // Convert flags to boolean with ADDE 0,0,Carry then compute 1 - bool.
4842 SDValue BoolCarry = DAG.getNode(
4843 Opcode: ARMISD::ADDE, DL, VTList: DAG.getVTList(VT1: VT, VT2: MVT::i32),
4844 N1: DAG.getConstant(Val: 0, DL, VT), N2: DAG.getConstant(Val: 0, DL, VT), N3: Flags);
4845 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: DAG.getConstant(Val: 1, DL, VT), N2: BoolCarry);
4846 }
4847
4848 // Now convert the carry flag into a boolean carry. We do this
4849 // using ARMISD::ADDE 0, 0, Carry
4850 return DAG.getNode(Opcode: ARMISD::ADDE, DL, VTList: DAG.getVTList(VT1: VT, VT2: MVT::i32),
4851 N1: DAG.getConstant(Val: 0, DL, VT), N2: DAG.getConstant(Val: 0, DL, VT),
4852 N3: Flags);
4853}
4854
4855// Value is 1 if 'V' bit is 1, else 0
4856static SDValue overflowFlagToValue(SDValue Flags, EVT VT, SelectionDAG &DAG) {
4857 SDLoc DL(Flags);
4858 SDValue Zero = DAG.getConstant(Val: 0, DL, VT);
4859 SDValue One = DAG.getConstant(Val: 1, DL, VT);
4860 SDValue ARMcc = DAG.getConstant(Val: ARMCC::VS, DL, VT: MVT::i32);
4861 return DAG.getNode(Opcode: ARMISD::CMOV, DL, VT, N1: Zero, N2: One, N3: ARMcc, N4: Flags);
4862}
4863
4864SDValue ARMTargetLowering::LowerALUO(SDValue Op, SelectionDAG &DAG) const {
4865 // Let legalize expand this if it isn't a legal type yet.
4866 if (!isTypeLegal(VT: Op.getValueType()))
4867 return SDValue();
4868
4869 SDValue LHS = Op.getOperand(i: 0);
4870 SDValue RHS = Op.getOperand(i: 1);
4871 SDLoc dl(Op);
4872
4873 EVT VT = Op.getValueType();
4874 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: MVT::i32);
4875 SDValue Value;
4876 SDValue Overflow;
4877 switch (Op.getOpcode()) {
4878 case ISD::UADDO:
4879 Value = DAG.getNode(Opcode: ARMISD::ADDC, DL: dl, VTList: VTs, N1: LHS, N2: RHS);
4880 // Convert the carry flag into a boolean value.
4881 Overflow = carryFlagToValue(Flags: Value.getValue(R: 1), VT, DAG, Invert: false);
4882 break;
4883 case ISD::USUBO:
4884 Value = DAG.getNode(Opcode: ARMISD::SUBC, DL: dl, VTList: VTs, N1: LHS, N2: RHS);
4885 // Convert the carry flag into a boolean value.
4886 Overflow = carryFlagToValue(Flags: Value.getValue(R: 1), VT, DAG, Invert: true);
4887 break;
4888 default: {
4889 // Handle other operations with getARMXALUOOp
4890 SDValue OverflowCmp, ARMcc;
4891 std::tie(args&: Value, args&: OverflowCmp) = getARMXALUOOp(Op, DAG, ARMcc);
4892 // We use 0 and 1 as false and true values.
4893 // ARMcc represents the "no overflow" condition (e.g., VC for signed ops).
4894 // CMOV operand order is (FalseVal, TrueVal), so we put 1 in FalseVal
4895 // position to get Overflow=1 when the "no overflow" condition is false.
4896 Overflow =
4897 DAG.getNode(Opcode: ARMISD::CMOV, DL: dl, VT: MVT::i32,
4898 N1: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32), // FalseVal: overflow
4899 N2: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32), // TrueVal: no overflow
4900 N3: ARMcc, N4: OverflowCmp);
4901 break;
4902 }
4903 }
4904
4905 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: dl, VTList: VTs, N1: Value, N2: Overflow);
4906}
4907
4908static SDValue LowerADDSUBSAT(SDValue Op, SelectionDAG &DAG,
4909 const ARMSubtarget *Subtarget) {
4910 EVT VT = Op.getValueType();
4911 if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP() || Subtarget->isThumb1Only())
4912 return SDValue();
4913 if (!VT.isSimple())
4914 return SDValue();
4915
4916 unsigned NewOpcode;
4917 switch (VT.getSimpleVT().SimpleTy) {
4918 default:
4919 return SDValue();
4920 case MVT::i8:
4921 switch (Op->getOpcode()) {
4922 case ISD::UADDSAT:
4923 NewOpcode = ARMISD::UQADD8b;
4924 break;
4925 case ISD::SADDSAT:
4926 NewOpcode = ARMISD::QADD8b;
4927 break;
4928 case ISD::USUBSAT:
4929 NewOpcode = ARMISD::UQSUB8b;
4930 break;
4931 case ISD::SSUBSAT:
4932 NewOpcode = ARMISD::QSUB8b;
4933 break;
4934 }
4935 break;
4936 case MVT::i16:
4937 switch (Op->getOpcode()) {
4938 case ISD::UADDSAT:
4939 NewOpcode = ARMISD::UQADD16b;
4940 break;
4941 case ISD::SADDSAT:
4942 NewOpcode = ARMISD::QADD16b;
4943 break;
4944 case ISD::USUBSAT:
4945 NewOpcode = ARMISD::UQSUB16b;
4946 break;
4947 case ISD::SSUBSAT:
4948 NewOpcode = ARMISD::QSUB16b;
4949 break;
4950 }
4951 break;
4952 }
4953
4954 SDLoc dl(Op);
4955 SDValue Add =
4956 DAG.getNode(Opcode: NewOpcode, DL: dl, VT: MVT::i32,
4957 N1: DAG.getSExtOrTrunc(Op: Op->getOperand(Num: 0), DL: dl, VT: MVT::i32),
4958 N2: DAG.getSExtOrTrunc(Op: Op->getOperand(Num: 1), DL: dl, VT: MVT::i32));
4959 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: Add);
4960}
4961
4962SDValue ARMTargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
4963 SDValue Cond = Op.getOperand(i: 0);
4964 SDValue SelectTrue = Op.getOperand(i: 1);
4965 SDValue SelectFalse = Op.getOperand(i: 2);
4966 SDLoc dl(Op);
4967 unsigned Opc = Cond.getOpcode();
4968
4969 if (Cond.getResNo() == 1 &&
4970 (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO ||
4971 Opc == ISD::USUBO)) {
4972 if (!isTypeLegal(VT: Cond->getValueType(ResNo: 0)))
4973 return SDValue();
4974
4975 SDValue Value, OverflowCmp;
4976 SDValue ARMcc;
4977 std::tie(args&: Value, args&: OverflowCmp) = getARMXALUOOp(Op: Cond, DAG, ARMcc);
4978 EVT VT = Op.getValueType();
4979
4980 return getCMOV(dl, VT, FalseVal: SelectTrue, TrueVal: SelectFalse, ARMcc, Flags: OverflowCmp, DAG);
4981 }
4982
4983 // Convert:
4984 //
4985 // (select (cmov 1, 0, cond), t, f) -> (cmov t, f, cond)
4986 // (select (cmov 0, 1, cond), t, f) -> (cmov f, t, cond)
4987 //
4988 if (Cond.getOpcode() == ARMISD::CMOV && Cond.hasOneUse()) {
4989 const ConstantSDNode *CMOVTrue =
4990 dyn_cast<ConstantSDNode>(Val: Cond.getOperand(i: 0));
4991 const ConstantSDNode *CMOVFalse =
4992 dyn_cast<ConstantSDNode>(Val: Cond.getOperand(i: 1));
4993
4994 if (CMOVTrue && CMOVFalse) {
4995 unsigned CMOVTrueVal = CMOVTrue->getZExtValue();
4996 unsigned CMOVFalseVal = CMOVFalse->getZExtValue();
4997
4998 SDValue True;
4999 SDValue False;
5000 if (CMOVTrueVal == 1 && CMOVFalseVal == 0) {
5001 True = SelectTrue;
5002 False = SelectFalse;
5003 } else if (CMOVTrueVal == 0 && CMOVFalseVal == 1) {
5004 True = SelectFalse;
5005 False = SelectTrue;
5006 }
5007
5008 if (True.getNode() && False.getNode())
5009 return getCMOV(dl, VT: Op.getValueType(), FalseVal: True, TrueVal: False, ARMcc: Cond.getOperand(i: 2),
5010 Flags: Cond.getOperand(i: 3), DAG);
5011 }
5012 }
5013
5014 return DAG.getSelectCC(DL: dl, LHS: Cond,
5015 RHS: DAG.getConstant(Val: 0, DL: dl, VT: Cond.getValueType()),
5016 True: SelectTrue, False: SelectFalse, Cond: ISD::SETNE);
5017}
5018
5019static void checkVSELConstraints(ISD::CondCode CC, ARMCC::CondCodes &CondCode,
5020 bool &swpCmpOps, bool &swpVselOps) {
5021 // Start by selecting the GE condition code for opcodes that return true for
5022 // 'equality'
5023 if (CC == ISD::SETUGE || CC == ISD::SETOGE || CC == ISD::SETOLE ||
5024 CC == ISD::SETULE || CC == ISD::SETGE || CC == ISD::SETLE)
5025 CondCode = ARMCC::GE;
5026
5027 // and GT for opcodes that return false for 'equality'.
5028 else if (CC == ISD::SETUGT || CC == ISD::SETOGT || CC == ISD::SETOLT ||
5029 CC == ISD::SETULT || CC == ISD::SETGT || CC == ISD::SETLT)
5030 CondCode = ARMCC::GT;
5031
5032 // Since we are constrained to GE/GT, if the opcode contains 'less', we need
5033 // to swap the compare operands.
5034 if (CC == ISD::SETOLE || CC == ISD::SETULE || CC == ISD::SETOLT ||
5035 CC == ISD::SETULT || CC == ISD::SETLE || CC == ISD::SETLT)
5036 swpCmpOps = true;
5037
5038 // Both GT and GE are ordered comparisons, and return false for 'unordered'.
5039 // If we have an unordered opcode, we need to swap the operands to the VSEL
5040 // instruction (effectively negating the condition).
5041 //
5042 // This also has the effect of swapping which one of 'less' or 'greater'
5043 // returns true, so we also swap the compare operands. It also switches
5044 // whether we return true for 'equality', so we compensate by picking the
5045 // opposite condition code to our original choice.
5046 if (CC == ISD::SETULE || CC == ISD::SETULT || CC == ISD::SETUGE ||
5047 CC == ISD::SETUGT) {
5048 swpCmpOps = !swpCmpOps;
5049 swpVselOps = !swpVselOps;
5050 CondCode = CondCode == ARMCC::GT ? ARMCC::GE : ARMCC::GT;
5051 }
5052
5053 // 'ordered' is 'anything but unordered', so use the VS condition code and
5054 // swap the VSEL operands.
5055 if (CC == ISD::SETO) {
5056 CondCode = ARMCC::VS;
5057 swpVselOps = true;
5058 }
5059
5060 // 'unordered or not equal' is 'anything but equal', so use the EQ condition
5061 // code and swap the VSEL operands. Also do this if we don't care about the
5062 // unordered case.
5063 if (CC == ISD::SETUNE || CC == ISD::SETNE) {
5064 CondCode = ARMCC::EQ;
5065 swpVselOps = true;
5066 }
5067}
5068
5069SDValue ARMTargetLowering::getCMOV(const SDLoc &dl, EVT VT, SDValue FalseVal,
5070 SDValue TrueVal, SDValue ARMcc,
5071 SDValue Flags, SelectionDAG &DAG) const {
5072 if (!Subtarget->hasFP64() && VT == MVT::f64) {
5073 FalseVal = DAG.getNode(Opcode: ARMISD::VMOVRRD, DL: dl,
5074 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N: FalseVal);
5075 TrueVal = DAG.getNode(Opcode: ARMISD::VMOVRRD, DL: dl,
5076 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N: TrueVal);
5077
5078 SDValue TrueLow = TrueVal.getValue(R: 0);
5079 SDValue TrueHigh = TrueVal.getValue(R: 1);
5080 SDValue FalseLow = FalseVal.getValue(R: 0);
5081 SDValue FalseHigh = FalseVal.getValue(R: 1);
5082
5083 SDValue Low = DAG.getNode(Opcode: ARMISD::CMOV, DL: dl, VT: MVT::i32, N1: FalseLow, N2: TrueLow,
5084 N3: ARMcc, N4: Flags);
5085 SDValue High = DAG.getNode(Opcode: ARMISD::CMOV, DL: dl, VT: MVT::i32, N1: FalseHigh, N2: TrueHigh,
5086 N3: ARMcc, N4: Flags);
5087
5088 return DAG.getNode(Opcode: ARMISD::VMOVDRR, DL: dl, VT: MVT::f64, N1: Low, N2: High);
5089 }
5090 return DAG.getNode(Opcode: ARMISD::CMOV, DL: dl, VT, N1: FalseVal, N2: TrueVal, N3: ARMcc, N4: Flags);
5091}
5092
5093static bool isGTorGE(ISD::CondCode CC) {
5094 return CC == ISD::SETGT || CC == ISD::SETGE;
5095}
5096
5097static bool isLTorLE(ISD::CondCode CC) {
5098 return CC == ISD::SETLT || CC == ISD::SETLE;
5099}
5100
5101// See if a conditional (LHS CC RHS ? TrueVal : FalseVal) is lower-saturating.
5102// All of these conditions (and their <= and >= counterparts) will do:
5103// x < k ? k : x
5104// x > k ? x : k
5105// k < x ? x : k
5106// k > x ? k : x
5107static bool isLowerSaturate(const SDValue LHS, const SDValue RHS,
5108 const SDValue TrueVal, const SDValue FalseVal,
5109 const ISD::CondCode CC, const SDValue K) {
5110 return (isGTorGE(CC) &&
5111 ((K == LHS && K == TrueVal) || (K == RHS && K == FalseVal))) ||
5112 (isLTorLE(CC) &&
5113 ((K == RHS && K == TrueVal) || (K == LHS && K == FalseVal)));
5114}
5115
5116// Check if two chained conditionals could be converted into SSAT or USAT.
5117//
5118// SSAT can replace a set of two conditional selectors that bound a number to an
5119// interval of type [k, ~k] when k + 1 is a power of 2. Here are some examples:
5120//
5121// x < -k ? -k : (x > k ? k : x)
5122// x < -k ? -k : (x < k ? x : k)
5123// x > -k ? (x > k ? k : x) : -k
5124// x < k ? (x < -k ? -k : x) : k
5125// etc.
5126//
5127// LLVM canonicalizes these to either a min(max()) or a max(min())
5128// pattern. This function tries to match one of these and will return a SSAT
5129// node if successful.
5130//
5131// USAT works similarly to SSAT but bounds on the interval [0, k] where k + 1
5132// is a power of 2.
5133static SDValue LowerSaturatingConditional(SDValue Op, SelectionDAG &DAG) {
5134 EVT VT = Op.getValueType();
5135 SDValue V1 = Op.getOperand(i: 0);
5136 SDValue K1 = Op.getOperand(i: 1);
5137 SDValue TrueVal1 = Op.getOperand(i: 2);
5138 SDValue FalseVal1 = Op.getOperand(i: 3);
5139 ISD::CondCode CC1 = cast<CondCodeSDNode>(Val: Op.getOperand(i: 4))->get();
5140
5141 const SDValue Op2 = isa<ConstantSDNode>(Val: TrueVal1) ? FalseVal1 : TrueVal1;
5142 if (Op2.getOpcode() != ISD::SELECT_CC)
5143 return SDValue();
5144
5145 SDValue V2 = Op2.getOperand(i: 0);
5146 SDValue K2 = Op2.getOperand(i: 1);
5147 SDValue TrueVal2 = Op2.getOperand(i: 2);
5148 SDValue FalseVal2 = Op2.getOperand(i: 3);
5149 ISD::CondCode CC2 = cast<CondCodeSDNode>(Val: Op2.getOperand(i: 4))->get();
5150
5151 SDValue V1Tmp = V1;
5152 SDValue V2Tmp = V2;
5153
5154 // Check that the registers and the constants match a max(min()) or min(max())
5155 // pattern
5156 if (V1Tmp != TrueVal1 || V2Tmp != TrueVal2 || K1 != FalseVal1 ||
5157 K2 != FalseVal2 ||
5158 !((isGTorGE(CC: CC1) && isLTorLE(CC: CC2)) || (isLTorLE(CC: CC1) && isGTorGE(CC: CC2))))
5159 return SDValue();
5160
5161 // Check that the constant in the lower-bound check is
5162 // the opposite of the constant in the upper-bound check
5163 // in 1's complement.
5164 if (!isa<ConstantSDNode>(Val: K1) || !isa<ConstantSDNode>(Val: K2))
5165 return SDValue();
5166
5167 int64_t Val1 = cast<ConstantSDNode>(Val&: K1)->getSExtValue();
5168 int64_t Val2 = cast<ConstantSDNode>(Val&: K2)->getSExtValue();
5169 int64_t PosVal = std::max(a: Val1, b: Val2);
5170 int64_t NegVal = std::min(a: Val1, b: Val2);
5171
5172 if (!((Val1 > Val2 && isLTorLE(CC: CC1)) || (Val1 < Val2 && isLTorLE(CC: CC2))) ||
5173 !isPowerOf2_64(Value: PosVal + 1))
5174 return SDValue();
5175
5176 // Handle the difference between USAT (unsigned) and SSAT (signed)
5177 // saturation
5178 // At this point, PosVal is guaranteed to be positive
5179 uint64_t K = PosVal;
5180 SDLoc dl(Op);
5181 if (Val1 == ~Val2)
5182 return DAG.getNode(Opcode: ARMISD::SSAT, DL: dl, VT, N1: V2Tmp,
5183 N2: DAG.getConstant(Val: llvm::countr_one(Value: K), DL: dl, VT));
5184 if (NegVal == 0)
5185 return DAG.getNode(Opcode: ARMISD::USAT, DL: dl, VT, N1: V2Tmp,
5186 N2: DAG.getConstant(Val: llvm::countr_one(Value: K), DL: dl, VT));
5187
5188 return SDValue();
5189}
5190
5191// Check if a condition of the type x < k ? k : x can be converted into a
5192// bit operation instead of conditional moves.
5193// Currently this is allowed given:
5194// - The conditions and values match up
5195// - k is 0 or -1 (all ones)
5196// This function will not check the last condition, thats up to the caller
5197// It returns true if the transformation can be made, and in such case
5198// returns x in V, and k in SatK.
5199static bool isLowerSaturatingConditional(const SDValue &Op, SDValue &V,
5200 SDValue &SatK)
5201{
5202 SDValue LHS = Op.getOperand(i: 0);
5203 SDValue RHS = Op.getOperand(i: 1);
5204 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 4))->get();
5205 SDValue TrueVal = Op.getOperand(i: 2);
5206 SDValue FalseVal = Op.getOperand(i: 3);
5207
5208 SDValue *K = isa<ConstantSDNode>(Val: LHS) ? &LHS : isa<ConstantSDNode>(Val: RHS)
5209 ? &RHS
5210 : nullptr;
5211
5212 // No constant operation in comparison, early out
5213 if (!K)
5214 return false;
5215
5216 SDValue KTmp = isa<ConstantSDNode>(Val: TrueVal) ? TrueVal : FalseVal;
5217 V = (KTmp == TrueVal) ? FalseVal : TrueVal;
5218 SDValue VTmp = (K && *K == LHS) ? RHS : LHS;
5219
5220 // If the constant on left and right side, or variable on left and right,
5221 // does not match, early out
5222 if (*K != KTmp || V != VTmp)
5223 return false;
5224
5225 if (isLowerSaturate(LHS, RHS, TrueVal, FalseVal, CC, K: *K)) {
5226 SatK = *K;
5227 return true;
5228 }
5229
5230 return false;
5231}
5232
5233bool ARMTargetLowering::isUnsupportedFloatingType(EVT VT) const {
5234 if (VT == MVT::f32)
5235 return !Subtarget->hasVFP2Base();
5236 if (VT == MVT::f64)
5237 return !Subtarget->hasFP64();
5238 if (VT == MVT::f16)
5239 return !Subtarget->hasFullFP16();
5240 return false;
5241}
5242
5243static SDValue matchCSET(unsigned &Opcode, bool &InvertCond, SDValue TrueVal,
5244 SDValue FalseVal, const ARMSubtarget *Subtarget) {
5245 ConstantSDNode *CFVal = dyn_cast<ConstantSDNode>(Val&: FalseVal);
5246 ConstantSDNode *CTVal = dyn_cast<ConstantSDNode>(Val&: TrueVal);
5247 if (!CFVal || !CTVal || !Subtarget->hasV8_1MMainlineOps())
5248 return SDValue();
5249
5250 unsigned TVal = CTVal->getZExtValue();
5251 unsigned FVal = CFVal->getZExtValue();
5252
5253 Opcode = 0;
5254 InvertCond = false;
5255 if (TVal == ~FVal) {
5256 Opcode = ARMISD::CSINV;
5257 } else if (TVal == ~FVal + 1) {
5258 Opcode = ARMISD::CSNEG;
5259 } else if (TVal + 1 == FVal) {
5260 Opcode = ARMISD::CSINC;
5261 } else if (TVal == FVal + 1) {
5262 Opcode = ARMISD::CSINC;
5263 std::swap(a&: TrueVal, b&: FalseVal);
5264 std::swap(a&: TVal, b&: FVal);
5265 InvertCond = !InvertCond;
5266 } else {
5267 return SDValue();
5268 }
5269
5270 // If one of the constants is cheaper than another, materialise the
5271 // cheaper one and let the csel generate the other.
5272 if (Opcode != ARMISD::CSINC &&
5273 HasLowerConstantMaterializationCost(Val1: FVal, Val2: TVal, Subtarget)) {
5274 std::swap(a&: TrueVal, b&: FalseVal);
5275 std::swap(a&: TVal, b&: FVal);
5276 InvertCond = !InvertCond;
5277 }
5278
5279 // Attempt to use ZR checking TVal is 0, possibly inverting the condition
5280 // to get there. CSINC not is invertable like the other two (~(~a) == a,
5281 // -(-a) == a, but (a+1)+1 != a).
5282 if (FVal == 0 && Opcode != ARMISD::CSINC) {
5283 std::swap(a&: TrueVal, b&: FalseVal);
5284 std::swap(a&: TVal, b&: FVal);
5285 InvertCond = !InvertCond;
5286 }
5287
5288 return TrueVal;
5289}
5290
5291SDValue ARMTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
5292 EVT VT = Op.getValueType();
5293 SDLoc dl(Op);
5294
5295 // Try to convert two saturating conditional selects into a single SSAT
5296 if ((!Subtarget->isThumb() && Subtarget->hasV6Ops()) || Subtarget->isThumb2())
5297 if (SDValue SatValue = LowerSaturatingConditional(Op, DAG))
5298 return SatValue;
5299
5300 // Try to convert expressions of the form x < k ? k : x (and similar forms)
5301 // into more efficient bit operations, which is possible when k is 0 or -1
5302 // On ARM and Thumb-2 which have flexible operand 2 this will result in
5303 // single instructions. On Thumb the shift and the bit operation will be two
5304 // instructions.
5305 // Only allow this transformation on full-width (32-bit) operations
5306 SDValue LowerSatConstant;
5307 SDValue SatValue;
5308 if (VT == MVT::i32 &&
5309 isLowerSaturatingConditional(Op, V&: SatValue, SatK&: LowerSatConstant)) {
5310 SDValue ShiftV = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: SatValue,
5311 N2: DAG.getConstant(Val: 31, DL: dl, VT));
5312 if (isNullConstant(V: LowerSatConstant)) {
5313 SDValue NotShiftV = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: ShiftV,
5314 N2: DAG.getAllOnesConstant(DL: dl, VT));
5315 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: SatValue, N2: NotShiftV);
5316 } else if (isAllOnesConstant(V: LowerSatConstant))
5317 return DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: SatValue, N2: ShiftV);
5318 }
5319
5320 SDValue LHS = Op.getOperand(i: 0);
5321 SDValue RHS = Op.getOperand(i: 1);
5322 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 4))->get();
5323 SDValue TrueVal = Op.getOperand(i: 2);
5324 SDValue FalseVal = Op.getOperand(i: 3);
5325 ConstantSDNode *CFVal = dyn_cast<ConstantSDNode>(Val&: FalseVal);
5326 ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(Val&: RHS);
5327 if (Op.getValueType().isInteger()) {
5328
5329 // Check for SMAX(lhs, 0) and SMIN(lhs, 0) patterns.
5330 // (SELECT_CC setgt, lhs, 0, lhs, 0) -> (BIC lhs, (SRA lhs, typesize-1))
5331 // (SELECT_CC setlt, lhs, 0, lhs, 0) -> (AND lhs, (SRA lhs, typesize-1))
5332 // Both require less instructions than compare and conditional select.
5333 if ((CC == ISD::SETGT || CC == ISD::SETLT) && LHS == TrueVal && RHSC &&
5334 RHSC->isZero() && CFVal && CFVal->isZero() &&
5335 LHS.getValueType() == RHS.getValueType()) {
5336 EVT VT = LHS.getValueType();
5337 SDValue Shift =
5338 DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: LHS,
5339 N2: DAG.getConstant(Val: VT.getSizeInBits() - 1, DL: dl, VT));
5340
5341 if (CC == ISD::SETGT)
5342 Shift = DAG.getNOT(DL: dl, Val: Shift, VT);
5343
5344 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: LHS, N2: Shift);
5345 }
5346
5347 // (SELECT_CC setlt, x, 0, 1, 0) -> SRL(x, bw-1)
5348 if (CC == ISD::SETLT && isNullConstant(V: RHS) && isOneConstant(V: TrueVal) &&
5349 isNullConstant(V: FalseVal) && LHS.getValueType() == VT)
5350 return DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: LHS,
5351 N2: DAG.getConstant(Val: VT.getSizeInBits() - 1, DL: dl, VT));
5352 }
5353
5354 if (LHS.getValueType() == MVT::i32) {
5355 unsigned Opcode;
5356 bool InvertCond;
5357 if (SDValue Op =
5358 matchCSET(Opcode, InvertCond, TrueVal, FalseVal, Subtarget)) {
5359 if (InvertCond)
5360 CC = ISD::getSetCCInverse(Operation: CC, Type: LHS.getValueType());
5361
5362 SDValue ARMcc;
5363 SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMcc, DAG, dl);
5364 EVT VT = Op.getValueType();
5365 return DAG.getNode(Opcode, DL: dl, VT, N1: Op, N2: Op, N3: ARMcc, N4: Cmp);
5366 }
5367 }
5368
5369 if (isUnsupportedFloatingType(VT: LHS.getValueType())) {
5370 softenSetCCOperands(DAG, VT: LHS.getValueType(), NewLHS&: LHS, NewRHS&: RHS, CCCode&: CC, DL: dl, OldLHS: LHS, OldRHS: RHS);
5371
5372 // If softenSetCCOperands only returned one value, we should compare it to
5373 // zero.
5374 if (!RHS.getNode()) {
5375 RHS = DAG.getConstant(Val: 0, DL: dl, VT: LHS.getValueType());
5376 CC = ISD::SETNE;
5377 }
5378 }
5379
5380 if (LHS.getValueType() == MVT::i32) {
5381 // Try to generate VSEL on ARMv8.
5382 // The VSEL instruction can't use all the usual ARM condition
5383 // codes: it only has two bits to select the condition code, so it's
5384 // constrained to use only GE, GT, VS and EQ.
5385 //
5386 // To implement all the various ISD::SETXXX opcodes, we sometimes need to
5387 // swap the operands of the previous compare instruction (effectively
5388 // inverting the compare condition, swapping 'less' and 'greater') and
5389 // sometimes need to swap the operands to the VSEL (which inverts the
5390 // condition in the sense of firing whenever the previous condition didn't)
5391 if (Subtarget->hasFPARMv8Base() && (TrueVal.getValueType() == MVT::f16 ||
5392 TrueVal.getValueType() == MVT::f32 ||
5393 TrueVal.getValueType() == MVT::f64)) {
5394 ARMCC::CondCodes CondCode = IntCCToARMCC(CC);
5395 if (CondCode == ARMCC::LT || CondCode == ARMCC::LE ||
5396 CondCode == ARMCC::VC || CondCode == ARMCC::NE) {
5397 CC = ISD::getSetCCInverse(Operation: CC, Type: LHS.getValueType());
5398 std::swap(a&: TrueVal, b&: FalseVal);
5399 }
5400 }
5401
5402 SDValue ARMcc;
5403 SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMcc, DAG, dl);
5404 // Choose GE over PL, which vsel does now support
5405 if (ARMcc->getAsZExtVal() == ARMCC::PL)
5406 ARMcc = DAG.getConstant(Val: ARMCC::GE, DL: dl, VT: MVT::i32);
5407 return getCMOV(dl, VT, FalseVal, TrueVal, ARMcc, Flags: Cmp, DAG);
5408 }
5409
5410 ARMCC::CondCodes CondCode, CondCode2;
5411 FPCCToARMCC(CC, CondCode, CondCode2);
5412
5413 // Normalize the fp compare. If RHS is zero we prefer to keep it there so we
5414 // match CMPFPw0 instead of CMPFP, though we don't do this for f16 because we
5415 // must use VSEL (limited condition codes), due to not having conditional f16
5416 // moves.
5417 if (Subtarget->hasFPARMv8Base() &&
5418 !(isFloatingPointZero(Op: RHS) && TrueVal.getValueType() != MVT::f16) &&
5419 (TrueVal.getValueType() == MVT::f16 ||
5420 TrueVal.getValueType() == MVT::f32 ||
5421 TrueVal.getValueType() == MVT::f64)) {
5422 bool swpCmpOps = false;
5423 bool swpVselOps = false;
5424 checkVSELConstraints(CC, CondCode, swpCmpOps, swpVselOps);
5425
5426 if (CondCode == ARMCC::GT || CondCode == ARMCC::GE ||
5427 CondCode == ARMCC::VS || CondCode == ARMCC::EQ) {
5428 if (swpCmpOps)
5429 std::swap(a&: LHS, b&: RHS);
5430 if (swpVselOps)
5431 std::swap(a&: TrueVal, b&: FalseVal);
5432 }
5433 }
5434
5435 SDValue ARMcc = DAG.getConstant(Val: CondCode, DL: dl, VT: MVT::i32);
5436 SDValue Cmp = getVFPCmp(LHS, RHS, DAG, dl);
5437 SDValue Result = getCMOV(dl, VT, FalseVal, TrueVal, ARMcc, Flags: Cmp, DAG);
5438 if (CondCode2 != ARMCC::AL) {
5439 SDValue ARMcc2 = DAG.getConstant(Val: CondCode2, DL: dl, VT: MVT::i32);
5440 Result = getCMOV(dl, VT, FalseVal: Result, TrueVal, ARMcc: ARMcc2, Flags: Cmp, DAG);
5441 }
5442 return Result;
5443}
5444
5445/// canChangeToInt - Given the fp compare operand, return true if it is suitable
5446/// to morph to an integer compare sequence.
5447static bool canChangeToInt(SDValue Op, bool &SeenZero,
5448 const ARMSubtarget *Subtarget) {
5449 SDNode *N = Op.getNode();
5450 if (!N->hasOneUse())
5451 // Otherwise it requires moving the value from fp to integer registers.
5452 return false;
5453 if (!N->getNumValues())
5454 return false;
5455 EVT VT = Op.getValueType();
5456 if (VT != MVT::f32 && !Subtarget->isFPBrccSlow())
5457 // f32 case is generally profitable. f64 case only makes sense when vcmpe +
5458 // vmrs are very slow, e.g. cortex-a8.
5459 return false;
5460
5461 if (isFloatingPointZero(Op)) {
5462 SeenZero = true;
5463 return true;
5464 }
5465 return ISD::isNormalLoad(N);
5466}
5467
5468static SDValue bitcastf32Toi32(SDValue Op, SelectionDAG &DAG) {
5469 if (isFloatingPointZero(Op))
5470 return DAG.getConstant(Val: 0, DL: SDLoc(Op), VT: MVT::i32);
5471
5472 if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(Val&: Op))
5473 return DAG.getLoad(VT: MVT::i32, dl: SDLoc(Op), Chain: Ld->getChain(), Ptr: Ld->getBasePtr(),
5474 PtrInfo: Ld->getPointerInfo(), Alignment: Ld->getAlign(),
5475 MMOFlags: Ld->getMemOperand()->getFlags());
5476
5477 llvm_unreachable("Unknown VFP cmp argument!");
5478}
5479
5480static void expandf64Toi32(SDValue Op, SelectionDAG &DAG,
5481 SDValue &RetVal1, SDValue &RetVal2) {
5482 SDLoc dl(Op);
5483
5484 if (isFloatingPointZero(Op)) {
5485 RetVal1 = DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32);
5486 RetVal2 = DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32);
5487 return;
5488 }
5489
5490 if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(Val&: Op)) {
5491 SDValue Ptr = Ld->getBasePtr();
5492 RetVal1 =
5493 DAG.getLoad(VT: MVT::i32, dl, Chain: Ld->getChain(), Ptr, PtrInfo: Ld->getPointerInfo(),
5494 Alignment: Ld->getAlign(), MMOFlags: Ld->getMemOperand()->getFlags());
5495
5496 EVT PtrType = Ptr.getValueType();
5497 SDValue NewPtr = DAG.getNode(Opcode: ISD::ADD, DL: dl,
5498 VT: PtrType, N1: Ptr, N2: DAG.getConstant(Val: 4, DL: dl, VT: PtrType));
5499 RetVal2 = DAG.getLoad(VT: MVT::i32, dl, Chain: Ld->getChain(), Ptr: NewPtr,
5500 PtrInfo: Ld->getPointerInfo().getWithOffset(O: 4),
5501 Alignment: commonAlignment(A: Ld->getAlign(), Offset: 4),
5502 MMOFlags: Ld->getMemOperand()->getFlags());
5503 return;
5504 }
5505
5506 llvm_unreachable("Unknown VFP cmp argument!");
5507}
5508
5509/// OptimizeVFPBrcond - With nnan and without daz, it's legal to optimize some
5510/// f32 and even f64 comparisons to integer ones.
5511SDValue
5512ARMTargetLowering::OptimizeVFPBrcond(SDValue Op, SelectionDAG &DAG) const {
5513 SDValue Chain = Op.getOperand(i: 0);
5514 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 1))->get();
5515 SDValue LHS = Op.getOperand(i: 2);
5516 SDValue RHS = Op.getOperand(i: 3);
5517 SDValue Dest = Op.getOperand(i: 4);
5518 SDLoc dl(Op);
5519
5520 bool LHSSeenZero = false;
5521 bool LHSOk = canChangeToInt(Op: LHS, SeenZero&: LHSSeenZero, Subtarget);
5522 bool RHSSeenZero = false;
5523 bool RHSOk = canChangeToInt(Op: RHS, SeenZero&: RHSSeenZero, Subtarget);
5524 if (LHSOk && RHSOk && (LHSSeenZero || RHSSeenZero)) {
5525 // If unsafe fp math optimization is enabled and there are no other uses of
5526 // the CMP operands, and the condition code is EQ or NE, we can optimize it
5527 // to an integer comparison.
5528 if (CC == ISD::SETOEQ)
5529 CC = ISD::SETEQ;
5530 else if (CC == ISD::SETUNE)
5531 CC = ISD::SETNE;
5532
5533 SDValue Mask = DAG.getConstant(Val: 0x7fffffff, DL: dl, VT: MVT::i32);
5534 SDValue ARMcc;
5535 if (LHS.getValueType() == MVT::f32) {
5536 LHS = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32,
5537 N1: bitcastf32Toi32(Op: LHS, DAG), N2: Mask);
5538 RHS = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32,
5539 N1: bitcastf32Toi32(Op: RHS, DAG), N2: Mask);
5540 SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMcc, DAG, dl);
5541 return DAG.getNode(Opcode: ARMISD::BRCOND, DL: dl, VT: MVT::Other, N1: Chain, N2: Dest, N3: ARMcc,
5542 N4: Cmp);
5543 }
5544
5545 SDValue LHS1, LHS2;
5546 SDValue RHS1, RHS2;
5547 expandf64Toi32(Op: LHS, DAG, RetVal1&: LHS1, RetVal2&: LHS2);
5548 expandf64Toi32(Op: RHS, DAG, RetVal1&: RHS1, RetVal2&: RHS2);
5549 LHS2 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32, N1: LHS2, N2: Mask);
5550 RHS2 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32, N1: RHS2, N2: Mask);
5551 ARMCC::CondCodes CondCode = IntCCToARMCC(CC);
5552 ARMcc = DAG.getConstant(Val: CondCode, DL: dl, VT: MVT::i32);
5553 SDValue Ops[] = { Chain, ARMcc, LHS1, LHS2, RHS1, RHS2, Dest };
5554 return DAG.getNode(Opcode: ARMISD::BCC_i64, DL: dl, VT: MVT::Other, Ops);
5555 }
5556
5557 return SDValue();
5558}
5559
5560// Generate CMP + CMOV for integer abs.
5561SDValue ARMTargetLowering::LowerABS(SDValue Op, SelectionDAG &DAG) const {
5562 SDLoc DL(Op);
5563
5564 SDValue Neg = DAG.getNegative(Val: Op.getOperand(i: 0), DL, VT: MVT::i32);
5565
5566 // Generate CMP & CMOV.
5567 SDValue Cmp = DAG.getNode(Opcode: ARMISD::CMP, DL, VT: FlagsVT, N1: Op.getOperand(i: 0),
5568 N2: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
5569 return DAG.getNode(Opcode: ARMISD::CMOV, DL, VT: MVT::i32, N1: Op.getOperand(i: 0), N2: Neg,
5570 N3: DAG.getConstant(Val: ARMCC::MI, DL, VT: MVT::i32), N4: Cmp);
5571}
5572
5573static SDValue getInvertedARMCondCode(SDValue ARMcc, SelectionDAG &DAG) {
5574 ARMCC::CondCodes CondCode =
5575 (ARMCC::CondCodes)cast<ConstantSDNode>(Val&: ARMcc)->getZExtValue();
5576 CondCode = ARMCC::getOppositeCondition(CC: CondCode);
5577 return DAG.getConstant(Val: CondCode, DL: SDLoc(ARMcc), VT: MVT::i32);
5578}
5579
5580SDValue ARMTargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
5581 SDValue Chain = Op.getOperand(i: 0);
5582 SDValue Cond = Op.getOperand(i: 1);
5583 SDValue Dest = Op.getOperand(i: 2);
5584 SDLoc dl(Op);
5585
5586 // Optimize {s|u}{add|sub|mul}.with.overflow feeding into a branch
5587 // instruction.
5588 unsigned Opc = Cond.getOpcode();
5589 bool OptimizeMul = (Opc == ISD::SMULO || Opc == ISD::UMULO) &&
5590 !Subtarget->isThumb1Only();
5591 if (Cond.getResNo() == 1 &&
5592 (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO ||
5593 Opc == ISD::USUBO || OptimizeMul)) {
5594 // Only lower legal XALUO ops.
5595 if (!isTypeLegal(VT: Cond->getValueType(ResNo: 0)))
5596 return SDValue();
5597
5598 // The actual operation with overflow check.
5599 SDValue Value, OverflowCmp;
5600 SDValue ARMcc;
5601 std::tie(args&: Value, args&: OverflowCmp) = getARMXALUOOp(Op: Cond, DAG, ARMcc);
5602
5603 // Reverse the condition code.
5604 ARMcc = getInvertedARMCondCode(ARMcc, DAG);
5605
5606 return DAG.getNode(Opcode: ARMISD::BRCOND, DL: dl, VT: MVT::Other, N1: Chain, N2: Dest, N3: ARMcc,
5607 N4: OverflowCmp);
5608 }
5609
5610 return SDValue();
5611}
5612
5613SDValue ARMTargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
5614 SDValue Chain = Op.getOperand(i: 0);
5615 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 1))->get();
5616 SDValue LHS = Op.getOperand(i: 2);
5617 SDValue RHS = Op.getOperand(i: 3);
5618 SDValue Dest = Op.getOperand(i: 4);
5619 SDLoc dl(Op);
5620
5621 if (isUnsupportedFloatingType(VT: LHS.getValueType())) {
5622 softenSetCCOperands(DAG, VT: LHS.getValueType(), NewLHS&: LHS, NewRHS&: RHS, CCCode&: CC, DL: dl, OldLHS: LHS, OldRHS: RHS);
5623
5624 // If softenSetCCOperands only returned one value, we should compare it to
5625 // zero.
5626 if (!RHS.getNode()) {
5627 RHS = DAG.getConstant(Val: 0, DL: dl, VT: LHS.getValueType());
5628 CC = ISD::SETNE;
5629 }
5630 }
5631
5632 // Optimize {s|u}{add|sub|mul}.with.overflow feeding into a branch
5633 // instruction.
5634 unsigned Opc = LHS.getOpcode();
5635 bool OptimizeMul = (Opc == ISD::SMULO || Opc == ISD::UMULO) &&
5636 !Subtarget->isThumb1Only();
5637 if (LHS.getResNo() == 1 && (isOneConstant(V: RHS) || isNullConstant(V: RHS)) &&
5638 (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO ||
5639 Opc == ISD::USUBO || OptimizeMul) &&
5640 (CC == ISD::SETEQ || CC == ISD::SETNE)) {
5641 // Only lower legal XALUO ops.
5642 if (!isTypeLegal(VT: LHS->getValueType(ResNo: 0)))
5643 return SDValue();
5644
5645 // The actual operation with overflow check.
5646 SDValue Value, OverflowCmp;
5647 SDValue ARMcc;
5648 std::tie(args&: Value, args&: OverflowCmp) = getARMXALUOOp(Op: LHS.getValue(R: 0), DAG, ARMcc);
5649
5650 if ((CC == ISD::SETNE) != isOneConstant(V: RHS)) {
5651 // Reverse the condition code.
5652 ARMcc = getInvertedARMCondCode(ARMcc, DAG);
5653 }
5654
5655 return DAG.getNode(Opcode: ARMISD::BRCOND, DL: dl, VT: MVT::Other, N1: Chain, N2: Dest, N3: ARMcc,
5656 N4: OverflowCmp);
5657 }
5658
5659 if (LHS.getValueType() == MVT::i32) {
5660 SDValue ARMcc;
5661 SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMcc, DAG, dl);
5662 return DAG.getNode(Opcode: ARMISD::BRCOND, DL: dl, VT: MVT::Other, N1: Chain, N2: Dest, N3: ARMcc, N4: Cmp);
5663 }
5664
5665 SDNodeFlags Flags = Op->getFlags();
5666 if (Flags.hasNoNaNs() &&
5667 DAG.getDenormalMode(VT: MVT::f32) == DenormalMode::getIEEE() &&
5668 DAG.getDenormalMode(VT: MVT::f64) == DenormalMode::getIEEE() &&
5669 (CC == ISD::SETEQ || CC == ISD::SETOEQ || CC == ISD::SETNE ||
5670 CC == ISD::SETUNE)) {
5671 if (SDValue Result = OptimizeVFPBrcond(Op, DAG))
5672 return Result;
5673 }
5674
5675 ARMCC::CondCodes CondCode, CondCode2;
5676 FPCCToARMCC(CC, CondCode, CondCode2);
5677
5678 SDValue ARMcc = DAG.getConstant(Val: CondCode, DL: dl, VT: MVT::i32);
5679 SDValue Cmp = getVFPCmp(LHS, RHS, DAG, dl);
5680 SDValue Ops[] = {Chain, Dest, ARMcc, Cmp};
5681 SDValue Res = DAG.getNode(Opcode: ARMISD::BRCOND, DL: dl, VT: MVT::Other, Ops);
5682 if (CondCode2 != ARMCC::AL) {
5683 ARMcc = DAG.getConstant(Val: CondCode2, DL: dl, VT: MVT::i32);
5684 SDValue Ops[] = {Res, Dest, ARMcc, Cmp};
5685 Res = DAG.getNode(Opcode: ARMISD::BRCOND, DL: dl, VT: MVT::Other, Ops);
5686 }
5687 return Res;
5688}
5689
5690SDValue ARMTargetLowering::LowerBR_JT(SDValue Op, SelectionDAG &DAG) const {
5691 SDValue Chain = Op.getOperand(i: 0);
5692 SDValue Table = Op.getOperand(i: 1);
5693 SDValue Index = Op.getOperand(i: 2);
5694 SDLoc dl(Op);
5695
5696 EVT PTy = getPointerTy(DL: DAG.getDataLayout());
5697 JumpTableSDNode *JT = cast<JumpTableSDNode>(Val&: Table);
5698 SDValue JTI = DAG.getTargetJumpTable(JTI: JT->getIndex(), VT: PTy);
5699 Table = DAG.getNode(Opcode: ARMISD::WrapperJT, DL: dl, VT: MVT::i32, Operand: JTI);
5700 Index = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: PTy, N1: Index, N2: DAG.getConstant(Val: 4, DL: dl, VT: PTy));
5701 SDValue Addr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PTy, N1: Table, N2: Index);
5702 if (Subtarget->isThumb2() || (Subtarget->hasV8MBaselineOps() && Subtarget->isThumb())) {
5703 // Thumb2 and ARMv8-M use a two-level jump. That is, it jumps into the jump table
5704 // which does another jump to the destination. This also makes it easier
5705 // to translate it to TBB / TBH later (Thumb2 only).
5706 // FIXME: This might not work if the function is extremely large.
5707 return DAG.getNode(Opcode: ARMISD::BR2_JT, DL: dl, VT: MVT::Other, N1: Chain,
5708 N2: Addr, N3: Op.getOperand(i: 2), N4: JTI);
5709 }
5710 if (isPositionIndependent() || Subtarget->isROPI()) {
5711 Addr =
5712 DAG.getLoad(VT: (EVT)MVT::i32, dl, Chain, Ptr: Addr,
5713 PtrInfo: MachinePointerInfo::getJumpTable(MF&: DAG.getMachineFunction()));
5714 Chain = Addr.getValue(R: 1);
5715 Addr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PTy, N1: Table, N2: Addr);
5716 return DAG.getNode(Opcode: ARMISD::BR_JT, DL: dl, VT: MVT::Other, N1: Chain, N2: Addr, N3: JTI);
5717 } else {
5718 Addr =
5719 DAG.getLoad(VT: PTy, dl, Chain, Ptr: Addr,
5720 PtrInfo: MachinePointerInfo::getJumpTable(MF&: DAG.getMachineFunction()));
5721 Chain = Addr.getValue(R: 1);
5722 return DAG.getNode(Opcode: ARMISD::BR_JT, DL: dl, VT: MVT::Other, N1: Chain, N2: Addr, N3: JTI);
5723 }
5724}
5725
5726static SDValue LowerVectorFP_TO_INT(SDValue Op, SelectionDAG &DAG) {
5727 EVT VT = Op.getValueType();
5728 SDLoc dl(Op);
5729
5730 if (Op.getValueType().getVectorElementType() == MVT::i32) {
5731 if (Op.getOperand(i: 0).getValueType().getVectorElementType() == MVT::f32)
5732 return Op;
5733 return DAG.UnrollVectorOp(N: Op.getNode());
5734 }
5735
5736 const bool HasFullFP16 = DAG.getSubtarget<ARMSubtarget>().hasFullFP16();
5737
5738 EVT NewTy;
5739 const EVT OpTy = Op.getOperand(i: 0).getValueType();
5740 if (OpTy == MVT::v4f32)
5741 NewTy = MVT::v4i32;
5742 else if (OpTy == MVT::v4f16 && HasFullFP16)
5743 NewTy = MVT::v4i16;
5744 else if (OpTy == MVT::v8f16 && HasFullFP16)
5745 NewTy = MVT::v8i16;
5746 else
5747 llvm_unreachable("Invalid type for custom lowering!");
5748
5749 if (VT != MVT::v4i16 && VT != MVT::v8i16)
5750 return DAG.UnrollVectorOp(N: Op.getNode());
5751
5752 Op = DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT: NewTy, Operand: Op.getOperand(i: 0));
5753 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: Op);
5754}
5755
5756SDValue ARMTargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const {
5757 EVT VT = Op.getValueType();
5758 if (VT.isVector())
5759 return LowerVectorFP_TO_INT(Op, DAG);
5760
5761 bool IsStrict = Op->isStrictFPOpcode();
5762 SDValue SrcVal = Op.getOperand(i: IsStrict ? 1 : 0);
5763
5764 if (isUnsupportedFloatingType(VT: SrcVal.getValueType())) {
5765 RTLIB::Libcall LC;
5766 if (Op.getOpcode() == ISD::FP_TO_SINT ||
5767 Op.getOpcode() == ISD::STRICT_FP_TO_SINT)
5768 LC = RTLIB::getFPTOSINT(OpVT: SrcVal.getValueType(),
5769 RetVT: Op.getValueType());
5770 else
5771 LC = RTLIB::getFPTOUINT(OpVT: SrcVal.getValueType(),
5772 RetVT: Op.getValueType());
5773 SDLoc Loc(Op);
5774 MakeLibCallOptions CallOptions;
5775 SDValue Chain = IsStrict ? Op.getOperand(i: 0) : SDValue();
5776 SDValue Result;
5777 std::tie(args&: Result, args&: Chain) = makeLibCall(DAG, LC, RetVT: Op.getValueType(), Ops: SrcVal,
5778 CallOptions, dl: Loc, Chain);
5779 return IsStrict ? DAG.getMergeValues(Ops: {Result, Chain}, dl: Loc) : Result;
5780 }
5781
5782 return Op;
5783}
5784
5785static SDValue LowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG,
5786 const ARMSubtarget *Subtarget) {
5787 EVT VT = Op.getValueType();
5788 EVT ToVT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
5789 EVT FromVT = Op.getOperand(i: 0).getValueType();
5790
5791 if (VT == MVT::i32 && ToVT == MVT::i32 && FromVT == MVT::f32)
5792 return Op;
5793 if (VT == MVT::i32 && ToVT == MVT::i32 && FromVT == MVT::f64 &&
5794 Subtarget->hasFP64())
5795 return Op;
5796 if (VT == MVT::i32 && ToVT == MVT::i32 && FromVT == MVT::f16 &&
5797 Subtarget->hasFullFP16())
5798 return Op;
5799 if (VT == MVT::v4i32 && ToVT == MVT::i32 && FromVT == MVT::v4f32 &&
5800 Subtarget->hasMVEFloatOps())
5801 return Op;
5802 if (VT == MVT::v8i16 && ToVT == MVT::i16 && FromVT == MVT::v8f16 &&
5803 Subtarget->hasMVEFloatOps())
5804 return Op;
5805
5806 if (FromVT != MVT::v4f32 && FromVT != MVT::v8f16)
5807 return SDValue();
5808
5809 SDLoc DL(Op);
5810 bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT_SAT;
5811 unsigned BW = ToVT.getScalarSizeInBits() - IsSigned;
5812 SDValue CVT = DAG.getNode(Opcode: Op.getOpcode(), DL, VT, N1: Op.getOperand(i: 0),
5813 N2: DAG.getValueType(VT.getScalarType()));
5814 SDValue Max = DAG.getNode(Opcode: IsSigned ? ISD::SMIN : ISD::UMIN, DL, VT, N1: CVT,
5815 N2: DAG.getConstant(Val: (1 << BW) - 1, DL, VT));
5816 if (IsSigned)
5817 Max = DAG.getNode(Opcode: ISD::SMAX, DL, VT, N1: Max,
5818 N2: DAG.getSignedConstant(Val: -(1 << BW), DL, VT));
5819 return Max;
5820}
5821
5822static SDValue LowerVectorINT_TO_FP(SDValue Op, SelectionDAG &DAG) {
5823 EVT VT = Op.getValueType();
5824 SDLoc dl(Op);
5825
5826 if (Op.getOperand(i: 0).getValueType().getVectorElementType() == MVT::i32) {
5827 if (VT.getVectorElementType() == MVT::f32)
5828 return Op;
5829 return DAG.UnrollVectorOp(N: Op.getNode());
5830 }
5831
5832 assert((Op.getOperand(0).getValueType() == MVT::v4i16 ||
5833 Op.getOperand(0).getValueType() == MVT::v8i16) &&
5834 "Invalid type for custom lowering!");
5835
5836 const bool HasFullFP16 = DAG.getSubtarget<ARMSubtarget>().hasFullFP16();
5837
5838 EVT DestVecType;
5839 if (VT == MVT::v4f32)
5840 DestVecType = MVT::v4i32;
5841 else if (VT == MVT::v4f16 && HasFullFP16)
5842 DestVecType = MVT::v4i16;
5843 else if (VT == MVT::v8f16 && HasFullFP16)
5844 DestVecType = MVT::v8i16;
5845 else
5846 return DAG.UnrollVectorOp(N: Op.getNode());
5847
5848 unsigned CastOpc;
5849 unsigned Opc;
5850 switch (Op.getOpcode()) {
5851 default: llvm_unreachable("Invalid opcode!");
5852 case ISD::SINT_TO_FP:
5853 CastOpc = ISD::SIGN_EXTEND;
5854 Opc = ISD::SINT_TO_FP;
5855 break;
5856 case ISD::UINT_TO_FP:
5857 CastOpc = ISD::ZERO_EXTEND;
5858 Opc = ISD::UINT_TO_FP;
5859 break;
5860 }
5861
5862 Op = DAG.getNode(Opcode: CastOpc, DL: dl, VT: DestVecType, Operand: Op.getOperand(i: 0));
5863 return DAG.getNode(Opcode: Opc, DL: dl, VT, Operand: Op);
5864}
5865
5866SDValue ARMTargetLowering::LowerINT_TO_FP(SDValue Op, SelectionDAG &DAG) const {
5867 EVT VT = Op.getValueType();
5868 if (VT.isVector())
5869 return LowerVectorINT_TO_FP(Op, DAG);
5870
5871 bool IsStrict = Op->isStrictFPOpcode();
5872 SDValue SrcVal = Op.getOperand(i: IsStrict ? 1 : 0);
5873
5874 if (isUnsupportedFloatingType(VT)) {
5875 RTLIB::Libcall LC;
5876 if (Op.getOpcode() == ISD::SINT_TO_FP ||
5877 Op.getOpcode() == ISD::STRICT_SINT_TO_FP)
5878 LC = RTLIB::getSINTTOFP(OpVT: SrcVal.getValueType(), RetVT: Op.getValueType());
5879 else
5880 LC = RTLIB::getUINTTOFP(OpVT: SrcVal.getValueType(), RetVT: Op.getValueType());
5881 SDLoc Loc(Op);
5882 MakeLibCallOptions CallOptions;
5883 SDValue Chain = IsStrict ? Op.getOperand(i: 0) : SDValue();
5884 SDValue Result;
5885 std::tie(args&: Result, args&: Chain) = makeLibCall(DAG, LC, RetVT: Op.getValueType(), Ops: SrcVal,
5886 CallOptions, dl: Loc, Chain);
5887 return IsStrict ? DAG.getMergeValues(Ops: {Result, Chain}, dl: Loc) : Result;
5888 }
5889
5890 return Op;
5891}
5892
5893SDValue ARMTargetLowering::LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const {
5894 // Implement fcopysign with a fabs and a conditional fneg.
5895 SDValue Tmp0 = Op.getOperand(i: 0);
5896 SDValue Tmp1 = Op.getOperand(i: 1);
5897 SDLoc dl(Op);
5898 EVT VT = Op.getValueType();
5899 EVT SrcVT = Tmp1.getValueType();
5900 bool InGPR = Tmp0.getOpcode() == ISD::BITCAST ||
5901 Tmp0.getOpcode() == ARMISD::VMOVDRR;
5902 bool UseNEON = !InGPR && Subtarget->hasNEON();
5903
5904 if (UseNEON) {
5905 // Use VBSL to copy the sign bit.
5906 unsigned EncodedVal = ARM_AM::createVMOVModImm(OpCmode: 0x6, Val: 0x80);
5907 SDValue Mask = DAG.getNode(Opcode: ARMISD::VMOVIMM, DL: dl, VT: MVT::v2i32,
5908 Operand: DAG.getTargetConstant(Val: EncodedVal, DL: dl, VT: MVT::i32));
5909 EVT OpVT = (VT == MVT::f32) ? MVT::v2i32 : MVT::v1i64;
5910 if (VT == MVT::f64)
5911 Mask = DAG.getNode(Opcode: ARMISD::VSHLIMM, DL: dl, VT: OpVT,
5912 N1: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: OpVT, Operand: Mask),
5913 N2: DAG.getConstant(Val: 32, DL: dl, VT: MVT::i32));
5914 else /*if (VT == MVT::f32)*/
5915 Tmp0 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: MVT::v2f32, Operand: Tmp0);
5916 if (SrcVT == MVT::f32) {
5917 Tmp1 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: MVT::v2f32, Operand: Tmp1);
5918 if (VT == MVT::f64)
5919 Tmp1 = DAG.getNode(Opcode: ARMISD::VSHLIMM, DL: dl, VT: OpVT,
5920 N1: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: OpVT, Operand: Tmp1),
5921 N2: DAG.getConstant(Val: 32, DL: dl, VT: MVT::i32));
5922 } else if (VT == MVT::f32)
5923 Tmp1 = DAG.getNode(Opcode: ARMISD::VSHRuIMM, DL: dl, VT: MVT::v1i64,
5924 N1: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v1i64, Operand: Tmp1),
5925 N2: DAG.getConstant(Val: 32, DL: dl, VT: MVT::i32));
5926 Tmp0 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: OpVT, Operand: Tmp0);
5927 Tmp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: OpVT, Operand: Tmp1);
5928
5929 SDValue AllOnes = DAG.getTargetConstant(Val: ARM_AM::createVMOVModImm(OpCmode: 0xe, Val: 0xff),
5930 DL: dl, VT: MVT::i32);
5931 AllOnes = DAG.getNode(Opcode: ARMISD::VMOVIMM, DL: dl, VT: MVT::v8i8, Operand: AllOnes);
5932 SDValue MaskNot = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: OpVT, N1: Mask,
5933 N2: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: OpVT, Operand: AllOnes));
5934
5935 SDValue Res = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: OpVT,
5936 N1: DAG.getNode(Opcode: ISD::AND, DL: dl, VT: OpVT, N1: Tmp1, N2: Mask),
5937 N2: DAG.getNode(Opcode: ISD::AND, DL: dl, VT: OpVT, N1: Tmp0, N2: MaskNot));
5938 if (VT == MVT::f32) {
5939 Res = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v2f32, Operand: Res);
5940 Res = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: MVT::f32, N1: Res,
5941 N2: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
5942 } else {
5943 Res = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::f64, Operand: Res);
5944 }
5945
5946 return Res;
5947 }
5948
5949 // Bitcast operand 1 to i32.
5950 if (SrcVT == MVT::f64)
5951 Tmp1 = DAG.getNode(Opcode: ARMISD::VMOVRRD, DL: dl, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
5952 N: Tmp1).getValue(R: 1);
5953 Tmp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: Tmp1);
5954
5955 // Or in the signbit with integer operations.
5956 SDValue Mask1 = DAG.getConstant(Val: 0x80000000, DL: dl, VT: MVT::i32);
5957 SDValue Mask2 = DAG.getConstant(Val: 0x7fffffff, DL: dl, VT: MVT::i32);
5958 Tmp1 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32, N1: Tmp1, N2: Mask1);
5959 if (VT == MVT::f32) {
5960 Tmp0 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32,
5961 N1: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: Tmp0), N2: Mask2);
5962 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::f32,
5963 Operand: DAG.getNode(Opcode: ISD::OR, DL: dl, VT: MVT::i32, N1: Tmp0, N2: Tmp1));
5964 }
5965
5966 // f64: Or the high part with signbit and then combine two parts.
5967 Tmp0 = DAG.getNode(Opcode: ARMISD::VMOVRRD, DL: dl, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
5968 N: Tmp0);
5969 SDValue Lo = Tmp0.getValue(R: 0);
5970 SDValue Hi = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32, N1: Tmp0.getValue(R: 1), N2: Mask2);
5971 Hi = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: MVT::i32, N1: Hi, N2: Tmp1);
5972 return DAG.getNode(Opcode: ARMISD::VMOVDRR, DL: dl, VT: MVT::f64, N1: Lo, N2: Hi);
5973}
5974
5975SDValue ARMTargetLowering::LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const{
5976 MachineFunction &MF = DAG.getMachineFunction();
5977 MachineFrameInfo &MFI = MF.getFrameInfo();
5978 MFI.setReturnAddressIsTaken(true);
5979
5980 EVT VT = Op.getValueType();
5981 SDLoc dl(Op);
5982 unsigned Depth = Op.getConstantOperandVal(i: 0);
5983 if (Depth) {
5984 SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
5985 SDValue Offset = DAG.getConstant(Val: 4, DL: dl, VT: MVT::i32);
5986 return DAG.getLoad(VT, dl, Chain: DAG.getEntryNode(),
5987 Ptr: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: FrameAddr, N2: Offset),
5988 PtrInfo: MachinePointerInfo());
5989 }
5990
5991 // Return LR, which contains the return address. Mark it an implicit live-in.
5992 Register Reg = MF.addLiveIn(PReg: ARM::LR, RC: getRegClassFor(VT: MVT::i32));
5993 return DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl, Reg, VT);
5994}
5995
5996SDValue ARMTargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const {
5997 const ARMBaseRegisterInfo &ARI =
5998 *static_cast<const ARMBaseRegisterInfo*>(RegInfo);
5999 MachineFunction &MF = DAG.getMachineFunction();
6000 MachineFrameInfo &MFI = MF.getFrameInfo();
6001 MFI.setFrameAddressIsTaken(true);
6002
6003 EVT VT = Op.getValueType();
6004 SDLoc dl(Op); // FIXME probably not meaningful
6005 unsigned Depth = Op.getConstantOperandVal(i: 0);
6006 Register FrameReg = ARI.getFrameRegister(MF);
6007 SDValue FrameAddr = DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl, Reg: FrameReg, VT);
6008 while (Depth--)
6009 FrameAddr = DAG.getLoad(VT, dl, Chain: DAG.getEntryNode(), Ptr: FrameAddr,
6010 PtrInfo: MachinePointerInfo());
6011 return FrameAddr;
6012}
6013
6014// FIXME? Maybe this could be a TableGen attribute on some registers and
6015// this table could be generated automatically from RegInfo.
6016Register ARMTargetLowering::getRegisterByName(const char* RegName, LLT VT,
6017 const MachineFunction &MF) const {
6018 return StringSwitch<Register>(RegName)
6019 .Case(S: "sp", Value: ARM::SP)
6020 .Default(Value: Register());
6021}
6022
6023// Result is 64 bit value so split into two 32 bit values and return as a
6024// pair of values.
6025static void ExpandREAD_REGISTER(SDNode *N, SmallVectorImpl<SDValue> &Results,
6026 SelectionDAG &DAG) {
6027 SDLoc DL(N);
6028
6029 // This function is only supposed to be called for i64 type destination.
6030 assert(N->getValueType(0) == MVT::i64
6031 && "ExpandREAD_REGISTER called for non-i64 type result.");
6032
6033 SDValue Read = DAG.getNode(Opcode: ISD::READ_REGISTER, DL,
6034 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32, VT3: MVT::Other),
6035 N1: N->getOperand(Num: 0),
6036 N2: N->getOperand(Num: 1));
6037
6038 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: MVT::i64, N1: Read.getValue(R: 0),
6039 N2: Read.getValue(R: 1)));
6040 Results.push_back(Elt: Read.getValue(R: 2)); // Chain
6041}
6042
6043/// \p BC is a bitcast that is about to be turned into a VMOVDRR.
6044/// When \p DstVT, the destination type of \p BC, is on the vector
6045/// register bank and the source of bitcast, \p Op, operates on the same bank,
6046/// it might be possible to combine them, such that everything stays on the
6047/// vector register bank.
6048/// \p return The node that would replace \p BT, if the combine
6049/// is possible.
6050static SDValue CombineVMOVDRRCandidateWithVecOp(const SDNode *BC,
6051 SelectionDAG &DAG) {
6052 SDValue Op = BC->getOperand(Num: 0);
6053 EVT DstVT = BC->getValueType(ResNo: 0);
6054
6055 // The only vector instruction that can produce a scalar (remember,
6056 // since the bitcast was about to be turned into VMOVDRR, the source
6057 // type is i64) from a vector is EXTRACT_VECTOR_ELT.
6058 // Moreover, we can do this combine only if there is one use.
6059 // Finally, if the destination type is not a vector, there is not
6060 // much point on forcing everything on the vector bank.
6061 if (!DstVT.isVector() || Op.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
6062 !Op.hasOneUse())
6063 return SDValue();
6064
6065 // If the index is not constant, we will introduce an additional
6066 // multiply that will stick.
6067 // Give up in that case.
6068 ConstantSDNode *Index = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1));
6069 if (!Index)
6070 return SDValue();
6071 unsigned DstNumElt = DstVT.getVectorNumElements();
6072
6073 // Compute the new index.
6074 const APInt &APIntIndex = Index->getAPIntValue();
6075 APInt NewIndex(APIntIndex.getBitWidth(), DstNumElt);
6076 NewIndex *= APIntIndex;
6077 // Check if the new constant index fits into i32.
6078 if (NewIndex.getBitWidth() > 32)
6079 return SDValue();
6080
6081 // vMTy bitcast(i64 extractelt vNi64 src, i32 index) ->
6082 // vMTy extractsubvector vNxMTy (bitcast vNi64 src), i32 index*M)
6083 SDLoc dl(Op);
6084 SDValue ExtractSrc = Op.getOperand(i: 0);
6085 EVT VecVT = EVT::getVectorVT(
6086 Context&: *DAG.getContext(), VT: DstVT.getScalarType(),
6087 NumElements: ExtractSrc.getValueType().getVectorNumElements() * DstNumElt);
6088 SDValue BitCast = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: VecVT, Operand: ExtractSrc);
6089 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: DstVT, N1: BitCast,
6090 N2: DAG.getConstant(Val: NewIndex.getZExtValue(), DL: dl, VT: MVT::i32));
6091}
6092
6093/// ExpandBITCAST - If the target supports VFP, this function is called to
6094/// expand a bit convert where either the source or destination type is i64 to
6095/// use a VMOVDRR or VMOVRRD node. This should not be done when the non-i64
6096/// operand type is illegal (e.g., v2f32 for a target that doesn't support
6097/// vectors), since the legalizer won't know what to do with that.
6098SDValue ARMTargetLowering::ExpandBITCAST(SDNode *N, SelectionDAG &DAG,
6099 const ARMSubtarget *Subtarget) const {
6100 SDLoc dl(N);
6101 SDValue Op = N->getOperand(Num: 0);
6102
6103 // This function is only supposed to be called for i16 and i64 types, either
6104 // as the source or destination of the bit convert.
6105 EVT SrcVT = Op.getValueType();
6106 EVT DstVT = N->getValueType(ResNo: 0);
6107
6108 if ((SrcVT == MVT::i16 || SrcVT == MVT::i32) &&
6109 (DstVT == MVT::f16 || DstVT == MVT::bf16))
6110 return MoveToHPR(dl: SDLoc(N), DAG, LocVT: MVT::i32, ValVT: DstVT.getSimpleVT(),
6111 Val: DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SDLoc(N), VT: MVT::i32, Operand: Op));
6112
6113 if ((DstVT == MVT::i16 || DstVT == MVT::i32) &&
6114 (SrcVT == MVT::f16 || SrcVT == MVT::bf16)) {
6115 if (Subtarget->hasFullFP16() && !Subtarget->hasBF16())
6116 Op = DAG.getBitcast(VT: MVT::f16, V: Op);
6117 return DAG.getNode(
6118 Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: DstVT,
6119 Operand: MoveFromHPR(dl: SDLoc(N), DAG, LocVT: MVT::i32, ValVT: SrcVT.getSimpleVT(), Val: Op));
6120 }
6121
6122 if (!(SrcVT == MVT::i64 || DstVT == MVT::i64))
6123 return SDValue();
6124
6125 // Turn i64->f64 into VMOVDRR.
6126 if (SrcVT == MVT::i64 && isTypeLegal(VT: DstVT)) {
6127 // Do not force values to GPRs (this is what VMOVDRR does for the inputs)
6128 // if we can combine the bitcast with its source.
6129 if (SDValue Val = CombineVMOVDRRCandidateWithVecOp(BC: N, DAG))
6130 return Val;
6131 SDValue Lo, Hi;
6132 std::tie(args&: Lo, args&: Hi) = DAG.SplitScalar(N: Op, DL: dl, LoVT: MVT::i32, HiVT: MVT::i32);
6133 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: DstVT,
6134 Operand: DAG.getNode(Opcode: ARMISD::VMOVDRR, DL: dl, VT: MVT::f64, N1: Lo, N2: Hi));
6135 }
6136
6137 // Turn f64->i64 into VMOVRRD.
6138 if (DstVT == MVT::i64 && isTypeLegal(VT: SrcVT)) {
6139 SDValue Cvt;
6140 if (DAG.getDataLayout().isBigEndian() && SrcVT.isVector() &&
6141 SrcVT.getVectorNumElements() > 1)
6142 Cvt = DAG.getNode(Opcode: ARMISD::VMOVRRD, DL: dl,
6143 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
6144 N: DAG.getNode(Opcode: ARMISD::VREV64, DL: dl, VT: SrcVT, Operand: Op));
6145 else
6146 Cvt = DAG.getNode(Opcode: ARMISD::VMOVRRD, DL: dl,
6147 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N: Op);
6148 // Merge the pieces into a single i64 value.
6149 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: MVT::i64, N1: Cvt, N2: Cvt.getValue(R: 1));
6150 }
6151
6152 return SDValue();
6153}
6154
6155/// getZeroVector - Returns a vector of specified type with all zero elements.
6156/// Zero vectors are used to represent vector negation and in those cases
6157/// will be implemented with the NEON VNEG instruction. However, VNEG does
6158/// not support i64 elements, so sometimes the zero vectors will need to be
6159/// explicitly constructed. Regardless, use a canonical VMOV to create the
6160/// zero vector.
6161static SDValue getZeroVector(EVT VT, SelectionDAG &DAG, const SDLoc &dl) {
6162 assert(VT.isVector() && "Expected a vector type");
6163 // The canonical modified immediate encoding of a zero vector is....0!
6164 SDValue EncodedVal = DAG.getTargetConstant(Val: 0, DL: dl, VT: MVT::i32);
6165 EVT VmovVT = VT.is128BitVector() ? MVT::v4i32 : MVT::v2i32;
6166 SDValue Vmov = DAG.getNode(Opcode: ARMISD::VMOVIMM, DL: dl, VT: VmovVT, Operand: EncodedVal);
6167 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: Vmov);
6168}
6169
6170/// LowerShiftRightParts - Lower SRA_PARTS, which returns two
6171/// i32 values and take a 2 x i32 value to shift plus a shift amount.
6172SDValue ARMTargetLowering::LowerShiftRightParts(SDValue Op,
6173 SelectionDAG &DAG) const {
6174 assert(Op.getNumOperands() == 3 && "Not a double-shift!");
6175 EVT VT = Op.getValueType();
6176 unsigned VTBits = VT.getSizeInBits();
6177 SDLoc dl(Op);
6178 SDValue ShOpLo = Op.getOperand(i: 0);
6179 SDValue ShOpHi = Op.getOperand(i: 1);
6180 SDValue ShAmt = Op.getOperand(i: 2);
6181 SDValue ARMcc;
6182 unsigned Opc = (Op.getOpcode() == ISD::SRA_PARTS) ? ISD::SRA : ISD::SRL;
6183
6184 assert(Op.getOpcode() == ISD::SRA_PARTS || Op.getOpcode() == ISD::SRL_PARTS);
6185
6186 SDValue RevShAmt = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32,
6187 N1: DAG.getConstant(Val: VTBits, DL: dl, VT: MVT::i32), N2: ShAmt);
6188 SDValue Tmp1 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: ShOpLo, N2: ShAmt);
6189 SDValue ExtraShAmt = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32, N1: ShAmt,
6190 N2: DAG.getConstant(Val: VTBits, DL: dl, VT: MVT::i32));
6191 SDValue Tmp2 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: ShOpHi, N2: RevShAmt);
6192 SDValue LoSmallShift = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp1, N2: Tmp2);
6193 SDValue LoBigShift = DAG.getNode(Opcode: Opc, DL: dl, VT, N1: ShOpHi, N2: ExtraShAmt);
6194 SDValue CmpLo = getARMCmp(LHS: ExtraShAmt, RHS: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32),
6195 CC: ISD::SETGE, ARMcc, DAG, dl);
6196 SDValue Lo =
6197 DAG.getNode(Opcode: ARMISD::CMOV, DL: dl, VT, N1: LoSmallShift, N2: LoBigShift, N3: ARMcc, N4: CmpLo);
6198
6199 SDValue HiSmallShift = DAG.getNode(Opcode: Opc, DL: dl, VT, N1: ShOpHi, N2: ShAmt);
6200 SDValue HiBigShift = Opc == ISD::SRA
6201 ? DAG.getNode(Opcode: Opc, DL: dl, VT, N1: ShOpHi,
6202 N2: DAG.getConstant(Val: VTBits - 1, DL: dl, VT))
6203 : DAG.getConstant(Val: 0, DL: dl, VT);
6204 SDValue CmpHi = getARMCmp(LHS: ExtraShAmt, RHS: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32),
6205 CC: ISD::SETGE, ARMcc, DAG, dl);
6206 SDValue Hi =
6207 DAG.getNode(Opcode: ARMISD::CMOV, DL: dl, VT, N1: HiSmallShift, N2: HiBigShift, N3: ARMcc, N4: CmpHi);
6208
6209 SDValue Ops[2] = { Lo, Hi };
6210 return DAG.getMergeValues(Ops, dl);
6211}
6212
6213/// LowerShiftLeftParts - Lower SHL_PARTS, which returns two
6214/// i32 values and take a 2 x i32 value to shift plus a shift amount.
6215SDValue ARMTargetLowering::LowerShiftLeftParts(SDValue Op,
6216 SelectionDAG &DAG) const {
6217 assert(Op.getNumOperands() == 3 && "Not a double-shift!");
6218 EVT VT = Op.getValueType();
6219 unsigned VTBits = VT.getSizeInBits();
6220 SDLoc dl(Op);
6221 SDValue ShOpLo = Op.getOperand(i: 0);
6222 SDValue ShOpHi = Op.getOperand(i: 1);
6223 SDValue ShAmt = Op.getOperand(i: 2);
6224 SDValue ARMcc;
6225
6226 assert(Op.getOpcode() == ISD::SHL_PARTS);
6227 SDValue RevShAmt = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32,
6228 N1: DAG.getConstant(Val: VTBits, DL: dl, VT: MVT::i32), N2: ShAmt);
6229 SDValue Tmp1 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: ShOpLo, N2: RevShAmt);
6230 SDValue Tmp2 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: ShOpHi, N2: ShAmt);
6231 SDValue HiSmallShift = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp1, N2: Tmp2);
6232
6233 SDValue ExtraShAmt = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32, N1: ShAmt,
6234 N2: DAG.getConstant(Val: VTBits, DL: dl, VT: MVT::i32));
6235 SDValue HiBigShift = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: ShOpLo, N2: ExtraShAmt);
6236 SDValue CmpHi = getARMCmp(LHS: ExtraShAmt, RHS: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32),
6237 CC: ISD::SETGE, ARMcc, DAG, dl);
6238 SDValue Hi =
6239 DAG.getNode(Opcode: ARMISD::CMOV, DL: dl, VT, N1: HiSmallShift, N2: HiBigShift, N3: ARMcc, N4: CmpHi);
6240
6241 SDValue CmpLo = getARMCmp(LHS: ExtraShAmt, RHS: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32),
6242 CC: ISD::SETGE, ARMcc, DAG, dl);
6243 SDValue LoSmallShift = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: ShOpLo, N2: ShAmt);
6244 SDValue Lo = DAG.getNode(Opcode: ARMISD::CMOV, DL: dl, VT, N1: LoSmallShift,
6245 N2: DAG.getConstant(Val: 0, DL: dl, VT), N3: ARMcc, N4: CmpLo);
6246
6247 SDValue Ops[2] = { Lo, Hi };
6248 return DAG.getMergeValues(Ops, dl);
6249}
6250
6251SDValue ARMTargetLowering::LowerGET_ROUNDING(SDValue Op,
6252 SelectionDAG &DAG) const {
6253 // The rounding mode is in bits 23:22 of the FPSCR.
6254 // The ARM rounding mode value to FLT_ROUNDS mapping is 0->1, 1->2, 2->3, 3->0
6255 // The formula we use to implement this is (((FPSCR + 1 << 22) >> 22) & 3)
6256 // so that the shift + and get folded into a bitfield extract.
6257 SDLoc dl(Op);
6258 SDValue Chain = Op.getOperand(i: 0);
6259 SDValue Ops[] = {Chain,
6260 DAG.getConstant(Val: Intrinsic::arm_get_fpscr, DL: dl, VT: MVT::i32)};
6261
6262 SDValue FPSCR =
6263 DAG.getNode(Opcode: ISD::INTRINSIC_W_CHAIN, DL: dl, ResultTys: {MVT::i32, MVT::Other}, Ops);
6264 Chain = FPSCR.getValue(R: 1);
6265 SDValue FltRounds = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::i32, N1: FPSCR,
6266 N2: DAG.getConstant(Val: 1U << 22, DL: dl, VT: MVT::i32));
6267 SDValue RMODE = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i32, N1: FltRounds,
6268 N2: DAG.getConstant(Val: 22, DL: dl, VT: MVT::i32));
6269 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32, N1: RMODE,
6270 N2: DAG.getConstant(Val: 3, DL: dl, VT: MVT::i32));
6271 return DAG.getMergeValues(Ops: {And, Chain}, dl);
6272}
6273
6274SDValue ARMTargetLowering::LowerSET_ROUNDING(SDValue Op,
6275 SelectionDAG &DAG) const {
6276 SDLoc DL(Op);
6277 SDValue Chain = Op->getOperand(Num: 0);
6278 SDValue RMValue = Op->getOperand(Num: 1);
6279
6280 // The rounding mode is in bits 23:22 of the FPSCR.
6281 // The llvm.set.rounding argument value to ARM rounding mode value mapping
6282 // is 0->3, 1->0, 2->1, 3->2. The formula we use to implement this is
6283 // ((arg - 1) & 3) << 22).
6284 //
6285 // It is expected that the argument of llvm.set.rounding is within the
6286 // segment [0, 3], so NearestTiesToAway (4) is not handled here. It is
6287 // responsibility of the code generated llvm.set.rounding to ensure this
6288 // condition.
6289
6290 // Calculate new value of FPSCR[23:22].
6291 RMValue = DAG.getNode(Opcode: ISD::SUB, DL, VT: MVT::i32, N1: RMValue,
6292 N2: DAG.getConstant(Val: 1, DL, VT: MVT::i32));
6293 RMValue = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: RMValue,
6294 N2: DAG.getConstant(Val: 0x3, DL, VT: MVT::i32));
6295 RMValue = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i32, N1: RMValue,
6296 N2: DAG.getConstant(Val: ARM::RoundingBitsPos, DL, VT: MVT::i32));
6297
6298 // Get current value of FPSCR.
6299 SDValue Ops[] = {Chain,
6300 DAG.getConstant(Val: Intrinsic::arm_get_fpscr, DL, VT: MVT::i32)};
6301 SDValue FPSCR =
6302 DAG.getNode(Opcode: ISD::INTRINSIC_W_CHAIN, DL, ResultTys: {MVT::i32, MVT::Other}, Ops);
6303 Chain = FPSCR.getValue(R: 1);
6304 FPSCR = FPSCR.getValue(R: 0);
6305
6306 // Put new rounding mode into FPSCR[23:22].
6307 const unsigned RMMask = ~(ARM::Rounding::rmMask << ARM::RoundingBitsPos);
6308 FPSCR = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: FPSCR,
6309 N2: DAG.getConstant(Val: RMMask, DL, VT: MVT::i32));
6310 FPSCR = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: FPSCR, N2: RMValue);
6311 SDValue Ops2[] = {
6312 Chain, DAG.getConstant(Val: Intrinsic::arm_set_fpscr, DL, VT: MVT::i32), FPSCR};
6313 return DAG.getNode(Opcode: ISD::INTRINSIC_VOID, DL, VT: MVT::Other, Ops: Ops2);
6314}
6315
6316SDValue ARMTargetLowering::LowerSET_FPMODE(SDValue Op,
6317 SelectionDAG &DAG) const {
6318 SDLoc DL(Op);
6319 SDValue Chain = Op->getOperand(Num: 0);
6320 SDValue Mode = Op->getOperand(Num: 1);
6321
6322 // Generate nodes to build:
6323 // FPSCR = (FPSCR & FPStatusBits) | (Mode & ~FPStatusBits)
6324 SDValue Ops[] = {Chain,
6325 DAG.getConstant(Val: Intrinsic::arm_get_fpscr, DL, VT: MVT::i32)};
6326 SDValue FPSCR =
6327 DAG.getNode(Opcode: ISD::INTRINSIC_W_CHAIN, DL, ResultTys: {MVT::i32, MVT::Other}, Ops);
6328 Chain = FPSCR.getValue(R: 1);
6329 FPSCR = FPSCR.getValue(R: 0);
6330
6331 SDValue FPSCRMasked =
6332 DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: FPSCR,
6333 N2: DAG.getConstant(Val: ARM::FPStatusBits, DL, VT: MVT::i32));
6334 SDValue InputMasked =
6335 DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: Mode,
6336 N2: DAG.getConstant(Val: ~ARM::FPStatusBits, DL, VT: MVT::i32));
6337 FPSCR = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: FPSCRMasked, N2: InputMasked);
6338
6339 SDValue Ops2[] = {
6340 Chain, DAG.getConstant(Val: Intrinsic::arm_set_fpscr, DL, VT: MVT::i32), FPSCR};
6341 return DAG.getNode(Opcode: ISD::INTRINSIC_VOID, DL, VT: MVT::Other, Ops: Ops2);
6342}
6343
6344SDValue ARMTargetLowering::LowerRESET_FPMODE(SDValue Op,
6345 SelectionDAG &DAG) const {
6346 SDLoc DL(Op);
6347 SDValue Chain = Op->getOperand(Num: 0);
6348
6349 // To get the default FP mode all control bits are cleared:
6350 // FPSCR = FPSCR & (FPStatusBits | FPReservedBits)
6351 SDValue Ops[] = {Chain,
6352 DAG.getConstant(Val: Intrinsic::arm_get_fpscr, DL, VT: MVT::i32)};
6353 SDValue FPSCR =
6354 DAG.getNode(Opcode: ISD::INTRINSIC_W_CHAIN, DL, ResultTys: {MVT::i32, MVT::Other}, Ops);
6355 Chain = FPSCR.getValue(R: 1);
6356 FPSCR = FPSCR.getValue(R: 0);
6357
6358 SDValue FPSCRMasked = DAG.getNode(
6359 Opcode: ISD::AND, DL, VT: MVT::i32, N1: FPSCR,
6360 N2: DAG.getConstant(Val: ARM::FPStatusBits | ARM::FPReservedBits, DL, VT: MVT::i32));
6361 SDValue Ops2[] = {Chain,
6362 DAG.getConstant(Val: Intrinsic::arm_set_fpscr, DL, VT: MVT::i32),
6363 FPSCRMasked};
6364 return DAG.getNode(Opcode: ISD::INTRINSIC_VOID, DL, VT: MVT::Other, Ops: Ops2);
6365}
6366
6367static SDValue LowerCTTZ(SDNode *N, SelectionDAG &DAG,
6368 const ARMSubtarget *ST) {
6369 SDLoc dl(N);
6370 EVT VT = N->getValueType(ResNo: 0);
6371 if (VT.isVector() && ST->hasNEON()) {
6372
6373 // Compute the least significant set bit: LSB = X & -X
6374 SDValue X = N->getOperand(Num: 0);
6375 SDValue NX = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: getZeroVector(VT, DAG, dl), N2: X);
6376 SDValue LSB = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: X, N2: NX);
6377
6378 EVT ElemTy = VT.getVectorElementType();
6379
6380 if (ElemTy == MVT::i8) {
6381 // Compute with: cttz(x) = ctpop(lsb - 1)
6382 SDValue One = DAG.getNode(Opcode: ARMISD::VMOVIMM, DL: dl, VT,
6383 Operand: DAG.getTargetConstant(Val: 1, DL: dl, VT: ElemTy));
6384 SDValue Bits = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: LSB, N2: One);
6385 return DAG.getNode(Opcode: ISD::CTPOP, DL: dl, VT, Operand: Bits);
6386 }
6387
6388 if ((ElemTy == MVT::i16 || ElemTy == MVT::i32) &&
6389 (N->getOpcode() == ISD::CTTZ_ZERO_POISON)) {
6390 // Compute with: cttz(x) = (width - 1) - ctlz(lsb), if x != 0
6391 unsigned NumBits = ElemTy.getSizeInBits();
6392 SDValue WidthMinus1 =
6393 DAG.getNode(Opcode: ARMISD::VMOVIMM, DL: dl, VT,
6394 Operand: DAG.getTargetConstant(Val: NumBits - 1, DL: dl, VT: ElemTy));
6395 SDValue CTLZ = DAG.getNode(Opcode: ISD::CTLZ, DL: dl, VT, Operand: LSB);
6396 return DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: WidthMinus1, N2: CTLZ);
6397 }
6398
6399 // Compute with: cttz(x) = ctpop(lsb - 1)
6400
6401 // Compute LSB - 1.
6402 SDValue Bits;
6403 if (ElemTy == MVT::i64) {
6404 // Load constant 0xffff'ffff'ffff'ffff to register.
6405 SDValue FF = DAG.getNode(Opcode: ARMISD::VMOVIMM, DL: dl, VT,
6406 Operand: DAG.getTargetConstant(Val: 0x1eff, DL: dl, VT: MVT::i32));
6407 Bits = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: LSB, N2: FF);
6408 } else {
6409 SDValue One = DAG.getNode(Opcode: ARMISD::VMOVIMM, DL: dl, VT,
6410 Operand: DAG.getTargetConstant(Val: 1, DL: dl, VT: ElemTy));
6411 Bits = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: LSB, N2: One);
6412 }
6413 return DAG.getNode(Opcode: ISD::CTPOP, DL: dl, VT, Operand: Bits);
6414 }
6415
6416 if (!ST->hasV6T2Ops())
6417 return SDValue();
6418
6419 SDValue rbit = DAG.getNode(Opcode: ISD::BITREVERSE, DL: dl, VT, Operand: N->getOperand(Num: 0));
6420 return DAG.getNode(Opcode: ISD::CTLZ, DL: dl, VT, Operand: rbit);
6421}
6422
6423static SDValue LowerCTPOP(SDNode *N, SelectionDAG &DAG,
6424 const ARMSubtarget *ST) {
6425 EVT VT = N->getValueType(ResNo: 0);
6426 SDLoc DL(N);
6427
6428 assert(ST->hasNEON() && "Custom ctpop lowering requires NEON.");
6429 assert((VT == MVT::v1i64 || VT == MVT::v2i64 || VT == MVT::v2i32 ||
6430 VT == MVT::v4i32 || VT == MVT::v4i16 || VT == MVT::v8i16) &&
6431 "Unexpected type for custom ctpop lowering");
6432
6433 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6434 EVT VT8Bit = VT.is64BitVector() ? MVT::v8i8 : MVT::v16i8;
6435 SDValue Res = DAG.getBitcast(VT: VT8Bit, V: N->getOperand(Num: 0));
6436 Res = DAG.getNode(Opcode: ISD::CTPOP, DL, VT: VT8Bit, Operand: Res);
6437
6438 // Widen v8i8/v16i8 CTPOP result to VT by repeatedly widening pairwise adds.
6439 unsigned EltSize = 8;
6440 unsigned NumElts = VT.is64BitVector() ? 8 : 16;
6441 while (EltSize != VT.getScalarSizeInBits()) {
6442 SmallVector<SDValue, 8> Ops;
6443 Ops.push_back(Elt: DAG.getConstant(Val: Intrinsic::arm_neon_vpaddlu, DL,
6444 VT: TLI.getPointerTy(DL: DAG.getDataLayout())));
6445 Ops.push_back(Elt: Res);
6446
6447 EltSize *= 2;
6448 NumElts /= 2;
6449 MVT WidenVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: EltSize), NumElements: NumElts);
6450 Res = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: WidenVT, Ops);
6451 }
6452
6453 return Res;
6454}
6455
6456/// Getvshiftimm - Check if this is a valid build_vector for the immediate
6457/// operand of a vector shift operation, where all the elements of the
6458/// build_vector must have the same constant integer value.
6459static bool getVShiftImm(SDValue Op, unsigned ElementBits, int64_t &Cnt) {
6460 // Ignore bit_converts.
6461 while (Op.getOpcode() == ISD::BITCAST)
6462 Op = Op.getOperand(i: 0);
6463 BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(Val: Op.getNode());
6464 APInt SplatBits, SplatUndef;
6465 unsigned SplatBitSize;
6466 bool HasAnyUndefs;
6467 if (!BVN ||
6468 !BVN->isConstantSplat(SplatValue&: SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs,
6469 MinSplatBits: ElementBits) ||
6470 SplatBitSize > ElementBits)
6471 return false;
6472 Cnt = SplatBits.getSExtValue();
6473 return true;
6474}
6475
6476/// isVShiftLImm - Check if this is a valid build_vector for the immediate
6477/// operand of a vector shift left operation. That value must be in the range:
6478/// 0 <= Value < ElementBits for a left shift; or
6479/// 0 <= Value <= ElementBits for a long left shift.
6480static bool isVShiftLImm(SDValue Op, EVT VT, bool isLong, int64_t &Cnt) {
6481 assert(VT.isVector() && "vector shift count is not a vector type");
6482 int64_t ElementBits = VT.getScalarSizeInBits();
6483 if (!getVShiftImm(Op, ElementBits, Cnt))
6484 return false;
6485 return (Cnt >= 0 && (isLong ? Cnt - 1 : Cnt) < ElementBits);
6486}
6487
6488/// isVShiftRImm - Check if this is a valid build_vector for the immediate
6489/// operand of a vector shift right operation. For a shift opcode, the value
6490/// is positive, but for an intrinsic the value count must be negative. The
6491/// absolute value must be in the range:
6492/// 1 <= |Value| <= ElementBits for a right shift; or
6493/// 1 <= |Value| <= ElementBits/2 for a narrow right shift.
6494static bool isVShiftRImm(SDValue Op, EVT VT, bool isNarrow, bool isIntrinsic,
6495 int64_t &Cnt) {
6496 assert(VT.isVector() && "vector shift count is not a vector type");
6497 int64_t ElementBits = VT.getScalarSizeInBits();
6498 if (!getVShiftImm(Op, ElementBits, Cnt))
6499 return false;
6500 if (!isIntrinsic)
6501 return (Cnt >= 1 && Cnt <= (isNarrow ? ElementBits / 2 : ElementBits));
6502 if (Cnt >= -(isNarrow ? ElementBits / 2 : ElementBits) && Cnt <= -1) {
6503 Cnt = -Cnt;
6504 return true;
6505 }
6506 return false;
6507}
6508
6509static SDValue LowerShift(SDNode *N, SelectionDAG &DAG,
6510 const ARMSubtarget *ST) {
6511 EVT VT = N->getValueType(ResNo: 0);
6512 SDLoc dl(N);
6513 int64_t Cnt;
6514
6515 if (!VT.isVector())
6516 return SDValue();
6517
6518 // We essentially have two forms here. Shift by an immediate and shift by a
6519 // vector register (there are also shift by a gpr, but that is just handled
6520 // with a tablegen pattern). We cannot easily match shift by an immediate in
6521 // tablegen so we do that here and generate a VSHLIMM/VSHRsIMM/VSHRuIMM.
6522 // For shifting by a vector, we don't have VSHR, only VSHL (which can be
6523 // signed or unsigned, and a negative shift indicates a shift right).
6524 if (N->getOpcode() == ISD::SHL) {
6525 if (isVShiftLImm(Op: N->getOperand(Num: 1), VT, isLong: false, Cnt))
6526 return DAG.getNode(Opcode: ARMISD::VSHLIMM, DL: dl, VT, N1: N->getOperand(Num: 0),
6527 N2: DAG.getConstant(Val: Cnt, DL: dl, VT: MVT::i32));
6528 return DAG.getNode(Opcode: ARMISD::VSHLu, DL: dl, VT, N1: N->getOperand(Num: 0),
6529 N2: N->getOperand(Num: 1));
6530 }
6531
6532 assert((N->getOpcode() == ISD::SRA || N->getOpcode() == ISD::SRL) &&
6533 "unexpected vector shift opcode");
6534
6535 if (isVShiftRImm(Op: N->getOperand(Num: 1), VT, isNarrow: false, isIntrinsic: false, Cnt)) {
6536 unsigned VShiftOpc =
6537 (N->getOpcode() == ISD::SRA ? ARMISD::VSHRsIMM : ARMISD::VSHRuIMM);
6538 return DAG.getNode(Opcode: VShiftOpc, DL: dl, VT, N1: N->getOperand(Num: 0),
6539 N2: DAG.getConstant(Val: Cnt, DL: dl, VT: MVT::i32));
6540 }
6541
6542 // Other right shifts we don't have operations for (we use a shift left by a
6543 // negative number).
6544 EVT ShiftVT = N->getOperand(Num: 1).getValueType();
6545 SDValue NegatedCount = DAG.getNode(
6546 Opcode: ISD::SUB, DL: dl, VT: ShiftVT, N1: getZeroVector(VT: ShiftVT, DAG, dl), N2: N->getOperand(Num: 1));
6547 unsigned VShiftOpc =
6548 (N->getOpcode() == ISD::SRA ? ARMISD::VSHLs : ARMISD::VSHLu);
6549 return DAG.getNode(Opcode: VShiftOpc, DL: dl, VT, N1: N->getOperand(Num: 0), N2: NegatedCount);
6550}
6551
6552static SDValue Expand64BitShift(SDNode *N, SelectionDAG &DAG,
6553 const ARMSubtarget *ST) {
6554 EVT VT = N->getValueType(ResNo: 0);
6555 SDLoc dl(N);
6556
6557 // We can get here for a node like i32 = ISD::SHL i32, i64
6558 if (VT != MVT::i64)
6559 return SDValue();
6560
6561 assert((N->getOpcode() == ISD::SRL || N->getOpcode() == ISD::SRA ||
6562 N->getOpcode() == ISD::SHL) &&
6563 "Unknown shift to lower!");
6564
6565 unsigned ShOpc = N->getOpcode();
6566 if (ST->hasMVEIntegerOps()) {
6567 SDValue ShAmt = N->getOperand(Num: 1);
6568 unsigned ShPartsOpc = ARMISD::LSLL;
6569 ConstantSDNode *Con = dyn_cast<ConstantSDNode>(Val&: ShAmt);
6570
6571 // If the shift amount is greater than 32 or has a greater bitwidth than 64
6572 // then do the default optimisation
6573 if ((!Con && ShAmt->getValueType(ResNo: 0).getSizeInBits() > 64) ||
6574 (Con && (Con->getAPIntValue() == 0 || Con->getAPIntValue().uge(RHS: 32))))
6575 return SDValue();
6576
6577 // Extract the lower 32 bits of the shift amount if it's not an i32
6578 if (ShAmt->getValueType(ResNo: 0) != MVT::i32)
6579 ShAmt = DAG.getZExtOrTrunc(Op: ShAmt, DL: dl, VT: MVT::i32);
6580
6581 if (ShOpc == ISD::SRL) {
6582 if (!Con)
6583 // There is no t2LSRLr instruction so negate and perform an lsll if the
6584 // shift amount is in a register, emulating a right shift.
6585 ShAmt = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32,
6586 N1: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32), N2: ShAmt);
6587 else
6588 // Else generate an lsrl on the immediate shift amount
6589 ShPartsOpc = ARMISD::LSRL;
6590 } else if (ShOpc == ISD::SRA)
6591 ShPartsOpc = ARMISD::ASRL;
6592
6593 // Split Lower/Upper 32 bits of the destination/source
6594 SDValue Lo, Hi;
6595 std::tie(args&: Lo, args&: Hi) =
6596 DAG.SplitScalar(N: N->getOperand(Num: 0), DL: dl, LoVT: MVT::i32, HiVT: MVT::i32);
6597 // Generate the shift operation as computed above
6598 Lo = DAG.getNode(Opcode: ShPartsOpc, DL: dl, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N1: Lo, N2: Hi,
6599 N3: ShAmt);
6600 // The upper 32 bits come from the second return value of lsll
6601 Hi = SDValue(Lo.getNode(), 1);
6602 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: MVT::i64, N1: Lo, N2: Hi);
6603 }
6604
6605 // We only lower SRA, SRL of 1 here, all others use generic lowering.
6606 if (!isOneConstant(V: N->getOperand(Num: 1)) || N->getOpcode() == ISD::SHL)
6607 return SDValue();
6608
6609 // If we are in thumb mode, we don't have RRX.
6610 if (ST->isThumb1Only())
6611 return SDValue();
6612
6613 // Okay, we have a 64-bit SRA or SRL of 1. Lower this to an RRX expr.
6614 SDValue Lo, Hi;
6615 std::tie(args&: Lo, args&: Hi) = DAG.SplitScalar(N: N->getOperand(Num: 0), DL: dl, LoVT: MVT::i32, HiVT: MVT::i32);
6616
6617 // First, build a LSRS1/ASRS1 op, which shifts the top part by one and
6618 // captures the shifted out bit into a carry flag.
6619 unsigned Opc = N->getOpcode() == ISD::SRL ? ARMISD::LSRS1 : ARMISD::ASRS1;
6620 Hi = DAG.getNode(Opcode: Opc, DL: dl, VTList: DAG.getVTList(VT1: MVT::i32, VT2: FlagsVT), N: Hi);
6621
6622 // The low part is an ARMISD::RRX operand, which shifts the carry in.
6623 Lo = DAG.getNode(Opcode: ARMISD::RRX, DL: dl, VT: MVT::i32, N1: Lo, N2: Hi.getValue(R: 1));
6624
6625 // Merge the pieces into a single i64 value.
6626 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: MVT::i64, N1: Lo, N2: Hi);
6627}
6628
6629static SDValue LowerVSETCC(SDValue Op, SelectionDAG &DAG,
6630 const ARMSubtarget *ST) {
6631 bool Invert = false;
6632 bool Swap = false;
6633 unsigned Opc = ARMCC::AL;
6634
6635 SDValue Op0 = Op.getOperand(i: 0);
6636 SDValue Op1 = Op.getOperand(i: 1);
6637 SDValue CC = Op.getOperand(i: 2);
6638 EVT VT = Op.getValueType();
6639 ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(Val&: CC)->get();
6640 SDLoc dl(Op);
6641
6642 EVT CmpVT;
6643 if (ST->hasNEON())
6644 CmpVT = Op0.getValueType().changeVectorElementTypeToInteger();
6645 else {
6646 assert(ST->hasMVEIntegerOps() &&
6647 "No hardware support for integer vector comparison!");
6648
6649 if (Op.getValueType().getVectorElementType() != MVT::i1)
6650 return SDValue();
6651
6652 // Make sure we expand floating point setcc to scalar if we do not have
6653 // mve.fp, so that we can handle them from there.
6654 if (Op0.getValueType().isFloatingPoint() && !ST->hasMVEFloatOps())
6655 return SDValue();
6656
6657 CmpVT = VT;
6658 }
6659
6660 if (Op0.getValueType().getVectorElementType() == MVT::i64 &&
6661 (SetCCOpcode == ISD::SETEQ || SetCCOpcode == ISD::SETNE)) {
6662 // Special-case integer 64-bit equality comparisons. They aren't legal,
6663 // but they can be lowered with a few vector instructions.
6664 unsigned CmpElements = CmpVT.getVectorNumElements() * 2;
6665 EVT SplitVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32, NumElements: CmpElements);
6666 SDValue CastOp0 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: SplitVT, Operand: Op0);
6667 SDValue CastOp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: SplitVT, Operand: Op1);
6668 SDValue Cmp = DAG.getNode(Opcode: ISD::SETCC, DL: dl, VT: SplitVT, N1: CastOp0, N2: CastOp1,
6669 N3: DAG.getCondCode(Cond: ISD::SETEQ));
6670 SDValue Reversed = DAG.getNode(Opcode: ARMISD::VREV64, DL: dl, VT: SplitVT, Operand: Cmp);
6671 SDValue Merged = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SplitVT, N1: Cmp, N2: Reversed);
6672 Merged = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: CmpVT, Operand: Merged);
6673 if (SetCCOpcode == ISD::SETNE)
6674 Merged = DAG.getNOT(DL: dl, Val: Merged, VT: CmpVT);
6675 Merged = DAG.getSExtOrTrunc(Op: Merged, DL: dl, VT);
6676 return Merged;
6677 }
6678
6679 if (CmpVT.getVectorElementType() == MVT::i64)
6680 // 64-bit comparisons are not legal in general.
6681 return SDValue();
6682
6683 if (Op1.getValueType().isFloatingPoint()) {
6684 switch (SetCCOpcode) {
6685 default: llvm_unreachable("Illegal FP comparison");
6686 case ISD::SETUNE:
6687 case ISD::SETNE:
6688 if (ST->hasMVEFloatOps()) {
6689 Opc = ARMCC::NE; break;
6690 } else {
6691 Invert = true; [[fallthrough]];
6692 }
6693 case ISD::SETOEQ:
6694 case ISD::SETEQ: Opc = ARMCC::EQ; break;
6695 case ISD::SETOLT:
6696 case ISD::SETLT: Swap = true; [[fallthrough]];
6697 case ISD::SETOGT:
6698 case ISD::SETGT: Opc = ARMCC::GT; break;
6699 case ISD::SETOLE:
6700 case ISD::SETLE: Swap = true; [[fallthrough]];
6701 case ISD::SETOGE:
6702 case ISD::SETGE: Opc = ARMCC::GE; break;
6703 case ISD::SETUGE: Swap = true; [[fallthrough]];
6704 case ISD::SETULE: Invert = true; Opc = ARMCC::GT; break;
6705 case ISD::SETUGT: Swap = true; [[fallthrough]];
6706 case ISD::SETULT: Invert = true; Opc = ARMCC::GE; break;
6707 case ISD::SETUEQ: Invert = true; [[fallthrough]];
6708 case ISD::SETONE: {
6709 // Expand this to (OLT | OGT).
6710 SDValue TmpOp0 = DAG.getNode(Opcode: ARMISD::VCMP, DL: dl, VT: CmpVT, N1: Op1, N2: Op0,
6711 N3: DAG.getConstant(Val: ARMCC::GT, DL: dl, VT: MVT::i32));
6712 SDValue TmpOp1 = DAG.getNode(Opcode: ARMISD::VCMP, DL: dl, VT: CmpVT, N1: Op0, N2: Op1,
6713 N3: DAG.getConstant(Val: ARMCC::GT, DL: dl, VT: MVT::i32));
6714 SDValue Result = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: CmpVT, N1: TmpOp0, N2: TmpOp1);
6715 if (Invert)
6716 Result = DAG.getNOT(DL: dl, Val: Result, VT);
6717 return Result;
6718 }
6719 case ISD::SETUO: Invert = true; [[fallthrough]];
6720 case ISD::SETO: {
6721 // Expand this to (OLT | OGE).
6722 SDValue TmpOp0 = DAG.getNode(Opcode: ARMISD::VCMP, DL: dl, VT: CmpVT, N1: Op1, N2: Op0,
6723 N3: DAG.getConstant(Val: ARMCC::GT, DL: dl, VT: MVT::i32));
6724 SDValue TmpOp1 = DAG.getNode(Opcode: ARMISD::VCMP, DL: dl, VT: CmpVT, N1: Op0, N2: Op1,
6725 N3: DAG.getConstant(Val: ARMCC::GE, DL: dl, VT: MVT::i32));
6726 SDValue Result = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: CmpVT, N1: TmpOp0, N2: TmpOp1);
6727 if (Invert)
6728 Result = DAG.getNOT(DL: dl, Val: Result, VT);
6729 return Result;
6730 }
6731 }
6732 } else {
6733 // Integer comparisons.
6734 switch (SetCCOpcode) {
6735 default: llvm_unreachable("Illegal integer comparison");
6736 case ISD::SETNE:
6737 if (ST->hasMVEIntegerOps()) {
6738 Opc = ARMCC::NE; break;
6739 } else {
6740 Invert = true; [[fallthrough]];
6741 }
6742 case ISD::SETEQ: Opc = ARMCC::EQ; break;
6743 case ISD::SETLT: Swap = true; [[fallthrough]];
6744 case ISD::SETGT: Opc = ARMCC::GT; break;
6745 case ISD::SETLE: Swap = true; [[fallthrough]];
6746 case ISD::SETGE: Opc = ARMCC::GE; break;
6747 case ISD::SETULT: Swap = true; [[fallthrough]];
6748 case ISD::SETUGT: Opc = ARMCC::HI; break;
6749 case ISD::SETULE: Swap = true; [[fallthrough]];
6750 case ISD::SETUGE: Opc = ARMCC::HS; break;
6751 }
6752
6753 // Detect VTST (Vector Test Bits) = icmp ne (and (op0, op1), zero).
6754 if (ST->hasNEON() && Opc == ARMCC::EQ) {
6755 SDValue AndOp;
6756 if (ISD::isBuildVectorAllZeros(N: Op1.getNode()))
6757 AndOp = Op0;
6758 else if (ISD::isBuildVectorAllZeros(N: Op0.getNode()))
6759 AndOp = Op1;
6760
6761 // Ignore bitconvert.
6762 if (AndOp.getNode() && AndOp.getOpcode() == ISD::BITCAST)
6763 AndOp = AndOp.getOperand(i: 0);
6764
6765 if (AndOp.getNode() && AndOp.getOpcode() == ISD::AND) {
6766 Op0 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: CmpVT, Operand: AndOp.getOperand(i: 0));
6767 Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: CmpVT, Operand: AndOp.getOperand(i: 1));
6768 SDValue Result = DAG.getNode(Opcode: ARMISD::VTST, DL: dl, VT: CmpVT, N1: Op0, N2: Op1);
6769 if (!Invert)
6770 Result = DAG.getNOT(DL: dl, Val: Result, VT);
6771 return Result;
6772 }
6773 }
6774 }
6775
6776 if (Swap)
6777 std::swap(a&: Op0, b&: Op1);
6778
6779 // If one of the operands is a constant vector zero, attempt to fold the
6780 // comparison to a specialized compare-against-zero form.
6781 if (ISD::isBuildVectorAllZeros(N: Op0.getNode()) &&
6782 (Opc == ARMCC::GE || Opc == ARMCC::GT || Opc == ARMCC::EQ ||
6783 Opc == ARMCC::NE)) {
6784 if (Opc == ARMCC::GE)
6785 Opc = ARMCC::LE;
6786 else if (Opc == ARMCC::GT)
6787 Opc = ARMCC::LT;
6788 std::swap(a&: Op0, b&: Op1);
6789 }
6790
6791 SDValue Result;
6792 if (ISD::isBuildVectorAllZeros(N: Op1.getNode()) &&
6793 (Opc == ARMCC::GE || Opc == ARMCC::GT || Opc == ARMCC::LE ||
6794 Opc == ARMCC::LT || Opc == ARMCC::NE || Opc == ARMCC::EQ))
6795 Result = DAG.getNode(Opcode: ARMISD::VCMPZ, DL: dl, VT: CmpVT, N1: Op0,
6796 N2: DAG.getConstant(Val: Opc, DL: dl, VT: MVT::i32));
6797 else
6798 Result = DAG.getNode(Opcode: ARMISD::VCMP, DL: dl, VT: CmpVT, N1: Op0, N2: Op1,
6799 N3: DAG.getConstant(Val: Opc, DL: dl, VT: MVT::i32));
6800
6801 Result = DAG.getSExtOrTrunc(Op: Result, DL: dl, VT);
6802
6803 if (Invert)
6804 Result = DAG.getNOT(DL: dl, Val: Result, VT);
6805
6806 return Result;
6807}
6808
6809static SDValue LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG) {
6810 SDValue LHS = Op.getOperand(i: 0);
6811 SDValue RHS = Op.getOperand(i: 1);
6812
6813 assert(LHS.getSimpleValueType().isInteger() && "SETCCCARRY is integer only.");
6814
6815 SDValue Carry = Op.getOperand(i: 2);
6816 SDValue Cond = Op.getOperand(i: 3);
6817 SDLoc DL(Op);
6818
6819 // ARMISD::SUBE expects a carry not a borrow like ISD::USUBO_CARRY so we
6820 // have to invert the carry first.
6821 SDValue InvCarry = valueToCarryFlag(Value: Carry, DAG, Invert: true);
6822
6823 SDVTList VTs = DAG.getVTList(VT1: LHS.getValueType(), VT2: MVT::i32);
6824 SDValue Cmp = DAG.getNode(Opcode: ARMISD::SUBE, DL, VTList: VTs, N1: LHS, N2: RHS, N3: InvCarry);
6825
6826 SDValue FVal = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
6827 SDValue TVal = DAG.getConstant(Val: 1, DL, VT: MVT::i32);
6828 SDValue ARMcc = DAG.getConstant(
6829 Val: IntCCToARMCC(CC: cast<CondCodeSDNode>(Val&: Cond)->get()), DL, VT: MVT::i32);
6830 return DAG.getNode(Opcode: ARMISD::CMOV, DL, VT: Op.getValueType(), N1: FVal, N2: TVal, N3: ARMcc,
6831 N4: Cmp.getValue(R: 1));
6832}
6833
6834/// isVMOVModifiedImm - Check if the specified splat value corresponds to a
6835/// valid vector constant for a NEON or MVE instruction with a "modified
6836/// immediate" operand (e.g., VMOV). If so, return the encoded value.
6837static SDValue isVMOVModifiedImm(uint64_t SplatBits, uint64_t SplatUndef,
6838 unsigned SplatBitSize, SelectionDAG &DAG,
6839 const SDLoc &dl, EVT &VT, EVT VectorVT,
6840 VMOVModImmType type) {
6841 unsigned OpCmode, Imm;
6842 bool is128Bits = VectorVT.is128BitVector();
6843
6844 // SplatBitSize is set to the smallest size that splats the vector, so a
6845 // zero vector will always have SplatBitSize == 8. However, NEON modified
6846 // immediate instructions others than VMOV do not support the 8-bit encoding
6847 // of a zero vector, and the default encoding of zero is supposed to be the
6848 // 32-bit version.
6849 if (SplatBits == 0)
6850 SplatBitSize = 32;
6851
6852 switch (SplatBitSize) {
6853 case 8:
6854 if (type != VMOVModImm)
6855 return SDValue();
6856 // Any 1-byte value is OK. Op=0, Cmode=1110.
6857 assert((SplatBits & ~0xff) == 0 && "one byte splat value is too big");
6858 OpCmode = 0xe;
6859 Imm = SplatBits;
6860 VT = is128Bits ? MVT::v16i8 : MVT::v8i8;
6861 break;
6862
6863 case 16:
6864 // NEON's 16-bit VMOV supports splat values where only one byte is nonzero.
6865 VT = is128Bits ? MVT::v8i16 : MVT::v4i16;
6866 if ((SplatBits & ~0xff) == 0) {
6867 // Value = 0x00nn: Op=x, Cmode=100x.
6868 OpCmode = 0x8;
6869 Imm = SplatBits;
6870 break;
6871 }
6872 if ((SplatBits & ~0xff00) == 0) {
6873 // Value = 0xnn00: Op=x, Cmode=101x.
6874 OpCmode = 0xa;
6875 Imm = SplatBits >> 8;
6876 break;
6877 }
6878 return SDValue();
6879
6880 case 32:
6881 // NEON's 32-bit VMOV supports splat values where:
6882 // * only one byte is nonzero, or
6883 // * the least significant byte is 0xff and the second byte is nonzero, or
6884 // * the least significant 2 bytes are 0xff and the third is nonzero.
6885 VT = is128Bits ? MVT::v4i32 : MVT::v2i32;
6886 if ((SplatBits & ~0xff) == 0) {
6887 // Value = 0x000000nn: Op=x, Cmode=000x.
6888 OpCmode = 0;
6889 Imm = SplatBits;
6890 break;
6891 }
6892 if ((SplatBits & ~0xff00) == 0) {
6893 // Value = 0x0000nn00: Op=x, Cmode=001x.
6894 OpCmode = 0x2;
6895 Imm = SplatBits >> 8;
6896 break;
6897 }
6898 if ((SplatBits & ~0xff0000) == 0) {
6899 // Value = 0x00nn0000: Op=x, Cmode=010x.
6900 OpCmode = 0x4;
6901 Imm = SplatBits >> 16;
6902 break;
6903 }
6904 if ((SplatBits & ~0xff000000) == 0) {
6905 // Value = 0xnn000000: Op=x, Cmode=011x.
6906 OpCmode = 0x6;
6907 Imm = SplatBits >> 24;
6908 break;
6909 }
6910
6911 // cmode == 0b1100 and cmode == 0b1101 are not supported for VORR or VBIC
6912 if (type == OtherModImm) return SDValue();
6913
6914 if ((SplatBits & ~0xffff) == 0 &&
6915 ((SplatBits | SplatUndef) & 0xff) == 0xff) {
6916 // Value = 0x0000nnff: Op=x, Cmode=1100.
6917 OpCmode = 0xc;
6918 Imm = SplatBits >> 8;
6919 break;
6920 }
6921
6922 // cmode == 0b1101 is not supported for MVE VMVN
6923 if (type == MVEVMVNModImm)
6924 return SDValue();
6925
6926 if ((SplatBits & ~0xffffff) == 0 &&
6927 ((SplatBits | SplatUndef) & 0xffff) == 0xffff) {
6928 // Value = 0x00nnffff: Op=x, Cmode=1101.
6929 OpCmode = 0xd;
6930 Imm = SplatBits >> 16;
6931 break;
6932 }
6933
6934 // Note: there are a few 32-bit splat values (specifically: 00ffff00,
6935 // ff000000, ff0000ff, and ffff00ff) that are valid for VMOV.I64 but not
6936 // VMOV.I32. A (very) minor optimization would be to replicate the value
6937 // and fall through here to test for a valid 64-bit splat. But, then the
6938 // caller would also need to check and handle the change in size.
6939 return SDValue();
6940
6941 case 64: {
6942 if (type != VMOVModImm)
6943 return SDValue();
6944 // NEON has a 64-bit VMOV splat where each byte is either 0 or 0xff.
6945 uint64_t BitMask = 0xff;
6946 unsigned ImmMask = 1;
6947 Imm = 0;
6948 for (int ByteNum = 0; ByteNum < 8; ++ByteNum) {
6949 if (((SplatBits | SplatUndef) & BitMask) == BitMask) {
6950 Imm |= ImmMask;
6951 } else if ((SplatBits & BitMask) != 0) {
6952 return SDValue();
6953 }
6954 BitMask <<= 8;
6955 ImmMask <<= 1;
6956 }
6957
6958 // Op=1, Cmode=1110.
6959 OpCmode = 0x1e;
6960 VT = is128Bits ? MVT::v2i64 : MVT::v1i64;
6961 break;
6962 }
6963
6964 default:
6965 llvm_unreachable("unexpected size for isVMOVModifiedImm");
6966 }
6967
6968 unsigned EncodedVal = ARM_AM::createVMOVModImm(OpCmode, Val: Imm);
6969 return DAG.getTargetConstant(Val: EncodedVal, DL: dl, VT: MVT::i32);
6970}
6971
6972SDValue ARMTargetLowering::LowerConstantFP(SDValue Op, SelectionDAG &DAG,
6973 const ARMSubtarget *ST) const {
6974 EVT VT = Op.getValueType();
6975 bool IsDouble = (VT == MVT::f64);
6976 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Val&: Op);
6977 const APFloat &FPVal = CFP->getValueAPF();
6978
6979 // Prevent floating-point constants from using literal loads
6980 // when execute-only is enabled.
6981 if (ST->genExecuteOnly()) {
6982 // We shouldn't trigger this for v6m execute-only
6983 assert((!ST->isThumb1Only() || ST->hasV8MBaselineOps()) &&
6984 "Unexpected architecture");
6985
6986 // If we can represent the constant as an immediate, don't lower it
6987 if (isFPImmLegal(Imm: FPVal, VT))
6988 return Op;
6989 // Otherwise, construct as integer, and move to float register
6990 APInt INTVal = FPVal.bitcastToAPInt();
6991 SDLoc DL(CFP);
6992 switch (VT.getSimpleVT().SimpleTy) {
6993 default:
6994 llvm_unreachable("Unknown floating point type!");
6995 break;
6996 case MVT::f64: {
6997 SDValue Lo = DAG.getConstant(Val: INTVal.trunc(width: 32), DL, VT: MVT::i32);
6998 SDValue Hi = DAG.getConstant(Val: INTVal.lshr(shiftAmt: 32).trunc(width: 32), DL, VT: MVT::i32);
6999 return DAG.getNode(Opcode: ARMISD::VMOVDRR, DL, VT: MVT::f64, N1: Lo, N2: Hi);
7000 }
7001 case MVT::f32:
7002 return DAG.getNode(Opcode: ARMISD::VMOVSR, DL, VT,
7003 Operand: DAG.getConstant(Val: INTVal, DL, VT: MVT::i32));
7004 }
7005 }
7006
7007 if (!ST->hasVFP3Base())
7008 return SDValue();
7009
7010 // Use the default (constant pool) lowering for double constants when we have
7011 // an SP-only FPU
7012 if (IsDouble && !Subtarget->hasFP64())
7013 return SDValue();
7014
7015 // Try splatting with a VMOV.f32...
7016 int ImmVal = IsDouble ? ARM_AM::getFP64Imm(FPImm: FPVal) : ARM_AM::getFP32Imm(FPImm: FPVal);
7017
7018 if (ImmVal != -1) {
7019 if (IsDouble || !ST->useNEONForSinglePrecisionFP()) {
7020 // We have code in place to select a valid ConstantFP already, no need to
7021 // do any mangling.
7022 return Op;
7023 }
7024
7025 // It's a float and we are trying to use NEON operations where
7026 // possible. Lower it to a splat followed by an extract.
7027 SDLoc DL(Op);
7028 SDValue NewVal = DAG.getTargetConstant(Val: ImmVal, DL, VT: MVT::i32);
7029 SDValue VecConstant = DAG.getNode(Opcode: ARMISD::VMOVFPIMM, DL, VT: MVT::v2f32,
7030 Operand: NewVal);
7031 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::f32, N1: VecConstant,
7032 N2: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
7033 }
7034
7035 // The rest of our options are NEON only, make sure that's allowed before
7036 // proceeding..
7037 if (!ST->hasNEON() || (!IsDouble && !ST->useNEONForSinglePrecisionFP()))
7038 return SDValue();
7039
7040 EVT VMovVT;
7041 uint64_t iVal = FPVal.bitcastToAPInt().getZExtValue();
7042
7043 // It wouldn't really be worth bothering for doubles except for one very
7044 // important value, which does happen to match: 0.0. So make sure we don't do
7045 // anything stupid.
7046 if (IsDouble && (iVal & 0xffffffff) != (iVal >> 32))
7047 return SDValue();
7048
7049 // Try a VMOV.i32 (FIXME: i8, i16, or i64 could work too).
7050 SDValue NewVal = isVMOVModifiedImm(SplatBits: iVal & 0xffffffffU, SplatUndef: 0, SplatBitSize: 32, DAG, dl: SDLoc(Op),
7051 VT&: VMovVT, VectorVT: VT, type: VMOVModImm);
7052 if (NewVal != SDValue()) {
7053 SDLoc DL(Op);
7054 SDValue VecConstant = DAG.getNode(Opcode: ARMISD::VMOVIMM, DL, VT: VMovVT,
7055 Operand: NewVal);
7056 if (IsDouble)
7057 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f64, Operand: VecConstant);
7058
7059 // It's a float: cast and extract a vector element.
7060 SDValue VecFConstant = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v2f32,
7061 Operand: VecConstant);
7062 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::f32, N1: VecFConstant,
7063 N2: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
7064 }
7065
7066 // Finally, try a VMVN.i32
7067 NewVal = isVMOVModifiedImm(SplatBits: ~iVal & 0xffffffffU, SplatUndef: 0, SplatBitSize: 32, DAG, dl: SDLoc(Op), VT&: VMovVT,
7068 VectorVT: VT, type: VMVNModImm);
7069 if (NewVal != SDValue()) {
7070 SDLoc DL(Op);
7071 SDValue VecConstant = DAG.getNode(Opcode: ARMISD::VMVNIMM, DL, VT: VMovVT, Operand: NewVal);
7072
7073 if (IsDouble)
7074 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f64, Operand: VecConstant);
7075
7076 // It's a float: cast and extract a vector element.
7077 SDValue VecFConstant = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v2f32,
7078 Operand: VecConstant);
7079 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::f32, N1: VecFConstant,
7080 N2: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
7081 }
7082
7083 return SDValue();
7084}
7085
7086// check if an VEXT instruction can handle the shuffle mask when the
7087// vector sources of the shuffle are the same.
7088static bool isSingletonVEXTMask(ArrayRef<int> M, EVT VT, unsigned &Imm) {
7089 unsigned NumElts = VT.getVectorNumElements();
7090
7091 // Assume that the first shuffle index is not UNDEF. Fail if it is.
7092 if (M[0] < 0)
7093 return false;
7094
7095 Imm = M[0];
7096
7097 // If this is a VEXT shuffle, the immediate value is the index of the first
7098 // element. The other shuffle indices must be the successive elements after
7099 // the first one.
7100 unsigned ExpectedElt = Imm;
7101 for (unsigned i = 1; i < NumElts; ++i) {
7102 // Increment the expected index. If it wraps around, just follow it
7103 // back to index zero and keep going.
7104 ++ExpectedElt;
7105 if (ExpectedElt == NumElts)
7106 ExpectedElt = 0;
7107
7108 if (M[i] < 0) continue; // ignore UNDEF indices
7109 if (ExpectedElt != static_cast<unsigned>(M[i]))
7110 return false;
7111 }
7112
7113 return true;
7114}
7115
7116static bool isVEXTMask(ArrayRef<int> M, EVT VT,
7117 bool &ReverseVEXT, unsigned &Imm) {
7118 unsigned NumElts = VT.getVectorNumElements();
7119 ReverseVEXT = false;
7120
7121 // Assume that the first shuffle index is not UNDEF. Fail if it is.
7122 if (M[0] < 0)
7123 return false;
7124
7125 Imm = M[0];
7126
7127 // If this is a VEXT shuffle, the immediate value is the index of the first
7128 // element. The other shuffle indices must be the successive elements after
7129 // the first one.
7130 unsigned ExpectedElt = Imm;
7131 for (unsigned i = 1; i < NumElts; ++i) {
7132 // Increment the expected index. If it wraps around, it may still be
7133 // a VEXT but the source vectors must be swapped.
7134 ExpectedElt += 1;
7135 if (ExpectedElt == NumElts * 2) {
7136 ExpectedElt = 0;
7137 ReverseVEXT = true;
7138 }
7139
7140 if (M[i] < 0) continue; // ignore UNDEF indices
7141 if (ExpectedElt != static_cast<unsigned>(M[i]))
7142 return false;
7143 }
7144
7145 // Adjust the index value if the source operands will be swapped.
7146 if (ReverseVEXT)
7147 Imm -= NumElts;
7148
7149 return true;
7150}
7151
7152static bool isVTBLMask(ArrayRef<int> M, EVT VT) {
7153 // We can handle <8 x i8> vector shuffles. If the index in the mask is out of
7154 // range, then 0 is placed into the resulting vector. So pretty much any mask
7155 // of 8 elements can work here.
7156 return VT == MVT::v8i8 && M.size() == 8;
7157}
7158
7159/// Check if \p ShuffleMask is a NEON two-result shuffle (VZIP, VUZP, VTRN),
7160/// and return the corresponding ARMISD opcode if it is, or 0 if it isn't.
7161static unsigned isNEONTwoResultShuffleMask(ArrayRef<int> ShuffleMask, EVT VT,
7162 unsigned &WhichResult,
7163 bool &isV_UNDEF) {
7164 isV_UNDEF = false;
7165 if (isVTRNMask(M: ShuffleMask, VT, WhichResult))
7166 return ARMISD::VTRN;
7167 if (isVUZPMask(M: ShuffleMask, VT, WhichResult))
7168 return ARMISD::VUZP;
7169 if (isVZIPMask(M: ShuffleMask, VT, WhichResult))
7170 return ARMISD::VZIP;
7171
7172 isV_UNDEF = true;
7173 if (isVTRN_v_undef_Mask(M: ShuffleMask, VT, WhichResult))
7174 return ARMISD::VTRN;
7175 if (isVUZP_v_undef_Mask(M: ShuffleMask, VT, WhichResult))
7176 return ARMISD::VUZP;
7177 if (isVZIP_v_undef_Mask(M: ShuffleMask, VT, WhichResult))
7178 return ARMISD::VZIP;
7179
7180 return 0;
7181}
7182
7183/// \return true if this is a reverse operation on an vector.
7184static bool isReverseMask(ArrayRef<int> M, EVT VT) {
7185 unsigned NumElts = VT.getVectorNumElements();
7186 // Make sure the mask has the right size.
7187 if (NumElts != M.size())
7188 return false;
7189
7190 // Look for <15, ..., 3, -1, 1, 0>.
7191 for (unsigned i = 0; i != NumElts; ++i)
7192 if (M[i] >= 0 && M[i] != (int) (NumElts - 1 - i))
7193 return false;
7194
7195 return true;
7196}
7197
7198static bool isTruncMask(ArrayRef<int> M, EVT VT, bool Top, bool SingleSource) {
7199 unsigned NumElts = VT.getVectorNumElements();
7200 // Make sure the mask has the right size.
7201 if (NumElts != M.size() || (VT != MVT::v8i16 && VT != MVT::v16i8))
7202 return false;
7203
7204 // Half-width truncation patterns (e.g. v4i32 -> v8i16):
7205 // !Top && SingleSource: <0, 2, 4, 6, 0, 2, 4, 6>
7206 // !Top && !SingleSource: <0, 2, 4, 6, 8, 10, 12, 14>
7207 // Top && SingleSource: <1, 3, 5, 7, 1, 3, 5, 7>
7208 // Top && !SingleSource: <1, 3, 5, 7, 9, 11, 13, 15>
7209 int Ofs = Top ? 1 : 0;
7210 int Upper = SingleSource ? 0 : NumElts;
7211 for (int i = 0, e = NumElts / 2; i != e; ++i) {
7212 if (M[i] >= 0 && M[i] != (i * 2) + Ofs)
7213 return false;
7214 if (M[i + e] >= 0 && M[i + e] != (i * 2) + Ofs + Upper)
7215 return false;
7216 }
7217 return true;
7218}
7219
7220static bool isVMOVNMask(ArrayRef<int> M, EVT VT, bool Top, bool SingleSource) {
7221 unsigned NumElts = VT.getVectorNumElements();
7222 // Make sure the mask has the right size.
7223 if (NumElts != M.size() || (VT != MVT::v8i16 && VT != MVT::v16i8))
7224 return false;
7225
7226 // If Top
7227 // Look for <0, N, 2, N+2, 4, N+4, ..>.
7228 // This inserts Input2 into Input1
7229 // else if not Top
7230 // Look for <0, N+1, 2, N+3, 4, N+5, ..>
7231 // This inserts Input1 into Input2
7232 unsigned Offset = Top ? 0 : 1;
7233 unsigned N = SingleSource ? 0 : NumElts;
7234 for (unsigned i = 0; i < NumElts; i += 2) {
7235 if (M[i] >= 0 && M[i] != (int)i)
7236 return false;
7237 if (M[i + 1] >= 0 && M[i + 1] != (int)(N + i + Offset))
7238 return false;
7239 }
7240
7241 return true;
7242}
7243
7244static bool isVMOVNTruncMask(ArrayRef<int> M, EVT ToVT, bool rev) {
7245 unsigned NumElts = ToVT.getVectorNumElements();
7246 if (NumElts != M.size())
7247 return false;
7248
7249 // Test if the Trunc can be convertible to a VMOVN with this shuffle. We are
7250 // looking for patterns of:
7251 // !rev: 0 N/2 1 N/2+1 2 N/2+2 ...
7252 // rev: N/2 0 N/2+1 1 N/2+2 2 ...
7253
7254 unsigned Off0 = rev ? NumElts / 2 : 0;
7255 unsigned Off1 = rev ? 0 : NumElts / 2;
7256 for (unsigned i = 0; i < NumElts; i += 2) {
7257 if (M[i] >= 0 && M[i] != (int)(Off0 + i / 2))
7258 return false;
7259 if (M[i + 1] >= 0 && M[i + 1] != (int)(Off1 + i / 2))
7260 return false;
7261 }
7262
7263 return true;
7264}
7265
7266// Reconstruct an MVE VCVT from a BuildVector of scalar fptrunc, all extracted
7267// from a pair of inputs. For example:
7268// BUILDVECTOR(FP_ROUND(EXTRACT_ELT(X, 0),
7269// FP_ROUND(EXTRACT_ELT(Y, 0),
7270// FP_ROUND(EXTRACT_ELT(X, 1),
7271// FP_ROUND(EXTRACT_ELT(Y, 1), ...)
7272static SDValue LowerBuildVectorOfFPTrunc(SDValue BV, SelectionDAG &DAG,
7273 const ARMSubtarget *ST) {
7274 assert(BV.getOpcode() == ISD::BUILD_VECTOR && "Unknown opcode!");
7275 if (!ST->hasMVEFloatOps())
7276 return SDValue();
7277
7278 SDLoc dl(BV);
7279 EVT VT = BV.getValueType();
7280 if (VT != MVT::v8f16)
7281 return SDValue();
7282
7283 // We are looking for a buildvector of fptrunc elements, where all the
7284 // elements are interleavingly extracted from two sources. Check the first two
7285 // items are valid enough and extract some info from them (they are checked
7286 // properly in the loop below).
7287 if (BV.getOperand(i: 0).getOpcode() != ISD::FP_ROUND ||
7288 BV.getOperand(i: 0).getOperand(i: 0).getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
7289 BV.getOperand(i: 0).getOperand(i: 0).getConstantOperandVal(i: 1) != 0)
7290 return SDValue();
7291 if (BV.getOperand(i: 1).getOpcode() != ISD::FP_ROUND ||
7292 BV.getOperand(i: 1).getOperand(i: 0).getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
7293 BV.getOperand(i: 1).getOperand(i: 0).getConstantOperandVal(i: 1) != 0)
7294 return SDValue();
7295 SDValue Op0 = BV.getOperand(i: 0).getOperand(i: 0).getOperand(i: 0);
7296 SDValue Op1 = BV.getOperand(i: 1).getOperand(i: 0).getOperand(i: 0);
7297 if (Op0.getValueType() != MVT::v4f32 || Op1.getValueType() != MVT::v4f32)
7298 return SDValue();
7299
7300 // Check all the values in the BuildVector line up with our expectations.
7301 for (unsigned i = 1; i < 4; i++) {
7302 auto Check = [](SDValue Trunc, SDValue Op, unsigned Idx) {
7303 return Trunc.getOpcode() == ISD::FP_ROUND &&
7304 Trunc.getOperand(i: 0).getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
7305 Trunc.getOperand(i: 0).getOperand(i: 0) == Op &&
7306 Trunc.getOperand(i: 0).getConstantOperandVal(i: 1) == Idx;
7307 };
7308 if (!Check(BV.getOperand(i: i * 2 + 0), Op0, i))
7309 return SDValue();
7310 if (!Check(BV.getOperand(i: i * 2 + 1), Op1, i))
7311 return SDValue();
7312 }
7313
7314 SDValue N1 = DAG.getNode(Opcode: ARMISD::VCVTN, DL: dl, VT, N1: DAG.getUNDEF(VT), N2: Op0,
7315 N3: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
7316 return DAG.getNode(Opcode: ARMISD::VCVTN, DL: dl, VT, N1, N2: Op1,
7317 N3: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
7318}
7319
7320// Reconstruct an MVE VCVT from a BuildVector of scalar fpext, all extracted
7321// from a single input on alternating lanes. For example:
7322// BUILDVECTOR(FP_ROUND(EXTRACT_ELT(X, 0),
7323// FP_ROUND(EXTRACT_ELT(X, 2),
7324// FP_ROUND(EXTRACT_ELT(X, 4), ...)
7325static SDValue LowerBuildVectorOfFPExt(SDValue BV, SelectionDAG &DAG,
7326 const ARMSubtarget *ST) {
7327 assert(BV.getOpcode() == ISD::BUILD_VECTOR && "Unknown opcode!");
7328 if (!ST->hasMVEFloatOps())
7329 return SDValue();
7330
7331 SDLoc dl(BV);
7332 EVT VT = BV.getValueType();
7333 if (VT != MVT::v4f32)
7334 return SDValue();
7335
7336 // We are looking for a buildvector of fptext elements, where all the
7337 // elements are alternating lanes from a single source. For example <0,2,4,6>
7338 // or <1,3,5,7>. Check the first two items are valid enough and extract some
7339 // info from them (they are checked properly in the loop below).
7340 if (BV.getOperand(i: 0).getOpcode() != ISD::FP_EXTEND ||
7341 BV.getOperand(i: 0).getOperand(i: 0).getOpcode() != ISD::EXTRACT_VECTOR_ELT)
7342 return SDValue();
7343 SDValue Op0 = BV.getOperand(i: 0).getOperand(i: 0).getOperand(i: 0);
7344 int Offset = BV.getOperand(i: 0).getOperand(i: 0).getConstantOperandVal(i: 1);
7345 if (Op0.getValueType() != MVT::v8f16 || (Offset != 0 && Offset != 1))
7346 return SDValue();
7347
7348 // Check all the values in the BuildVector line up with our expectations.
7349 for (unsigned i = 1; i < 4; i++) {
7350 auto Check = [](SDValue Trunc, SDValue Op, unsigned Idx) {
7351 return Trunc.getOpcode() == ISD::FP_EXTEND &&
7352 Trunc.getOperand(i: 0).getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
7353 Trunc.getOperand(i: 0).getOperand(i: 0) == Op &&
7354 Trunc.getOperand(i: 0).getConstantOperandVal(i: 1) == Idx;
7355 };
7356 if (!Check(BV.getOperand(i), Op0, 2 * i + Offset))
7357 return SDValue();
7358 }
7359
7360 return DAG.getNode(Opcode: ARMISD::VCVTL, DL: dl, VT, N1: Op0,
7361 N2: DAG.getConstant(Val: Offset, DL: dl, VT: MVT::i32));
7362}
7363
7364// If N is an integer constant that can be moved into a register in one
7365// instruction, return an SDValue of such a constant (will become a MOV
7366// instruction). Otherwise return null.
7367static SDValue IsSingleInstrConstant(SDValue N, SelectionDAG &DAG,
7368 const ARMSubtarget *ST, const SDLoc &dl) {
7369 uint64_t Val;
7370 if (!isa<ConstantSDNode>(Val: N))
7371 return SDValue();
7372 Val = N->getAsZExtVal();
7373
7374 if (ST->isThumb1Only()) {
7375 if (Val <= 255 || ~Val <= 255)
7376 return DAG.getConstant(Val, DL: dl, VT: MVT::i32);
7377 } else {
7378 if (ARM_AM::getSOImmVal(Arg: Val) != -1 || ARM_AM::getSOImmVal(Arg: ~Val) != -1)
7379 return DAG.getConstant(Val, DL: dl, VT: MVT::i32);
7380 }
7381 return SDValue();
7382}
7383
7384static SDValue LowerBUILD_VECTOR_i1(SDValue Op, SelectionDAG &DAG,
7385 const ARMSubtarget *ST) {
7386 SDLoc dl(Op);
7387 EVT VT = Op.getValueType();
7388
7389 assert(ST->hasMVEIntegerOps() && "LowerBUILD_VECTOR_i1 called without MVE!");
7390
7391 unsigned NumElts = VT.getVectorNumElements();
7392 unsigned BoolMask;
7393 unsigned BitsPerBool;
7394 if (NumElts == 2) {
7395 BitsPerBool = 8;
7396 BoolMask = 0xff;
7397 } else if (NumElts == 4) {
7398 BitsPerBool = 4;
7399 BoolMask = 0xf;
7400 } else if (NumElts == 8) {
7401 BitsPerBool = 2;
7402 BoolMask = 0x3;
7403 } else if (NumElts == 16) {
7404 BitsPerBool = 1;
7405 BoolMask = 0x1;
7406 } else
7407 return SDValue();
7408
7409 // If this is a single value copied into all lanes (a splat), we can just sign
7410 // extend that single value
7411 SDValue FirstOp = Op.getOperand(i: 0);
7412 if (!isa<ConstantSDNode>(Val: FirstOp) &&
7413 llvm::all_of(Range: llvm::drop_begin(RangeOrContainer: Op->ops()), P: [&FirstOp](const SDUse &U) {
7414 return U.get().isUndef() || U.get() == FirstOp;
7415 })) {
7416 SDValue Ext = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl, VT: MVT::i32, N1: FirstOp,
7417 N2: DAG.getValueType(MVT::i1));
7418 return DAG.getNode(Opcode: ARMISD::PREDICATE_CAST, DL: dl, VT: Op.getValueType(), Operand: Ext);
7419 }
7420
7421 // First create base with bits set where known
7422 unsigned Bits32 = 0;
7423 for (unsigned i = 0; i < NumElts; ++i) {
7424 SDValue V = Op.getOperand(i);
7425 if (!isa<ConstantSDNode>(Val: V) && !V.isUndef())
7426 continue;
7427 bool BitSet = V.isUndef() ? false : V->getAsZExtVal();
7428 if (BitSet)
7429 Bits32 |= BoolMask << (i * BitsPerBool);
7430 }
7431
7432 // Add in unknown nodes
7433 SDValue Base = DAG.getNode(Opcode: ARMISD::PREDICATE_CAST, DL: dl, VT,
7434 Operand: DAG.getConstant(Val: Bits32, DL: dl, VT: MVT::i32));
7435 for (unsigned i = 0; i < NumElts; ++i) {
7436 SDValue V = Op.getOperand(i);
7437 if (isa<ConstantSDNode>(Val: V) || V.isUndef())
7438 continue;
7439 Base = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT, N1: Base, N2: V,
7440 N3: DAG.getConstant(Val: i, DL: dl, VT: MVT::i32));
7441 }
7442
7443 return Base;
7444}
7445
7446static SDValue LowerBUILD_VECTORToVIDUP(SDValue Op, SelectionDAG &DAG,
7447 const ARMSubtarget *ST) {
7448 if (!ST->hasMVEIntegerOps())
7449 return SDValue();
7450
7451 // We are looking for a buildvector where each element is Op[0] + i*N
7452 EVT VT = Op.getValueType();
7453 SDValue Op0 = Op.getOperand(i: 0);
7454 unsigned NumElts = VT.getVectorNumElements();
7455
7456 // Get the increment value from operand 1
7457 SDValue Op1 = Op.getOperand(i: 1);
7458 if (Op1.getOpcode() != ISD::ADD || Op1.getOperand(i: 0) != Op0 ||
7459 !isa<ConstantSDNode>(Val: Op1.getOperand(i: 1)))
7460 return SDValue();
7461 unsigned N = Op1.getConstantOperandVal(i: 1);
7462 if (N != 1 && N != 2 && N != 4 && N != 8)
7463 return SDValue();
7464
7465 // Check that each other operand matches
7466 for (unsigned I = 2; I < NumElts; I++) {
7467 SDValue OpI = Op.getOperand(i: I);
7468 if (OpI.getOpcode() != ISD::ADD || OpI.getOperand(i: 0) != Op0 ||
7469 !isa<ConstantSDNode>(Val: OpI.getOperand(i: 1)) ||
7470 OpI.getConstantOperandVal(i: 1) != I * N)
7471 return SDValue();
7472 }
7473
7474 SDLoc DL(Op);
7475 return DAG.getNode(Opcode: ARMISD::VIDUP, DL, VTList: DAG.getVTList(VT1: VT, VT2: MVT::i32), N1: Op0,
7476 N2: DAG.getConstant(Val: N, DL, VT: MVT::i32));
7477}
7478
7479// Returns true if the operation N can be treated as qr instruction variant at
7480// operand Op.
7481static bool IsQRMVEInstruction(const SDNode *N, const SDNode *Op) {
7482 switch (N->getOpcode()) {
7483 case ISD::ADD:
7484 case ISD::MUL:
7485 case ISD::SADDSAT:
7486 case ISD::UADDSAT:
7487 case ISD::AVGFLOORS:
7488 case ISD::AVGFLOORU:
7489 return true;
7490 case ISD::SUB:
7491 case ISD::SSUBSAT:
7492 case ISD::USUBSAT:
7493 return N->getOperand(Num: 1).getNode() == Op;
7494 case ISD::INTRINSIC_WO_CHAIN:
7495 switch (N->getConstantOperandVal(Num: 0)) {
7496 case Intrinsic::arm_mve_add_predicated:
7497 case Intrinsic::arm_mve_mul_predicated:
7498 case Intrinsic::arm_mve_qadd_predicated:
7499 case Intrinsic::arm_mve_vhadd:
7500 case Intrinsic::arm_mve_hadd_predicated:
7501 case Intrinsic::arm_mve_vqdmulh:
7502 case Intrinsic::arm_mve_qdmulh_predicated:
7503 case Intrinsic::arm_mve_vqrdmulh:
7504 case Intrinsic::arm_mve_qrdmulh_predicated:
7505 case Intrinsic::arm_mve_vqdmull:
7506 case Intrinsic::arm_mve_vqdmull_predicated:
7507 return true;
7508 case Intrinsic::arm_mve_sub_predicated:
7509 case Intrinsic::arm_mve_qsub_predicated:
7510 case Intrinsic::arm_mve_vhsub:
7511 case Intrinsic::arm_mve_hsub_predicated:
7512 return N->getOperand(Num: 2).getNode() == Op;
7513 default:
7514 return false;
7515 }
7516 default:
7517 return false;
7518 }
7519}
7520
7521// If this is a case we can't handle, return null and let the default
7522// expansion code take care of it.
7523SDValue ARMTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
7524 const ARMSubtarget *ST) const {
7525 BuildVectorSDNode *BVN = cast<BuildVectorSDNode>(Val: Op.getNode());
7526 SDLoc dl(Op);
7527 EVT VT = Op.getValueType();
7528
7529 if (ST->hasMVEIntegerOps() && VT.getScalarSizeInBits() == 1)
7530 return LowerBUILD_VECTOR_i1(Op, DAG, ST);
7531
7532 if (SDValue R = LowerBUILD_VECTORToVIDUP(Op, DAG, ST))
7533 return R;
7534
7535 APInt SplatBits, SplatUndef;
7536 unsigned SplatBitSize;
7537 bool HasAnyUndefs;
7538 if (BVN->isConstantSplat(SplatValue&: SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
7539 if (SplatUndef.isAllOnes())
7540 return DAG.getUNDEF(VT);
7541
7542 // If all the users of this constant splat are qr instruction variants,
7543 // generate a vdup of the constant.
7544 if (ST->hasMVEIntegerOps() && VT.getScalarSizeInBits() == SplatBitSize &&
7545 (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32) &&
7546 all_of(Range: BVN->users(),
7547 P: [BVN](const SDNode *U) { return IsQRMVEInstruction(N: U, Op: BVN); })) {
7548 EVT DupVT = SplatBitSize == 32 ? MVT::v4i32
7549 : SplatBitSize == 16 ? MVT::v8i16
7550 : MVT::v16i8;
7551 SDValue Const = DAG.getConstant(Val: SplatBits.getZExtValue(), DL: dl, VT: MVT::i32);
7552 SDValue VDup = DAG.getNode(Opcode: ARMISD::VDUP, DL: dl, VT: DupVT, Operand: Const);
7553 return DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT, Operand: VDup);
7554 }
7555
7556 if ((ST->hasNEON() && SplatBitSize <= 64) ||
7557 (ST->hasMVEIntegerOps() && SplatBitSize <= 64)) {
7558 // Check if an immediate VMOV works.
7559 EVT VmovVT;
7560 SDValue Val =
7561 isVMOVModifiedImm(SplatBits: SplatBits.getZExtValue(), SplatUndef: SplatUndef.getZExtValue(),
7562 SplatBitSize, DAG, dl, VT&: VmovVT, VectorVT: VT, type: VMOVModImm);
7563
7564 if (Val.getNode()) {
7565 SDValue Vmov = DAG.getNode(Opcode: ARMISD::VMOVIMM, DL: dl, VT: VmovVT, Operand: Val);
7566 return DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT, Operand: Vmov);
7567 }
7568
7569 // Try an immediate VMVN.
7570 uint64_t NegatedImm = (~SplatBits).getZExtValue();
7571 Val = isVMOVModifiedImm(
7572 SplatBits: NegatedImm, SplatUndef: SplatUndef.getZExtValue(), SplatBitSize, DAG, dl, VT&: VmovVT,
7573 VectorVT: VT, type: ST->hasMVEIntegerOps() ? MVEVMVNModImm : VMVNModImm);
7574 if (Val.getNode()) {
7575 SDValue Vmov = DAG.getNode(Opcode: ARMISD::VMVNIMM, DL: dl, VT: VmovVT, Operand: Val);
7576 return DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT, Operand: Vmov);
7577 }
7578
7579 // Use vmov.f32 to materialize other v2f32 and v4f32 splats.
7580 if ((VT == MVT::v2f32 || VT == MVT::v4f32) && SplatBitSize == 32) {
7581 int ImmVal = ARM_AM::getFP32Imm(Imm: SplatBits);
7582 if (ImmVal != -1) {
7583 SDValue Val = DAG.getTargetConstant(Val: ImmVal, DL: dl, VT: MVT::i32);
7584 return DAG.getNode(Opcode: ARMISD::VMOVFPIMM, DL: dl, VT, Operand: Val);
7585 }
7586 }
7587
7588 // If we are under MVE, generate a VDUP(constant), bitcast to the original
7589 // type.
7590 if (ST->hasMVEIntegerOps() &&
7591 (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32)) {
7592 EVT DupVT = SplatBitSize == 32 ? MVT::v4i32
7593 : SplatBitSize == 16 ? MVT::v8i16
7594 : MVT::v16i8;
7595 SDValue Const = DAG.getConstant(Val: SplatBits.getZExtValue(), DL: dl, VT: MVT::i32);
7596 SDValue VDup = DAG.getNode(Opcode: ARMISD::VDUP, DL: dl, VT: DupVT, Operand: Const);
7597 return DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT, Operand: VDup);
7598 }
7599 }
7600 }
7601
7602 // Scan through the operands to see if only one value is used.
7603 //
7604 // As an optimisation, even if more than one value is used it may be more
7605 // profitable to splat with one value then change some lanes.
7606 //
7607 // Heuristically we decide to do this if the vector has a "dominant" value,
7608 // defined as splatted to more than half of the lanes.
7609 unsigned NumElts = VT.getVectorNumElements();
7610 bool isOnlyLowElement = true;
7611 bool usesOnlyOneValue = true;
7612 bool hasDominantValue = false;
7613 bool isConstant = true;
7614
7615 // Map of the number of times a particular SDValue appears in the
7616 // element list.
7617 DenseMap<SDValue, unsigned> ValueCounts;
7618 SDValue Value;
7619 for (unsigned i = 0; i < NumElts; ++i) {
7620 SDValue V = Op.getOperand(i);
7621 if (V.isUndef())
7622 continue;
7623 if (i > 0)
7624 isOnlyLowElement = false;
7625 if (!isa<ConstantFPSDNode>(Val: V) && !isa<ConstantSDNode>(Val: V))
7626 isConstant = false;
7627
7628 unsigned &Count = ValueCounts[V];
7629
7630 // Is this value dominant? (takes up more than half of the lanes)
7631 if (++Count > (NumElts / 2)) {
7632 hasDominantValue = true;
7633 Value = V;
7634 }
7635 }
7636 if (ValueCounts.size() != 1)
7637 usesOnlyOneValue = false;
7638 if (!Value.getNode() && !ValueCounts.empty())
7639 Value = ValueCounts.begin()->first;
7640
7641 if (ValueCounts.empty())
7642 return DAG.getUNDEF(VT);
7643
7644 // Loads are better lowered with insert_vector_elt/ARMISD::BUILD_VECTOR.
7645 // Keep going if we are hitting this case.
7646 if (isOnlyLowElement && !ISD::isNormalLoad(N: Value.getNode()) &&
7647 (VT != MVT::v8f16 || ST->hasFullFP16()))
7648 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT, Operand: Value);
7649
7650 unsigned EltSize = VT.getScalarSizeInBits();
7651
7652 // Use VDUP for non-constant splats. For f32 constant splats, reduce to
7653 // i32 and try again.
7654 if (hasDominantValue && EltSize <= 32) {
7655 if (!isConstant) {
7656 SDValue N;
7657
7658 // If we are VDUPing a value that comes directly from a vector, that will
7659 // cause an unnecessary move to and from a GPR, where instead we could
7660 // just use VDUPLANE. We can only do this if the lane being extracted
7661 // is at a constant index, as the VDUP from lane instructions only have
7662 // constant-index forms.
7663 ConstantSDNode *constIndex;
7664 if (Value->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
7665 (constIndex = dyn_cast<ConstantSDNode>(Val: Value->getOperand(Num: 1)))) {
7666 // We need to create a new undef vector to use for the VDUPLANE if the
7667 // size of the vector from which we get the value is different than the
7668 // size of the vector that we need to create. We will insert the element
7669 // such that the register coalescer will remove unnecessary copies.
7670 if (VT != Value->getOperand(Num: 0).getValueType()) {
7671 unsigned index = constIndex->getAPIntValue().getLimitedValue() %
7672 VT.getVectorNumElements();
7673 N = DAG.getNode(Opcode: ARMISD::VDUPLANE, DL: dl, VT,
7674 N1: DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT, N1: DAG.getUNDEF(VT),
7675 N2: Value, N3: DAG.getConstant(Val: index, DL: dl, VT: MVT::i32)),
7676 N2: DAG.getConstant(Val: index, DL: dl, VT: MVT::i32));
7677 } else
7678 N = DAG.getNode(Opcode: ARMISD::VDUPLANE, DL: dl, VT,
7679 N1: Value->getOperand(Num: 0), N2: Value->getOperand(Num: 1));
7680 } else
7681 N = DAG.getNode(Opcode: ARMISD::VDUP, DL: dl, VT, Operand: Value);
7682
7683 if (!usesOnlyOneValue) {
7684 // The dominant value was splatted as 'N', but we now have to insert
7685 // all differing elements.
7686 for (unsigned I = 0; I < NumElts; ++I) {
7687 if (Op.getOperand(i: I) == Value)
7688 continue;
7689 SmallVector<SDValue, 3> Ops;
7690 Ops.push_back(Elt: N);
7691 Ops.push_back(Elt: Op.getOperand(i: I));
7692 Ops.push_back(Elt: DAG.getConstant(Val: I, DL: dl, VT: MVT::i32));
7693 N = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT, Ops);
7694 }
7695 }
7696 return N;
7697 }
7698 if (VT.getVectorElementType().isFloatingPoint()) {
7699 SmallVector<SDValue, 8> Ops;
7700 MVT FVT = VT.getVectorElementType().getSimpleVT();
7701 assert(FVT == MVT::f32 || FVT == MVT::f16);
7702 MVT IVT = (FVT == MVT::f32) ? MVT::i32 : MVT::i16;
7703 for (unsigned i = 0; i < NumElts; ++i)
7704 Ops.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: IVT,
7705 Operand: Op.getOperand(i)));
7706 EVT VecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: IVT, NumElements: NumElts);
7707 SDValue Val = DAG.getBuildVector(VT: VecVT, DL: dl, Ops);
7708 Val = LowerBUILD_VECTOR(Op: Val, DAG, ST);
7709 if (Val.getNode())
7710 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: Val);
7711 }
7712 if (usesOnlyOneValue) {
7713 SDValue Val = IsSingleInstrConstant(N: Value, DAG, ST, dl);
7714 if (isConstant && Val.getNode())
7715 return DAG.getNode(Opcode: ARMISD::VDUP, DL: dl, VT, Operand: Val);
7716 }
7717 }
7718
7719 // If all elements are constants and the case above didn't get hit, fall back
7720 // to the default expansion, which will generate a load from the constant
7721 // pool.
7722 if (isConstant)
7723 return SDValue();
7724
7725 // Reconstruct the BUILDVECTOR to one of the legal shuffles (such as vext and
7726 // vmovn). Empirical tests suggest this is rarely worth it for vectors of
7727 // length <= 2.
7728 if (NumElts >= 4)
7729 if (SDValue shuffle = ReconstructShuffle(Op, DAG))
7730 return shuffle;
7731
7732 // Attempt to turn a buildvector of scalar fptrunc's or fpext's back into
7733 // VCVT's
7734 if (SDValue VCVT = LowerBuildVectorOfFPTrunc(BV: Op, DAG, ST: Subtarget))
7735 return VCVT;
7736 if (SDValue VCVT = LowerBuildVectorOfFPExt(BV: Op, DAG, ST: Subtarget))
7737 return VCVT;
7738
7739 if (ST->hasNEON() && VT.is128BitVector() && VT != MVT::v2f64 && VT != MVT::v4f32) {
7740 // If we haven't found an efficient lowering, try splitting a 128-bit vector
7741 // into two 64-bit vectors; we might discover a better way to lower it.
7742 SmallVector<SDValue, 64> Ops(Op->op_begin(), Op->op_begin() + NumElts);
7743 EVT ExtVT = VT.getVectorElementType();
7744 EVT HVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ExtVT, NumElements: NumElts / 2);
7745 SDValue Lower = DAG.getBuildVector(VT: HVT, DL: dl, Ops: ArrayRef(&Ops[0], NumElts / 2));
7746 if (Lower.getOpcode() == ISD::BUILD_VECTOR)
7747 Lower = LowerBUILD_VECTOR(Op: Lower, DAG, ST);
7748 SDValue Upper =
7749 DAG.getBuildVector(VT: HVT, DL: dl, Ops: ArrayRef(&Ops[NumElts / 2], NumElts / 2));
7750 if (Upper.getOpcode() == ISD::BUILD_VECTOR)
7751 Upper = LowerBUILD_VECTOR(Op: Upper, DAG, ST);
7752 if (Lower && Upper)
7753 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT, N1: Lower, N2: Upper);
7754 }
7755
7756 // Vectors with 32- or 64-bit elements can be built by directly assigning
7757 // the subregisters. Lower it to an ARMISD::BUILD_VECTOR so the operands
7758 // will be legalized.
7759 if (EltSize >= 32) {
7760 // Do the expansion with floating-point types, since that is what the VFP
7761 // registers are defined to use, and since i64 is not legal.
7762 EVT EltVT = EVT::getFloatingPointVT(BitWidth: EltSize);
7763 EVT VecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, NumElements: NumElts);
7764 SmallVector<SDValue, 8> Ops;
7765 for (unsigned i = 0; i < NumElts; ++i)
7766 Ops.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: EltVT, Operand: Op.getOperand(i)));
7767 SDValue Val = DAG.getNode(Opcode: ARMISD::BUILD_VECTOR, DL: dl, VT: VecVT, Ops);
7768 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: Val);
7769 }
7770
7771 // If all else fails, just use a sequence of INSERT_VECTOR_ELT when we
7772 // know the default expansion would otherwise fall back on something even
7773 // worse. For a vector with one or two non-undef values, that's
7774 // scalar_to_vector for the elements followed by a shuffle (provided the
7775 // shuffle is valid for the target) and materialization element by element
7776 // on the stack followed by a load for everything else.
7777 if ((!isConstant && !usesOnlyOneValue) ||
7778 (VT == MVT::v8f16 && !ST->hasFullFP16())) {
7779 SDValue Vec = DAG.getUNDEF(VT);
7780 for (unsigned i = 0 ; i < NumElts; ++i) {
7781 SDValue V = Op.getOperand(i);
7782 if (V.isUndef())
7783 continue;
7784 SDValue LaneIdx = DAG.getConstant(Val: i, DL: dl, VT: MVT::i32);
7785 Vec = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT, N1: Vec, N2: V, N3: LaneIdx);
7786 }
7787 return Vec;
7788 }
7789
7790 return SDValue();
7791}
7792
7793// Gather data to see if the operation can be modelled as a
7794// shuffle in combination with VEXTs.
7795SDValue ARMTargetLowering::ReconstructShuffle(SDValue Op,
7796 SelectionDAG &DAG) const {
7797 assert(Op.getOpcode() == ISD::BUILD_VECTOR && "Unknown opcode!");
7798 SDLoc dl(Op);
7799 EVT VT = Op.getValueType();
7800 unsigned NumElts = VT.getVectorNumElements();
7801
7802 struct ShuffleSourceInfo {
7803 SDValue Vec;
7804 unsigned MinElt = std::numeric_limits<unsigned>::max();
7805 unsigned MaxElt = 0;
7806
7807 // We may insert some combination of BITCASTs and VEXT nodes to force Vec to
7808 // be compatible with the shuffle we intend to construct. As a result
7809 // ShuffleVec will be some sliding window into the original Vec.
7810 SDValue ShuffleVec;
7811
7812 // Code should guarantee that element i in Vec starts at element "WindowBase
7813 // + i * WindowScale in ShuffleVec".
7814 int WindowBase = 0;
7815 int WindowScale = 1;
7816
7817 ShuffleSourceInfo(SDValue Vec) : Vec(Vec), ShuffleVec(Vec) {}
7818
7819 bool operator ==(SDValue OtherVec) { return Vec == OtherVec; }
7820 };
7821
7822 // First gather all vectors used as an immediate source for this BUILD_VECTOR
7823 // node.
7824 SmallVector<ShuffleSourceInfo, 2> Sources;
7825 for (unsigned i = 0; i < NumElts; ++i) {
7826 SDValue V = Op.getOperand(i);
7827 if (V.isUndef())
7828 continue;
7829 else if (V.getOpcode() != ISD::EXTRACT_VECTOR_ELT) {
7830 // A shuffle can only come from building a vector from various
7831 // elements of other vectors.
7832 return SDValue();
7833 } else if (!isa<ConstantSDNode>(Val: V.getOperand(i: 1))) {
7834 // Furthermore, shuffles require a constant mask, whereas extractelts
7835 // accept variable indices.
7836 return SDValue();
7837 }
7838
7839 // Add this element source to the list if it's not already there.
7840 SDValue SourceVec = V.getOperand(i: 0);
7841 auto Source = llvm::find(Range&: Sources, Val: SourceVec);
7842 if (Source == Sources.end())
7843 Source = Sources.insert(I: Sources.end(), Elt: ShuffleSourceInfo(SourceVec));
7844
7845 // Update the minimum and maximum lane number seen.
7846 unsigned EltNo = V.getConstantOperandVal(i: 1);
7847 Source->MinElt = std::min(a: Source->MinElt, b: EltNo);
7848 Source->MaxElt = std::max(a: Source->MaxElt, b: EltNo);
7849 }
7850
7851 // Currently only do something sane when at most two source vectors
7852 // are involved.
7853 if (Sources.size() > 2)
7854 return SDValue();
7855
7856 // Find out the smallest element size among result and two sources, and use
7857 // it as element size to build the shuffle_vector.
7858 EVT SmallestEltTy = VT.getVectorElementType();
7859 for (auto &Source : Sources) {
7860 EVT SrcEltTy = Source.Vec.getValueType().getVectorElementType();
7861 if (SrcEltTy.bitsLT(VT: SmallestEltTy))
7862 SmallestEltTy = SrcEltTy;
7863 }
7864 unsigned ResMultiplier =
7865 VT.getScalarSizeInBits() / SmallestEltTy.getSizeInBits();
7866 NumElts = VT.getSizeInBits() / SmallestEltTy.getSizeInBits();
7867 EVT ShuffleVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: SmallestEltTy, NumElements: NumElts);
7868
7869 // If the source vector is too wide or too narrow, we may nevertheless be able
7870 // to construct a compatible shuffle either by concatenating it with UNDEF or
7871 // extracting a suitable range of elements.
7872 for (auto &Src : Sources) {
7873 EVT SrcVT = Src.ShuffleVec.getValueType();
7874
7875 uint64_t SrcVTSize = SrcVT.getFixedSizeInBits();
7876 uint64_t VTSize = VT.getFixedSizeInBits();
7877 if (SrcVTSize == VTSize)
7878 continue;
7879
7880 // This stage of the search produces a source with the same element type as
7881 // the original, but with a total width matching the BUILD_VECTOR output.
7882 EVT EltVT = SrcVT.getVectorElementType();
7883 unsigned NumSrcElts = VTSize / EltVT.getFixedSizeInBits();
7884 EVT DestVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, NumElements: NumSrcElts);
7885
7886 if (SrcVTSize < VTSize) {
7887 if (2 * SrcVTSize != VTSize)
7888 return SDValue();
7889 // We can pad out the smaller vector for free, so if it's part of a
7890 // shuffle...
7891 Src.ShuffleVec =
7892 DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: DestVT, N1: Src.ShuffleVec,
7893 N2: DAG.getUNDEF(VT: Src.ShuffleVec.getValueType()));
7894 continue;
7895 }
7896
7897 if (SrcVTSize != 2 * VTSize)
7898 return SDValue();
7899
7900 if (Src.MaxElt - Src.MinElt >= NumSrcElts) {
7901 // Span too large for a VEXT to cope
7902 return SDValue();
7903 }
7904
7905 if (Src.MinElt >= NumSrcElts) {
7906 // The extraction can just take the second half
7907 Src.ShuffleVec =
7908 DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: DestVT, N1: Src.ShuffleVec,
7909 N2: DAG.getConstant(Val: NumSrcElts, DL: dl, VT: MVT::i32));
7910 Src.WindowBase = -NumSrcElts;
7911 } else if (Src.MaxElt < NumSrcElts) {
7912 // The extraction can just take the first half
7913 Src.ShuffleVec =
7914 DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: DestVT, N1: Src.ShuffleVec,
7915 N2: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
7916 } else {
7917 // An actual VEXT is needed
7918 SDValue VEXTSrc1 =
7919 DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: DestVT, N1: Src.ShuffleVec,
7920 N2: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
7921 SDValue VEXTSrc2 =
7922 DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: DestVT, N1: Src.ShuffleVec,
7923 N2: DAG.getConstant(Val: NumSrcElts, DL: dl, VT: MVT::i32));
7924
7925 Src.ShuffleVec = DAG.getNode(Opcode: ARMISD::VEXT, DL: dl, VT: DestVT, N1: VEXTSrc1,
7926 N2: VEXTSrc2,
7927 N3: DAG.getConstant(Val: Src.MinElt, DL: dl, VT: MVT::i32));
7928 Src.WindowBase = -Src.MinElt;
7929 }
7930 }
7931
7932 // Another possible incompatibility occurs from the vector element types. We
7933 // can fix this by bitcasting the source vectors to the same type we intend
7934 // for the shuffle.
7935 for (auto &Src : Sources) {
7936 EVT SrcEltTy = Src.ShuffleVec.getValueType().getVectorElementType();
7937 if (SrcEltTy == SmallestEltTy)
7938 continue;
7939 assert(ShuffleVT.getVectorElementType() == SmallestEltTy);
7940 Src.ShuffleVec = DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT: ShuffleVT, Operand: Src.ShuffleVec);
7941 Src.WindowScale = SrcEltTy.getSizeInBits() / SmallestEltTy.getSizeInBits();
7942 Src.WindowBase *= Src.WindowScale;
7943 }
7944
7945 // Final check before we try to actually produce a shuffle.
7946 LLVM_DEBUG({
7947 for (auto Src : Sources)
7948 assert(Src.ShuffleVec.getValueType() == ShuffleVT);
7949 });
7950
7951 // The stars all align, our next step is to produce the mask for the shuffle.
7952 SmallVector<int, 8> Mask(ShuffleVT.getVectorNumElements(), -1);
7953 int BitsPerShuffleLane = ShuffleVT.getScalarSizeInBits();
7954 for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) {
7955 SDValue Entry = Op.getOperand(i);
7956 if (Entry.isUndef())
7957 continue;
7958
7959 auto Src = llvm::find(Range&: Sources, Val: Entry.getOperand(i: 0));
7960 int EltNo = cast<ConstantSDNode>(Val: Entry.getOperand(i: 1))->getSExtValue();
7961
7962 // EXTRACT_VECTOR_ELT performs an implicit any_ext; BUILD_VECTOR an implicit
7963 // trunc. So only std::min(SrcBits, DestBits) actually get defined in this
7964 // segment.
7965 EVT OrigEltTy = Entry.getOperand(i: 0).getValueType().getVectorElementType();
7966 int BitsDefined = std::min(a: OrigEltTy.getScalarSizeInBits(),
7967 b: VT.getScalarSizeInBits());
7968 int LanesDefined = BitsDefined / BitsPerShuffleLane;
7969
7970 // This source is expected to fill ResMultiplier lanes of the final shuffle,
7971 // starting at the appropriate offset.
7972 int *LaneMask = &Mask[i * ResMultiplier];
7973
7974 int ExtractBase = EltNo * Src->WindowScale + Src->WindowBase;
7975 ExtractBase += NumElts * (Src - Sources.begin());
7976 for (int j = 0; j < LanesDefined; ++j)
7977 LaneMask[j] = ExtractBase + j;
7978 }
7979
7980
7981 // We can't handle more than two sources. This should have already
7982 // been checked before this point.
7983 assert(Sources.size() <= 2 && "Too many sources!");
7984
7985 SDValue ShuffleOps[] = { DAG.getUNDEF(VT: ShuffleVT), DAG.getUNDEF(VT: ShuffleVT) };
7986 for (unsigned i = 0; i < Sources.size(); ++i)
7987 ShuffleOps[i] = Sources[i].ShuffleVec;
7988
7989 SDValue Shuffle = buildLegalVectorShuffle(VT: ShuffleVT, DL: dl, N0: ShuffleOps[0],
7990 N1: ShuffleOps[1], Mask, DAG);
7991 if (!Shuffle)
7992 return SDValue();
7993 return DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT, Operand: Shuffle);
7994}
7995
7996enum ShuffleOpCodes {
7997 OP_COPY = 0, // Copy, used for things like <u,u,u,3> to say it is <0,1,2,3>
7998 OP_VREV,
7999 OP_VDUP0,
8000 OP_VDUP1,
8001 OP_VDUP2,
8002 OP_VDUP3,
8003 OP_VEXT1,
8004 OP_VEXT2,
8005 OP_VEXT3,
8006 OP_VUZPL, // VUZP, left result
8007 OP_VUZPR, // VUZP, right result
8008 OP_VZIPL, // VZIP, left result
8009 OP_VZIPR, // VZIP, right result
8010 OP_VTRNL, // VTRN, left result
8011 OP_VTRNR // VTRN, right result
8012};
8013
8014static bool isLegalMVEShuffleOp(unsigned PFEntry) {
8015 unsigned OpNum = (PFEntry >> 26) & 0x0F;
8016 switch (OpNum) {
8017 case OP_COPY:
8018 case OP_VREV:
8019 case OP_VDUP0:
8020 case OP_VDUP1:
8021 case OP_VDUP2:
8022 case OP_VDUP3:
8023 return true;
8024 }
8025 return false;
8026}
8027
8028/// isShuffleMaskLegal - Targets can use this to indicate that they only
8029/// support *some* VECTOR_SHUFFLE operations, those with specific masks.
8030/// By default, if a target supports the VECTOR_SHUFFLE node, all mask values
8031/// are assumed to be legal.
8032bool ARMTargetLowering::isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const {
8033 if (VT.getVectorNumElements() == 4 &&
8034 (VT.is128BitVector() || VT.is64BitVector())) {
8035 unsigned PFIndexes[4];
8036 for (unsigned i = 0; i != 4; ++i) {
8037 if (M[i] < 0)
8038 PFIndexes[i] = 8;
8039 else
8040 PFIndexes[i] = M[i];
8041 }
8042
8043 // Compute the index in the perfect shuffle table.
8044 unsigned PFTableIndex =
8045 PFIndexes[0]*9*9*9+PFIndexes[1]*9*9+PFIndexes[2]*9+PFIndexes[3];
8046 unsigned PFEntry = PerfectShuffleTable[PFTableIndex];
8047 unsigned Cost = (PFEntry >> 30);
8048
8049 if (Cost <= 4 && (Subtarget->hasNEON() || isLegalMVEShuffleOp(PFEntry)))
8050 return true;
8051 }
8052
8053 bool ReverseVEXT, isV_UNDEF;
8054 unsigned Imm, WhichResult;
8055
8056 unsigned EltSize = VT.getScalarSizeInBits();
8057 if (EltSize >= 32 ||
8058 ShuffleVectorSDNode::isSplatMask(Mask: M) ||
8059 ShuffleVectorInst::isIdentityMask(Mask: M, NumSrcElts: M.size()) ||
8060 isVREVMask(M, VT, BlockSize: 64) ||
8061 isVREVMask(M, VT, BlockSize: 32) ||
8062 isVREVMask(M, VT, BlockSize: 16))
8063 return true;
8064 else if (Subtarget->hasNEON() &&
8065 (isVEXTMask(M, VT, ReverseVEXT, Imm) ||
8066 isVTBLMask(M, VT) ||
8067 isNEONTwoResultShuffleMask(ShuffleMask: M, VT, WhichResult, isV_UNDEF)))
8068 return true;
8069 else if ((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) &&
8070 isReverseMask(M, VT))
8071 return true;
8072 else if (Subtarget->hasMVEIntegerOps() &&
8073 (isVMOVNMask(M, VT, Top: true, SingleSource: false) ||
8074 isVMOVNMask(M, VT, Top: false, SingleSource: false) || isVMOVNMask(M, VT, Top: true, SingleSource: true)))
8075 return true;
8076 else if (Subtarget->hasMVEIntegerOps() &&
8077 (isTruncMask(M, VT, Top: false, SingleSource: false) ||
8078 isTruncMask(M, VT, Top: false, SingleSource: true) ||
8079 isTruncMask(M, VT, Top: true, SingleSource: false) || isTruncMask(M, VT, Top: true, SingleSource: true)))
8080 return true;
8081 else
8082 return false;
8083}
8084
8085/// GeneratePerfectShuffle - Given an entry in the perfect-shuffle table, emit
8086/// the specified operations to build the shuffle.
8087static SDValue GeneratePerfectShuffle(unsigned PFEntry, SDValue LHS,
8088 SDValue RHS, SelectionDAG &DAG,
8089 const SDLoc &dl) {
8090 unsigned OpNum = (PFEntry >> 26) & 0x0F;
8091 unsigned LHSID = (PFEntry >> 13) & ((1 << 13)-1);
8092 unsigned RHSID = (PFEntry >> 0) & ((1 << 13)-1);
8093
8094 if (OpNum == OP_COPY) {
8095 if (LHSID == (1*9+2)*9+3) return LHS;
8096 assert(LHSID == ((4*9+5)*9+6)*9+7 && "Illegal OP_COPY!");
8097 return RHS;
8098 }
8099
8100 SDValue OpLHS, OpRHS;
8101 OpLHS = GeneratePerfectShuffle(PFEntry: PerfectShuffleTable[LHSID], LHS, RHS, DAG, dl);
8102 OpRHS = GeneratePerfectShuffle(PFEntry: PerfectShuffleTable[RHSID], LHS, RHS, DAG, dl);
8103 EVT VT = OpLHS.getValueType();
8104
8105 switch (OpNum) {
8106 default: llvm_unreachable("Unknown shuffle opcode!");
8107 case OP_VREV:
8108 // VREV divides the vector in half and swaps within the half.
8109 if (VT.getScalarSizeInBits() == 32)
8110 return DAG.getNode(Opcode: ARMISD::VREV64, DL: dl, VT, Operand: OpLHS);
8111 // vrev <4 x i16> -> VREV32
8112 if (VT.getScalarSizeInBits() == 16)
8113 return DAG.getNode(Opcode: ARMISD::VREV32, DL: dl, VT, Operand: OpLHS);
8114 // vrev <4 x i8> -> VREV16
8115 assert(VT.getScalarSizeInBits() == 8);
8116 return DAG.getNode(Opcode: ARMISD::VREV16, DL: dl, VT, Operand: OpLHS);
8117 case OP_VDUP0:
8118 case OP_VDUP1:
8119 case OP_VDUP2:
8120 case OP_VDUP3:
8121 return DAG.getNode(Opcode: ARMISD::VDUPLANE, DL: dl, VT,
8122 N1: OpLHS, N2: DAG.getConstant(Val: OpNum-OP_VDUP0, DL: dl, VT: MVT::i32));
8123 case OP_VEXT1:
8124 case OP_VEXT2:
8125 case OP_VEXT3:
8126 return DAG.getNode(Opcode: ARMISD::VEXT, DL: dl, VT,
8127 N1: OpLHS, N2: OpRHS,
8128 N3: DAG.getConstant(Val: OpNum - OP_VEXT1 + 1, DL: dl, VT: MVT::i32));
8129 case OP_VUZPL:
8130 case OP_VUZPR:
8131 return DAG.getNode(Opcode: ARMISD::VUZP, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: VT),
8132 N1: OpLHS, N2: OpRHS).getValue(R: OpNum-OP_VUZPL);
8133 case OP_VZIPL:
8134 case OP_VZIPR:
8135 return DAG.getNode(Opcode: ARMISD::VZIP, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: VT),
8136 N1: OpLHS, N2: OpRHS).getValue(R: OpNum-OP_VZIPL);
8137 case OP_VTRNL:
8138 case OP_VTRNR:
8139 return DAG.getNode(Opcode: ARMISD::VTRN, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: VT),
8140 N1: OpLHS, N2: OpRHS).getValue(R: OpNum-OP_VTRNL);
8141 }
8142}
8143
8144static SDValue LowerVECTOR_SHUFFLEv8i8(SDValue Op,
8145 ArrayRef<int> ShuffleMask,
8146 SelectionDAG &DAG) {
8147 // Check to see if we can use the VTBL instruction.
8148 SDValue V1 = Op.getOperand(i: 0);
8149 SDValue V2 = Op.getOperand(i: 1);
8150 SDLoc DL(Op);
8151
8152 SmallVector<SDValue, 8> VTBLMask;
8153 for (int I : ShuffleMask)
8154 VTBLMask.push_back(Elt: DAG.getSignedConstant(Val: I, DL, VT: MVT::i32));
8155
8156 if (V2.getNode()->isUndef())
8157 return DAG.getNode(Opcode: ARMISD::VTBL1, DL, VT: MVT::v8i8, N1: V1,
8158 N2: DAG.getBuildVector(VT: MVT::v8i8, DL, Ops: VTBLMask));
8159
8160 return DAG.getNode(Opcode: ARMISD::VTBL2, DL, VT: MVT::v8i8, N1: V1, N2: V2,
8161 N3: DAG.getBuildVector(VT: MVT::v8i8, DL, Ops: VTBLMask));
8162}
8163
8164static SDValue LowerReverse_VECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) {
8165 SDLoc DL(Op);
8166 EVT VT = Op.getValueType();
8167
8168 assert((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) &&
8169 "Expect an v8i16/v16i8 type");
8170 SDValue OpLHS = DAG.getNode(Opcode: ARMISD::VREV64, DL, VT, Operand: Op.getOperand(i: 0));
8171 // For a v16i8 type: After the VREV, we have got <7, ..., 0, 15, ..., 8>. Now,
8172 // extract the first 8 bytes into the top double word and the last 8 bytes
8173 // into the bottom double word, through a new vector shuffle that will be
8174 // turned into a VEXT on Neon, or a couple of VMOVDs on MVE.
8175 std::vector<int> NewMask;
8176 for (unsigned i = 0; i < VT.getVectorNumElements() / 2; i++)
8177 NewMask.push_back(x: VT.getVectorNumElements() / 2 + i);
8178 for (unsigned i = 0; i < VT.getVectorNumElements() / 2; i++)
8179 NewMask.push_back(x: i);
8180 return DAG.getVectorShuffle(VT, dl: DL, N1: OpLHS, N2: OpLHS, Mask: NewMask);
8181}
8182
8183static EVT getVectorTyFromPredicateVector(EVT VT) {
8184 switch (VT.getSimpleVT().SimpleTy) {
8185 case MVT::v2i1:
8186 return MVT::v2f64;
8187 case MVT::v4i1:
8188 return MVT::v4i32;
8189 case MVT::v8i1:
8190 return MVT::v8i16;
8191 case MVT::v16i1:
8192 return MVT::v16i8;
8193 default:
8194 llvm_unreachable("Unexpected vector predicate type");
8195 }
8196}
8197
8198static SDValue PromoteMVEPredVector(SDLoc dl, SDValue Pred, EVT VT,
8199 SelectionDAG &DAG) {
8200 // Converting from boolean predicates to integers involves creating a vector
8201 // of all ones or all zeroes and selecting the lanes based upon the real
8202 // predicate.
8203 SDValue AllOnes =
8204 DAG.getTargetConstant(Val: ARM_AM::createVMOVModImm(OpCmode: 0xe, Val: 0xff), DL: dl, VT: MVT::i32);
8205 AllOnes = DAG.getNode(Opcode: ARMISD::VMOVIMM, DL: dl, VT: MVT::v16i8, Operand: AllOnes);
8206
8207 SDValue AllZeroes =
8208 DAG.getTargetConstant(Val: ARM_AM::createVMOVModImm(OpCmode: 0xe, Val: 0x0), DL: dl, VT: MVT::i32);
8209 AllZeroes = DAG.getNode(Opcode: ARMISD::VMOVIMM, DL: dl, VT: MVT::v16i8, Operand: AllZeroes);
8210
8211 // Get full vector type from predicate type
8212 EVT NewVT = getVectorTyFromPredicateVector(VT);
8213
8214 SDValue RecastV1;
8215 // If the real predicate is an v8i1 or v4i1 (not v16i1) then we need to recast
8216 // this to a v16i1. This cannot be done with an ordinary bitcast because the
8217 // sizes are not the same. We have to use a MVE specific PREDICATE_CAST node,
8218 // since we know in hardware the sizes are really the same.
8219 if (VT != MVT::v16i1)
8220 RecastV1 = DAG.getNode(Opcode: ARMISD::PREDICATE_CAST, DL: dl, VT: MVT::v16i1, Operand: Pred);
8221 else
8222 RecastV1 = Pred;
8223
8224 // Select either all ones or zeroes depending upon the real predicate bits.
8225 SDValue PredAsVector =
8226 DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: MVT::v16i8, N1: RecastV1, N2: AllOnes, N3: AllZeroes);
8227
8228 // Recast our new predicate-as-integer v16i8 vector into something
8229 // appropriate for the shuffle, i.e. v4i32 for a real v4i1 predicate.
8230 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVT, Operand: PredAsVector);
8231}
8232
8233static SDValue LowerVECTOR_SHUFFLE_i1(SDValue Op, SelectionDAG &DAG,
8234 const ARMSubtarget *ST) {
8235 EVT VT = Op.getValueType();
8236 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val: Op.getNode());
8237 ArrayRef<int> ShuffleMask = SVN->getMask();
8238
8239 assert(ST->hasMVEIntegerOps() &&
8240 "No support for vector shuffle of boolean predicates");
8241
8242 SDValue V1 = Op.getOperand(i: 0);
8243 SDValue V2 = Op.getOperand(i: 1);
8244 SDLoc dl(Op);
8245 if (isReverseMask(M: ShuffleMask, VT)) {
8246 SDValue cast = DAG.getNode(Opcode: ARMISD::PREDICATE_CAST, DL: dl, VT: MVT::i32, Operand: V1);
8247 SDValue rbit = DAG.getNode(Opcode: ISD::BITREVERSE, DL: dl, VT: MVT::i32, Operand: cast);
8248 SDValue srl = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i32, N1: rbit,
8249 N2: DAG.getConstant(Val: 16, DL: dl, VT: MVT::i32));
8250 return DAG.getNode(Opcode: ARMISD::PREDICATE_CAST, DL: dl, VT, Operand: srl);
8251 }
8252
8253 // Until we can come up with optimised cases for every single vector
8254 // shuffle in existence we have chosen the least painful strategy. This is
8255 // to essentially promote the boolean predicate to a 8-bit integer, where
8256 // each predicate represents a byte. Then we fall back on a normal integer
8257 // vector shuffle and convert the result back into a predicate vector. In
8258 // many cases the generated code might be even better than scalar code
8259 // operating on bits. Just imagine trying to shuffle 8 arbitrary 2-bit
8260 // fields in a register into 8 other arbitrary 2-bit fields!
8261 SDValue PredAsVector1 = PromoteMVEPredVector(dl, Pred: V1, VT, DAG);
8262 EVT NewVT = PredAsVector1.getValueType();
8263 SDValue PredAsVector2 = V2.isUndef() ? DAG.getUNDEF(VT: NewVT)
8264 : PromoteMVEPredVector(dl, Pred: V2, VT, DAG);
8265 assert(PredAsVector2.getValueType() == NewVT &&
8266 "Expected identical vector type in expanded i1 shuffle!");
8267
8268 // Do the shuffle!
8269 SDValue Shuffled = DAG.getVectorShuffle(VT: NewVT, dl, N1: PredAsVector1,
8270 N2: PredAsVector2, Mask: ShuffleMask);
8271
8272 // Now return the result of comparing the shuffled vector with zero,
8273 // which will generate a real predicate, i.e. v4i1, v8i1 or v16i1. For a v2i1
8274 // we convert to a v4i1 compare to fill in the two halves of the i64 as i32s.
8275 if (VT == MVT::v2i1) {
8276 SDValue BC = DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT: MVT::v4i32, Operand: Shuffled);
8277 SDValue Cmp = DAG.getNode(Opcode: ARMISD::VCMPZ, DL: dl, VT: MVT::v4i1, N1: BC,
8278 N2: DAG.getConstant(Val: ARMCC::NE, DL: dl, VT: MVT::i32));
8279 return DAG.getNode(Opcode: ARMISD::PREDICATE_CAST, DL: dl, VT: MVT::v2i1, Operand: Cmp);
8280 }
8281 return DAG.getNode(Opcode: ARMISD::VCMPZ, DL: dl, VT, N1: Shuffled,
8282 N2: DAG.getConstant(Val: ARMCC::NE, DL: dl, VT: MVT::i32));
8283}
8284
8285static SDValue LowerVECTOR_SHUFFLEUsingMovs(SDValue Op,
8286 ArrayRef<int> ShuffleMask,
8287 SelectionDAG &DAG) {
8288 // Attempt to lower the vector shuffle using as many whole register movs as
8289 // possible. This is useful for types smaller than 32bits, which would
8290 // often otherwise become a series for grp movs.
8291 SDLoc dl(Op);
8292 EVT VT = Op.getValueType();
8293 if (VT.getScalarSizeInBits() >= 32)
8294 return SDValue();
8295
8296 assert((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) &&
8297 "Unexpected vector type");
8298 int NumElts = VT.getVectorNumElements();
8299 int QuarterSize = NumElts / 4;
8300 // The four final parts of the vector, as i32's
8301 SDValue Parts[4];
8302
8303 // Look for full lane vmovs like <0,1,2,3> or <u,5,6,7> etc, (but not
8304 // <u,u,u,u>), returning the vmov lane index
8305 auto getMovIdx = [](ArrayRef<int> ShuffleMask, int Start, int Length) {
8306 // Detect which mov lane this would be from the first non-undef element.
8307 int MovIdx = -1;
8308 for (int i = 0; i < Length; i++) {
8309 if (ShuffleMask[Start + i] >= 0) {
8310 if (ShuffleMask[Start + i] % Length != i)
8311 return -1;
8312 MovIdx = ShuffleMask[Start + i] / Length;
8313 break;
8314 }
8315 }
8316 // If all items are undef, leave this for other combines
8317 if (MovIdx == -1)
8318 return -1;
8319 // Check the remaining values are the correct part of the same mov
8320 for (int i = 1; i < Length; i++) {
8321 if (ShuffleMask[Start + i] >= 0 &&
8322 (ShuffleMask[Start + i] / Length != MovIdx ||
8323 ShuffleMask[Start + i] % Length != i))
8324 return -1;
8325 }
8326 return MovIdx;
8327 };
8328
8329 for (int Part = 0; Part < 4; ++Part) {
8330 // Does this part look like a mov
8331 int Elt = getMovIdx(ShuffleMask, Part * QuarterSize, QuarterSize);
8332 if (Elt != -1) {
8333 SDValue Input = Op->getOperand(Num: 0);
8334 if (Elt >= 4) {
8335 Input = Op->getOperand(Num: 1);
8336 Elt -= 4;
8337 }
8338 SDValue BitCast = DAG.getBitcast(VT: MVT::v4f32, V: Input);
8339 Parts[Part] = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: MVT::f32, N1: BitCast,
8340 N2: DAG.getConstant(Val: Elt, DL: dl, VT: MVT::i32));
8341 }
8342 }
8343
8344 // Nothing interesting found, just return
8345 if (!Parts[0] && !Parts[1] && !Parts[2] && !Parts[3])
8346 return SDValue();
8347
8348 // The other parts need to be built with the old shuffle vector, cast to a
8349 // v4i32 and extract_vector_elts
8350 if (!Parts[0] || !Parts[1] || !Parts[2] || !Parts[3]) {
8351 SmallVector<int, 16> NewShuffleMask;
8352 for (int Part = 0; Part < 4; ++Part)
8353 for (int i = 0; i < QuarterSize; i++)
8354 NewShuffleMask.push_back(
8355 Elt: Parts[Part] ? -1 : ShuffleMask[Part * QuarterSize + i]);
8356 SDValue NewShuffle = DAG.getVectorShuffle(
8357 VT, dl, N1: Op->getOperand(Num: 0), N2: Op->getOperand(Num: 1), Mask: NewShuffleMask);
8358 SDValue BitCast = DAG.getBitcast(VT: MVT::v4f32, V: NewShuffle);
8359
8360 for (int Part = 0; Part < 4; ++Part)
8361 if (!Parts[Part])
8362 Parts[Part] = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: MVT::f32,
8363 N1: BitCast, N2: DAG.getConstant(Val: Part, DL: dl, VT: MVT::i32));
8364 }
8365 // Build a vector out of the various parts and bitcast it back to the original
8366 // type.
8367 SDValue NewVec = DAG.getNode(Opcode: ARMISD::BUILD_VECTOR, DL: dl, VT: MVT::v4f32, Ops: Parts);
8368 return DAG.getBitcast(VT, V: NewVec);
8369}
8370
8371static SDValue LowerVECTOR_SHUFFLEUsingOneOff(SDValue Op,
8372 ArrayRef<int> ShuffleMask,
8373 SelectionDAG &DAG) {
8374 SDValue V1 = Op.getOperand(i: 0);
8375 SDValue V2 = Op.getOperand(i: 1);
8376 EVT VT = Op.getValueType();
8377 unsigned NumElts = VT.getVectorNumElements();
8378
8379 // An One-Off Identity mask is one that is mostly an identity mask from as
8380 // single source but contains a single element out-of-place, either from a
8381 // different vector or from another position in the same vector. As opposed to
8382 // lowering this via a ARMISD::BUILD_VECTOR we can generate an extract/insert
8383 // pair directly.
8384 auto isOneOffIdentityMask = [](ArrayRef<int> Mask, EVT VT, int BaseOffset,
8385 int &OffElement) {
8386 OffElement = -1;
8387 int NonUndef = 0;
8388 for (int i = 0, NumMaskElts = Mask.size(); i < NumMaskElts; ++i) {
8389 if (Mask[i] == -1)
8390 continue;
8391 NonUndef++;
8392 if (Mask[i] != i + BaseOffset) {
8393 if (OffElement == -1)
8394 OffElement = i;
8395 else
8396 return false;
8397 }
8398 }
8399 return NonUndef > 2 && OffElement != -1;
8400 };
8401 int OffElement;
8402 SDValue VInput;
8403 if (isOneOffIdentityMask(ShuffleMask, VT, 0, OffElement))
8404 VInput = V1;
8405 else if (isOneOffIdentityMask(ShuffleMask, VT, NumElts, OffElement))
8406 VInput = V2;
8407 else
8408 return SDValue();
8409
8410 SDLoc dl(Op);
8411 EVT SVT = VT.getScalarType() == MVT::i8 || VT.getScalarType() == MVT::i16
8412 ? MVT::i32
8413 : VT.getScalarType();
8414 SDValue Elt = DAG.getNode(
8415 Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: SVT,
8416 N1: ShuffleMask[OffElement] < (int)NumElts ? V1 : V2,
8417 N2: DAG.getVectorIdxConstant(Val: ShuffleMask[OffElement] % NumElts, DL: dl));
8418 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT, N1: VInput, N2: Elt,
8419 N3: DAG.getVectorIdxConstant(Val: OffElement % NumElts, DL: dl));
8420}
8421
8422static SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG,
8423 const ARMSubtarget *ST) {
8424 SDValue V1 = Op.getOperand(i: 0);
8425 SDValue V2 = Op.getOperand(i: 1);
8426 SDLoc dl(Op);
8427 EVT VT = Op.getValueType();
8428 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val: Op.getNode());
8429 unsigned EltSize = VT.getScalarSizeInBits();
8430
8431 if (ST->hasMVEIntegerOps() && EltSize == 1)
8432 return LowerVECTOR_SHUFFLE_i1(Op, DAG, ST);
8433
8434 // Convert shuffles that are directly supported on NEON to target-specific
8435 // DAG nodes, instead of keeping them as shuffles and matching them again
8436 // during code selection. This is more efficient and avoids the possibility
8437 // of inconsistencies between legalization and selection.
8438 // FIXME: floating-point vectors should be canonicalized to integer vectors
8439 // of the same time so that they get CSEd properly.
8440 ArrayRef<int> ShuffleMask = SVN->getMask();
8441
8442 if (EltSize <= 32) {
8443 if (SVN->isSplat()) {
8444 int Lane = SVN->getSplatIndex();
8445 // If this is undef splat, generate it via "just" vdup, if possible.
8446 if (Lane == -1) Lane = 0;
8447
8448 // Test if V1 is a SCALAR_TO_VECTOR.
8449 if (Lane == 0 && V1.getOpcode() == ISD::SCALAR_TO_VECTOR) {
8450 return DAG.getNode(Opcode: ARMISD::VDUP, DL: dl, VT, Operand: V1.getOperand(i: 0));
8451 }
8452 // Test if V1 is a BUILD_VECTOR which is equivalent to a SCALAR_TO_VECTOR
8453 // (and probably will turn into a SCALAR_TO_VECTOR once legalization
8454 // reaches it).
8455 if (Lane == 0 && V1.getOpcode() == ISD::BUILD_VECTOR &&
8456 !isa<ConstantSDNode>(Val: V1.getOperand(i: 0))) {
8457 bool IsScalarToVector = true;
8458 for (unsigned i = 1, e = V1.getNumOperands(); i != e; ++i)
8459 if (!V1.getOperand(i).isUndef()) {
8460 IsScalarToVector = false;
8461 break;
8462 }
8463 if (IsScalarToVector)
8464 return DAG.getNode(Opcode: ARMISD::VDUP, DL: dl, VT, Operand: V1.getOperand(i: 0));
8465 }
8466 return DAG.getNode(Opcode: ARMISD::VDUPLANE, DL: dl, VT, N1: V1,
8467 N2: DAG.getConstant(Val: Lane, DL: dl, VT: MVT::i32));
8468 }
8469
8470 bool ReverseVEXT = false;
8471 unsigned Imm = 0;
8472 if (ST->hasNEON() && isVEXTMask(M: ShuffleMask, VT, ReverseVEXT, Imm)) {
8473 if (ReverseVEXT)
8474 std::swap(a&: V1, b&: V2);
8475 return DAG.getNode(Opcode: ARMISD::VEXT, DL: dl, VT, N1: V1, N2: V2,
8476 N3: DAG.getConstant(Val: Imm, DL: dl, VT: MVT::i32));
8477 }
8478
8479 if (isVREVMask(M: ShuffleMask, VT, BlockSize: 64))
8480 return DAG.getNode(Opcode: ARMISD::VREV64, DL: dl, VT, Operand: V1);
8481 if (isVREVMask(M: ShuffleMask, VT, BlockSize: 32))
8482 return DAG.getNode(Opcode: ARMISD::VREV32, DL: dl, VT, Operand: V1);
8483 if (isVREVMask(M: ShuffleMask, VT, BlockSize: 16))
8484 return DAG.getNode(Opcode: ARMISD::VREV16, DL: dl, VT, Operand: V1);
8485
8486 if (ST->hasNEON() && V2->isUndef() && isSingletonVEXTMask(M: ShuffleMask, VT, Imm)) {
8487 return DAG.getNode(Opcode: ARMISD::VEXT, DL: dl, VT, N1: V1, N2: V1,
8488 N3: DAG.getConstant(Val: Imm, DL: dl, VT: MVT::i32));
8489 }
8490
8491 // Check for Neon shuffles that modify both input vectors in place.
8492 // If both results are used, i.e., if there are two shuffles with the same
8493 // source operands and with masks corresponding to both results of one of
8494 // these operations, DAG memoization will ensure that a single node is
8495 // used for both shuffles.
8496 unsigned WhichResult = 0;
8497 bool isV_UNDEF = false;
8498 if (ST->hasNEON()) {
8499 if (unsigned ShuffleOpc = isNEONTwoResultShuffleMask(
8500 ShuffleMask, VT, WhichResult, isV_UNDEF)) {
8501 if (isV_UNDEF)
8502 V2 = V1;
8503 return DAG.getNode(Opcode: ShuffleOpc, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: VT), N1: V1, N2: V2)
8504 .getValue(R: WhichResult);
8505 }
8506 }
8507 if (ST->hasMVEIntegerOps()) {
8508 if (isVMOVNMask(M: ShuffleMask, VT, Top: false, SingleSource: false))
8509 return DAG.getNode(Opcode: ARMISD::VMOVN, DL: dl, VT, N1: V2, N2: V1,
8510 N3: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
8511 if (isVMOVNMask(M: ShuffleMask, VT, Top: true, SingleSource: false))
8512 return DAG.getNode(Opcode: ARMISD::VMOVN, DL: dl, VT, N1: V1, N2: V2,
8513 N3: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
8514 if (isVMOVNMask(M: ShuffleMask, VT, Top: true, SingleSource: true))
8515 return DAG.getNode(Opcode: ARMISD::VMOVN, DL: dl, VT, N1: V1, N2: V1,
8516 N3: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
8517 }
8518
8519 // Also check for these shuffles through CONCAT_VECTORS: we canonicalize
8520 // shuffles that produce a result larger than their operands with:
8521 // shuffle(concat(v1, undef), concat(v2, undef))
8522 // ->
8523 // shuffle(concat(v1, v2), undef)
8524 // because we can access quad vectors (see PerformVECTOR_SHUFFLECombine).
8525 //
8526 // This is useful in the general case, but there are special cases where
8527 // native shuffles produce larger results: the two-result ops.
8528 //
8529 // Look through the concat when lowering them:
8530 // shuffle(concat(v1, v2), undef)
8531 // ->
8532 // concat(VZIP(v1, v2):0, :1)
8533 //
8534 if (ST->hasNEON() && V1->getOpcode() == ISD::CONCAT_VECTORS && V2->isUndef()) {
8535 SDValue SubV1 = V1->getOperand(Num: 0);
8536 SDValue SubV2 = V1->getOperand(Num: 1);
8537 EVT SubVT = SubV1.getValueType();
8538
8539 // We expect these to have been canonicalized to -1.
8540 assert(llvm::all_of(ShuffleMask, [&](int i) {
8541 return i < (int)VT.getVectorNumElements();
8542 }) && "Unexpected shuffle index into UNDEF operand!");
8543
8544 if (unsigned ShuffleOpc = isNEONTwoResultShuffleMask(
8545 ShuffleMask, VT: SubVT, WhichResult, isV_UNDEF)) {
8546 if (isV_UNDEF)
8547 SubV2 = SubV1;
8548 assert((WhichResult == 0) &&
8549 "In-place shuffle of concat can only have one result!");
8550 SDValue Res = DAG.getNode(Opcode: ShuffleOpc, DL: dl, VTList: DAG.getVTList(VT1: SubVT, VT2: SubVT),
8551 N1: SubV1, N2: SubV2);
8552 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT, N1: Res.getValue(R: 0),
8553 N2: Res.getValue(R: 1));
8554 }
8555 }
8556 }
8557
8558 if (ST->hasMVEIntegerOps() && EltSize <= 32 &&
8559 (ST->hasFullFP16() || VT != MVT::v8f16)) {
8560 if (SDValue V = LowerVECTOR_SHUFFLEUsingOneOff(Op, ShuffleMask, DAG))
8561 return V;
8562
8563 for (bool Top : {false, true}) {
8564 for (bool SingleSource : {false, true}) {
8565 if (isTruncMask(M: ShuffleMask, VT, Top, SingleSource)) {
8566 MVT FromSVT = MVT::getIntegerVT(BitWidth: EltSize * 2);
8567 MVT FromVT = MVT::getVectorVT(VT: FromSVT, NumElements: ShuffleMask.size() / 2);
8568 SDValue Lo = DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT: FromVT, Operand: V1);
8569 SDValue Hi = DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT: FromVT,
8570 Operand: SingleSource ? V1 : V2);
8571 if (Top) {
8572 SDValue Amt = DAG.getConstant(Val: EltSize, DL: dl, VT: FromVT);
8573 Lo = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: FromVT, N1: Lo, N2: Amt);
8574 Hi = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: FromVT, N1: Hi, N2: Amt);
8575 }
8576 return DAG.getNode(Opcode: ARMISD::MVETRUNC, DL: dl, VT, N1: Lo, N2: Hi);
8577 }
8578 }
8579 }
8580 }
8581
8582 // If the shuffle is not directly supported and it has 4 elements, use
8583 // the PerfectShuffle-generated table to synthesize it from other shuffles.
8584 unsigned NumElts = VT.getVectorNumElements();
8585 if (NumElts == 4) {
8586 unsigned PFIndexes[4];
8587 for (unsigned i = 0; i != 4; ++i) {
8588 if (ShuffleMask[i] < 0)
8589 PFIndexes[i] = 8;
8590 else
8591 PFIndexes[i] = ShuffleMask[i];
8592 }
8593
8594 // Compute the index in the perfect shuffle table.
8595 unsigned PFTableIndex =
8596 PFIndexes[0]*9*9*9+PFIndexes[1]*9*9+PFIndexes[2]*9+PFIndexes[3];
8597 unsigned PFEntry = PerfectShuffleTable[PFTableIndex];
8598 unsigned Cost = (PFEntry >> 30);
8599
8600 if (Cost <= 4) {
8601 if (ST->hasNEON())
8602 return GeneratePerfectShuffle(PFEntry, LHS: V1, RHS: V2, DAG, dl);
8603 else if (isLegalMVEShuffleOp(PFEntry)) {
8604 unsigned LHSID = (PFEntry >> 13) & ((1 << 13)-1);
8605 unsigned RHSID = (PFEntry >> 0) & ((1 << 13)-1);
8606 unsigned PFEntryLHS = PerfectShuffleTable[LHSID];
8607 unsigned PFEntryRHS = PerfectShuffleTable[RHSID];
8608 if (isLegalMVEShuffleOp(PFEntry: PFEntryLHS) && isLegalMVEShuffleOp(PFEntry: PFEntryRHS))
8609 return GeneratePerfectShuffle(PFEntry, LHS: V1, RHS: V2, DAG, dl);
8610 }
8611 }
8612 }
8613
8614 // Implement shuffles with 32- or 64-bit elements as ARMISD::BUILD_VECTORs.
8615 if (EltSize >= 32) {
8616 // Do the expansion with floating-point types, since that is what the VFP
8617 // registers are defined to use, and since i64 is not legal.
8618 EVT EltVT = EVT::getFloatingPointVT(BitWidth: EltSize);
8619 EVT VecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, NumElements: NumElts);
8620 V1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: VecVT, Operand: V1);
8621 V2 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: VecVT, Operand: V2);
8622 SmallVector<SDValue, 8> Ops;
8623 for (unsigned i = 0; i < NumElts; ++i) {
8624 if (ShuffleMask[i] < 0)
8625 Ops.push_back(Elt: DAG.getUNDEF(VT: EltVT));
8626 else
8627 Ops.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT,
8628 N1: ShuffleMask[i] < (int)NumElts ? V1 : V2,
8629 N2: DAG.getConstant(Val: ShuffleMask[i] & (NumElts-1),
8630 DL: dl, VT: MVT::i32)));
8631 }
8632 SDValue Val = DAG.getNode(Opcode: ARMISD::BUILD_VECTOR, DL: dl, VT: VecVT, Ops);
8633 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: Val);
8634 }
8635
8636 if ((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) &&
8637 isReverseMask(M: ShuffleMask, VT))
8638 return LowerReverse_VECTOR_SHUFFLE(Op, DAG);
8639
8640 if (ST->hasNEON() && VT == MVT::v8i8)
8641 if (SDValue NewOp = LowerVECTOR_SHUFFLEv8i8(Op, ShuffleMask, DAG))
8642 return NewOp;
8643
8644 if (ST->hasMVEIntegerOps())
8645 if (SDValue NewOp = LowerVECTOR_SHUFFLEUsingMovs(Op, ShuffleMask, DAG))
8646 return NewOp;
8647
8648 // Lower v8f16 via v8i16 to avoid invalid f16 nodes.
8649 if (VT == MVT::v8f16 && !ST->hasFullFP16()) {
8650 SDValue BC0 =
8651 DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT: MVT::v8i16, Operand: Op.getOperand(i: 0));
8652 SDValue BC1 =
8653 DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT: MVT::v8i16, Operand: Op.getOperand(i: 1));
8654 SDValue Shuf = DAG.getVectorShuffle(VT: MVT::v8i16, dl, N1: BC0, N2: BC1, Mask: ShuffleMask);
8655 return DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT, Operand: Shuf);
8656 }
8657
8658 return SDValue();
8659}
8660
8661static SDValue LowerINSERT_VECTOR_ELT_i1(SDValue Op, SelectionDAG &DAG,
8662 const ARMSubtarget *ST) {
8663 EVT VecVT = Op.getOperand(i: 0).getValueType();
8664 SDLoc dl(Op);
8665
8666 assert(ST->hasMVEIntegerOps() &&
8667 "LowerINSERT_VECTOR_ELT_i1 called without MVE!");
8668
8669 SDValue Conv =
8670 DAG.getNode(Opcode: ARMISD::PREDICATE_CAST, DL: dl, VT: MVT::i32, Operand: Op->getOperand(Num: 0));
8671 unsigned Lane = Op.getConstantOperandVal(i: 2);
8672 unsigned LaneWidth =
8673 getVectorTyFromPredicateVector(VT: VecVT).getScalarSizeInBits() / 8;
8674 unsigned Mask = ((1 << LaneWidth) - 1) << Lane * LaneWidth;
8675 SDValue Ext = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl, VT: MVT::i32,
8676 N1: Op.getOperand(i: 1), N2: DAG.getValueType(MVT::i1));
8677 SDValue BFI = DAG.getNode(Opcode: ARMISD::BFI, DL: dl, VT: MVT::i32, N1: Conv, N2: Ext,
8678 N3: DAG.getConstant(Val: ~Mask, DL: dl, VT: MVT::i32));
8679 return DAG.getNode(Opcode: ARMISD::PREDICATE_CAST, DL: dl, VT: Op.getValueType(), Operand: BFI);
8680}
8681
8682SDValue ARMTargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
8683 SelectionDAG &DAG) const {
8684 // INSERT_VECTOR_ELT is legal only for immediate indexes.
8685 SDValue Lane = Op.getOperand(i: 2);
8686 if (!isa<ConstantSDNode>(Val: Lane))
8687 return SDValue();
8688
8689 SDValue Elt = Op.getOperand(i: 1);
8690 EVT EltVT = Elt.getValueType();
8691
8692 if (Subtarget->hasMVEIntegerOps() &&
8693 Op.getValueType().getScalarSizeInBits() == 1)
8694 return LowerINSERT_VECTOR_ELT_i1(Op, DAG, ST: Subtarget);
8695
8696 if (getTypeAction(Context&: *DAG.getContext(), VT: EltVT) ==
8697 TargetLowering::TypeSoftPromoteHalf) {
8698 // INSERT_VECTOR_ELT doesn't want f16 operands promoting to f32,
8699 // but the type system will try to do that if we don't intervene.
8700 // Reinterpret any such vector-element insertion as one with the
8701 // corresponding integer types.
8702
8703 SDLoc dl(Op);
8704
8705 EVT IEltVT = MVT::getIntegerVT(BitWidth: EltVT.getScalarSizeInBits());
8706 assert(getTypeAction(*DAG.getContext(), IEltVT) !=
8707 TargetLowering::TypeSoftPromoteHalf);
8708
8709 SDValue VecIn = Op.getOperand(i: 0);
8710 EVT VecVT = VecIn.getValueType();
8711 EVT IVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: IEltVT,
8712 NumElements: VecVT.getVectorNumElements());
8713
8714 SDValue IElt = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: IEltVT, Operand: Elt);
8715 SDValue IVecIn = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: IVecVT, Operand: VecIn);
8716 SDValue IVecOut = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: IVecVT,
8717 N1: IVecIn, N2: IElt, N3: Lane);
8718 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: VecVT, Operand: IVecOut);
8719 }
8720
8721 return Op;
8722}
8723
8724static SDValue LowerEXTRACT_VECTOR_ELT_i1(SDValue Op, SelectionDAG &DAG,
8725 const ARMSubtarget *ST) {
8726 EVT VecVT = Op.getOperand(i: 0).getValueType();
8727 SDLoc dl(Op);
8728
8729 assert(ST->hasMVEIntegerOps() &&
8730 "LowerINSERT_VECTOR_ELT_i1 called without MVE!");
8731
8732 SDValue Conv =
8733 DAG.getNode(Opcode: ARMISD::PREDICATE_CAST, DL: dl, VT: MVT::i32, Operand: Op->getOperand(Num: 0));
8734 unsigned Lane = Op.getConstantOperandVal(i: 1);
8735 unsigned LaneWidth =
8736 getVectorTyFromPredicateVector(VT: VecVT).getScalarSizeInBits() / 8;
8737 SDValue Shift = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i32, N1: Conv,
8738 N2: DAG.getConstant(Val: Lane * LaneWidth, DL: dl, VT: MVT::i32));
8739 return Shift;
8740}
8741
8742static SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG,
8743 const ARMSubtarget *ST) {
8744 // EXTRACT_VECTOR_ELT is legal only for immediate indexes.
8745 SDValue Lane = Op.getOperand(i: 1);
8746 if (!isa<ConstantSDNode>(Val: Lane))
8747 return SDValue();
8748
8749 SDValue Vec = Op.getOperand(i: 0);
8750 EVT VT = Vec.getValueType();
8751
8752 if (ST->hasMVEIntegerOps() && VT.getScalarSizeInBits() == 1)
8753 return LowerEXTRACT_VECTOR_ELT_i1(Op, DAG, ST);
8754
8755 if (Op.getValueType() == MVT::i32 && Vec.getScalarValueSizeInBits() < 32) {
8756 SDLoc dl(Op);
8757 return DAG.getNode(Opcode: ARMISD::VGETLANEu, DL: dl, VT: MVT::i32, N1: Vec, N2: Lane);
8758 }
8759
8760 return Op;
8761}
8762
8763static SDValue LowerCONCAT_VECTORS_i1(SDValue Op, SelectionDAG &DAG,
8764 const ARMSubtarget *ST) {
8765 SDLoc dl(Op);
8766 assert(Op.getValueType().getScalarSizeInBits() == 1 &&
8767 "Unexpected custom CONCAT_VECTORS lowering");
8768 assert(isPowerOf2_32(Op.getNumOperands()) &&
8769 "Unexpected custom CONCAT_VECTORS lowering");
8770 assert(ST->hasMVEIntegerOps() &&
8771 "CONCAT_VECTORS lowering only supported for MVE");
8772
8773 auto ConcatPair = [&](SDValue V1, SDValue V2) {
8774 EVT Op1VT = V1.getValueType();
8775 EVT Op2VT = V2.getValueType();
8776 assert(Op1VT == Op2VT && "Operand types don't match!");
8777 assert((Op1VT == MVT::v2i1 || Op1VT == MVT::v4i1 || Op1VT == MVT::v8i1) &&
8778 "Unexpected i1 concat operations!");
8779 EVT VT = Op1VT.getDoubleNumVectorElementsVT(Context&: *DAG.getContext());
8780
8781 SDValue NewV1 = PromoteMVEPredVector(dl, Pred: V1, VT: Op1VT, DAG);
8782 SDValue NewV2 = PromoteMVEPredVector(dl, Pred: V2, VT: Op2VT, DAG);
8783
8784 // We now have Op1 + Op2 promoted to vectors of integers, where v8i1 gets
8785 // promoted to v8i16, etc.
8786 MVT ElType =
8787 getVectorTyFromPredicateVector(VT).getScalarType().getSimpleVT();
8788 unsigned NumElts = 2 * Op1VT.getVectorNumElements();
8789
8790 EVT ConcatVT = MVT::getVectorVT(VT: ElType, NumElements: NumElts);
8791 if (Op1VT == MVT::v4i1 || Op1VT == MVT::v8i1) {
8792 // Use MVETRUNC to truncate the combined NewV1::NewV2 into the smaller
8793 // ConcatVT.
8794 SDValue ConVec =
8795 DAG.getNode(Opcode: ARMISD::MVETRUNC, DL: dl, VT: ConcatVT, N1: NewV1, N2: NewV2);
8796 return DAG.getNode(Opcode: ARMISD::VCMPZ, DL: dl, VT, N1: ConVec,
8797 N2: DAG.getConstant(Val: ARMCC::NE, DL: dl, VT: MVT::i32));
8798 }
8799
8800 // Extract the vector elements from Op1 and Op2 one by one and truncate them
8801 // to be the right size for the destination. For example, if Op1 is v4i1
8802 // then the promoted vector is v4i32. The result of concatenation gives a
8803 // v8i1, which when promoted is v8i16. That means each i32 element from Op1
8804 // needs truncating to i16 and inserting in the result.
8805 auto ExtractInto = [&DAG, &dl](SDValue NewV, SDValue ConVec, unsigned &j) {
8806 EVT NewVT = NewV.getValueType();
8807 EVT ConcatVT = ConVec.getValueType();
8808 unsigned ExtScale = 1;
8809 if (NewVT == MVT::v2f64) {
8810 NewV = DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT: MVT::v4i32, Operand: NewV);
8811 ExtScale = 2;
8812 }
8813 for (unsigned i = 0, e = NewVT.getVectorNumElements(); i < e; i++, j++) {
8814 SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: MVT::i32, N1: NewV,
8815 N2: DAG.getIntPtrConstant(Val: i * ExtScale, DL: dl));
8816 ConVec = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: ConcatVT, N1: ConVec, N2: Elt,
8817 N3: DAG.getConstant(Val: j, DL: dl, VT: MVT::i32));
8818 }
8819 return ConVec;
8820 };
8821 unsigned j = 0;
8822 SDValue ConVec = DAG.getNode(Opcode: ISD::UNDEF, DL: dl, VT: ConcatVT);
8823 ConVec = ExtractInto(NewV1, ConVec, j);
8824 ConVec = ExtractInto(NewV2, ConVec, j);
8825
8826 // Now return the result of comparing the subvector with zero, which will
8827 // generate a real predicate, i.e. v4i1, v8i1 or v16i1.
8828 return DAG.getNode(Opcode: ARMISD::VCMPZ, DL: dl, VT, N1: ConVec,
8829 N2: DAG.getConstant(Val: ARMCC::NE, DL: dl, VT: MVT::i32));
8830 };
8831
8832 // Concat each pair of subvectors and pack into the lower half of the array.
8833 SmallVector<SDValue> ConcatOps(Op->ops());
8834 while (ConcatOps.size() > 1) {
8835 for (unsigned I = 0, E = ConcatOps.size(); I != E; I += 2) {
8836 SDValue V1 = ConcatOps[I];
8837 SDValue V2 = ConcatOps[I + 1];
8838 ConcatOps[I / 2] = ConcatPair(V1, V2);
8839 }
8840 ConcatOps.resize(N: ConcatOps.size() / 2);
8841 }
8842 return ConcatOps[0];
8843}
8844
8845static SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG,
8846 const ARMSubtarget *ST) {
8847 EVT VT = Op->getValueType(ResNo: 0);
8848 if (ST->hasMVEIntegerOps() && VT.getScalarSizeInBits() == 1)
8849 return LowerCONCAT_VECTORS_i1(Op, DAG, ST);
8850
8851 // The only time a CONCAT_VECTORS operation can have legal types is when
8852 // two 64-bit vectors are concatenated to a 128-bit vector.
8853 assert(Op.getValueType().is128BitVector() && Op.getNumOperands() == 2 &&
8854 "unexpected CONCAT_VECTORS");
8855 SDLoc dl(Op);
8856 SDValue Val = DAG.getUNDEF(VT: MVT::v2f64);
8857 SDValue Op0 = Op.getOperand(i: 0);
8858 SDValue Op1 = Op.getOperand(i: 1);
8859 if (!Op0.isUndef())
8860 Val = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: MVT::v2f64, N1: Val,
8861 N2: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::f64, Operand: Op0),
8862 N3: DAG.getIntPtrConstant(Val: 0, DL: dl));
8863 if (!Op1.isUndef())
8864 Val = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: MVT::v2f64, N1: Val,
8865 N2: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::f64, Operand: Op1),
8866 N3: DAG.getIntPtrConstant(Val: 1, DL: dl));
8867 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: Op.getValueType(), Operand: Val);
8868}
8869
8870static SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, SelectionDAG &DAG,
8871 const ARMSubtarget *ST) {
8872 SDValue V1 = Op.getOperand(i: 0);
8873 SDValue V2 = Op.getOperand(i: 1);
8874 SDLoc dl(Op);
8875 EVT VT = Op.getValueType();
8876 EVT Op1VT = V1.getValueType();
8877 unsigned NumElts = VT.getVectorNumElements();
8878 unsigned Index = V2->getAsZExtVal();
8879
8880 assert(VT.getScalarSizeInBits() == 1 &&
8881 "Unexpected custom EXTRACT_SUBVECTOR lowering");
8882 assert(ST->hasMVEIntegerOps() &&
8883 "EXTRACT_SUBVECTOR lowering only supported for MVE");
8884
8885 SDValue NewV1 = PromoteMVEPredVector(dl, Pred: V1, VT: Op1VT, DAG);
8886
8887 // We now have Op1 promoted to a vector of integers, where v8i1 gets
8888 // promoted to v8i16, etc.
8889
8890 MVT ElType = getVectorTyFromPredicateVector(VT).getScalarType().getSimpleVT();
8891
8892 if (NumElts == 2) {
8893 EVT SubVT = MVT::v4i32;
8894 SDValue SubVec = DAG.getNode(Opcode: ISD::UNDEF, DL: dl, VT: SubVT);
8895 for (unsigned i = Index, j = 0; i < (Index + NumElts); i++, j += 2) {
8896 SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: MVT::i32, N1: NewV1,
8897 N2: DAG.getIntPtrConstant(Val: i, DL: dl));
8898 SubVec = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: SubVT, N1: SubVec, N2: Elt,
8899 N3: DAG.getConstant(Val: j, DL: dl, VT: MVT::i32));
8900 SubVec = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: SubVT, N1: SubVec, N2: Elt,
8901 N3: DAG.getConstant(Val: j + 1, DL: dl, VT: MVT::i32));
8902 }
8903 SDValue Cmp = DAG.getNode(Opcode: ARMISD::VCMPZ, DL: dl, VT: MVT::v4i1, N1: SubVec,
8904 N2: DAG.getConstant(Val: ARMCC::NE, DL: dl, VT: MVT::i32));
8905 return DAG.getNode(Opcode: ARMISD::PREDICATE_CAST, DL: dl, VT: MVT::v2i1, Operand: Cmp);
8906 }
8907
8908 EVT SubVT = MVT::getVectorVT(VT: ElType, NumElements: NumElts);
8909 SDValue SubVec = DAG.getNode(Opcode: ISD::UNDEF, DL: dl, VT: SubVT);
8910 for (unsigned i = Index, j = 0; i < (Index + NumElts); i++, j++) {
8911 SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: MVT::i32, N1: NewV1,
8912 N2: DAG.getIntPtrConstant(Val: i, DL: dl));
8913 SubVec = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: SubVT, N1: SubVec, N2: Elt,
8914 N3: DAG.getConstant(Val: j, DL: dl, VT: MVT::i32));
8915 }
8916
8917 // Now return the result of comparing the subvector with zero,
8918 // which will generate a real predicate, i.e. v4i1, v8i1 or v16i1.
8919 return DAG.getNode(Opcode: ARMISD::VCMPZ, DL: dl, VT, N1: SubVec,
8920 N2: DAG.getConstant(Val: ARMCC::NE, DL: dl, VT: MVT::i32));
8921}
8922
8923// Turn a truncate into a predicate (an i1 vector) into icmp(and(x, 1), 0).
8924static SDValue LowerTruncatei1(SDNode *N, SelectionDAG &DAG,
8925 const ARMSubtarget *ST) {
8926 assert(ST->hasMVEIntegerOps() && "Expected MVE!");
8927 EVT VT = N->getValueType(ResNo: 0);
8928 assert((VT == MVT::v16i1 || VT == MVT::v8i1 || VT == MVT::v4i1) &&
8929 "Expected a vector i1 type!");
8930 SDValue Op = N->getOperand(Num: 0);
8931 EVT FromVT = Op.getValueType();
8932 SDLoc DL(N);
8933
8934 SDValue And =
8935 DAG.getNode(Opcode: ISD::AND, DL, VT: FromVT, N1: Op, N2: DAG.getConstant(Val: 1, DL, VT: FromVT));
8936 return DAG.getNode(Opcode: ISD::SETCC, DL, VT, N1: And, N2: DAG.getConstant(Val: 0, DL, VT: FromVT),
8937 N3: DAG.getCondCode(Cond: ISD::SETNE));
8938}
8939
8940static SDValue LowerTruncate(SDNode *N, SelectionDAG &DAG,
8941 const ARMSubtarget *Subtarget) {
8942 if (!Subtarget->hasMVEIntegerOps())
8943 return SDValue();
8944
8945 EVT ToVT = N->getValueType(ResNo: 0);
8946 if (ToVT.getScalarType() == MVT::i1)
8947 return LowerTruncatei1(N, DAG, ST: Subtarget);
8948
8949 // MVE does not have a single instruction to perform the truncation of a v4i32
8950 // into the lower half of a v8i16, in the same way that a NEON vmovn would.
8951 // Most of the instructions in MVE follow the 'Beats' system, where moving
8952 // values from different lanes is usually something that the instructions
8953 // avoid.
8954 //
8955 // Instead it has top/bottom instructions such as VMOVLT/B and VMOVNT/B,
8956 // which take a the top/bottom half of a larger lane and extend it (or do the
8957 // opposite, truncating into the top/bottom lane from a larger lane). Note
8958 // that because of the way we widen lanes, a v4i16 is really a v4i32 using the
8959 // bottom 16bits from each vector lane. This works really well with T/B
8960 // instructions, but that doesn't extend to v8i32->v8i16 where the lanes need
8961 // to move order.
8962 //
8963 // But truncates and sext/zext are always going to be fairly common from llvm.
8964 // We have several options for how to deal with them:
8965 // - Wherever possible combine them into an instruction that makes them
8966 // "free". This includes loads/stores, which can perform the trunc as part
8967 // of the memory operation. Or certain shuffles that can be turned into
8968 // VMOVN/VMOVL.
8969 // - Lane Interleaving to transform blocks surrounded by ext/trunc. So
8970 // trunc(mul(sext(a), sext(b))) may become
8971 // VMOVNT(VMUL(VMOVLB(a), VMOVLB(b)), VMUL(VMOVLT(a), VMOVLT(b))). (Which in
8972 // this case can use VMULL). This is performed in the
8973 // MVELaneInterleavingPass.
8974 // - Otherwise we have an option. By default we would expand the
8975 // zext/sext/trunc into a series of lane extract/inserts going via GPR
8976 // registers. One for each vector lane in the vector. This can obviously be
8977 // very expensive.
8978 // - The other option is to use the fact that loads/store can extend/truncate
8979 // to turn a trunc into two truncating stack stores and a stack reload. This
8980 // becomes 3 back-to-back memory operations, but at least that is less than
8981 // all the insert/extracts.
8982 //
8983 // In order to do the last, we convert certain trunc's into MVETRUNC, which
8984 // are either optimized where they can be, or eventually lowered into stack
8985 // stores/loads. This prevents us from splitting a v8i16 trunc into two stores
8986 // two early, where other instructions would be better, and stops us from
8987 // having to reconstruct multiple buildvector shuffles into loads/stores.
8988 if (ToVT != MVT::v8i16 && ToVT != MVT::v16i8)
8989 return SDValue();
8990 EVT FromVT = N->getOperand(Num: 0).getValueType();
8991 if (FromVT != MVT::v8i32 && FromVT != MVT::v16i16)
8992 return SDValue();
8993
8994 SDValue Lo, Hi;
8995 std::tie(args&: Lo, args&: Hi) = DAG.SplitVectorOperand(N, OpNo: 0);
8996 SDLoc DL(N);
8997 return DAG.getNode(Opcode: ARMISD::MVETRUNC, DL, VT: ToVT, N1: Lo, N2: Hi);
8998}
8999
9000static SDValue LowerVectorExtend(SDNode *N, SelectionDAG &DAG,
9001 const ARMSubtarget *Subtarget) {
9002 if (!Subtarget->hasMVEIntegerOps())
9003 return SDValue();
9004
9005 // See LowerTruncate above for an explanation of MVEEXT/MVETRUNC.
9006
9007 EVT ToVT = N->getValueType(ResNo: 0);
9008 if (ToVT != MVT::v16i32 && ToVT != MVT::v8i32 && ToVT != MVT::v16i16)
9009 return SDValue();
9010 SDValue Op = N->getOperand(Num: 0);
9011 EVT FromVT = Op.getValueType();
9012 if (FromVT != MVT::v8i16 && FromVT != MVT::v16i8)
9013 return SDValue();
9014
9015 SDLoc DL(N);
9016 EVT ExtVT = ToVT.getHalfNumVectorElementsVT(Context&: *DAG.getContext());
9017 if (ToVT.getScalarType() == MVT::i32 && FromVT.getScalarType() == MVT::i8)
9018 ExtVT = MVT::v8i16;
9019
9020 unsigned Opcode =
9021 N->getOpcode() == ISD::SIGN_EXTEND ? ARMISD::MVESEXT : ARMISD::MVEZEXT;
9022 SDValue Ext = DAG.getNode(Opcode, DL, VTList: DAG.getVTList(VT1: ExtVT, VT2: ExtVT), N: Op);
9023 SDValue Ext1 = Ext.getValue(R: 1);
9024
9025 if (ToVT.getScalarType() == MVT::i32 && FromVT.getScalarType() == MVT::i8) {
9026 Ext = DAG.getNode(Opcode: N->getOpcode(), DL, VT: MVT::v8i32, Operand: Ext);
9027 Ext1 = DAG.getNode(Opcode: N->getOpcode(), DL, VT: MVT::v8i32, Operand: Ext1);
9028 }
9029
9030 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: ToVT, N1: Ext, N2: Ext1);
9031}
9032
9033/// isExtendedBUILD_VECTOR - Check if N is a constant BUILD_VECTOR where each
9034/// element has been zero/sign-extended, depending on the isSigned parameter,
9035/// from an integer type half its size.
9036static bool isExtendedBUILD_VECTOR(SDNode *N, SelectionDAG &DAG,
9037 bool isSigned) {
9038 // A v2i64 BUILD_VECTOR will have been legalized to a BITCAST from v4i32.
9039 EVT VT = N->getValueType(ResNo: 0);
9040 if (VT == MVT::v2i64 && N->getOpcode() == ISD::BITCAST) {
9041 SDNode *BVN = N->getOperand(Num: 0).getNode();
9042 if (BVN->getValueType(ResNo: 0) != MVT::v4i32 ||
9043 BVN->getOpcode() != ISD::BUILD_VECTOR)
9044 return false;
9045 unsigned LoElt = DAG.getDataLayout().isBigEndian() ? 1 : 0;
9046 unsigned HiElt = 1 - LoElt;
9047 ConstantSDNode *Lo0 = dyn_cast<ConstantSDNode>(Val: BVN->getOperand(Num: LoElt));
9048 ConstantSDNode *Hi0 = dyn_cast<ConstantSDNode>(Val: BVN->getOperand(Num: HiElt));
9049 ConstantSDNode *Lo1 = dyn_cast<ConstantSDNode>(Val: BVN->getOperand(Num: LoElt+2));
9050 ConstantSDNode *Hi1 = dyn_cast<ConstantSDNode>(Val: BVN->getOperand(Num: HiElt+2));
9051 if (!Lo0 || !Hi0 || !Lo1 || !Hi1)
9052 return false;
9053 if (isSigned) {
9054 if (Hi0->getSExtValue() == Lo0->getSExtValue() >> 32 &&
9055 Hi1->getSExtValue() == Lo1->getSExtValue() >> 32)
9056 return true;
9057 } else {
9058 if (Hi0->isZero() && Hi1->isZero())
9059 return true;
9060 }
9061 return false;
9062 }
9063
9064 if (N->getOpcode() != ISD::BUILD_VECTOR)
9065 return false;
9066
9067 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
9068 SDNode *Elt = N->getOperand(Num: i).getNode();
9069 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val: Elt)) {
9070 unsigned EltSize = VT.getScalarSizeInBits();
9071 unsigned HalfSize = EltSize / 2;
9072 if (isSigned) {
9073 if (!isIntN(N: HalfSize, x: C->getSExtValue()))
9074 return false;
9075 } else {
9076 if (!isUIntN(N: HalfSize, x: C->getZExtValue()))
9077 return false;
9078 }
9079 continue;
9080 }
9081 return false;
9082 }
9083
9084 return true;
9085}
9086
9087/// isSignExtended - Check if a node is a vector value that is sign-extended
9088/// or a constant BUILD_VECTOR with sign-extended elements.
9089static bool isSignExtended(SDNode *N, SelectionDAG &DAG) {
9090 if (N->getOpcode() == ISD::SIGN_EXTEND || ISD::isSEXTLoad(N))
9091 return true;
9092 if (isExtendedBUILD_VECTOR(N, DAG, isSigned: true))
9093 return true;
9094 return false;
9095}
9096
9097/// isZeroExtended - Check if a node is a vector value that is zero-extended (or
9098/// any-extended) or a constant BUILD_VECTOR with zero-extended elements.
9099static bool isZeroExtended(SDNode *N, SelectionDAG &DAG) {
9100 if (N->getOpcode() == ISD::ZERO_EXTEND || N->getOpcode() == ISD::ANY_EXTEND ||
9101 ISD::isZEXTLoad(N))
9102 return true;
9103 if (isExtendedBUILD_VECTOR(N, DAG, isSigned: false))
9104 return true;
9105 return false;
9106}
9107
9108static EVT getExtensionTo64Bits(const EVT &OrigVT) {
9109 if (OrigVT.getSizeInBits() >= 64)
9110 return OrigVT;
9111
9112 assert(OrigVT.isSimple() && "Expecting a simple value type");
9113
9114 MVT::SimpleValueType OrigSimpleTy = OrigVT.getSimpleVT().SimpleTy;
9115 switch (OrigSimpleTy) {
9116 default: llvm_unreachable("Unexpected Vector Type");
9117 case MVT::v2i8:
9118 case MVT::v2i16:
9119 return MVT::v2i32;
9120 case MVT::v4i8:
9121 return MVT::v4i16;
9122 }
9123}
9124
9125/// AddRequiredExtensionForVMULL - Add a sign/zero extension to extend the total
9126/// value size to 64 bits. We need a 64-bit D register as an operand to VMULL.
9127/// We insert the required extension here to get the vector to fill a D register.
9128static SDValue AddRequiredExtensionForVMULL(SDValue N, SelectionDAG &DAG,
9129 const EVT &OrigTy,
9130 const EVT &ExtTy,
9131 unsigned ExtOpcode) {
9132 // The vector originally had a size of OrigTy. It was then extended to ExtTy.
9133 // We expect the ExtTy to be 128-bits total. If the OrigTy is less than
9134 // 64-bits we need to insert a new extension so that it will be 64-bits.
9135 assert(ExtTy.is128BitVector() && "Unexpected extension size");
9136 if (OrigTy.getSizeInBits() >= 64)
9137 return N;
9138
9139 // Must extend size to at least 64 bits to be used as an operand for VMULL.
9140 EVT NewVT = getExtensionTo64Bits(OrigVT: OrigTy);
9141
9142 return DAG.getNode(Opcode: ExtOpcode, DL: SDLoc(N), VT: NewVT, Operand: N);
9143}
9144
9145/// SkipLoadExtensionForVMULL - return a load of the original vector size that
9146/// does not do any sign/zero extension. If the original vector is less
9147/// than 64 bits, an appropriate extension will be added after the load to
9148/// reach a total size of 64 bits. We have to add the extension separately
9149/// because ARM does not have a sign/zero extending load for vectors.
9150static SDValue SkipLoadExtensionForVMULL(LoadSDNode *LD, SelectionDAG& DAG) {
9151 EVT ExtendedTy = getExtensionTo64Bits(OrigVT: LD->getMemoryVT());
9152
9153 // The load already has the right type.
9154 if (ExtendedTy == LD->getMemoryVT())
9155 return DAG.getLoad(VT: LD->getMemoryVT(), dl: SDLoc(LD), Chain: LD->getChain(),
9156 Ptr: LD->getBasePtr(), PtrInfo: LD->getPointerInfo(), Alignment: LD->getAlign(),
9157 MMOFlags: LD->getMemOperand()->getFlags());
9158
9159 // We need to create a zextload/sextload. We cannot just create a load
9160 // followed by a zext/zext node because LowerMUL is also run during normal
9161 // operation legalization where we can't create illegal types.
9162 return DAG.getExtLoad(ExtType: LD->getExtensionType(), dl: SDLoc(LD), VT: ExtendedTy,
9163 Chain: LD->getChain(), Ptr: LD->getBasePtr(), PtrInfo: LD->getPointerInfo(),
9164 MemVT: LD->getMemoryVT(), Alignment: LD->getAlign(),
9165 MMOFlags: LD->getMemOperand()->getFlags());
9166}
9167
9168/// SkipExtensionForVMULL - For a node that is a SIGN_EXTEND, ZERO_EXTEND,
9169/// ANY_EXTEND, extending load, or BUILD_VECTOR with extended elements, return
9170/// the unextended value. The unextended vector should be 64 bits so that it can
9171/// be used as an operand to a VMULL instruction. If the original vector size
9172/// before extension is less than 64 bits we add a an extension to resize
9173/// the vector to 64 bits.
9174static SDValue SkipExtensionForVMULL(SDNode *N, SelectionDAG &DAG) {
9175 if (N->getOpcode() == ISD::SIGN_EXTEND ||
9176 N->getOpcode() == ISD::ZERO_EXTEND || N->getOpcode() == ISD::ANY_EXTEND)
9177 return AddRequiredExtensionForVMULL(N: N->getOperand(Num: 0), DAG,
9178 OrigTy: N->getOperand(Num: 0)->getValueType(ResNo: 0),
9179 ExtTy: N->getValueType(ResNo: 0),
9180 ExtOpcode: N->getOpcode());
9181
9182 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Val: N)) {
9183 assert((ISD::isSEXTLoad(LD) || ISD::isZEXTLoad(LD)) &&
9184 "Expected extending load");
9185
9186 SDValue newLoad = SkipLoadExtensionForVMULL(LD, DAG);
9187 DAG.ReplaceAllUsesOfValueWith(From: SDValue(LD, 1), To: newLoad.getValue(R: 1));
9188 unsigned Opcode = ISD::isSEXTLoad(N: LD) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
9189 SDValue extLoad =
9190 DAG.getNode(Opcode, DL: SDLoc(newLoad), VT: LD->getValueType(ResNo: 0), Operand: newLoad);
9191 DAG.ReplaceAllUsesOfValueWith(From: SDValue(LD, 0), To: extLoad);
9192
9193 return newLoad;
9194 }
9195
9196 // Otherwise, the value must be a BUILD_VECTOR. For v2i64, it will
9197 // have been legalized as a BITCAST from v4i32.
9198 if (N->getOpcode() == ISD::BITCAST) {
9199 SDNode *BVN = N->getOperand(Num: 0).getNode();
9200 assert(BVN->getOpcode() == ISD::BUILD_VECTOR &&
9201 BVN->getValueType(0) == MVT::v4i32 && "expected v4i32 BUILD_VECTOR");
9202 unsigned LowElt = DAG.getDataLayout().isBigEndian() ? 1 : 0;
9203 return DAG.getBuildVector(
9204 VT: MVT::v2i32, DL: SDLoc(N),
9205 Ops: {BVN->getOperand(Num: LowElt), BVN->getOperand(Num: LowElt + 2)});
9206 }
9207 // Construct a new BUILD_VECTOR with elements truncated to half the size.
9208 assert(N->getOpcode() == ISD::BUILD_VECTOR && "expected BUILD_VECTOR");
9209 EVT VT = N->getValueType(ResNo: 0);
9210 unsigned EltSize = VT.getScalarSizeInBits() / 2;
9211 unsigned NumElts = VT.getVectorNumElements();
9212 MVT TruncVT = MVT::getIntegerVT(BitWidth: EltSize);
9213 SmallVector<SDValue, 8> Ops;
9214 SDLoc dl(N);
9215 for (unsigned i = 0; i != NumElts; ++i) {
9216 const APInt &CInt = N->getConstantOperandAPInt(Num: i);
9217 // Element types smaller than 32 bits are not legal, so use i32 elements.
9218 // The values are implicitly truncated so sext vs. zext doesn't matter.
9219 Ops.push_back(Elt: DAG.getConstant(Val: CInt.zextOrTrunc(width: 32), DL: dl, VT: MVT::i32));
9220 }
9221 return DAG.getBuildVector(VT: MVT::getVectorVT(VT: TruncVT, NumElements: NumElts), DL: dl, Ops);
9222}
9223
9224static bool isAddSubSExt(SDNode *N, SelectionDAG &DAG) {
9225 unsigned Opcode = N->getOpcode();
9226 if (Opcode == ISD::ADD || Opcode == ISD::SUB) {
9227 SDNode *N0 = N->getOperand(Num: 0).getNode();
9228 SDNode *N1 = N->getOperand(Num: 1).getNode();
9229 return N0->hasOneUse() && N1->hasOneUse() &&
9230 isSignExtended(N: N0, DAG) && isSignExtended(N: N1, DAG);
9231 }
9232 return false;
9233}
9234
9235static bool isAddSubZExt(SDNode *N, SelectionDAG &DAG) {
9236 unsigned Opcode = N->getOpcode();
9237 if (Opcode == ISD::ADD || Opcode == ISD::SUB) {
9238 SDNode *N0 = N->getOperand(Num: 0).getNode();
9239 SDNode *N1 = N->getOperand(Num: 1).getNode();
9240 return N0->hasOneUse() && N1->hasOneUse() &&
9241 isZeroExtended(N: N0, DAG) && isZeroExtended(N: N1, DAG);
9242 }
9243 return false;
9244}
9245
9246static SDValue LowerMUL(SDValue Op, SelectionDAG &DAG) {
9247 // Multiplications are only custom-lowered for 128-bit vectors so that
9248 // VMULL can be detected. Otherwise v2i64 multiplications are not legal.
9249 EVT VT = Op.getValueType();
9250 assert(VT.is128BitVector() && VT.isInteger() &&
9251 "unexpected type for custom-lowering ISD::MUL");
9252 SDNode *N0 = Op.getOperand(i: 0).getNode();
9253 SDNode *N1 = Op.getOperand(i: 1).getNode();
9254 unsigned NewOpc = 0;
9255 bool isMLA = false;
9256 bool isN0SExt = isSignExtended(N: N0, DAG);
9257 bool isN1SExt = isSignExtended(N: N1, DAG);
9258 if (isN0SExt && isN1SExt)
9259 NewOpc = ARMISD::VMULLs;
9260 else {
9261 bool isN0ZExt = isZeroExtended(N: N0, DAG);
9262 bool isN1ZExt = isZeroExtended(N: N1, DAG);
9263 if (isN0ZExt && isN1ZExt)
9264 NewOpc = ARMISD::VMULLu;
9265 else if (isN1SExt || isN1ZExt) {
9266 // Look for (s/zext A + s/zext B) * (s/zext C). We want to turn these
9267 // into (s/zext A * s/zext C) + (s/zext B * s/zext C)
9268 if (isN1SExt && isAddSubSExt(N: N0, DAG)) {
9269 NewOpc = ARMISD::VMULLs;
9270 isMLA = true;
9271 } else if (isN1ZExt && isAddSubZExt(N: N0, DAG)) {
9272 NewOpc = ARMISD::VMULLu;
9273 isMLA = true;
9274 } else if (isN0ZExt && isAddSubZExt(N: N1, DAG)) {
9275 std::swap(a&: N0, b&: N1);
9276 NewOpc = ARMISD::VMULLu;
9277 isMLA = true;
9278 }
9279 }
9280
9281 if (!NewOpc) {
9282 if (VT == MVT::v2i64)
9283 // Fall through to expand this. It is not legal.
9284 return SDValue();
9285 else
9286 // Other vector multiplications are legal.
9287 return Op;
9288 }
9289 }
9290
9291 // Legalize to a VMULL instruction.
9292 SDLoc DL(Op);
9293 SDValue Op0;
9294 SDValue Op1 = SkipExtensionForVMULL(N: N1, DAG);
9295 if (!isMLA) {
9296 Op0 = SkipExtensionForVMULL(N: N0, DAG);
9297 assert(Op0.getValueType().is64BitVector() &&
9298 Op1.getValueType().is64BitVector() &&
9299 "unexpected types for extended operands to VMULL");
9300 return DAG.getNode(Opcode: NewOpc, DL, VT, N1: Op0, N2: Op1);
9301 }
9302
9303 // Optimizing (zext A + zext B) * C, to (VMULL A, C) + (VMULL B, C) during
9304 // isel lowering to take advantage of no-stall back to back vmul + vmla.
9305 // vmull q0, d4, d6
9306 // vmlal q0, d5, d6
9307 // is faster than
9308 // vaddl q0, d4, d5
9309 // vmovl q1, d6
9310 // vmul q0, q0, q1
9311 SDValue N00 = SkipExtensionForVMULL(N: N0->getOperand(Num: 0).getNode(), DAG);
9312 SDValue N01 = SkipExtensionForVMULL(N: N0->getOperand(Num: 1).getNode(), DAG);
9313 EVT Op1VT = Op1.getValueType();
9314 return DAG.getNode(Opcode: N0->getOpcode(), DL, VT,
9315 N1: DAG.getNode(Opcode: NewOpc, DL, VT,
9316 N1: DAG.getNode(Opcode: ISD::BITCAST, DL, VT: Op1VT, Operand: N00), N2: Op1),
9317 N2: DAG.getNode(Opcode: NewOpc, DL, VT,
9318 N1: DAG.getNode(Opcode: ISD::BITCAST, DL, VT: Op1VT, Operand: N01), N2: Op1));
9319}
9320
9321static SDValue LowerSDIV_v4i8(SDValue X, SDValue Y, const SDLoc &dl,
9322 SelectionDAG &DAG) {
9323 // TODO: Should this propagate fast-math-flags?
9324
9325 // Convert to float
9326 // float4 xf = vcvt_f32_s32(vmovl_s16(a.lo));
9327 // float4 yf = vcvt_f32_s32(vmovl_s16(b.lo));
9328 X = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: MVT::v4i32, Operand: X);
9329 Y = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: MVT::v4i32, Operand: Y);
9330 X = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: MVT::v4f32, Operand: X);
9331 Y = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: MVT::v4f32, Operand: Y);
9332 // Get reciprocal estimate.
9333 // float4 recip = vrecpeq_f32(yf);
9334 Y = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: MVT::v4f32,
9335 N1: DAG.getConstant(Val: Intrinsic::arm_neon_vrecpe, DL: dl, VT: MVT::i32),
9336 N2: Y);
9337 // Because char has a smaller range than uchar, we can actually get away
9338 // without any newton steps. This requires that we use a weird bias
9339 // of 0xb000, however (again, this has been exhaustively tested).
9340 // float4 result = as_float4(as_int4(xf*recip) + 0xb000);
9341 X = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::v4f32, N1: X, N2: Y);
9342 X = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v4i32, Operand: X);
9343 Y = DAG.getConstant(Val: 0xb000, DL: dl, VT: MVT::v4i32);
9344 X = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::v4i32, N1: X, N2: Y);
9345 X = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v4f32, Operand: X);
9346 // Convert back to short.
9347 X = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: dl, VT: MVT::v4i32, Operand: X);
9348 X = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::v4i16, Operand: X);
9349 return X;
9350}
9351
9352static SDValue LowerSDIV_v4i16(SDValue N0, SDValue N1, const SDLoc &dl,
9353 SelectionDAG &DAG) {
9354 // TODO: Should this propagate fast-math-flags?
9355
9356 SDValue N2;
9357 // Convert to float.
9358 // float4 yf = vcvt_f32_s32(vmovl_s16(y));
9359 // float4 xf = vcvt_f32_s32(vmovl_s16(x));
9360 N0 = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: MVT::v4i32, Operand: N0);
9361 N1 = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: MVT::v4i32, Operand: N1);
9362 N0 = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: MVT::v4f32, Operand: N0);
9363 N1 = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: MVT::v4f32, Operand: N1);
9364
9365 // Use reciprocal estimate and one refinement step.
9366 // float4 recip = vrecpeq_f32(yf);
9367 // recip *= vrecpsq_f32(yf, recip);
9368 N2 = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: MVT::v4f32,
9369 N1: DAG.getConstant(Val: Intrinsic::arm_neon_vrecpe, DL: dl, VT: MVT::i32),
9370 N2: N1);
9371 N1 = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: MVT::v4f32,
9372 N1: DAG.getConstant(Val: Intrinsic::arm_neon_vrecps, DL: dl, VT: MVT::i32),
9373 N2: N1, N3: N2);
9374 N2 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::v4f32, N1, N2);
9375 // Because short has a smaller range than ushort, we can actually get away
9376 // with only a single newton step. This requires that we use a weird bias
9377 // of 89, however (again, this has been exhaustively tested).
9378 // float4 result = as_float4(as_int4(xf*recip) + 0x89);
9379 N0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::v4f32, N1: N0, N2);
9380 N0 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v4i32, Operand: N0);
9381 N1 = DAG.getConstant(Val: 0x89, DL: dl, VT: MVT::v4i32);
9382 N0 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::v4i32, N1: N0, N2: N1);
9383 N0 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v4f32, Operand: N0);
9384 // Convert back to integer and return.
9385 // return vmovn_s32(vcvt_s32_f32(result));
9386 N0 = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: dl, VT: MVT::v4i32, Operand: N0);
9387 N0 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::v4i16, Operand: N0);
9388 return N0;
9389}
9390
9391static SDValue LowerSDIV(SDValue Op, SelectionDAG &DAG,
9392 const ARMSubtarget *ST) {
9393 EVT VT = Op.getValueType();
9394 assert((VT == MVT::v4i16 || VT == MVT::v8i8) &&
9395 "unexpected type for custom-lowering ISD::SDIV");
9396
9397 SDLoc dl(Op);
9398 SDValue N0 = Op.getOperand(i: 0);
9399 SDValue N1 = Op.getOperand(i: 1);
9400 SDValue N2, N3;
9401
9402 if (VT == MVT::v8i8) {
9403 N0 = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: MVT::v8i16, Operand: N0);
9404 N1 = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: MVT::v8i16, Operand: N1);
9405
9406 N2 = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: MVT::v4i16, N1: N0,
9407 N2: DAG.getIntPtrConstant(Val: 4, DL: dl));
9408 N3 = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: MVT::v4i16, N1,
9409 N2: DAG.getIntPtrConstant(Val: 4, DL: dl));
9410 N0 = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: MVT::v4i16, N1: N0,
9411 N2: DAG.getIntPtrConstant(Val: 0, DL: dl));
9412 N1 = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: MVT::v4i16, N1,
9413 N2: DAG.getIntPtrConstant(Val: 0, DL: dl));
9414
9415 N0 = LowerSDIV_v4i8(X: N0, Y: N1, dl, DAG); // v4i16
9416 N2 = LowerSDIV_v4i8(X: N2, Y: N3, dl, DAG); // v4i16
9417
9418 N0 = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: MVT::v8i16, N1: N0, N2);
9419 N0 = LowerCONCAT_VECTORS(Op: N0, DAG, ST);
9420
9421 N0 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::v8i8, Operand: N0);
9422 return N0;
9423 }
9424 return LowerSDIV_v4i16(N0, N1, dl, DAG);
9425}
9426
9427static SDValue LowerUDIV(SDValue Op, SelectionDAG &DAG,
9428 const ARMSubtarget *ST) {
9429 // TODO: Should this propagate fast-math-flags?
9430 EVT VT = Op.getValueType();
9431 assert((VT == MVT::v4i16 || VT == MVT::v8i8) &&
9432 "unexpected type for custom-lowering ISD::UDIV");
9433
9434 SDLoc dl(Op);
9435 SDValue N0 = Op.getOperand(i: 0);
9436 SDValue N1 = Op.getOperand(i: 1);
9437 SDValue N2, N3;
9438
9439 if (VT == MVT::v8i8) {
9440 N0 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: MVT::v8i16, Operand: N0);
9441 N1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: MVT::v8i16, Operand: N1);
9442
9443 N2 = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: MVT::v4i16, N1: N0,
9444 N2: DAG.getIntPtrConstant(Val: 4, DL: dl));
9445 N3 = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: MVT::v4i16, N1,
9446 N2: DAG.getIntPtrConstant(Val: 4, DL: dl));
9447 N0 = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: MVT::v4i16, N1: N0,
9448 N2: DAG.getIntPtrConstant(Val: 0, DL: dl));
9449 N1 = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: MVT::v4i16, N1,
9450 N2: DAG.getIntPtrConstant(Val: 0, DL: dl));
9451
9452 N0 = LowerSDIV_v4i16(N0, N1, dl, DAG); // v4i16
9453 N2 = LowerSDIV_v4i16(N0: N2, N1: N3, dl, DAG); // v4i16
9454
9455 N0 = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: MVT::v8i16, N1: N0, N2);
9456 N0 = LowerCONCAT_VECTORS(Op: N0, DAG, ST);
9457
9458 N0 = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: MVT::v8i8,
9459 N1: DAG.getConstant(Val: Intrinsic::arm_neon_vqmovnsu, DL: dl,
9460 VT: MVT::i32),
9461 N2: N0);
9462 return N0;
9463 }
9464
9465 // v4i16 sdiv ... Convert to float.
9466 // float4 yf = vcvt_f32_s32(vmovl_u16(y));
9467 // float4 xf = vcvt_f32_s32(vmovl_u16(x));
9468 N0 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: MVT::v4i32, Operand: N0);
9469 N1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: MVT::v4i32, Operand: N1);
9470 N0 = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: MVT::v4f32, Operand: N0);
9471 SDValue BN1 = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: MVT::v4f32, Operand: N1);
9472
9473 // Use reciprocal estimate and two refinement steps.
9474 // float4 recip = vrecpeq_f32(yf);
9475 // recip *= vrecpsq_f32(yf, recip);
9476 // recip *= vrecpsq_f32(yf, recip);
9477 N2 = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: MVT::v4f32,
9478 N1: DAG.getConstant(Val: Intrinsic::arm_neon_vrecpe, DL: dl, VT: MVT::i32),
9479 N2: BN1);
9480 N1 = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: MVT::v4f32,
9481 N1: DAG.getConstant(Val: Intrinsic::arm_neon_vrecps, DL: dl, VT: MVT::i32),
9482 N2: BN1, N3: N2);
9483 N2 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::v4f32, N1, N2);
9484 N1 = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: MVT::v4f32,
9485 N1: DAG.getConstant(Val: Intrinsic::arm_neon_vrecps, DL: dl, VT: MVT::i32),
9486 N2: BN1, N3: N2);
9487 N2 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::v4f32, N1, N2);
9488 // Simply multiplying by the reciprocal estimate can leave us a few ulps
9489 // too low, so we add 2 ulps (exhaustive testing shows that this is enough,
9490 // and that it will never cause us to return an answer too large).
9491 // float4 result = as_float4(as_int4(xf*recip) + 2);
9492 N0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::v4f32, N1: N0, N2);
9493 N0 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v4i32, Operand: N0);
9494 N1 = DAG.getConstant(Val: 2, DL: dl, VT: MVT::v4i32);
9495 N0 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::v4i32, N1: N0, N2: N1);
9496 N0 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v4f32, Operand: N0);
9497 // Convert back to integer and return.
9498 // return vmovn_u32(vcvt_s32_f32(result));
9499 N0 = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: dl, VT: MVT::v4i32, Operand: N0);
9500 N0 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::v4i16, Operand: N0);
9501 return N0;
9502}
9503
9504static SDValue LowerADDSUBO_CARRY(SDValue Op, SelectionDAG &DAG,
9505 unsigned Opcode, bool IsSigned) {
9506 EVT VT0 = Op.getValue(R: 0).getValueType();
9507 EVT VT1 = Op.getValue(R: 1).getValueType();
9508
9509 bool InvertCarry = Opcode == ARMISD::SUBE;
9510 SDValue OpLHS = Op.getOperand(i: 0);
9511 SDValue OpRHS = Op.getOperand(i: 1);
9512 SDValue OpCarryIn = valueToCarryFlag(Value: Op.getOperand(i: 2), DAG, Invert: InvertCarry);
9513
9514 SDLoc DL(Op);
9515
9516 SDValue Result = DAG.getNode(Opcode, DL, VTList: DAG.getVTList(VT1: VT0, VT2: MVT::i32), N1: OpLHS,
9517 N2: OpRHS, N3: OpCarryIn);
9518
9519 SDValue OutFlag =
9520 IsSigned ? overflowFlagToValue(Flags: Result.getValue(R: 1), VT: VT1, DAG)
9521 : carryFlagToValue(Flags: Result.getValue(R: 1), VT: VT1, DAG, Invert: InvertCarry);
9522
9523 return DAG.getMergeValues(Ops: {Result, OutFlag}, dl: DL);
9524}
9525
9526SDValue ARMTargetLowering::LowerWindowsDIVLibCall(SDValue Op, SelectionDAG &DAG,
9527 bool Signed,
9528 SDValue &Chain) const {
9529 EVT VT = Op.getValueType();
9530 assert((VT == MVT::i32 || VT == MVT::i64) &&
9531 "unexpected type for custom lowering DIV");
9532 SDLoc dl(Op);
9533
9534 const auto &DL = DAG.getDataLayout();
9535 RTLIB::Libcall LC;
9536 if (Signed)
9537 LC = VT == MVT::i32 ? RTLIB::SDIVREM_I32 : RTLIB::SDIVREM_I64;
9538 else
9539 LC = VT == MVT::i32 ? RTLIB::UDIVREM_I32 : RTLIB::UDIVREM_I64;
9540
9541 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
9542 SDValue ES = DAG.getExternalSymbol(LCImpl, VT: getPointerTy(DL));
9543
9544 ARMTargetLowering::ArgListTy Args;
9545
9546 for (auto AI : {1, 0}) {
9547 SDValue Operand = Op.getOperand(i: AI);
9548 Args.emplace_back(args&: Operand,
9549 args: Operand.getValueType().getTypeForEVT(Context&: *DAG.getContext()));
9550 }
9551
9552 CallLoweringInfo CLI(DAG);
9553 CLI.setDebugLoc(dl).setChain(Chain).setCallee(
9554 CC: DAG.getLibcalls().getLibcallImplCallingConv(Call: LCImpl),
9555 ResultType: VT.getTypeForEVT(Context&: *DAG.getContext()), Target: ES, ArgsList: std::move(Args));
9556
9557 return LowerCallTo(CLI).first;
9558}
9559
9560// This is a code size optimisation: return the original SDIV node to
9561// DAGCombiner when we don't want to expand SDIV into a sequence of
9562// instructions, and an empty node otherwise which will cause the
9563// SDIV to be expanded in DAGCombine.
9564SDValue
9565ARMTargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
9566 SelectionDAG &DAG,
9567 SmallVectorImpl<SDNode *> &Created) const {
9568 // TODO: Support SREM
9569 if (N->getOpcode() != ISD::SDIV)
9570 return SDValue();
9571
9572 const auto &ST = DAG.getSubtarget<ARMSubtarget>();
9573 const bool MinSize = ST.hasMinSize();
9574 const bool HasDivide = ST.isThumb() ? ST.hasDivideInThumbMode()
9575 : ST.hasDivideInARMMode();
9576
9577 // Don't touch vector types; rewriting this may lead to scalarizing
9578 // the int divs.
9579 if (N->getOperand(Num: 0).getValueType().isVector())
9580 return SDValue();
9581
9582 // Bail if MinSize is not set, and also for both ARM and Thumb mode we need
9583 // hwdiv support for this to be really profitable.
9584 if (!(MinSize && HasDivide))
9585 return SDValue();
9586
9587 // ARM mode is a bit simpler than Thumb: we can handle large power
9588 // of 2 immediates with 1 mov instruction; no further checks required,
9589 // just return the sdiv node.
9590 if (!ST.isThumb())
9591 return SDValue(N, 0);
9592
9593 // In Thumb mode, immediates larger than 128 need a wide 4-byte MOV,
9594 // and thus lose the code size benefits of a MOVS that requires only 2.
9595 // TargetTransformInfo and 'getIntImmCodeSizeCost' could be helpful here,
9596 // but as it's doing exactly this, it's not worth the trouble to get TTI.
9597 if (Divisor.sgt(RHS: 128))
9598 return SDValue();
9599
9600 return SDValue(N, 0);
9601}
9602
9603SDValue ARMTargetLowering::LowerDIV_Windows(SDValue Op, SelectionDAG &DAG,
9604 bool Signed) const {
9605 assert(Op.getValueType() == MVT::i32 &&
9606 "unexpected type for custom lowering DIV");
9607 SDLoc dl(Op);
9608
9609 SDValue DBZCHK = DAG.getNode(Opcode: ARMISD::WIN__DBZCHK, DL: dl, VT: MVT::Other,
9610 N1: DAG.getEntryNode(), N2: Op.getOperand(i: 1));
9611
9612 return LowerWindowsDIVLibCall(Op, DAG, Signed, Chain&: DBZCHK);
9613}
9614
9615static SDValue WinDBZCheckDenominator(SelectionDAG &DAG, SDNode *N, SDValue InChain) {
9616 SDLoc DL(N);
9617 SDValue Op = N->getOperand(Num: 1);
9618 if (N->getValueType(ResNo: 0) == MVT::i32)
9619 return DAG.getNode(Opcode: ARMISD::WIN__DBZCHK, DL, VT: MVT::Other, N1: InChain, N2: Op);
9620 SDValue Lo, Hi;
9621 std::tie(args&: Lo, args&: Hi) = DAG.SplitScalar(N: Op, DL, LoVT: MVT::i32, HiVT: MVT::i32);
9622 return DAG.getNode(Opcode: ARMISD::WIN__DBZCHK, DL, VT: MVT::Other, N1: InChain,
9623 N2: DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: Lo, N2: Hi));
9624}
9625
9626void ARMTargetLowering::ExpandDIV_Windows(
9627 SDValue Op, SelectionDAG &DAG, bool Signed,
9628 SmallVectorImpl<SDValue> &Results) const {
9629 const auto &DL = DAG.getDataLayout();
9630
9631 assert(Op.getValueType() == MVT::i64 &&
9632 "unexpected type for custom lowering DIV");
9633 SDLoc dl(Op);
9634
9635 SDValue DBZCHK = WinDBZCheckDenominator(DAG, N: Op.getNode(), InChain: DAG.getEntryNode());
9636
9637 SDValue Result = LowerWindowsDIVLibCall(Op, DAG, Signed, Chain&: DBZCHK);
9638
9639 SDValue Lower = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i32, Operand: Result);
9640 SDValue Upper = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i64, N1: Result,
9641 N2: DAG.getConstant(Val: 32, DL: dl, VT: getPointerTy(DL)));
9642 Upper = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i32, Operand: Upper);
9643
9644 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: MVT::i64, N1: Lower, N2: Upper));
9645}
9646
9647std::pair<SDValue, SDValue>
9648ARMTargetLowering::LowerAEABIUnalignedLoad(SDValue Op,
9649 SelectionDAG &DAG) const {
9650 // If we have an unaligned load from a i32 or i64 that would normally be
9651 // split into separate ldrb's, we can use the __aeabi_uread4/__aeabi_uread8
9652 // functions instead.
9653 LoadSDNode *LD = cast<LoadSDNode>(Val: Op.getNode());
9654 EVT MemVT = LD->getMemoryVT();
9655 if (MemVT != MVT::i32 && MemVT != MVT::i64)
9656 return std::make_pair(x: SDValue(), y: SDValue());
9657
9658 const auto &MF = DAG.getMachineFunction();
9659 unsigned AS = LD->getAddressSpace();
9660 Align Alignment = LD->getAlign();
9661 const DataLayout &DL = DAG.getDataLayout();
9662 bool AllowsUnaligned = Subtarget->allowsUnalignedMem();
9663 RTLIB::Libcall LC =
9664 (MemVT == MVT::i32) ? RTLIB::AEABI_UREAD4 : RTLIB::AEABI_UREAD8;
9665
9666 if (MF.getFunction().hasMinSize() && !AllowsUnaligned &&
9667 Alignment <= llvm::Align(2) && DAG.getLibcalls().getLibcallImpl(Call: LC)) {
9668 MakeLibCallOptions Opts;
9669 SDLoc dl(Op);
9670
9671 auto Pair = makeLibCall(DAG, LC, RetVT: MemVT.getSimpleVT(), Ops: LD->getBasePtr(),
9672 CallOptions: Opts, dl, Chain: LD->getChain());
9673
9674 // If necessary, extend the node to 64bit
9675 if (LD->getExtensionType() != ISD::NON_EXTLOAD) {
9676 unsigned ExtType = LD->getExtensionType() == ISD::SEXTLOAD
9677 ? ISD::SIGN_EXTEND
9678 : ISD::ZERO_EXTEND;
9679 SDValue EN = DAG.getNode(Opcode: ExtType, DL: dl, VT: LD->getValueType(ResNo: 0), Operand: Pair.first);
9680 Pair.first = EN;
9681 }
9682 return Pair;
9683 }
9684
9685 // Default expand to individual loads
9686 if (!allowsMemoryAccess(Context&: *DAG.getContext(), DL, VT: MemVT, AddrSpace: AS, Alignment))
9687 return expandUnalignedLoad(LD, DAG);
9688 return std::make_pair(x: SDValue(), y: SDValue());
9689}
9690
9691SDValue ARMTargetLowering::LowerAEABIUnalignedStore(SDValue Op,
9692 SelectionDAG &DAG) const {
9693 // If we have an unaligned store to a i32 or i64 that would normally be
9694 // split into separate ldrb's, we can use the __aeabi_uwrite4/__aeabi_uwrite8
9695 // functions instead.
9696 StoreSDNode *ST = cast<StoreSDNode>(Val: Op.getNode());
9697 EVT MemVT = ST->getMemoryVT();
9698 if (MemVT != MVT::i32 && MemVT != MVT::i64)
9699 return SDValue();
9700
9701 const auto &MF = DAG.getMachineFunction();
9702 unsigned AS = ST->getAddressSpace();
9703 Align Alignment = ST->getAlign();
9704 const DataLayout &DL = DAG.getDataLayout();
9705 bool AllowsUnaligned = Subtarget->allowsUnalignedMem();
9706 RTLIB::Libcall LC =
9707 (MemVT == MVT::i32) ? RTLIB::AEABI_UWRITE4 : RTLIB::AEABI_UWRITE8;
9708
9709 if (MF.getFunction().hasMinSize() && !AllowsUnaligned &&
9710 Alignment <= llvm::Align(2) && DAG.getLibcalls().getLibcallImpl(Call: LC)) {
9711
9712 SDLoc dl(Op);
9713
9714 // If necessary, trunc the value to 32bit
9715 SDValue StoreVal = ST->getOperand(Num: 1);
9716 if (ST->isTruncatingStore())
9717 StoreVal = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MemVT, Operand: ST->getOperand(Num: 1));
9718
9719 MakeLibCallOptions Opts;
9720 auto CallResult =
9721 makeLibCall(DAG, LC, RetVT: MVT::isVoid, Ops: {StoreVal, ST->getBasePtr()}, CallOptions: Opts,
9722 dl, Chain: ST->getChain());
9723
9724 return CallResult.second;
9725 }
9726
9727 // Default expand to individual stores
9728 if (!allowsMemoryAccess(Context&: *DAG.getContext(), DL, VT: MemVT, AddrSpace: AS, Alignment))
9729 return expandUnalignedStore(ST, DAG);
9730 return SDValue();
9731}
9732
9733static SDValue LowerPredicateLoad(SDValue Op, SelectionDAG &DAG) {
9734 LoadSDNode *LD = cast<LoadSDNode>(Val: Op.getNode());
9735 EVT MemVT = LD->getMemoryVT();
9736 assert((MemVT == MVT::v2i1 || MemVT == MVT::v4i1 || MemVT == MVT::v8i1 ||
9737 MemVT == MVT::v16i1) &&
9738 "Expected a predicate type!");
9739 assert(MemVT == Op.getValueType());
9740 assert(LD->getExtensionType() == ISD::NON_EXTLOAD &&
9741 "Expected a non-extending load");
9742 assert(LD->isUnindexed() && "Expected a unindexed load");
9743
9744 // The basic MVE VLDR on a v2i1/v4i1/v8i1 actually loads the entire 16bit
9745 // predicate, with the "v4i1" bits spread out over the 16 bits loaded. We
9746 // need to make sure that 8/4/2 bits are actually loaded into the correct
9747 // place, which means loading the value and then shuffling the values into
9748 // the bottom bits of the predicate.
9749 // Equally, VLDR for an v16i1 will actually load 32bits (so will be incorrect
9750 // for BE).
9751 // Speaking of BE, apparently the rest of llvm will assume a reverse order to
9752 // a natural VMSR(load), so needs to be reversed.
9753
9754 SDLoc dl(Op);
9755 SDValue Load = DAG.getExtLoad(
9756 ExtType: ISD::EXTLOAD, dl, VT: MVT::i32, Chain: LD->getChain(), Ptr: LD->getBasePtr(),
9757 MemVT: EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: MemVT.getSizeInBits()),
9758 MMO: LD->getMemOperand());
9759 SDValue Val = Load;
9760 if (DAG.getDataLayout().isBigEndian())
9761 Val = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i32,
9762 N1: DAG.getNode(Opcode: ISD::BITREVERSE, DL: dl, VT: MVT::i32, Operand: Load),
9763 N2: DAG.getConstant(Val: 32 - MemVT.getSizeInBits(), DL: dl, VT: MVT::i32));
9764 SDValue Pred = DAG.getNode(Opcode: ARMISD::PREDICATE_CAST, DL: dl, VT: MVT::v16i1, Operand: Val);
9765 if (MemVT != MVT::v16i1)
9766 Pred = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: MemVT, N1: Pred,
9767 N2: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
9768 return DAG.getMergeValues(Ops: {Pred, Load.getValue(R: 1)}, dl);
9769}
9770
9771void ARMTargetLowering::LowerLOAD(SDNode *N, SmallVectorImpl<SDValue> &Results,
9772 SelectionDAG &DAG) const {
9773 LoadSDNode *LD = cast<LoadSDNode>(Val: N);
9774 EVT MemVT = LD->getMemoryVT();
9775
9776 if (MemVT == MVT::i64 && Subtarget->hasV5TEOps() &&
9777 !Subtarget->isThumb1Only() && LD->isVolatile() &&
9778 LD->getAlign() >= Subtarget->getDualLoadStoreAlignment()) {
9779 assert(LD->isUnindexed() && "Loads should be unindexed at this point.");
9780 SDLoc dl(N);
9781 SDValue Result = DAG.getMemIntrinsicNode(
9782 Opcode: ARMISD::LDRD, dl, VTList: DAG.getVTList(VTs: {MVT::i32, MVT::i32, MVT::Other}),
9783 Ops: {LD->getChain(), LD->getBasePtr()}, MemVT, MMO: LD->getMemOperand());
9784 SDValue Lo = Result.getValue(R: DAG.getDataLayout().isLittleEndian() ? 0 : 1);
9785 SDValue Hi = Result.getValue(R: DAG.getDataLayout().isLittleEndian() ? 1 : 0);
9786 SDValue Pair = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: MVT::i64, N1: Lo, N2: Hi);
9787 Results.append(IL: {Pair, Result.getValue(R: 2)});
9788 } else if (MemVT == MVT::i32 || MemVT == MVT::i64) {
9789 auto Pair = LowerAEABIUnalignedLoad(Op: SDValue(N, 0), DAG);
9790 if (Pair.first) {
9791 Results.push_back(Elt: Pair.first);
9792 Results.push_back(Elt: Pair.second);
9793 }
9794 }
9795}
9796
9797static SDValue LowerPredicateStore(SDValue Op, SelectionDAG &DAG) {
9798 StoreSDNode *ST = cast<StoreSDNode>(Val: Op.getNode());
9799 EVT MemVT = ST->getMemoryVT();
9800 assert((MemVT == MVT::v2i1 || MemVT == MVT::v4i1 || MemVT == MVT::v8i1 ||
9801 MemVT == MVT::v16i1) &&
9802 "Expected a predicate type!");
9803 assert(MemVT == ST->getValue().getValueType());
9804 assert(!ST->isTruncatingStore() && "Expected a non-extending store");
9805 assert(ST->isUnindexed() && "Expected a unindexed store");
9806
9807 // Only store the v2i1 or v4i1 or v8i1 worth of bits, via a buildvector with
9808 // top bits unset and a scalar store.
9809 SDLoc dl(Op);
9810 SDValue Build = ST->getValue();
9811 if (MemVT != MVT::v16i1) {
9812 SmallVector<SDValue, 16> Ops;
9813 for (unsigned I = 0; I < MemVT.getVectorNumElements(); I++) {
9814 unsigned Elt = DAG.getDataLayout().isBigEndian()
9815 ? MemVT.getVectorNumElements() - I - 1
9816 : I;
9817 Ops.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: MVT::i32, N1: Build,
9818 N2: DAG.getConstant(Val: Elt, DL: dl, VT: MVT::i32)));
9819 }
9820 for (unsigned I = MemVT.getVectorNumElements(); I < 16; I++)
9821 Ops.push_back(Elt: DAG.getUNDEF(VT: MVT::i32));
9822 Build = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: dl, VT: MVT::v16i1, Ops);
9823 }
9824 SDValue GRP = DAG.getNode(Opcode: ARMISD::PREDICATE_CAST, DL: dl, VT: MVT::i32, Operand: Build);
9825 if (MemVT == MVT::v16i1 && DAG.getDataLayout().isBigEndian())
9826 GRP = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i32,
9827 N1: DAG.getNode(Opcode: ISD::BITREVERSE, DL: dl, VT: MVT::i32, Operand: GRP),
9828 N2: DAG.getConstant(Val: 16, DL: dl, VT: MVT::i32));
9829 return DAG.getTruncStore(
9830 Chain: ST->getChain(), dl, Val: GRP, Ptr: ST->getBasePtr(),
9831 SVT: EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: MemVT.getSizeInBits()),
9832 MMO: ST->getMemOperand());
9833}
9834
9835SDValue ARMTargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG,
9836 const ARMSubtarget *Subtarget) const {
9837 StoreSDNode *ST = cast<StoreSDNode>(Val: Op.getNode());
9838 EVT MemVT = ST->getMemoryVT();
9839
9840 if (MemVT == MVT::i64 && Subtarget->hasV5TEOps() &&
9841 !Subtarget->isThumb1Only() && ST->isVolatile() &&
9842 ST->getAlign() >= Subtarget->getDualLoadStoreAlignment()) {
9843 assert(ST->isUnindexed() && "Stores should be unindexed at this point.");
9844 SDNode *N = Op.getNode();
9845 SDLoc dl(N);
9846
9847 SDValue Lo = DAG.getNode(
9848 Opcode: ISD::EXTRACT_ELEMENT, DL: dl, VT: MVT::i32, N1: ST->getValue(),
9849 N2: DAG.getTargetConstant(Val: DAG.getDataLayout().isLittleEndian() ? 0 : 1, DL: dl,
9850 VT: MVT::i32));
9851 SDValue Hi = DAG.getNode(
9852 Opcode: ISD::EXTRACT_ELEMENT, DL: dl, VT: MVT::i32, N1: ST->getValue(),
9853 N2: DAG.getTargetConstant(Val: DAG.getDataLayout().isLittleEndian() ? 1 : 0, DL: dl,
9854 VT: MVT::i32));
9855
9856 return DAG.getMemIntrinsicNode(Opcode: ARMISD::STRD, dl, VTList: DAG.getVTList(VT: MVT::Other),
9857 Ops: {ST->getChain(), Lo, Hi, ST->getBasePtr()},
9858 MemVT, MMO: ST->getMemOperand());
9859 } else if (Subtarget->hasMVEIntegerOps() &&
9860 ((MemVT == MVT::v2i1 || MemVT == MVT::v4i1 || MemVT == MVT::v8i1 ||
9861 MemVT == MVT::v16i1))) {
9862 return LowerPredicateStore(Op, DAG);
9863 } else if (MemVT == MVT::i32 || MemVT == MVT::i64) {
9864 return LowerAEABIUnalignedStore(Op, DAG);
9865 }
9866 return SDValue();
9867}
9868
9869static bool isZeroVector(SDValue N) {
9870 return (ISD::isBuildVectorAllZeros(N: N.getNode()) ||
9871 (N->getOpcode() == ARMISD::VMOVIMM &&
9872 isNullConstant(V: N->getOperand(Num: 0))));
9873}
9874
9875static SDValue LowerMLOAD(SDValue Op, SelectionDAG &DAG) {
9876 MaskedLoadSDNode *N = cast<MaskedLoadSDNode>(Val: Op.getNode());
9877 MVT VT = Op.getSimpleValueType();
9878 SDValue Mask = N->getMask();
9879 SDValue PassThru = N->getPassThru();
9880 SDLoc dl(Op);
9881
9882 if (isZeroVector(N: PassThru))
9883 return Op;
9884
9885 // MVE Masked loads use zero as the passthru value. Here we convert undef to
9886 // zero too, and other values are lowered to a select.
9887 SDValue ZeroVec = DAG.getNode(Opcode: ARMISD::VMOVIMM, DL: dl, VT,
9888 Operand: DAG.getTargetConstant(Val: 0, DL: dl, VT: MVT::i32));
9889 SDValue NewLoad = DAG.getMaskedLoad(
9890 VT, dl, Chain: N->getChain(), Base: N->getBasePtr(), Offset: N->getOffset(), Mask, Src0: ZeroVec,
9891 MemVT: N->getMemoryVT(), MMO: N->getMemOperand(), AM: N->getAddressingMode(),
9892 N->getExtensionType(), IsExpanding: N->isExpandingLoad());
9893 SDValue Combo = NewLoad;
9894 bool PassThruIsCastZero = (PassThru.getOpcode() == ISD::BITCAST ||
9895 PassThru.getOpcode() == ARMISD::VECTOR_REG_CAST) &&
9896 isZeroVector(N: PassThru->getOperand(Num: 0));
9897 if (!PassThru.isUndef() && !PassThruIsCastZero)
9898 Combo = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT, N1: Mask, N2: NewLoad, N3: PassThru);
9899 return DAG.getMergeValues(Ops: {Combo, NewLoad.getValue(R: 1)}, dl);
9900}
9901
9902static SDValue LowerVecReduce(SDValue Op, SelectionDAG &DAG,
9903 const ARMSubtarget *ST) {
9904 if (!ST->hasMVEIntegerOps())
9905 return SDValue();
9906
9907 SDLoc dl(Op);
9908 unsigned BaseOpcode = 0;
9909 switch (Op->getOpcode()) {
9910 default: llvm_unreachable("Expected VECREDUCE opcode");
9911 case ISD::VECREDUCE_FADD: BaseOpcode = ISD::FADD; break;
9912 case ISD::VECREDUCE_FMUL: BaseOpcode = ISD::FMUL; break;
9913 case ISD::VECREDUCE_MUL: BaseOpcode = ISD::MUL; break;
9914 case ISD::VECREDUCE_AND: BaseOpcode = ISD::AND; break;
9915 case ISD::VECREDUCE_OR: BaseOpcode = ISD::OR; break;
9916 case ISD::VECREDUCE_XOR: BaseOpcode = ISD::XOR; break;
9917 case ISD::VECREDUCE_FMAX: BaseOpcode = ISD::FMAXNUM; break;
9918 case ISD::VECREDUCE_FMIN: BaseOpcode = ISD::FMINNUM; break;
9919 }
9920
9921 SDValue Op0 = Op->getOperand(Num: 0);
9922 EVT VT = Op0.getValueType();
9923 EVT EltVT = VT.getVectorElementType();
9924 unsigned NumElts = VT.getVectorNumElements();
9925 unsigned NumActiveLanes = NumElts;
9926
9927 assert((NumActiveLanes == 16 || NumActiveLanes == 8 || NumActiveLanes == 4 ||
9928 NumActiveLanes == 2) &&
9929 "Only expected a power 2 vector size");
9930
9931 // Use Mul(X, Rev(X)) until 4 items remain. Going down to 4 vector elements
9932 // allows us to easily extract vector elements from the lanes.
9933 while (NumActiveLanes > 4) {
9934 unsigned RevOpcode = NumActiveLanes == 16 ? ARMISD::VREV16 : ARMISD::VREV32;
9935 SDValue Rev = DAG.getNode(Opcode: RevOpcode, DL: dl, VT, Operand: Op0);
9936 Op0 = DAG.getNode(Opcode: BaseOpcode, DL: dl, VT, N1: Op0, N2: Rev);
9937 NumActiveLanes /= 2;
9938 }
9939
9940 SDValue Res;
9941 if (NumActiveLanes == 4) {
9942 // The remaining 4 elements are summed sequentially
9943 SDValue Ext0 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Op0,
9944 N2: DAG.getConstant(Val: 0 * NumElts / 4, DL: dl, VT: MVT::i32));
9945 SDValue Ext1 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Op0,
9946 N2: DAG.getConstant(Val: 1 * NumElts / 4, DL: dl, VT: MVT::i32));
9947 SDValue Ext2 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Op0,
9948 N2: DAG.getConstant(Val: 2 * NumElts / 4, DL: dl, VT: MVT::i32));
9949 SDValue Ext3 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Op0,
9950 N2: DAG.getConstant(Val: 3 * NumElts / 4, DL: dl, VT: MVT::i32));
9951 SDValue Res0 = DAG.getNode(Opcode: BaseOpcode, DL: dl, VT: EltVT, N1: Ext0, N2: Ext1, Flags: Op->getFlags());
9952 SDValue Res1 = DAG.getNode(Opcode: BaseOpcode, DL: dl, VT: EltVT, N1: Ext2, N2: Ext3, Flags: Op->getFlags());
9953 Res = DAG.getNode(Opcode: BaseOpcode, DL: dl, VT: EltVT, N1: Res0, N2: Res1, Flags: Op->getFlags());
9954 } else {
9955 SDValue Ext0 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Op0,
9956 N2: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
9957 SDValue Ext1 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Op0,
9958 N2: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
9959 Res = DAG.getNode(Opcode: BaseOpcode, DL: dl, VT: EltVT, N1: Ext0, N2: Ext1, Flags: Op->getFlags());
9960 }
9961
9962 // Result type may be wider than element type.
9963 if (EltVT != Op->getValueType(ResNo: 0))
9964 Res = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: Op->getValueType(ResNo: 0), Operand: Res);
9965 return Res;
9966}
9967
9968static SDValue LowerVecReduceF(SDValue Op, SelectionDAG &DAG,
9969 const ARMSubtarget *ST) {
9970 if (!ST->hasMVEFloatOps())
9971 return SDValue();
9972 return LowerVecReduce(Op, DAG, ST);
9973}
9974
9975static SDValue LowerVecReduceMinMax(SDValue Op, SelectionDAG &DAG,
9976 const ARMSubtarget *ST) {
9977 if (!ST->hasNEON())
9978 return SDValue();
9979
9980 SDLoc dl(Op);
9981 SDValue Op0 = Op->getOperand(Num: 0);
9982 EVT VT = Op0.getValueType();
9983 EVT EltVT = VT.getVectorElementType();
9984
9985 unsigned PairwiseIntrinsic = 0;
9986 switch (Op->getOpcode()) {
9987 default:
9988 llvm_unreachable("Expected VECREDUCE opcode");
9989 case ISD::VECREDUCE_UMIN:
9990 PairwiseIntrinsic = Intrinsic::arm_neon_vpminu;
9991 break;
9992 case ISD::VECREDUCE_UMAX:
9993 PairwiseIntrinsic = Intrinsic::arm_neon_vpmaxu;
9994 break;
9995 case ISD::VECREDUCE_SMIN:
9996 PairwiseIntrinsic = Intrinsic::arm_neon_vpmins;
9997 break;
9998 case ISD::VECREDUCE_SMAX:
9999 PairwiseIntrinsic = Intrinsic::arm_neon_vpmaxs;
10000 break;
10001 }
10002 SDValue PairwiseOp = DAG.getConstant(Val: PairwiseIntrinsic, DL: dl, VT: MVT::i32);
10003
10004 unsigned NumElts = VT.getVectorNumElements();
10005 unsigned NumActiveLanes = NumElts;
10006
10007 assert((NumActiveLanes == 16 || NumActiveLanes == 8 || NumActiveLanes == 4 ||
10008 NumActiveLanes == 2) &&
10009 "Only expected a power 2 vector size");
10010
10011 // Split 128-bit vectors, since vpmin/max takes 2 64-bit vectors.
10012 if (VT.is128BitVector()) {
10013 SDValue Lo, Hi;
10014 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Op0, DL: dl);
10015 VT = Lo.getValueType();
10016 Op0 = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT, Ops: {PairwiseOp, Lo, Hi});
10017 NumActiveLanes /= 2;
10018 }
10019
10020 // Use pairwise reductions until one lane remains
10021 while (NumActiveLanes > 1) {
10022 Op0 = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT, Ops: {PairwiseOp, Op0, Op0});
10023 NumActiveLanes /= 2;
10024 }
10025
10026 SDValue Res = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Op0,
10027 N2: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
10028
10029 // Result type may be wider than element type.
10030 if (EltVT != Op.getValueType()) {
10031 unsigned Extend = 0;
10032 switch (Op->getOpcode()) {
10033 default:
10034 llvm_unreachable("Expected VECREDUCE opcode");
10035 case ISD::VECREDUCE_UMIN:
10036 case ISD::VECREDUCE_UMAX:
10037 Extend = ISD::ZERO_EXTEND;
10038 break;
10039 case ISD::VECREDUCE_SMIN:
10040 case ISD::VECREDUCE_SMAX:
10041 Extend = ISD::SIGN_EXTEND;
10042 break;
10043 }
10044 Res = DAG.getNode(Opcode: Extend, DL: dl, VT: Op.getValueType(), Operand: Res);
10045 }
10046 return Res;
10047}
10048
10049static SDValue LowerAtomicLoadStore(SDValue Op, SelectionDAG &DAG) {
10050 if (isStrongerThanMonotonic(AO: cast<AtomicSDNode>(Val&: Op)->getSuccessOrdering()))
10051 // Acquire/Release load/store is not legal for targets without a dmb or
10052 // equivalent available.
10053 return SDValue();
10054
10055 // Monotonic load/store is legal for all targets.
10056 return Op;
10057}
10058
10059static void ReplaceREADCYCLECOUNTER(SDNode *N,
10060 SmallVectorImpl<SDValue> &Results,
10061 SelectionDAG &DAG,
10062 const ARMSubtarget *Subtarget) {
10063 SDLoc DL(N);
10064 // Under Power Management extensions, the cycle-count is:
10065 // mrc p15, #0, <Rt>, c9, c13, #0
10066 SDValue Ops[] = { N->getOperand(Num: 0), // Chain
10067 DAG.getTargetConstant(Val: Intrinsic::arm_mrc, DL, VT: MVT::i32),
10068 DAG.getTargetConstant(Val: 15, DL, VT: MVT::i32),
10069 DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32),
10070 DAG.getTargetConstant(Val: 9, DL, VT: MVT::i32),
10071 DAG.getTargetConstant(Val: 13, DL, VT: MVT::i32),
10072 DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32)
10073 };
10074
10075 SDValue Cycles32 = DAG.getNode(Opcode: ISD::INTRINSIC_W_CHAIN, DL,
10076 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other), Ops);
10077 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: MVT::i64, N1: Cycles32,
10078 N2: DAG.getConstant(Val: 0, DL, VT: MVT::i32)));
10079 Results.push_back(Elt: Cycles32.getValue(R: 1));
10080}
10081
10082static SDValue createGPRPairNode2xi32(SelectionDAG &DAG, SDValue V0,
10083 SDValue V1) {
10084 SDLoc dl(V0.getNode());
10085 SDValue RegClass =
10086 DAG.getTargetConstant(Val: ARM::GPRPairRegClassID, DL: dl, VT: MVT::i32);
10087 SDValue SubReg0 = DAG.getTargetConstant(Val: ARM::gsub_0, DL: dl, VT: MVT::i32);
10088 SDValue SubReg1 = DAG.getTargetConstant(Val: ARM::gsub_1, DL: dl, VT: MVT::i32);
10089 const SDValue Ops[] = {RegClass, V0, SubReg0, V1, SubReg1};
10090 return SDValue(
10091 DAG.getMachineNode(Opcode: TargetOpcode::REG_SEQUENCE, dl, VT: MVT::Untyped, Ops), 0);
10092}
10093
10094static SDValue createGPRPairNodei64(SelectionDAG &DAG, SDValue V) {
10095 SDLoc dl(V.getNode());
10096 auto [VLo, VHi] = DAG.SplitScalar(N: V, DL: dl, LoVT: MVT::i32, HiVT: MVT::i32);
10097 bool isBigEndian = DAG.getDataLayout().isBigEndian();
10098 if (isBigEndian)
10099 std::swap(a&: VLo, b&: VHi);
10100 return createGPRPairNode2xi32(DAG, V0: VLo, V1: VHi);
10101}
10102
10103static void ReplaceCMP_SWAP_64Results(SDNode *N,
10104 SmallVectorImpl<SDValue> &Results,
10105 SelectionDAG &DAG) {
10106 assert(N->getValueType(0) == MVT::i64 &&
10107 "AtomicCmpSwap on types less than 64 should be legal");
10108 SDValue Ops[] = {
10109 createGPRPairNode2xi32(DAG, V0: N->getOperand(Num: 1),
10110 V1: DAG.getUNDEF(VT: MVT::i32)), // pointer, temp
10111 createGPRPairNodei64(DAG, V: N->getOperand(Num: 2)), // expected
10112 createGPRPairNodei64(DAG, V: N->getOperand(Num: 3)), // new
10113 N->getOperand(Num: 0), // chain in
10114 };
10115 SDNode *CmpSwap = DAG.getMachineNode(
10116 Opcode: ARM::CMP_SWAP_64, dl: SDLoc(N),
10117 VTs: DAG.getVTList(VT1: MVT::Untyped, VT2: MVT::Untyped, VT3: MVT::Other), Ops);
10118
10119 MachineMemOperand *MemOp = cast<MemSDNode>(Val: N)->getMemOperand();
10120 DAG.setNodeMemRefs(N: cast<MachineSDNode>(Val: CmpSwap), NewMemRefs: {MemOp});
10121
10122 bool isBigEndian = DAG.getDataLayout().isBigEndian();
10123
10124 SDValue Lo =
10125 DAG.getTargetExtractSubreg(SRIdx: isBigEndian ? ARM::gsub_1 : ARM::gsub_0,
10126 DL: SDLoc(N), VT: MVT::i32, Operand: SDValue(CmpSwap, 0));
10127 SDValue Hi =
10128 DAG.getTargetExtractSubreg(SRIdx: isBigEndian ? ARM::gsub_0 : ARM::gsub_1,
10129 DL: SDLoc(N), VT: MVT::i32, Operand: SDValue(CmpSwap, 0));
10130 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: SDLoc(N), VT: MVT::i64, N1: Lo, N2: Hi));
10131 Results.push_back(Elt: SDValue(CmpSwap, 2));
10132}
10133
10134SDValue ARMTargetLowering::LowerFSETCC(SDValue Op, SelectionDAG &DAG) const {
10135 SDLoc dl(Op);
10136 EVT VT = Op.getValueType();
10137 SDValue Chain = Op.getOperand(i: 0);
10138 SDValue LHS = Op.getOperand(i: 1);
10139 SDValue RHS = Op.getOperand(i: 2);
10140 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 3))->get();
10141 bool IsSignaling = Op.getOpcode() == ISD::STRICT_FSETCCS;
10142
10143 // If we don't have instructions of this float type then soften to a libcall
10144 // and use SETCC instead.
10145 if (isUnsupportedFloatingType(VT: LHS.getValueType())) {
10146 softenSetCCOperands(DAG, VT: LHS.getValueType(), NewLHS&: LHS, NewRHS&: RHS, CCCode&: CC, DL: dl, OldLHS: LHS, OldRHS: RHS,
10147 Chain, IsSignaling);
10148 if (!RHS.getNode()) {
10149 RHS = DAG.getConstant(Val: 0, DL: dl, VT: LHS.getValueType());
10150 CC = ISD::SETNE;
10151 }
10152 SDValue Result = DAG.getNode(Opcode: ISD::SETCC, DL: dl, VT, N1: LHS, N2: RHS,
10153 N3: DAG.getCondCode(Cond: CC));
10154 return DAG.getMergeValues(Ops: {Result, Chain}, dl);
10155 }
10156
10157 ARMCC::CondCodes CondCode, CondCode2;
10158 FPCCToARMCC(CC, CondCode, CondCode2);
10159
10160 SDValue True = DAG.getConstant(Val: 1, DL: dl, VT);
10161 SDValue False = DAG.getConstant(Val: 0, DL: dl, VT);
10162 SDValue ARMcc = DAG.getConstant(Val: CondCode, DL: dl, VT: MVT::i32);
10163 SDValue Cmp = getVFPCmp(LHS, RHS, DAG, dl, Signaling: IsSignaling);
10164 SDValue Result = getCMOV(dl, VT, FalseVal: False, TrueVal: True, ARMcc, Flags: Cmp, DAG);
10165 if (CondCode2 != ARMCC::AL) {
10166 ARMcc = DAG.getConstant(Val: CondCode2, DL: dl, VT: MVT::i32);
10167 Result = getCMOV(dl, VT, FalseVal: Result, TrueVal: True, ARMcc, Flags: Cmp, DAG);
10168 }
10169 return DAG.getMergeValues(Ops: {Result, Chain}, dl);
10170}
10171
10172SDValue ARMTargetLowering::LowerSPONENTRY(SDValue Op, SelectionDAG &DAG) const {
10173 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
10174
10175 EVT VT = getPointerTy(DL: DAG.getDataLayout());
10176 int FI = MFI.CreateFixedObject(Size: 4, SPOffset: 0, IsImmutable: false);
10177 return DAG.getFrameIndex(FI, VT);
10178}
10179
10180SDValue ARMTargetLowering::LowerFP_TO_BF16(SDValue Op,
10181 SelectionDAG &DAG) const {
10182 SDLoc DL(Op);
10183 MakeLibCallOptions CallOptions;
10184 MVT SVT = Op.getOperand(i: 0).getSimpleValueType();
10185 RTLIB::Libcall LC = RTLIB::getFPROUND(OpVT: SVT, RetVT: MVT::bf16);
10186 SDValue Res =
10187 makeLibCall(DAG, LC, RetVT: MVT::f32, Ops: Op.getOperand(i: 0), CallOptions, dl: DL).first;
10188 return DAG.getBitcast(VT: MVT::i32, V: Res);
10189}
10190
10191SDValue ARMTargetLowering::LowerCMP(SDValue Op, SelectionDAG &DAG) const {
10192 SDLoc dl(Op);
10193 SDValue LHS = Op.getOperand(i: 0);
10194 SDValue RHS = Op.getOperand(i: 1);
10195
10196 // Determine if this is signed or unsigned comparison
10197 bool IsSigned = (Op.getOpcode() == ISD::SCMP);
10198
10199 // Special case for Thumb1 UCMP only
10200 if (!IsSigned && Subtarget->isThumb1Only()) {
10201 // For Thumb unsigned comparison, use this sequence:
10202 // subs r2, r0, r1 ; r2 = LHS - RHS, sets flags
10203 // sbc r2, r2 ; r2 = r2 - r2 - !carry
10204 // cmp r1, r0 ; compare RHS with LHS
10205 // sbc r1, r1 ; r1 = r1 - r1 - !carry
10206 // subs r0, r2, r1 ; r0 = r2 - r1 (final result)
10207
10208 // First subtraction: LHS - RHS
10209 SDValue Sub1WithFlags = DAG.getNode(
10210 Opcode: ARMISD::SUBC, DL: dl, VTList: DAG.getVTList(VT1: MVT::i32, VT2: FlagsVT), N1: LHS, N2: RHS);
10211 SDValue Sub1Result = Sub1WithFlags.getValue(R: 0);
10212 SDValue Flags1 = Sub1WithFlags.getValue(R: 1);
10213
10214 // SUBE: Sub1Result - Sub1Result - !carry
10215 // This gives 0 if LHS >= RHS (unsigned), -1 if LHS < RHS (unsigned)
10216 SDValue Sbc1 =
10217 DAG.getNode(Opcode: ARMISD::SUBE, DL: dl, VTList: DAG.getVTList(VT1: MVT::i32, VT2: FlagsVT),
10218 N1: Sub1Result, N2: Sub1Result, N3: Flags1);
10219 SDValue Sbc1Result = Sbc1.getValue(R: 0);
10220
10221 // Second comparison: RHS vs LHS (reverse comparison)
10222 SDValue CmpFlags = DAG.getNode(Opcode: ARMISD::CMP, DL: dl, VT: FlagsVT, N1: RHS, N2: LHS);
10223
10224 // SUBE: RHS - RHS - !carry
10225 // This gives 0 if RHS <= LHS (unsigned), -1 if RHS > LHS (unsigned)
10226 SDValue Sbc2 = DAG.getNode(
10227 Opcode: ARMISD::SUBE, DL: dl, VTList: DAG.getVTList(VT1: MVT::i32, VT2: FlagsVT), N1: RHS, N2: RHS, N3: CmpFlags);
10228 SDValue Sbc2Result = Sbc2.getValue(R: 0);
10229
10230 // Final subtraction: Sbc1Result - Sbc2Result (no flags needed)
10231 SDValue Result =
10232 DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32, N1: Sbc1Result, N2: Sbc2Result);
10233 if (Op.getValueType() != MVT::i32)
10234 Result = DAG.getSExtOrTrunc(Op: Result, DL: dl, VT: Op.getValueType());
10235
10236 return Result;
10237 }
10238
10239 // For the ARM assembly pattern:
10240 // subs r0, r0, r1 ; subtract RHS from LHS and set flags
10241 // movgt r0, #1 ; if LHS > RHS, set result to 1 (GT for signed, HI for
10242 // unsigned) mvnlt r0, #0 ; if LHS < RHS, set result to -1 (LT for
10243 // signed, LO for unsigned)
10244 // ; if LHS == RHS, result remains 0 from the subs
10245
10246 // Optimization: if RHS is a subtraction against 0, use ADDC instead of SUBC
10247 unsigned Opcode = ARMISD::SUBC;
10248
10249 // Check if RHS is a subtraction against 0: (0 - X)
10250 if (RHS.getOpcode() == ISD::SUB) {
10251 SDValue SubLHS = RHS.getOperand(i: 0);
10252 SDValue SubRHS = RHS.getOperand(i: 1);
10253
10254 // Check if it's 0 - X
10255 if (isNullConstant(V: SubLHS)) {
10256 bool CanUseAdd = false;
10257 if (IsSigned) {
10258 // For SCMP: only if X is known to never be INT_MIN (to avoid overflow)
10259 if (RHS->getFlags().hasNoSignedWrap() || !DAG.computeKnownBits(Op: SubRHS)
10260 .getSignedMinValue()
10261 .isMinSignedValue()) {
10262 CanUseAdd = true;
10263 }
10264 } else {
10265 // For UCMP: only if X is known to never be zero
10266 if (DAG.isKnownNeverZero(Op: SubRHS)) {
10267 CanUseAdd = true;
10268 }
10269 }
10270
10271 if (CanUseAdd) {
10272 Opcode = ARMISD::ADDC;
10273 RHS = SubRHS; // Replace RHS with X, so we do LHS + X instead of
10274 // LHS - (0 - X)
10275 }
10276 }
10277 }
10278
10279 // Generate the operation with flags
10280 SDValue OpWithFlags =
10281 DAG.getNode(Opcode, DL: dl, VTList: DAG.getVTList(VT1: MVT::i32, VT2: FlagsVT), N1: LHS, N2: RHS);
10282
10283 SDValue OpResult = OpWithFlags.getValue(R: 0);
10284 SDValue Flags = OpWithFlags.getValue(R: 1);
10285
10286 // Constants for conditional moves
10287 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32);
10288 SDValue MinusOne = DAG.getAllOnesConstant(DL: dl, VT: MVT::i32);
10289
10290 // Select condition codes based on signed vs unsigned
10291 ARMCC::CondCodes GTCond = IsSigned ? ARMCC::GT : ARMCC::HI;
10292 ARMCC::CondCodes LTCond = IsSigned ? ARMCC::LT : ARMCC::LO;
10293
10294 // First conditional move: if greater than, set to 1
10295 SDValue GTCondValue = DAG.getConstant(Val: GTCond, DL: dl, VT: MVT::i32);
10296 SDValue Result1 = DAG.getNode(Opcode: ARMISD::CMOV, DL: dl, VT: MVT::i32, N1: OpResult, N2: One,
10297 N3: GTCondValue, N4: Flags);
10298
10299 // Second conditional move: if less than, set to -1
10300 SDValue LTCondValue = DAG.getConstant(Val: LTCond, DL: dl, VT: MVT::i32);
10301 SDValue Result2 = DAG.getNode(Opcode: ARMISD::CMOV, DL: dl, VT: MVT::i32, N1: Result1, N2: MinusOne,
10302 N3: LTCondValue, N4: Flags);
10303
10304 if (Op.getValueType() != MVT::i32)
10305 Result2 = DAG.getSExtOrTrunc(Op: Result2, DL: dl, VT: Op.getValueType());
10306
10307 return Result2;
10308}
10309
10310SDValue ARMTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
10311 LLVM_DEBUG(dbgs() << "Lowering node: "; Op.dump());
10312 switch (Op.getOpcode()) {
10313 default: llvm_unreachable("Don't know how to custom lower this!");
10314 case ISD::WRITE_REGISTER: return LowerWRITE_REGISTER(Op, DAG);
10315 case ISD::ConstantPool: return LowerConstantPool(Op, DAG);
10316 case ISD::BlockAddress: return LowerBlockAddress(Op, DAG);
10317 case ISD::GlobalAddress: return LowerGlobalAddress(Op, DAG);
10318 case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG);
10319 case ISD::SELECT: return LowerSELECT(Op, DAG);
10320 case ISD::SELECT_CC: return LowerSELECT_CC(Op, DAG);
10321 case ISD::BRCOND: return LowerBRCOND(Op, DAG);
10322 case ISD::BR_CC: return LowerBR_CC(Op, DAG);
10323 case ISD::BR_JT: return LowerBR_JT(Op, DAG);
10324 case ISD::VASTART: return LowerVASTART(Op, DAG);
10325 case ISD::ATOMIC_FENCE: return LowerATOMIC_FENCE(Op, DAG, Subtarget);
10326 case ISD::PREFETCH: return LowerPREFETCH(Op, DAG, Subtarget);
10327 case ISD::STRICT_UINT_TO_FP:
10328 case ISD::STRICT_SINT_TO_FP:
10329 case ISD::SINT_TO_FP:
10330 case ISD::UINT_TO_FP: return LowerINT_TO_FP(Op, DAG);
10331 case ISD::STRICT_FP_TO_SINT:
10332 case ISD::STRICT_FP_TO_UINT:
10333 case ISD::FP_TO_SINT:
10334 case ISD::FP_TO_UINT: return LowerFP_TO_INT(Op, DAG);
10335 case ISD::FP_TO_SINT_SAT:
10336 case ISD::FP_TO_UINT_SAT: return LowerFP_TO_INT_SAT(Op, DAG, Subtarget);
10337 case ISD::FCOPYSIGN: return LowerFCOPYSIGN(Op, DAG);
10338 case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG);
10339 case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG);
10340 case ISD::EH_SJLJ_SETJMP: return LowerEH_SJLJ_SETJMP(Op, DAG);
10341 case ISD::EH_SJLJ_LONGJMP: return LowerEH_SJLJ_LONGJMP(Op, DAG);
10342 case ISD::EH_SJLJ_SETUP_DISPATCH: return LowerEH_SJLJ_SETUP_DISPATCH(Op, DAG);
10343 case ISD::INTRINSIC_VOID: return LowerINTRINSIC_VOID(Op, DAG, Subtarget);
10344 case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG,
10345 Subtarget);
10346 case ISD::BITCAST: return ExpandBITCAST(N: Op.getNode(), DAG, Subtarget);
10347 case ISD::SHL:
10348 case ISD::SRL:
10349 case ISD::SRA: return LowerShift(N: Op.getNode(), DAG, ST: Subtarget);
10350 case ISD::SREM: return LowerREM(N: Op.getNode(), DAG);
10351 case ISD::UREM: return LowerREM(N: Op.getNode(), DAG);
10352 case ISD::SHL_PARTS: return LowerShiftLeftParts(Op, DAG);
10353 case ISD::SRL_PARTS:
10354 case ISD::SRA_PARTS: return LowerShiftRightParts(Op, DAG);
10355 case ISD::CTTZ:
10356 case ISD::CTTZ_ZERO_POISON: return LowerCTTZ(N: Op.getNode(), DAG, ST: Subtarget);
10357 case ISD::CTPOP: return LowerCTPOP(N: Op.getNode(), DAG, ST: Subtarget);
10358 case ISD::SETCC: return LowerVSETCC(Op, DAG, ST: Subtarget);
10359 case ISD::SETCCCARRY: return LowerSETCCCARRY(Op, DAG);
10360 case ISD::ConstantFP: return LowerConstantFP(Op, DAG, ST: Subtarget);
10361 case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG, ST: Subtarget);
10362 case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG, ST: Subtarget);
10363 case ISD::EXTRACT_SUBVECTOR: return LowerEXTRACT_SUBVECTOR(Op, DAG, ST: Subtarget);
10364 case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG);
10365 case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG, ST: Subtarget);
10366 case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG, ST: Subtarget);
10367 case ISD::TRUNCATE: return LowerTruncate(N: Op.getNode(), DAG, Subtarget);
10368 case ISD::SIGN_EXTEND:
10369 case ISD::ZERO_EXTEND: return LowerVectorExtend(N: Op.getNode(), DAG, Subtarget);
10370 case ISD::GET_ROUNDING: return LowerGET_ROUNDING(Op, DAG);
10371 case ISD::SET_ROUNDING: return LowerSET_ROUNDING(Op, DAG);
10372 case ISD::SET_FPMODE:
10373 return LowerSET_FPMODE(Op, DAG);
10374 case ISD::RESET_FPMODE:
10375 return LowerRESET_FPMODE(Op, DAG);
10376 case ISD::MUL: return LowerMUL(Op, DAG);
10377 case ISD::SDIV:
10378 if (getTargetMachine().getTargetTriple().isOSWindows() &&
10379 !Op.getValueType().isVector())
10380 return LowerDIV_Windows(Op, DAG, /* Signed */ true);
10381 return LowerSDIV(Op, DAG, ST: Subtarget);
10382 case ISD::UDIV:
10383 if (getTargetMachine().getTargetTriple().isOSWindows() &&
10384 !Op.getValueType().isVector())
10385 return LowerDIV_Windows(Op, DAG, /* Signed */ false);
10386 return LowerUDIV(Op, DAG, ST: Subtarget);
10387 case ISD::UADDO_CARRY:
10388 return LowerADDSUBO_CARRY(Op, DAG, Opcode: ARMISD::ADDE, IsSigned: false /*unsigned*/);
10389 case ISD::USUBO_CARRY:
10390 return LowerADDSUBO_CARRY(Op, DAG, Opcode: ARMISD::SUBE, IsSigned: false /*unsigned*/);
10391 case ISD::SADDO_CARRY:
10392 return LowerADDSUBO_CARRY(Op, DAG, Opcode: ARMISD::ADDE, IsSigned: true /*signed*/);
10393 case ISD::SSUBO_CARRY:
10394 return LowerADDSUBO_CARRY(Op, DAG, Opcode: ARMISD::SUBE, IsSigned: true /*signed*/);
10395 case ISD::UADDO:
10396 case ISD::USUBO:
10397 case ISD::UMULO:
10398 case ISD::SADDO:
10399 case ISD::SSUBO:
10400 case ISD::SMULO:
10401 return LowerALUO(Op, DAG);
10402 case ISD::SADDSAT:
10403 case ISD::SSUBSAT:
10404 case ISD::UADDSAT:
10405 case ISD::USUBSAT:
10406 return LowerADDSUBSAT(Op, DAG, Subtarget);
10407 case ISD::LOAD: {
10408 auto *LD = cast<LoadSDNode>(Val&: Op);
10409 EVT MemVT = LD->getMemoryVT();
10410 if (Subtarget->hasMVEIntegerOps() &&
10411 (MemVT == MVT::v2i1 || MemVT == MVT::v4i1 || MemVT == MVT::v8i1 ||
10412 MemVT == MVT::v16i1))
10413 return LowerPredicateLoad(Op, DAG);
10414
10415 auto Pair = LowerAEABIUnalignedLoad(Op, DAG);
10416 if (Pair.first)
10417 return DAG.getMergeValues(Ops: {Pair.first, Pair.second}, dl: SDLoc(Pair.first));
10418 return SDValue();
10419 }
10420 case ISD::STORE:
10421 return LowerSTORE(Op, DAG, Subtarget);
10422 case ISD::MLOAD:
10423 return LowerMLOAD(Op, DAG);
10424 case ISD::VECREDUCE_MUL:
10425 case ISD::VECREDUCE_AND:
10426 case ISD::VECREDUCE_OR:
10427 case ISD::VECREDUCE_XOR:
10428 return LowerVecReduce(Op, DAG, ST: Subtarget);
10429 case ISD::VECREDUCE_FADD:
10430 case ISD::VECREDUCE_FMUL:
10431 case ISD::VECREDUCE_FMIN:
10432 case ISD::VECREDUCE_FMAX:
10433 return LowerVecReduceF(Op, DAG, ST: Subtarget);
10434 case ISD::VECREDUCE_UMIN:
10435 case ISD::VECREDUCE_UMAX:
10436 case ISD::VECREDUCE_SMIN:
10437 case ISD::VECREDUCE_SMAX:
10438 return LowerVecReduceMinMax(Op, DAG, ST: Subtarget);
10439 case ISD::ATOMIC_LOAD:
10440 case ISD::ATOMIC_STORE:
10441 return LowerAtomicLoadStore(Op, DAG);
10442 case ISD::SDIVREM:
10443 case ISD::UDIVREM: return LowerDivRem(Op, DAG);
10444 case ISD::DYNAMIC_STACKALLOC:
10445 if (getTargetMachine().getTargetTriple().isOSWindows())
10446 return LowerDYNAMIC_STACKALLOC(Op, DAG);
10447 llvm_unreachable("Don't know how to custom lower this!");
10448 case ISD::STRICT_FP_ROUND:
10449 case ISD::FP_ROUND: return LowerFP_ROUND(Op, DAG);
10450 case ISD::STRICT_FP_EXTEND:
10451 case ISD::FP_EXTEND: return LowerFP_EXTEND(Op, DAG);
10452 case ISD::STRICT_FSETCC:
10453 case ISD::STRICT_FSETCCS: return LowerFSETCC(Op, DAG);
10454 case ISD::SPONENTRY:
10455 return LowerSPONENTRY(Op, DAG);
10456 case ISD::FP_TO_BF16:
10457 return LowerFP_TO_BF16(Op, DAG);
10458 case ARMISD::WIN__DBZCHK: return SDValue();
10459 case ISD::UCMP:
10460 case ISD::SCMP:
10461 return LowerCMP(Op, DAG);
10462 case ISD::ABS:
10463 return LowerABS(Op, DAG);
10464 case ISD::STRICT_LROUND:
10465 case ISD::STRICT_LLROUND:
10466 case ISD::STRICT_LRINT:
10467 case ISD::STRICT_LLRINT: {
10468 assert((Op.getOperand(1).getValueType() == MVT::f16 ||
10469 Op.getOperand(1).getValueType() == MVT::bf16) &&
10470 "Expected custom lowering of rounding operations only for f16");
10471 SDLoc DL(Op);
10472 SDValue Ext = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL, ResultTys: {MVT::f32, MVT::Other},
10473 Ops: {Op.getOperand(i: 0), Op.getOperand(i: 1)});
10474 return DAG.getNode(Opcode: Op.getOpcode(), DL, ResultTys: {Op.getValueType(), MVT::Other},
10475 Ops: {Ext.getValue(R: 1), Ext.getValue(R: 0)});
10476 }
10477 }
10478}
10479
10480static void ReplaceLongIntrinsic(SDNode *N, SmallVectorImpl<SDValue> &Results,
10481 SelectionDAG &DAG) {
10482 unsigned IntNo = N->getConstantOperandVal(Num: 0);
10483 unsigned Opc = 0;
10484 if (IntNo == Intrinsic::arm_smlald)
10485 Opc = ARMISD::SMLALD;
10486 else if (IntNo == Intrinsic::arm_smlaldx)
10487 Opc = ARMISD::SMLALDX;
10488 else if (IntNo == Intrinsic::arm_smlsld)
10489 Opc = ARMISD::SMLSLD;
10490 else if (IntNo == Intrinsic::arm_smlsldx)
10491 Opc = ARMISD::SMLSLDX;
10492 else
10493 return;
10494
10495 SDLoc dl(N);
10496 SDValue Lo, Hi;
10497 std::tie(args&: Lo, args&: Hi) = DAG.SplitScalar(N: N->getOperand(Num: 3), DL: dl, LoVT: MVT::i32, HiVT: MVT::i32);
10498
10499 SDValue LongMul = DAG.getNode(Opcode: Opc, DL: dl,
10500 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
10501 N1: N->getOperand(Num: 1), N2: N->getOperand(Num: 2),
10502 N3: Lo, N4: Hi);
10503 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: MVT::i64,
10504 N1: LongMul.getValue(R: 0), N2: LongMul.getValue(R: 1)));
10505}
10506
10507/// ReplaceNodeResults - Replace the results of node with an illegal result
10508/// type with new values built out of custom code.
10509void ARMTargetLowering::ReplaceNodeResults(SDNode *N,
10510 SmallVectorImpl<SDValue> &Results,
10511 SelectionDAG &DAG) const {
10512 SDValue Res;
10513 switch (N->getOpcode()) {
10514 default:
10515 llvm_unreachable("Don't know how to custom expand this!");
10516 case ISD::READ_REGISTER:
10517 ExpandREAD_REGISTER(N, Results, DAG);
10518 break;
10519 case ISD::BITCAST:
10520 Res = ExpandBITCAST(N, DAG, Subtarget);
10521 break;
10522 case ISD::SRL:
10523 case ISD::SRA:
10524 case ISD::SHL:
10525 Res = Expand64BitShift(N, DAG, ST: Subtarget);
10526 break;
10527 case ISD::SREM:
10528 case ISD::UREM:
10529 Res = LowerREM(N, DAG);
10530 break;
10531 case ISD::SDIVREM:
10532 case ISD::UDIVREM:
10533 Res = LowerDivRem(Op: SDValue(N, 0), DAG);
10534 assert(Res.getNumOperands() == 2 && "DivRem needs two values");
10535 Results.push_back(Elt: Res.getValue(R: 0));
10536 Results.push_back(Elt: Res.getValue(R: 1));
10537 return;
10538 case ISD::SADDSAT:
10539 case ISD::SSUBSAT:
10540 case ISD::UADDSAT:
10541 case ISD::USUBSAT:
10542 Res = LowerADDSUBSAT(Op: SDValue(N, 0), DAG, Subtarget);
10543 break;
10544 case ISD::READCYCLECOUNTER:
10545 ReplaceREADCYCLECOUNTER(N, Results, DAG, Subtarget);
10546 return;
10547 case ISD::UDIV:
10548 case ISD::SDIV:
10549 assert(getTargetMachine().getTargetTriple().isOSWindows() &&
10550 "can only expand DIV on Windows");
10551 return ExpandDIV_Windows(Op: SDValue(N, 0), DAG, Signed: N->getOpcode() == ISD::SDIV,
10552 Results);
10553 case ISD::ATOMIC_CMP_SWAP:
10554 ReplaceCMP_SWAP_64Results(N, Results, DAG);
10555 return;
10556 case ISD::INTRINSIC_WO_CHAIN:
10557 return ReplaceLongIntrinsic(N, Results, DAG);
10558 case ISD::LOAD:
10559 LowerLOAD(N, Results, DAG);
10560 break;
10561 case ISD::STORE:
10562 Res = LowerAEABIUnalignedStore(Op: SDValue(N, 0), DAG);
10563 break;
10564 case ISD::TRUNCATE:
10565 Res = LowerTruncate(N, DAG, Subtarget);
10566 break;
10567 case ISD::SIGN_EXTEND:
10568 case ISD::ZERO_EXTEND:
10569 Res = LowerVectorExtend(N, DAG, Subtarget);
10570 break;
10571 case ISD::FP_TO_SINT_SAT:
10572 case ISD::FP_TO_UINT_SAT:
10573 Res = LowerFP_TO_INT_SAT(Op: SDValue(N, 0), DAG, Subtarget);
10574 break;
10575 }
10576 if (Res.getNode())
10577 Results.push_back(Elt: Res);
10578}
10579
10580//===----------------------------------------------------------------------===//
10581// ARM Scheduler Hooks
10582//===----------------------------------------------------------------------===//
10583
10584/// SetupEntryBlockForSjLj - Insert code into the entry block that creates and
10585/// registers the function context.
10586void ARMTargetLowering::SetupEntryBlockForSjLj(MachineInstr &MI,
10587 MachineBasicBlock *MBB,
10588 MachineBasicBlock *DispatchBB,
10589 int FI) const {
10590 assert(!Subtarget->isROPI() && !Subtarget->isRWPI() &&
10591 "ROPI/RWPI not currently supported with SjLj");
10592 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
10593 DebugLoc dl = MI.getDebugLoc();
10594 MachineFunction *MF = MBB->getParent();
10595 MachineRegisterInfo *MRI = &MF->getRegInfo();
10596 MachineConstantPool *MCP = MF->getConstantPool();
10597 ARMFunctionInfo *AFI = MF->getInfo<ARMFunctionInfo>();
10598 const Function &F = MF->getFunction();
10599
10600 bool isThumb = Subtarget->isThumb();
10601 bool isThumb2 = Subtarget->isThumb2();
10602
10603 unsigned PCLabelId = AFI->createPICLabelUId();
10604 unsigned PCAdj = (isThumb || isThumb2) ? 4 : 8;
10605 ARMConstantPoolValue *CPV =
10606 ARMConstantPoolMBB::Create(C&: F.getContext(), mbb: DispatchBB, ID: PCLabelId, PCAdj);
10607 unsigned CPI = MCP->getConstantPoolIndex(V: CPV, Alignment: Align(4));
10608
10609 const TargetRegisterClass *TRC = isThumb ? &ARM::tGPRRegClass
10610 : &ARM::GPRRegClass;
10611
10612 // Grab constant pool and fixed stack memory operands.
10613 MachineMemOperand *CPMMO =
10614 MF->getMachineMemOperand(PtrInfo: MachinePointerInfo::getConstantPool(MF&: *MF),
10615 F: MachineMemOperand::MOLoad, Size: 4, BaseAlignment: Align(4));
10616
10617 MachineMemOperand *FIMMOSt =
10618 MF->getMachineMemOperand(PtrInfo: MachinePointerInfo::getFixedStack(MF&: *MF, FI),
10619 F: MachineMemOperand::MOStore, Size: 4, BaseAlignment: Align(4));
10620
10621 // Load the address of the dispatch MBB into the jump buffer.
10622 if (isThumb2) {
10623 // Incoming value: jbuf
10624 // ldr.n r5, LCPI1_1
10625 // orr r5, r5, #1
10626 // add r5, pc
10627 // str r5, [$jbuf, #+4] ; &jbuf[1]
10628 Register NewVReg1 = MRI->createVirtualRegister(RegClass: TRC);
10629 BuildMI(BB&: *MBB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: ARM::t2LDRpci), DestReg: NewVReg1)
10630 .addConstantPoolIndex(Idx: CPI)
10631 .addMemOperand(MMO: CPMMO)
10632 .add(MOs: predOps(Pred: ARMCC::AL));
10633 // Set the low bit because of thumb mode.
10634 Register NewVReg2 = MRI->createVirtualRegister(RegClass: TRC);
10635 BuildMI(BB&: *MBB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: ARM::t2ORRri), DestReg: NewVReg2)
10636 .addReg(RegNo: NewVReg1)
10637 .addImm(Val: 0x01)
10638 .add(MOs: predOps(Pred: ARMCC::AL))
10639 .add(MO: condCodeOp());
10640 Register NewVReg3 = MRI->createVirtualRegister(RegClass: TRC);
10641 BuildMI(BB&: *MBB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: ARM::tPICADD), DestReg: NewVReg3)
10642 .addReg(RegNo: NewVReg2)
10643 .addImm(Val: PCLabelId);
10644 BuildMI(BB&: *MBB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: ARM::t2STRi12))
10645 .addReg(RegNo: NewVReg3)
10646 .addFrameIndex(Idx: FI)
10647 .addImm(Val: 36) // &jbuf[1] :: pc
10648 .addMemOperand(MMO: FIMMOSt)
10649 .add(MOs: predOps(Pred: ARMCC::AL));
10650 } else if (isThumb) {
10651 // Incoming value: jbuf
10652 // ldr.n r1, LCPI1_4
10653 // add r1, pc
10654 // mov r2, #1
10655 // orrs r1, r2
10656 // add r2, $jbuf, #+4 ; &jbuf[1]
10657 // str r1, [r2]
10658 Register NewVReg1 = MRI->createVirtualRegister(RegClass: TRC);
10659 BuildMI(BB&: *MBB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: ARM::tLDRpci), DestReg: NewVReg1)
10660 .addConstantPoolIndex(Idx: CPI)
10661 .addMemOperand(MMO: CPMMO)
10662 .add(MOs: predOps(Pred: ARMCC::AL));
10663 Register NewVReg2 = MRI->createVirtualRegister(RegClass: TRC);
10664 BuildMI(BB&: *MBB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: ARM::tPICADD), DestReg: NewVReg2)
10665 .addReg(RegNo: NewVReg1)
10666 .addImm(Val: PCLabelId);
10667 // Set the low bit because of thumb mode.
10668 Register NewVReg3 = MRI->createVirtualRegister(RegClass: TRC);
10669 BuildMI(BB&: *MBB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: ARM::tMOVi8), DestReg: NewVReg3)
10670 .addReg(RegNo: ARM::CPSR, Flags: RegState::Define)
10671 .addImm(Val: 1)
10672 .add(MOs: predOps(Pred: ARMCC::AL));
10673 Register NewVReg4 = MRI->createVirtualRegister(RegClass: TRC);
10674 BuildMI(BB&: *MBB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: ARM::tORR), DestReg: NewVReg4)
10675 .addReg(RegNo: ARM::CPSR, Flags: RegState::Define)
10676 .addReg(RegNo: NewVReg2)
10677 .addReg(RegNo: NewVReg3)
10678 .add(MOs: predOps(Pred: ARMCC::AL));
10679 Register NewVReg5 = MRI->createVirtualRegister(RegClass: TRC);
10680 BuildMI(BB&: *MBB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: ARM::tADDframe), DestReg: NewVReg5)
10681 .addFrameIndex(Idx: FI)
10682 .addImm(Val: 36); // &jbuf[1] :: pc
10683 BuildMI(BB&: *MBB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: ARM::tSTRi))
10684 .addReg(RegNo: NewVReg4)
10685 .addReg(RegNo: NewVReg5)
10686 .addImm(Val: 0)
10687 .addMemOperand(MMO: FIMMOSt)
10688 .add(MOs: predOps(Pred: ARMCC::AL));
10689 } else {
10690 // Incoming value: jbuf
10691 // ldr r1, LCPI1_1
10692 // add r1, pc, r1
10693 // str r1, [$jbuf, #+4] ; &jbuf[1]
10694 Register NewVReg1 = MRI->createVirtualRegister(RegClass: TRC);
10695 BuildMI(BB&: *MBB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: ARM::LDRi12), DestReg: NewVReg1)
10696 .addConstantPoolIndex(Idx: CPI)
10697 .addImm(Val: 0)
10698 .addMemOperand(MMO: CPMMO)
10699 .add(MOs: predOps(Pred: ARMCC::AL));
10700 Register NewVReg2 = MRI->createVirtualRegister(RegClass: TRC);
10701 BuildMI(BB&: *MBB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: ARM::PICADD), DestReg: NewVReg2)
10702 .addReg(RegNo: NewVReg1)
10703 .addImm(Val: PCLabelId)
10704 .add(MOs: predOps(Pred: ARMCC::AL));
10705 BuildMI(BB&: *MBB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: ARM::STRi12))
10706 .addReg(RegNo: NewVReg2)
10707 .addFrameIndex(Idx: FI)
10708 .addImm(Val: 36) // &jbuf[1] :: pc
10709 .addMemOperand(MMO: FIMMOSt)
10710 .add(MOs: predOps(Pred: ARMCC::AL));
10711 }
10712}
10713
10714void ARMTargetLowering::EmitSjLjDispatchBlock(MachineInstr &MI,
10715 MachineBasicBlock *MBB) const {
10716 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
10717 DebugLoc dl = MI.getDebugLoc();
10718 MachineFunction *MF = MBB->getParent();
10719 MachineRegisterInfo *MRI = &MF->getRegInfo();
10720 MachineFrameInfo &MFI = MF->getFrameInfo();
10721 int FI = MFI.getFunctionContextIndex();
10722
10723 const TargetRegisterClass *TRC = Subtarget->isThumb() ? &ARM::tGPRRegClass
10724 : &ARM::GPRnopcRegClass;
10725
10726 // Get a mapping of the call site numbers to all of the landing pads they're
10727 // associated with.
10728 DenseMap<unsigned, SmallVector<MachineBasicBlock*, 2>> CallSiteNumToLPad;
10729 unsigned MaxCSNum = 0;
10730 for (MachineBasicBlock &BB : *MF) {
10731 if (!BB.isEHPad())
10732 continue;
10733
10734 // FIXME: We should assert that the EH_LABEL is the first MI in the landing
10735 // pad.
10736 for (MachineInstr &II : BB) {
10737 if (!II.isEHLabel())
10738 continue;
10739
10740 MCSymbol *Sym = II.getOperand(i: 0).getMCSymbol();
10741 if (!MF->hasCallSiteLandingPad(Sym)) continue;
10742
10743 SmallVectorImpl<unsigned> &CallSiteIdxs = MF->getCallSiteLandingPad(Sym);
10744 for (unsigned Idx : CallSiteIdxs) {
10745 CallSiteNumToLPad[Idx].push_back(Elt: &BB);
10746 MaxCSNum = std::max(a: MaxCSNum, b: Idx);
10747 }
10748 break;
10749 }
10750 }
10751
10752 // Get an ordered list of the machine basic blocks for the jump table.
10753 std::vector<MachineBasicBlock*> LPadList;
10754 SmallPtrSet<MachineBasicBlock*, 32> InvokeBBs;
10755 LPadList.reserve(n: CallSiteNumToLPad.size());
10756 for (unsigned I = 1; I <= MaxCSNum; ++I) {
10757 SmallVectorImpl<MachineBasicBlock*> &MBBList = CallSiteNumToLPad[I];
10758 for (MachineBasicBlock *MBB : MBBList) {
10759 LPadList.push_back(x: MBB);
10760 InvokeBBs.insert_range(R: MBB->predecessors());
10761 }
10762 }
10763
10764 assert(!LPadList.empty() &&
10765 "No landing pad destinations for the dispatch jump table!");
10766
10767 // Create the jump table and associated information.
10768 MachineJumpTableInfo *JTI =
10769 MF->getOrCreateJumpTableInfo(JTEntryKind: MachineJumpTableInfo::EK_Inline);
10770 unsigned MJTI = JTI->createJumpTableIndex(DestBBs: LPadList);
10771
10772 // Create the MBBs for the dispatch code.
10773
10774 // Shove the dispatch's address into the return slot in the function context.
10775 MachineBasicBlock *DispatchBB = MF->CreateMachineBasicBlock();
10776 DispatchBB->setIsEHPad();
10777
10778 MachineBasicBlock *TrapBB = MF->CreateMachineBasicBlock();
10779
10780 BuildMI(BB: TrapBB, MIMD: dl, MCID: TII->get(Opcode: Subtarget->isThumb() ? ARM::tTRAP : ARM::TRAP));
10781 DispatchBB->addSuccessor(Succ: TrapBB);
10782
10783 MachineBasicBlock *DispContBB = MF->CreateMachineBasicBlock();
10784 DispatchBB->addSuccessor(Succ: DispContBB);
10785
10786 // Insert and MBBs.
10787 MF->insert(MBBI: MF->end(), MBB: DispatchBB);
10788 MF->insert(MBBI: MF->end(), MBB: DispContBB);
10789 MF->insert(MBBI: MF->end(), MBB: TrapBB);
10790
10791 // Insert code into the entry block that creates and registers the function
10792 // context.
10793 SetupEntryBlockForSjLj(MI, MBB, DispatchBB, FI);
10794
10795 MachineMemOperand *FIMMOLd = MF->getMachineMemOperand(
10796 PtrInfo: MachinePointerInfo::getFixedStack(MF&: *MF, FI),
10797 F: MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile, Size: 4, BaseAlignment: Align(4));
10798
10799 MachineInstrBuilder MIB;
10800 MIB = BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::Int_eh_sjlj_dispatchsetup));
10801
10802 const ARMBaseInstrInfo *AII = static_cast<const ARMBaseInstrInfo*>(TII);
10803 const ARMBaseRegisterInfo &RI = AII->getRegisterInfo();
10804
10805 // Add a register mask with no preserved registers. This results in all
10806 // registers being marked as clobbered. This can't work if the dispatch block
10807 // is in a Thumb1 function and is linked with ARM code which uses the FP
10808 // registers, as there is no way to preserve the FP registers in Thumb1 mode.
10809 MIB.addRegMask(Mask: RI.getSjLjDispatchPreservedMask(MF: *MF));
10810
10811 bool IsPositionIndependent = isPositionIndependent();
10812 unsigned NumLPads = LPadList.size();
10813 if (Subtarget->isThumb2()) {
10814 Register NewVReg1 = MRI->createVirtualRegister(RegClass: TRC);
10815 BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::t2LDRi12), DestReg: NewVReg1)
10816 .addFrameIndex(Idx: FI)
10817 .addImm(Val: 4)
10818 .addMemOperand(MMO: FIMMOLd)
10819 .add(MOs: predOps(Pred: ARMCC::AL));
10820
10821 if (NumLPads < 256) {
10822 BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::t2CMPri))
10823 .addReg(RegNo: NewVReg1)
10824 .addImm(Val: LPadList.size())
10825 .add(MOs: predOps(Pred: ARMCC::AL));
10826 } else {
10827 Register VReg1 = MRI->createVirtualRegister(RegClass: TRC);
10828 BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::t2MOVi16), DestReg: VReg1)
10829 .addImm(Val: NumLPads & 0xFFFF)
10830 .add(MOs: predOps(Pred: ARMCC::AL));
10831
10832 unsigned VReg2 = VReg1;
10833 if ((NumLPads & 0xFFFF0000) != 0) {
10834 VReg2 = MRI->createVirtualRegister(RegClass: TRC);
10835 BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::t2MOVTi16), DestReg: VReg2)
10836 .addReg(RegNo: VReg1)
10837 .addImm(Val: NumLPads >> 16)
10838 .add(MOs: predOps(Pred: ARMCC::AL));
10839 }
10840
10841 BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::t2CMPrr))
10842 .addReg(RegNo: NewVReg1)
10843 .addReg(RegNo: VReg2)
10844 .add(MOs: predOps(Pred: ARMCC::AL));
10845 }
10846
10847 BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::t2Bcc))
10848 .addMBB(MBB: TrapBB)
10849 .addImm(Val: ARMCC::HI)
10850 .addReg(RegNo: ARM::CPSR);
10851
10852 Register NewVReg3 = MRI->createVirtualRegister(RegClass: TRC);
10853 BuildMI(BB: DispContBB, MIMD: dl, MCID: TII->get(Opcode: ARM::t2LEApcrelJT), DestReg: NewVReg3)
10854 .addJumpTableIndex(Idx: MJTI)
10855 .add(MOs: predOps(Pred: ARMCC::AL));
10856
10857 Register NewVReg4 = MRI->createVirtualRegister(RegClass: TRC);
10858 BuildMI(BB: DispContBB, MIMD: dl, MCID: TII->get(Opcode: ARM::t2ADDrs), DestReg: NewVReg4)
10859 .addReg(RegNo: NewVReg3)
10860 .addReg(RegNo: NewVReg1)
10861 .addImm(Val: ARM_AM::getSORegOpc(ShOp: ARM_AM::lsl, Imm: 2))
10862 .add(MOs: predOps(Pred: ARMCC::AL))
10863 .add(MO: condCodeOp());
10864
10865 BuildMI(BB: DispContBB, MIMD: dl, MCID: TII->get(Opcode: ARM::t2BR_JT))
10866 .addReg(RegNo: NewVReg4)
10867 .addReg(RegNo: NewVReg1)
10868 .addJumpTableIndex(Idx: MJTI);
10869 } else if (Subtarget->isThumb()) {
10870 Register NewVReg1 = MRI->createVirtualRegister(RegClass: TRC);
10871 BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::tLDRspi), DestReg: NewVReg1)
10872 .addFrameIndex(Idx: FI)
10873 .addImm(Val: 1)
10874 .addMemOperand(MMO: FIMMOLd)
10875 .add(MOs: predOps(Pred: ARMCC::AL));
10876
10877 if (NumLPads < 256) {
10878 BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::tCMPi8))
10879 .addReg(RegNo: NewVReg1)
10880 .addImm(Val: NumLPads)
10881 .add(MOs: predOps(Pred: ARMCC::AL));
10882 } else {
10883 MachineConstantPool *ConstantPool = MF->getConstantPool();
10884 Type *Int32Ty = Type::getInt32Ty(C&: MF->getFunction().getContext());
10885 const Constant *C = ConstantInt::get(Ty: Int32Ty, V: NumLPads);
10886
10887 // MachineConstantPool wants an explicit alignment.
10888 Align Alignment = MF->getDataLayout().getPrefTypeAlign(Ty: Int32Ty);
10889 unsigned Idx = ConstantPool->getConstantPoolIndex(C, Alignment);
10890
10891 Register VReg1 = MRI->createVirtualRegister(RegClass: TRC);
10892 BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::tLDRpci))
10893 .addReg(RegNo: VReg1, Flags: RegState::Define)
10894 .addConstantPoolIndex(Idx)
10895 .add(MOs: predOps(Pred: ARMCC::AL));
10896 BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::tCMPr))
10897 .addReg(RegNo: NewVReg1)
10898 .addReg(RegNo: VReg1)
10899 .add(MOs: predOps(Pred: ARMCC::AL));
10900 }
10901
10902 BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::tBcc))
10903 .addMBB(MBB: TrapBB)
10904 .addImm(Val: ARMCC::HI)
10905 .addReg(RegNo: ARM::CPSR);
10906
10907 Register NewVReg2 = MRI->createVirtualRegister(RegClass: TRC);
10908 BuildMI(BB: DispContBB, MIMD: dl, MCID: TII->get(Opcode: ARM::tLSLri), DestReg: NewVReg2)
10909 .addReg(RegNo: ARM::CPSR, Flags: RegState::Define)
10910 .addReg(RegNo: NewVReg1)
10911 .addImm(Val: 2)
10912 .add(MOs: predOps(Pred: ARMCC::AL));
10913
10914 Register NewVReg3 = MRI->createVirtualRegister(RegClass: TRC);
10915 BuildMI(BB: DispContBB, MIMD: dl, MCID: TII->get(Opcode: ARM::tLEApcrelJT), DestReg: NewVReg3)
10916 .addJumpTableIndex(Idx: MJTI)
10917 .add(MOs: predOps(Pred: ARMCC::AL));
10918
10919 Register NewVReg4 = MRI->createVirtualRegister(RegClass: TRC);
10920 BuildMI(BB: DispContBB, MIMD: dl, MCID: TII->get(Opcode: ARM::tADDrr), DestReg: NewVReg4)
10921 .addReg(RegNo: ARM::CPSR, Flags: RegState::Define)
10922 .addReg(RegNo: NewVReg2)
10923 .addReg(RegNo: NewVReg3)
10924 .add(MOs: predOps(Pred: ARMCC::AL));
10925
10926 MachineMemOperand *JTMMOLd =
10927 MF->getMachineMemOperand(PtrInfo: MachinePointerInfo::getJumpTable(MF&: *MF),
10928 F: MachineMemOperand::MOLoad, Size: 4, BaseAlignment: Align(4));
10929
10930 Register NewVReg5 = MRI->createVirtualRegister(RegClass: TRC);
10931 BuildMI(BB: DispContBB, MIMD: dl, MCID: TII->get(Opcode: ARM::tLDRi), DestReg: NewVReg5)
10932 .addReg(RegNo: NewVReg4)
10933 .addImm(Val: 0)
10934 .addMemOperand(MMO: JTMMOLd)
10935 .add(MOs: predOps(Pred: ARMCC::AL));
10936
10937 unsigned NewVReg6 = NewVReg5;
10938 if (IsPositionIndependent) {
10939 NewVReg6 = MRI->createVirtualRegister(RegClass: TRC);
10940 BuildMI(BB: DispContBB, MIMD: dl, MCID: TII->get(Opcode: ARM::tADDrr), DestReg: NewVReg6)
10941 .addReg(RegNo: ARM::CPSR, Flags: RegState::Define)
10942 .addReg(RegNo: NewVReg5)
10943 .addReg(RegNo: NewVReg3)
10944 .add(MOs: predOps(Pred: ARMCC::AL));
10945 }
10946
10947 BuildMI(BB: DispContBB, MIMD: dl, MCID: TII->get(Opcode: ARM::tBR_JTr))
10948 .addReg(RegNo: NewVReg6)
10949 .addJumpTableIndex(Idx: MJTI);
10950 } else {
10951 Register NewVReg1 = MRI->createVirtualRegister(RegClass: TRC);
10952 BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::LDRi12), DestReg: NewVReg1)
10953 .addFrameIndex(Idx: FI)
10954 .addImm(Val: 4)
10955 .addMemOperand(MMO: FIMMOLd)
10956 .add(MOs: predOps(Pred: ARMCC::AL));
10957
10958 if (NumLPads < 256) {
10959 BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::CMPri))
10960 .addReg(RegNo: NewVReg1)
10961 .addImm(Val: NumLPads)
10962 .add(MOs: predOps(Pred: ARMCC::AL));
10963 } else if (Subtarget->hasV6T2Ops() && isUInt<16>(x: NumLPads)) {
10964 Register VReg1 = MRI->createVirtualRegister(RegClass: TRC);
10965 BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::MOVi16), DestReg: VReg1)
10966 .addImm(Val: NumLPads & 0xFFFF)
10967 .add(MOs: predOps(Pred: ARMCC::AL));
10968
10969 unsigned VReg2 = VReg1;
10970 if ((NumLPads & 0xFFFF0000) != 0) {
10971 VReg2 = MRI->createVirtualRegister(RegClass: TRC);
10972 BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::MOVTi16), DestReg: VReg2)
10973 .addReg(RegNo: VReg1)
10974 .addImm(Val: NumLPads >> 16)
10975 .add(MOs: predOps(Pred: ARMCC::AL));
10976 }
10977
10978 BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::CMPrr))
10979 .addReg(RegNo: NewVReg1)
10980 .addReg(RegNo: VReg2)
10981 .add(MOs: predOps(Pred: ARMCC::AL));
10982 } else {
10983 MachineConstantPool *ConstantPool = MF->getConstantPool();
10984 Type *Int32Ty = Type::getInt32Ty(C&: MF->getFunction().getContext());
10985 const Constant *C = ConstantInt::get(Ty: Int32Ty, V: NumLPads);
10986
10987 // MachineConstantPool wants an explicit alignment.
10988 Align Alignment = MF->getDataLayout().getPrefTypeAlign(Ty: Int32Ty);
10989 unsigned Idx = ConstantPool->getConstantPoolIndex(C, Alignment);
10990
10991 Register VReg1 = MRI->createVirtualRegister(RegClass: TRC);
10992 BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::LDRcp))
10993 .addReg(RegNo: VReg1, Flags: RegState::Define)
10994 .addConstantPoolIndex(Idx)
10995 .addImm(Val: 0)
10996 .add(MOs: predOps(Pred: ARMCC::AL));
10997 BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::CMPrr))
10998 .addReg(RegNo: NewVReg1)
10999 .addReg(RegNo: VReg1)
11000 .add(MOs: predOps(Pred: ARMCC::AL));
11001 }
11002
11003 BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::Bcc))
11004 .addMBB(MBB: TrapBB)
11005 .addImm(Val: ARMCC::HI)
11006 .addReg(RegNo: ARM::CPSR);
11007
11008 Register NewVReg3 = MRI->createVirtualRegister(RegClass: TRC);
11009 BuildMI(BB: DispContBB, MIMD: dl, MCID: TII->get(Opcode: ARM::MOVsi), DestReg: NewVReg3)
11010 .addReg(RegNo: NewVReg1)
11011 .addImm(Val: ARM_AM::getSORegOpc(ShOp: ARM_AM::lsl, Imm: 2))
11012 .add(MOs: predOps(Pred: ARMCC::AL))
11013 .add(MO: condCodeOp());
11014 Register NewVReg4 = MRI->createVirtualRegister(RegClass: TRC);
11015 BuildMI(BB: DispContBB, MIMD: dl, MCID: TII->get(Opcode: ARM::LEApcrelJT), DestReg: NewVReg4)
11016 .addJumpTableIndex(Idx: MJTI)
11017 .add(MOs: predOps(Pred: ARMCC::AL));
11018
11019 MachineMemOperand *JTMMOLd =
11020 MF->getMachineMemOperand(PtrInfo: MachinePointerInfo::getJumpTable(MF&: *MF),
11021 F: MachineMemOperand::MOLoad, Size: 4, BaseAlignment: Align(4));
11022 Register NewVReg5 = MRI->createVirtualRegister(RegClass: TRC);
11023 BuildMI(BB: DispContBB, MIMD: dl, MCID: TII->get(Opcode: ARM::LDRrs), DestReg: NewVReg5)
11024 .addReg(RegNo: NewVReg3)
11025 .addReg(RegNo: NewVReg4)
11026 .addImm(Val: 0)
11027 .addMemOperand(MMO: JTMMOLd)
11028 .add(MOs: predOps(Pred: ARMCC::AL));
11029
11030 if (IsPositionIndependent) {
11031 BuildMI(BB: DispContBB, MIMD: dl, MCID: TII->get(Opcode: ARM::BR_JTadd))
11032 .addReg(RegNo: NewVReg5)
11033 .addReg(RegNo: NewVReg4)
11034 .addJumpTableIndex(Idx: MJTI);
11035 } else {
11036 BuildMI(BB: DispContBB, MIMD: dl, MCID: TII->get(Opcode: ARM::BR_JTr))
11037 .addReg(RegNo: NewVReg5)
11038 .addJumpTableIndex(Idx: MJTI);
11039 }
11040 }
11041
11042 // Add the jump table entries as successors to the MBB.
11043 SmallPtrSet<MachineBasicBlock*, 8> SeenMBBs;
11044 for (MachineBasicBlock *CurMBB : LPadList) {
11045 if (SeenMBBs.insert(Ptr: CurMBB).second)
11046 DispContBB->addSuccessor(Succ: CurMBB);
11047 }
11048
11049 // N.B. the order the invoke BBs are processed in doesn't matter here.
11050 const MCPhysReg *SavedRegs = RI.getCalleeSavedRegs(MF);
11051 SmallVector<MachineBasicBlock*, 64> MBBLPads;
11052 for (MachineBasicBlock *BB : InvokeBBs) {
11053
11054 // Remove the landing pad successor from the invoke block and replace it
11055 // with the new dispatch block.
11056 SmallVector<MachineBasicBlock*, 4> Successors(BB->successors());
11057 while (!Successors.empty()) {
11058 MachineBasicBlock *SMBB = Successors.pop_back_val();
11059 if (SMBB->isEHPad()) {
11060 BB->removeSuccessor(Succ: SMBB);
11061 MBBLPads.push_back(Elt: SMBB);
11062 }
11063 }
11064
11065 BB->addSuccessor(Succ: DispatchBB, Prob: BranchProbability::getZero());
11066 BB->normalizeSuccProbs();
11067
11068 // Find the invoke call and mark all of the callee-saved registers as
11069 // 'implicit defined' so that they're spilled. This prevents code from
11070 // moving instructions to before the EH block, where they will never be
11071 // executed.
11072 for (MachineBasicBlock::reverse_iterator
11073 II = BB->rbegin(), IE = BB->rend(); II != IE; ++II) {
11074 if (!II->isCall()) continue;
11075
11076 DenseSet<unsigned> DefRegs;
11077 for (MachineInstr::mop_iterator
11078 OI = II->operands_begin(), OE = II->operands_end();
11079 OI != OE; ++OI) {
11080 if (!OI->isReg()) continue;
11081 DefRegs.insert(V: OI->getReg());
11082 }
11083
11084 MachineInstrBuilder MIB(*MF, &*II);
11085
11086 for (unsigned i = 0; SavedRegs[i] != 0; ++i) {
11087 unsigned Reg = SavedRegs[i];
11088 if (Subtarget->isThumb2() &&
11089 !ARM::tGPRRegClass.contains(Reg) &&
11090 !ARM::hGPRRegClass.contains(Reg))
11091 continue;
11092 if (Subtarget->isThumb1Only() && !ARM::tGPRRegClass.contains(Reg))
11093 continue;
11094 if (!Subtarget->isThumb() && !ARM::GPRRegClass.contains(Reg))
11095 continue;
11096 if (!DefRegs.contains(V: Reg))
11097 MIB.addReg(RegNo: Reg, Flags: RegState::ImplicitDefine | RegState::Dead);
11098 }
11099
11100 break;
11101 }
11102 }
11103
11104 // Mark all former landing pads as non-landing pads. The dispatch is the only
11105 // landing pad now.
11106 for (MachineBasicBlock *MBBLPad : MBBLPads)
11107 MBBLPad->setIsEHPad(false);
11108
11109 // The instruction is gone now.
11110 MI.eraseFromParent();
11111}
11112
11113static
11114MachineBasicBlock *OtherSucc(MachineBasicBlock *MBB, MachineBasicBlock *Succ) {
11115 for (MachineBasicBlock *S : MBB->successors())
11116 if (S != Succ)
11117 return S;
11118 llvm_unreachable("Expecting a BB with two successors!");
11119}
11120
11121/// Return the load opcode for a given load size. If load size >= 8,
11122/// neon opcode will be returned.
11123static unsigned getLdOpcode(unsigned LdSize, bool IsThumb1, bool IsThumb2) {
11124 if (LdSize >= 8)
11125 return LdSize == 16 ? ARM::VLD1q32wb_fixed
11126 : LdSize == 8 ? ARM::VLD1d32wb_fixed : 0;
11127 if (IsThumb1)
11128 return LdSize == 4 ? ARM::tLDRi
11129 : LdSize == 2 ? ARM::tLDRHi
11130 : LdSize == 1 ? ARM::tLDRBi : 0;
11131 if (IsThumb2)
11132 return LdSize == 4 ? ARM::t2LDR_POST
11133 : LdSize == 2 ? ARM::t2LDRH_POST
11134 : LdSize == 1 ? ARM::t2LDRB_POST : 0;
11135 return LdSize == 4 ? ARM::LDR_POST_IMM
11136 : LdSize == 2 ? ARM::LDRH_POST
11137 : LdSize == 1 ? ARM::LDRB_POST_IMM : 0;
11138}
11139
11140/// Return the store opcode for a given store size. If store size >= 8,
11141/// neon opcode will be returned.
11142static unsigned getStOpcode(unsigned StSize, bool IsThumb1, bool IsThumb2) {
11143 if (StSize >= 8)
11144 return StSize == 16 ? ARM::VST1q32wb_fixed
11145 : StSize == 8 ? ARM::VST1d32wb_fixed : 0;
11146 if (IsThumb1)
11147 return StSize == 4 ? ARM::tSTRi
11148 : StSize == 2 ? ARM::tSTRHi
11149 : StSize == 1 ? ARM::tSTRBi : 0;
11150 if (IsThumb2)
11151 return StSize == 4 ? ARM::t2STR_POST
11152 : StSize == 2 ? ARM::t2STRH_POST
11153 : StSize == 1 ? ARM::t2STRB_POST : 0;
11154 return StSize == 4 ? ARM::STR_POST_IMM
11155 : StSize == 2 ? ARM::STRH_POST
11156 : StSize == 1 ? ARM::STRB_POST_IMM : 0;
11157}
11158
11159/// Emit a post-increment load operation with given size. The instructions
11160/// will be added to BB at Pos.
11161static void emitPostLd(MachineBasicBlock *BB, MachineBasicBlock::iterator Pos,
11162 const TargetInstrInfo *TII, const DebugLoc &dl,
11163 unsigned LdSize, unsigned Data, unsigned AddrIn,
11164 unsigned AddrOut, bool IsThumb1, bool IsThumb2) {
11165 unsigned LdOpc = getLdOpcode(LdSize, IsThumb1, IsThumb2);
11166 assert(LdOpc != 0 && "Should have a load opcode");
11167 if (LdSize >= 8) {
11168 BuildMI(BB&: *BB, I: Pos, MIMD: dl, MCID: TII->get(Opcode: LdOpc), DestReg: Data)
11169 .addReg(RegNo: AddrOut, Flags: RegState::Define)
11170 .addReg(RegNo: AddrIn)
11171 .addImm(Val: 0)
11172 .add(MOs: predOps(Pred: ARMCC::AL));
11173 } else if (IsThumb1) {
11174 // load + update AddrIn
11175 BuildMI(BB&: *BB, I: Pos, MIMD: dl, MCID: TII->get(Opcode: LdOpc), DestReg: Data)
11176 .addReg(RegNo: AddrIn)
11177 .addImm(Val: 0)
11178 .add(MOs: predOps(Pred: ARMCC::AL));
11179 BuildMI(BB&: *BB, I: Pos, MIMD: dl, MCID: TII->get(Opcode: ARM::tADDi8), DestReg: AddrOut)
11180 .add(MO: t1CondCodeOp())
11181 .addReg(RegNo: AddrIn)
11182 .addImm(Val: LdSize)
11183 .add(MOs: predOps(Pred: ARMCC::AL));
11184 } else if (IsThumb2) {
11185 BuildMI(BB&: *BB, I: Pos, MIMD: dl, MCID: TII->get(Opcode: LdOpc), DestReg: Data)
11186 .addReg(RegNo: AddrOut, Flags: RegState::Define)
11187 .addReg(RegNo: AddrIn)
11188 .addImm(Val: LdSize)
11189 .add(MOs: predOps(Pred: ARMCC::AL));
11190 } else { // arm
11191 BuildMI(BB&: *BB, I: Pos, MIMD: dl, MCID: TII->get(Opcode: LdOpc), DestReg: Data)
11192 .addReg(RegNo: AddrOut, Flags: RegState::Define)
11193 .addReg(RegNo: AddrIn)
11194 .addReg(RegNo: 0)
11195 .addImm(Val: LdSize)
11196 .add(MOs: predOps(Pred: ARMCC::AL));
11197 }
11198}
11199
11200/// Emit a post-increment store operation with given size. The instructions
11201/// will be added to BB at Pos.
11202static void emitPostSt(MachineBasicBlock *BB, MachineBasicBlock::iterator Pos,
11203 const TargetInstrInfo *TII, const DebugLoc &dl,
11204 unsigned StSize, unsigned Data, unsigned AddrIn,
11205 unsigned AddrOut, bool IsThumb1, bool IsThumb2) {
11206 unsigned StOpc = getStOpcode(StSize, IsThumb1, IsThumb2);
11207 assert(StOpc != 0 && "Should have a store opcode");
11208 if (StSize >= 8) {
11209 BuildMI(BB&: *BB, I: Pos, MIMD: dl, MCID: TII->get(Opcode: StOpc), DestReg: AddrOut)
11210 .addReg(RegNo: AddrIn)
11211 .addImm(Val: 0)
11212 .addReg(RegNo: Data)
11213 .add(MOs: predOps(Pred: ARMCC::AL));
11214 } else if (IsThumb1) {
11215 // store + update AddrIn
11216 BuildMI(BB&: *BB, I: Pos, MIMD: dl, MCID: TII->get(Opcode: StOpc))
11217 .addReg(RegNo: Data)
11218 .addReg(RegNo: AddrIn)
11219 .addImm(Val: 0)
11220 .add(MOs: predOps(Pred: ARMCC::AL));
11221 BuildMI(BB&: *BB, I: Pos, MIMD: dl, MCID: TII->get(Opcode: ARM::tADDi8), DestReg: AddrOut)
11222 .add(MO: t1CondCodeOp())
11223 .addReg(RegNo: AddrIn)
11224 .addImm(Val: StSize)
11225 .add(MOs: predOps(Pred: ARMCC::AL));
11226 } else if (IsThumb2) {
11227 BuildMI(BB&: *BB, I: Pos, MIMD: dl, MCID: TII->get(Opcode: StOpc), DestReg: AddrOut)
11228 .addReg(RegNo: Data)
11229 .addReg(RegNo: AddrIn)
11230 .addImm(Val: StSize)
11231 .add(MOs: predOps(Pred: ARMCC::AL));
11232 } else { // arm
11233 BuildMI(BB&: *BB, I: Pos, MIMD: dl, MCID: TII->get(Opcode: StOpc), DestReg: AddrOut)
11234 .addReg(RegNo: Data)
11235 .addReg(RegNo: AddrIn)
11236 .addReg(RegNo: 0)
11237 .addImm(Val: StSize)
11238 .add(MOs: predOps(Pred: ARMCC::AL));
11239 }
11240}
11241
11242MachineBasicBlock *
11243ARMTargetLowering::EmitStructByval(MachineInstr &MI,
11244 MachineBasicBlock *BB) const {
11245 // This pseudo instruction has 3 operands: dst, src, size
11246 // We expand it to a loop if size > Subtarget->getMaxInlineSizeThreshold().
11247 // Otherwise, we will generate unrolled scalar copies.
11248 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
11249 const BasicBlock *LLVM_BB = BB->getBasicBlock();
11250 MachineFunction::iterator It = ++BB->getIterator();
11251
11252 Register dest = MI.getOperand(i: 0).getReg();
11253 Register src = MI.getOperand(i: 1).getReg();
11254 unsigned SizeVal = MI.getOperand(i: 2).getImm();
11255 unsigned Alignment = MI.getOperand(i: 3).getImm();
11256 DebugLoc dl = MI.getDebugLoc();
11257
11258 MachineFunction *MF = BB->getParent();
11259 MachineRegisterInfo &MRI = MF->getRegInfo();
11260 unsigned UnitSize = 0;
11261 const TargetRegisterClass *TRC = nullptr;
11262 const TargetRegisterClass *VecTRC = nullptr;
11263
11264 bool IsThumb1 = Subtarget->isThumb1Only();
11265 bool IsThumb2 = Subtarget->isThumb2();
11266 bool IsThumb = Subtarget->isThumb();
11267
11268 if (Alignment & 1) {
11269 UnitSize = 1;
11270 } else if (Alignment & 2) {
11271 UnitSize = 2;
11272 } else {
11273 // Check whether we can use NEON instructions.
11274 if (!MF->getFunction().hasFnAttribute(Kind: Attribute::NoImplicitFloat) &&
11275 Subtarget->hasNEON()) {
11276 if ((Alignment % 16 == 0) && SizeVal >= 16)
11277 UnitSize = 16;
11278 else if ((Alignment % 8 == 0) && SizeVal >= 8)
11279 UnitSize = 8;
11280 }
11281 // Can't use NEON instructions.
11282 if (UnitSize == 0)
11283 UnitSize = 4;
11284 }
11285
11286 // Select the correct opcode and register class for unit size load/store
11287 bool IsNeon = UnitSize >= 8;
11288 TRC = IsThumb ? &ARM::tGPRRegClass : &ARM::GPRRegClass;
11289 if (IsNeon)
11290 VecTRC = UnitSize == 16 ? &ARM::DPairRegClass
11291 : UnitSize == 8 ? &ARM::DPRRegClass
11292 : nullptr;
11293
11294 unsigned BytesLeft = SizeVal % UnitSize;
11295 unsigned LoopSize = SizeVal - BytesLeft;
11296
11297 if (SizeVal <= Subtarget->getMaxInlineSizeThreshold()) {
11298 // Use LDR and STR to copy.
11299 // [scratch, srcOut] = LDR_POST(srcIn, UnitSize)
11300 // [destOut] = STR_POST(scratch, destIn, UnitSize)
11301 unsigned srcIn = src;
11302 unsigned destIn = dest;
11303 for (unsigned i = 0; i < LoopSize; i+=UnitSize) {
11304 Register srcOut = MRI.createVirtualRegister(RegClass: TRC);
11305 Register destOut = MRI.createVirtualRegister(RegClass: TRC);
11306 Register scratch = MRI.createVirtualRegister(RegClass: IsNeon ? VecTRC : TRC);
11307 emitPostLd(BB, Pos: MI, TII, dl, LdSize: UnitSize, Data: scratch, AddrIn: srcIn, AddrOut: srcOut,
11308 IsThumb1, IsThumb2);
11309 emitPostSt(BB, Pos: MI, TII, dl, StSize: UnitSize, Data: scratch, AddrIn: destIn, AddrOut: destOut,
11310 IsThumb1, IsThumb2);
11311 srcIn = srcOut;
11312 destIn = destOut;
11313 }
11314
11315 // Handle the leftover bytes with LDRB and STRB.
11316 // [scratch, srcOut] = LDRB_POST(srcIn, 1)
11317 // [destOut] = STRB_POST(scratch, destIn, 1)
11318 for (unsigned i = 0; i < BytesLeft; i++) {
11319 Register srcOut = MRI.createVirtualRegister(RegClass: TRC);
11320 Register destOut = MRI.createVirtualRegister(RegClass: TRC);
11321 Register scratch = MRI.createVirtualRegister(RegClass: TRC);
11322 emitPostLd(BB, Pos: MI, TII, dl, LdSize: 1, Data: scratch, AddrIn: srcIn, AddrOut: srcOut,
11323 IsThumb1, IsThumb2);
11324 emitPostSt(BB, Pos: MI, TII, dl, StSize: 1, Data: scratch, AddrIn: destIn, AddrOut: destOut,
11325 IsThumb1, IsThumb2);
11326 srcIn = srcOut;
11327 destIn = destOut;
11328 }
11329 MI.eraseFromParent(); // The instruction is gone now.
11330 return BB;
11331 }
11332
11333 // Expand the pseudo op to a loop.
11334 // thisMBB:
11335 // ...
11336 // movw varEnd, # --> with thumb2
11337 // movt varEnd, #
11338 // ldrcp varEnd, idx --> without thumb2
11339 // fallthrough --> loopMBB
11340 // loopMBB:
11341 // PHI varPhi, varEnd, varLoop
11342 // PHI srcPhi, src, srcLoop
11343 // PHI destPhi, dst, destLoop
11344 // [scratch, srcLoop] = LDR_POST(srcPhi, UnitSize)
11345 // [destLoop] = STR_POST(scratch, destPhi, UnitSize)
11346 // subs varLoop, varPhi, #UnitSize
11347 // bne loopMBB
11348 // fallthrough --> exitMBB
11349 // exitMBB:
11350 // epilogue to handle left-over bytes
11351 // [scratch, srcOut] = LDRB_POST(srcLoop, 1)
11352 // [destOut] = STRB_POST(scratch, destLoop, 1)
11353 MachineBasicBlock *loopMBB = MF->CreateMachineBasicBlock(BB: LLVM_BB);
11354 MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(BB: LLVM_BB);
11355 MF->insert(MBBI: It, MBB: loopMBB);
11356 MF->insert(MBBI: It, MBB: exitMBB);
11357
11358 // Set the call frame size on entry to the new basic blocks.
11359 unsigned CallFrameSize = TII->getCallFrameSizeAt(MI);
11360 loopMBB->setCallFrameSize(CallFrameSize);
11361 exitMBB->setCallFrameSize(CallFrameSize);
11362
11363 // Transfer the remainder of BB and its successor edges to exitMBB.
11364 exitMBB->splice(Where: exitMBB->begin(), Other: BB,
11365 From: std::next(x: MachineBasicBlock::iterator(MI)), To: BB->end());
11366 exitMBB->transferSuccessorsAndUpdatePHIs(FromMBB: BB);
11367
11368 // Load an immediate to varEnd.
11369 Register varEnd = MRI.createVirtualRegister(RegClass: TRC);
11370 if (Subtarget->useMovt()) {
11371 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: IsThumb ? ARM::t2MOVi32imm : ARM::MOVi32imm),
11372 DestReg: varEnd)
11373 .addImm(Val: LoopSize);
11374 } else if (Subtarget->genExecuteOnly()) {
11375 assert(IsThumb && "Non-thumb expected to have used movt");
11376 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: ARM::tMOVi32imm), DestReg: varEnd).addImm(Val: LoopSize);
11377 } else {
11378 MachineConstantPool *ConstantPool = MF->getConstantPool();
11379 Type *Int32Ty = Type::getInt32Ty(C&: MF->getFunction().getContext());
11380 const Constant *C = ConstantInt::get(Ty: Int32Ty, V: LoopSize);
11381
11382 // MachineConstantPool wants an explicit alignment.
11383 Align Alignment = MF->getDataLayout().getPrefTypeAlign(Ty: Int32Ty);
11384 unsigned Idx = ConstantPool->getConstantPoolIndex(C, Alignment);
11385 MachineMemOperand *CPMMO =
11386 MF->getMachineMemOperand(PtrInfo: MachinePointerInfo::getConstantPool(MF&: *MF),
11387 F: MachineMemOperand::MOLoad, Size: 4, BaseAlignment: Align(4));
11388
11389 if (IsThumb)
11390 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: ARM::tLDRpci))
11391 .addReg(RegNo: varEnd, Flags: RegState::Define)
11392 .addConstantPoolIndex(Idx)
11393 .add(MOs: predOps(Pred: ARMCC::AL))
11394 .addMemOperand(MMO: CPMMO);
11395 else
11396 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: ARM::LDRcp))
11397 .addReg(RegNo: varEnd, Flags: RegState::Define)
11398 .addConstantPoolIndex(Idx)
11399 .addImm(Val: 0)
11400 .add(MOs: predOps(Pred: ARMCC::AL))
11401 .addMemOperand(MMO: CPMMO);
11402 }
11403 BB->addSuccessor(Succ: loopMBB);
11404
11405 // Generate the loop body:
11406 // varPhi = PHI(varLoop, varEnd)
11407 // srcPhi = PHI(srcLoop, src)
11408 // destPhi = PHI(destLoop, dst)
11409 MachineBasicBlock *entryBB = BB;
11410 BB = loopMBB;
11411 Register varLoop = MRI.createVirtualRegister(RegClass: TRC);
11412 Register varPhi = MRI.createVirtualRegister(RegClass: TRC);
11413 Register srcLoop = MRI.createVirtualRegister(RegClass: TRC);
11414 Register srcPhi = MRI.createVirtualRegister(RegClass: TRC);
11415 Register destLoop = MRI.createVirtualRegister(RegClass: TRC);
11416 Register destPhi = MRI.createVirtualRegister(RegClass: TRC);
11417
11418 BuildMI(BB&: *BB, I: BB->begin(), MIMD: dl, MCID: TII->get(Opcode: ARM::PHI), DestReg: varPhi)
11419 .addReg(RegNo: varLoop).addMBB(MBB: loopMBB)
11420 .addReg(RegNo: varEnd).addMBB(MBB: entryBB);
11421 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: ARM::PHI), DestReg: srcPhi)
11422 .addReg(RegNo: srcLoop).addMBB(MBB: loopMBB)
11423 .addReg(RegNo: src).addMBB(MBB: entryBB);
11424 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: ARM::PHI), DestReg: destPhi)
11425 .addReg(RegNo: destLoop).addMBB(MBB: loopMBB)
11426 .addReg(RegNo: dest).addMBB(MBB: entryBB);
11427
11428 // [scratch, srcLoop] = LDR_POST(srcPhi, UnitSize)
11429 // [destLoop] = STR_POST(scratch, destPhi, UnitSiz)
11430 Register scratch = MRI.createVirtualRegister(RegClass: IsNeon ? VecTRC : TRC);
11431 emitPostLd(BB, Pos: BB->end(), TII, dl, LdSize: UnitSize, Data: scratch, AddrIn: srcPhi, AddrOut: srcLoop,
11432 IsThumb1, IsThumb2);
11433 emitPostSt(BB, Pos: BB->end(), TII, dl, StSize: UnitSize, Data: scratch, AddrIn: destPhi, AddrOut: destLoop,
11434 IsThumb1, IsThumb2);
11435
11436 // Decrement loop variable by UnitSize.
11437 if (IsThumb1) {
11438 BuildMI(BB&: *BB, I: BB->end(), MIMD: dl, MCID: TII->get(Opcode: ARM::tSUBi8), DestReg: varLoop)
11439 .add(MO: t1CondCodeOp())
11440 .addReg(RegNo: varPhi)
11441 .addImm(Val: UnitSize)
11442 .add(MOs: predOps(Pred: ARMCC::AL));
11443 } else {
11444 MachineInstrBuilder MIB =
11445 BuildMI(BB&: *BB, I: BB->end(), MIMD: dl,
11446 MCID: TII->get(Opcode: IsThumb2 ? ARM::t2SUBri : ARM::SUBri), DestReg: varLoop);
11447 MIB.addReg(RegNo: varPhi)
11448 .addImm(Val: UnitSize)
11449 .add(MOs: predOps(Pred: ARMCC::AL))
11450 .add(MO: condCodeOp());
11451 MIB->getOperand(i: 5).setReg(ARM::CPSR);
11452 MIB->getOperand(i: 5).setIsDef(true);
11453 }
11454 BuildMI(BB&: *BB, I: BB->end(), MIMD: dl,
11455 MCID: TII->get(Opcode: IsThumb1 ? ARM::tBcc : IsThumb2 ? ARM::t2Bcc : ARM::Bcc))
11456 .addMBB(MBB: loopMBB).addImm(Val: ARMCC::NE).addReg(RegNo: ARM::CPSR);
11457
11458 // loopMBB can loop back to loopMBB or fall through to exitMBB.
11459 BB->addSuccessor(Succ: loopMBB);
11460 BB->addSuccessor(Succ: exitMBB);
11461
11462 // Add epilogue to handle BytesLeft.
11463 BB = exitMBB;
11464 auto StartOfExit = exitMBB->begin();
11465
11466 // [scratch, srcOut] = LDRB_POST(srcLoop, 1)
11467 // [destOut] = STRB_POST(scratch, destLoop, 1)
11468 unsigned srcIn = srcLoop;
11469 unsigned destIn = destLoop;
11470 for (unsigned i = 0; i < BytesLeft; i++) {
11471 Register srcOut = MRI.createVirtualRegister(RegClass: TRC);
11472 Register destOut = MRI.createVirtualRegister(RegClass: TRC);
11473 Register scratch = MRI.createVirtualRegister(RegClass: TRC);
11474 emitPostLd(BB, Pos: StartOfExit, TII, dl, LdSize: 1, Data: scratch, AddrIn: srcIn, AddrOut: srcOut,
11475 IsThumb1, IsThumb2);
11476 emitPostSt(BB, Pos: StartOfExit, TII, dl, StSize: 1, Data: scratch, AddrIn: destIn, AddrOut: destOut,
11477 IsThumb1, IsThumb2);
11478 srcIn = srcOut;
11479 destIn = destOut;
11480 }
11481
11482 MI.eraseFromParent(); // The instruction is gone now.
11483 return BB;
11484}
11485
11486MachineBasicBlock *
11487ARMTargetLowering::EmitLowered__chkstk(MachineInstr &MI,
11488 MachineBasicBlock *MBB) const {
11489 const TargetMachine &TM = getTargetMachine();
11490 const TargetInstrInfo &TII = *Subtarget->getInstrInfo();
11491 DebugLoc DL = MI.getDebugLoc();
11492
11493 assert(TM.getTargetTriple().isOSWindows() &&
11494 "__chkstk is only supported on Windows");
11495 assert(Subtarget->isThumb2() && "Windows on ARM requires Thumb-2 mode");
11496
11497 // __chkstk takes the number of words to allocate on the stack in R4, and
11498 // returns the stack adjustment in number of bytes in R4. This will not
11499 // clober any other registers (other than the obvious lr).
11500 //
11501 // Although, technically, IP should be considered a register which may be
11502 // clobbered, the call itself will not touch it. Windows on ARM is a pure
11503 // thumb-2 environment, so there is no interworking required. As a result, we
11504 // do not expect a veneer to be emitted by the linker, clobbering IP.
11505 //
11506 // Each module receives its own copy of __chkstk, so no import thunk is
11507 // required, again, ensuring that IP is not clobbered.
11508 //
11509 // Finally, although some linkers may theoretically provide a trampoline for
11510 // out of range calls (which is quite common due to a 32M range limitation of
11511 // branches for Thumb), we can generate the long-call version via
11512 // -mcmodel=large, alleviating the need for the trampoline which may clobber
11513 // IP.
11514
11515 RTLIB::LibcallImpl ChkStkLibcall = getLibcallImpl(Call: RTLIB::STACK_PROBE);
11516 if (ChkStkLibcall == RTLIB::Unsupported)
11517 reportFatalUsageError(reason: "no available implementation of __chkstk");
11518
11519 const char *ChkStk = getLibcallImplName(Call: ChkStkLibcall).data();
11520 switch (TM.getCodeModel()) {
11521 case CodeModel::Tiny:
11522 llvm_unreachable("Tiny code model not available on ARM.");
11523 case CodeModel::Small:
11524 case CodeModel::Medium:
11525 case CodeModel::Kernel:
11526 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: ARM::tBL))
11527 .add(MOs: predOps(Pred: ARMCC::AL))
11528 .addExternalSymbol(FnName: ChkStk)
11529 .setOperandDead(3) // implicit-def $lr
11530 .addReg(RegNo: ARM::R4, Flags: RegState::Implicit | RegState::Kill)
11531 .addReg(RegNo: ARM::R4, Flags: RegState::Implicit | RegState::Define)
11532 .addReg(RegNo: ARM::R12,
11533 Flags: RegState::Implicit | RegState::Define | RegState::Dead)
11534 .addReg(RegNo: ARM::CPSR,
11535 Flags: RegState::Implicit | RegState::Define | RegState::Dead);
11536 break;
11537 case CodeModel::Large: {
11538 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
11539 Register Reg = MRI.createVirtualRegister(RegClass: &ARM::rGPRRegClass);
11540
11541 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: ARM::t2MOVi32imm), DestReg: Reg)
11542 .addExternalSymbol(FnName: ChkStk);
11543 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: gettBLXrOpcode(MF: *MBB->getParent())))
11544 .add(MOs: predOps(Pred: ARMCC::AL))
11545 .addReg(RegNo: Reg)
11546 .setOperandDead(3) // implicit-def $lr
11547 .addReg(RegNo: ARM::R4, Flags: RegState::Implicit | RegState::Kill)
11548 .addReg(RegNo: ARM::R4, Flags: RegState::Implicit | RegState::Define)
11549 .addReg(RegNo: ARM::R12,
11550 Flags: RegState::Implicit | RegState::Define | RegState::Dead)
11551 .addReg(RegNo: ARM::CPSR,
11552 Flags: RegState::Implicit | RegState::Define | RegState::Dead);
11553 break;
11554 }
11555 }
11556
11557 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: ARM::t2SUBrr), DestReg: ARM::SP)
11558 .addReg(RegNo: ARM::SP, Flags: RegState::Kill)
11559 .addReg(RegNo: ARM::R4, Flags: RegState::Kill)
11560 .setMIFlags(MachineInstr::FrameSetup)
11561 .add(MOs: predOps(Pred: ARMCC::AL))
11562 .add(MO: condCodeOp());
11563
11564 MI.eraseFromParent();
11565 return MBB;
11566}
11567
11568MachineBasicBlock *
11569ARMTargetLowering::EmitLowered__dbzchk(MachineInstr &MI,
11570 MachineBasicBlock *MBB) const {
11571 DebugLoc DL = MI.getDebugLoc();
11572 MachineFunction *MF = MBB->getParent();
11573 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
11574
11575 MachineBasicBlock *ContBB = MF->CreateMachineBasicBlock();
11576 MF->insert(MBBI: ++MBB->getIterator(), MBB: ContBB);
11577 ContBB->splice(Where: ContBB->begin(), Other: MBB,
11578 From: std::next(x: MachineBasicBlock::iterator(MI)), To: MBB->end());
11579 ContBB->transferSuccessorsAndUpdatePHIs(FromMBB: MBB);
11580 MBB->addSuccessor(Succ: ContBB);
11581
11582 MachineBasicBlock *TrapBB = MF->CreateMachineBasicBlock();
11583 BuildMI(BB: TrapBB, MIMD: DL, MCID: TII->get(Opcode: ARM::t__brkdiv0));
11584 MF->push_back(MBB: TrapBB);
11585 MBB->addSuccessor(Succ: TrapBB);
11586
11587 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: ARM::tCMPi8))
11588 .addReg(RegNo: MI.getOperand(i: 0).getReg())
11589 .addImm(Val: 0)
11590 .add(MOs: predOps(Pred: ARMCC::AL));
11591 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: ARM::t2Bcc))
11592 .addMBB(MBB: TrapBB)
11593 .addImm(Val: ARMCC::EQ)
11594 .addReg(RegNo: ARM::CPSR);
11595
11596 MI.eraseFromParent();
11597 return ContBB;
11598}
11599
11600// The CPSR operand of SelectItr might be missing a kill marker
11601// because there were multiple uses of CPSR, and ISel didn't know
11602// which to mark. Figure out whether SelectItr should have had a
11603// kill marker, and set it if it should. Returns the correct kill
11604// marker value.
11605static bool checkAndUpdateCPSRKill(MachineBasicBlock::iterator SelectItr,
11606 MachineBasicBlock* BB,
11607 const TargetRegisterInfo* TRI) {
11608 // Scan forward through BB for a use/def of CPSR.
11609 MachineBasicBlock::iterator miI(std::next(x: SelectItr));
11610 for (MachineBasicBlock::iterator miE = BB->end(); miI != miE; ++miI) {
11611 const MachineInstr& mi = *miI;
11612 if (mi.readsRegister(Reg: ARM::CPSR, /*TRI=*/nullptr))
11613 return false;
11614 if (mi.definesRegister(Reg: ARM::CPSR, /*TRI=*/nullptr))
11615 break; // Should have kill-flag - update below.
11616 }
11617
11618 // If we hit the end of the block, check whether CPSR is live into a
11619 // successor.
11620 if (miI == BB->end()) {
11621 for (MachineBasicBlock *Succ : BB->successors())
11622 if (Succ->isLiveIn(Reg: ARM::CPSR))
11623 return false;
11624 }
11625
11626 // We found a def, or hit the end of the basic block and CPSR wasn't live
11627 // out. SelectMI should have a kill flag on CPSR.
11628 SelectItr->addRegisterKilled(IncomingReg: ARM::CPSR, RegInfo: TRI);
11629 return true;
11630}
11631
11632/// Adds logic in loop entry MBB to calculate loop iteration count and adds
11633/// t2WhileLoopSetup and t2WhileLoopStart to generate WLS loop
11634static Register genTPEntry(MachineBasicBlock *TpEntry,
11635 MachineBasicBlock *TpLoopBody,
11636 MachineBasicBlock *TpExit, Register OpSizeReg,
11637 const TargetInstrInfo *TII, DebugLoc Dl,
11638 MachineRegisterInfo &MRI) {
11639 // Calculates loop iteration count = ceil(n/16) = (n + 15) >> 4.
11640 Register AddDestReg = MRI.createVirtualRegister(RegClass: &ARM::rGPRRegClass);
11641 BuildMI(BB: TpEntry, MIMD: Dl, MCID: TII->get(Opcode: ARM::t2ADDri), DestReg: AddDestReg)
11642 .addUse(RegNo: OpSizeReg)
11643 .addImm(Val: 15)
11644 .add(MOs: predOps(Pred: ARMCC::AL))
11645 .addReg(RegNo: 0);
11646
11647 Register LsrDestReg = MRI.createVirtualRegister(RegClass: &ARM::rGPRRegClass);
11648 BuildMI(BB: TpEntry, MIMD: Dl, MCID: TII->get(Opcode: ARM::t2LSRri), DestReg: LsrDestReg)
11649 .addUse(RegNo: AddDestReg)
11650 .addImm(Val: 4)
11651 .add(MOs: predOps(Pred: ARMCC::AL))
11652 .addReg(RegNo: 0);
11653
11654 Register TotalIterationsReg = MRI.createVirtualRegister(RegClass: &ARM::GPRlrRegClass);
11655 BuildMI(BB: TpEntry, MIMD: Dl, MCID: TII->get(Opcode: ARM::t2WhileLoopSetup), DestReg: TotalIterationsReg)
11656 .addUse(RegNo: LsrDestReg);
11657
11658 BuildMI(BB: TpEntry, MIMD: Dl, MCID: TII->get(Opcode: ARM::t2WhileLoopStart))
11659 .addUse(RegNo: TotalIterationsReg)
11660 .addMBB(MBB: TpExit);
11661
11662 BuildMI(BB: TpEntry, MIMD: Dl, MCID: TII->get(Opcode: ARM::t2B))
11663 .addMBB(MBB: TpLoopBody)
11664 .add(MOs: predOps(Pred: ARMCC::AL));
11665
11666 return TotalIterationsReg;
11667}
11668
11669/// Adds logic in the loopBody MBB to generate MVE_VCTP, t2DoLoopDec and
11670/// t2DoLoopEnd. These are used by later passes to generate tail predicated
11671/// loops.
11672static void genTPLoopBody(MachineBasicBlock *TpLoopBody,
11673 MachineBasicBlock *TpEntry, MachineBasicBlock *TpExit,
11674 const TargetInstrInfo *TII, DebugLoc Dl,
11675 MachineRegisterInfo &MRI, Register OpSrcReg,
11676 Register OpDestReg, Register ElementCountReg,
11677 Register TotalIterationsReg, bool IsMemcpy) {
11678 // First insert 4 PHI nodes for: Current pointer to Src (if memcpy), Dest
11679 // array, loop iteration counter, predication counter.
11680
11681 Register SrcPhiReg, CurrSrcReg;
11682 if (IsMemcpy) {
11683 // Current position in the src array
11684 SrcPhiReg = MRI.createVirtualRegister(RegClass: &ARM::rGPRRegClass);
11685 CurrSrcReg = MRI.createVirtualRegister(RegClass: &ARM::rGPRRegClass);
11686 BuildMI(BB: TpLoopBody, MIMD: Dl, MCID: TII->get(Opcode: ARM::PHI), DestReg: SrcPhiReg)
11687 .addUse(RegNo: OpSrcReg)
11688 .addMBB(MBB: TpEntry)
11689 .addUse(RegNo: CurrSrcReg)
11690 .addMBB(MBB: TpLoopBody);
11691 }
11692
11693 // Current position in the dest array
11694 Register DestPhiReg = MRI.createVirtualRegister(RegClass: &ARM::rGPRRegClass);
11695 Register CurrDestReg = MRI.createVirtualRegister(RegClass: &ARM::rGPRRegClass);
11696 BuildMI(BB: TpLoopBody, MIMD: Dl, MCID: TII->get(Opcode: ARM::PHI), DestReg: DestPhiReg)
11697 .addUse(RegNo: OpDestReg)
11698 .addMBB(MBB: TpEntry)
11699 .addUse(RegNo: CurrDestReg)
11700 .addMBB(MBB: TpLoopBody);
11701
11702 // Current loop counter
11703 Register LoopCounterPhiReg = MRI.createVirtualRegister(RegClass: &ARM::GPRlrRegClass);
11704 Register RemainingLoopIterationsReg =
11705 MRI.createVirtualRegister(RegClass: &ARM::GPRlrRegClass);
11706 BuildMI(BB: TpLoopBody, MIMD: Dl, MCID: TII->get(Opcode: ARM::PHI), DestReg: LoopCounterPhiReg)
11707 .addUse(RegNo: TotalIterationsReg)
11708 .addMBB(MBB: TpEntry)
11709 .addUse(RegNo: RemainingLoopIterationsReg)
11710 .addMBB(MBB: TpLoopBody);
11711
11712 // Predication counter
11713 Register PredCounterPhiReg = MRI.createVirtualRegister(RegClass: &ARM::rGPRRegClass);
11714 Register RemainingElementsReg = MRI.createVirtualRegister(RegClass: &ARM::rGPRRegClass);
11715 BuildMI(BB: TpLoopBody, MIMD: Dl, MCID: TII->get(Opcode: ARM::PHI), DestReg: PredCounterPhiReg)
11716 .addUse(RegNo: ElementCountReg)
11717 .addMBB(MBB: TpEntry)
11718 .addUse(RegNo: RemainingElementsReg)
11719 .addMBB(MBB: TpLoopBody);
11720
11721 // Pass predication counter to VCTP
11722 Register VccrReg = MRI.createVirtualRegister(RegClass: &ARM::VCCRRegClass);
11723 BuildMI(BB: TpLoopBody, MIMD: Dl, MCID: TII->get(Opcode: ARM::MVE_VCTP8), DestReg: VccrReg)
11724 .addUse(RegNo: PredCounterPhiReg)
11725 .addImm(Val: ARMVCC::None)
11726 .addReg(RegNo: 0)
11727 .addReg(RegNo: 0);
11728
11729 BuildMI(BB: TpLoopBody, MIMD: Dl, MCID: TII->get(Opcode: ARM::t2SUBri), DestReg: RemainingElementsReg)
11730 .addUse(RegNo: PredCounterPhiReg)
11731 .addImm(Val: 16)
11732 .add(MOs: predOps(Pred: ARMCC::AL))
11733 .addReg(RegNo: 0);
11734
11735 // VLDRB (only if memcpy) and VSTRB instructions, predicated using VPR
11736 Register SrcValueReg;
11737 if (IsMemcpy) {
11738 SrcValueReg = MRI.createVirtualRegister(RegClass: &ARM::MQPRRegClass);
11739 BuildMI(BB: TpLoopBody, MIMD: Dl, MCID: TII->get(Opcode: ARM::MVE_VLDRBU8_post))
11740 .addDef(RegNo: CurrSrcReg)
11741 .addDef(RegNo: SrcValueReg)
11742 .addReg(RegNo: SrcPhiReg)
11743 .addImm(Val: 16)
11744 .addImm(Val: ARMVCC::Then)
11745 .addUse(RegNo: VccrReg)
11746 .addReg(RegNo: 0);
11747 } else
11748 SrcValueReg = OpSrcReg;
11749
11750 BuildMI(BB: TpLoopBody, MIMD: Dl, MCID: TII->get(Opcode: ARM::MVE_VSTRBU8_post))
11751 .addDef(RegNo: CurrDestReg)
11752 .addUse(RegNo: SrcValueReg)
11753 .addReg(RegNo: DestPhiReg)
11754 .addImm(Val: 16)
11755 .addImm(Val: ARMVCC::Then)
11756 .addUse(RegNo: VccrReg)
11757 .addReg(RegNo: 0);
11758
11759 // Add the pseudoInstrs for decrementing the loop counter and marking the
11760 // end:t2DoLoopDec and t2DoLoopEnd
11761 BuildMI(BB: TpLoopBody, MIMD: Dl, MCID: TII->get(Opcode: ARM::t2LoopDec), DestReg: RemainingLoopIterationsReg)
11762 .addUse(RegNo: LoopCounterPhiReg)
11763 .addImm(Val: 1);
11764
11765 BuildMI(BB: TpLoopBody, MIMD: Dl, MCID: TII->get(Opcode: ARM::t2LoopEnd))
11766 .addUse(RegNo: RemainingLoopIterationsReg)
11767 .addMBB(MBB: TpLoopBody);
11768
11769 BuildMI(BB: TpLoopBody, MIMD: Dl, MCID: TII->get(Opcode: ARM::t2B))
11770 .addMBB(MBB: TpExit)
11771 .add(MOs: predOps(Pred: ARMCC::AL));
11772}
11773
11774bool ARMTargetLowering::supportKCFIBundles() const {
11775 // KCFI is supported in all ARM/Thumb modes
11776 return true;
11777}
11778
11779MachineInstr *
11780ARMTargetLowering::EmitKCFICheck(MachineBasicBlock &MBB,
11781 MachineBasicBlock::instr_iterator &MBBI,
11782 const TargetInstrInfo *TII) const {
11783 assert(MBBI->isCall() && MBBI->getCFIType() &&
11784 "Invalid call instruction for a KCFI check");
11785
11786 MachineOperand *TargetOp = nullptr;
11787 switch (MBBI->getOpcode()) {
11788 // ARM mode opcodes
11789 case ARM::BLX:
11790 case ARM::BLX_pred:
11791 case ARM::BLX_noip:
11792 case ARM::BLX_pred_noip:
11793 case ARM::BX_CALL:
11794 TargetOp = &MBBI->getOperand(i: 0);
11795 break;
11796 case ARM::TCRETURNri:
11797 case ARM::TCRETURNrinotr12:
11798 case ARM::TAILJMPr:
11799 case ARM::TAILJMPr4:
11800 TargetOp = &MBBI->getOperand(i: 0);
11801 break;
11802 // Thumb mode opcodes (Thumb1 and Thumb2)
11803 // Note: Most Thumb call instructions have predicate operands before the
11804 // target register Format: tBLXr pred, predreg, target_register, ...
11805 case ARM::tBLXr: // Thumb1/Thumb2: BLX register (requires V5T)
11806 case ARM::tBLXr_noip: // Thumb1/Thumb2: BLX register, no IP clobber
11807 case ARM::tBX_CALL: // Thumb1 only: BX call (push LR, BX)
11808 TargetOp = &MBBI->getOperand(i: 2);
11809 break;
11810 // Tail call instructions don't have predicates, target is operand 0
11811 case ARM::tTAILJMPr: // Thumb1/Thumb2: Tail call via register
11812 TargetOp = &MBBI->getOperand(i: 0);
11813 break;
11814 default:
11815 llvm_unreachable("Unexpected CFI call opcode");
11816 }
11817
11818 assert(TargetOp && TargetOp->isReg() && "Invalid target operand");
11819 TargetOp->setIsRenamable(false);
11820
11821 // Select the appropriate KCFI_CHECK variant based on the instruction set
11822 unsigned KCFICheckOpcode;
11823 if (Subtarget->isThumb()) {
11824 if (Subtarget->isThumb2()) {
11825 KCFICheckOpcode = ARM::KCFI_CHECK_Thumb2;
11826 } else {
11827 KCFICheckOpcode = ARM::KCFI_CHECK_Thumb1;
11828 }
11829 } else {
11830 KCFICheckOpcode = ARM::KCFI_CHECK_ARM;
11831 }
11832
11833 return BuildMI(BB&: MBB, I: MBBI, MIMD: MBBI->getDebugLoc(), MCID: TII->get(Opcode: KCFICheckOpcode))
11834 .addReg(RegNo: TargetOp->getReg())
11835 .addImm(Val: MBBI->getCFIType())
11836 .getInstr();
11837}
11838
11839MachineBasicBlock *
11840ARMTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
11841 MachineBasicBlock *BB) const {
11842 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
11843 DebugLoc dl = MI.getDebugLoc();
11844 bool isThumb2 = Subtarget->isThumb2();
11845 switch (MI.getOpcode()) {
11846 default: {
11847 MI.print(OS&: errs());
11848 llvm_unreachable("Unexpected instr type to insert");
11849 }
11850
11851 // Thumb1 post-indexed loads are really just single-register LDMs.
11852 case ARM::tLDR_postidx: {
11853 MachineOperand Def(MI.getOperand(i: 1));
11854 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: ARM::tLDMIA_UPD))
11855 .add(MO: Def) // Rn_wb
11856 .add(MO: MI.getOperand(i: 2)) // Rn
11857 .add(MO: MI.getOperand(i: 3)) // PredImm
11858 .add(MO: MI.getOperand(i: 4)) // PredReg
11859 .add(MO: MI.getOperand(i: 0)) // Rt
11860 .cloneMemRefs(OtherMI: MI);
11861 MI.eraseFromParent();
11862 return BB;
11863 }
11864
11865 case ARM::MVE_MEMCPYLOOPINST:
11866 case ARM::MVE_MEMSETLOOPINST: {
11867
11868 // Transformation below expands MVE_MEMCPYLOOPINST/MVE_MEMSETLOOPINST Pseudo
11869 // into a Tail Predicated (TP) Loop. It adds the instructions to calculate
11870 // the iteration count =ceil(size_in_bytes/16)) in the TP entry block and
11871 // adds the relevant instructions in the TP loop Body for generation of a
11872 // WLSTP loop.
11873
11874 // Below is relevant portion of the CFG after the transformation.
11875 // The Machine Basic Blocks are shown along with branch conditions (in
11876 // brackets). Note that TP entry/exit MBBs depict the entry/exit of this
11877 // portion of the CFG and may not necessarily be the entry/exit of the
11878 // function.
11879
11880 // (Relevant) CFG after transformation:
11881 // TP entry MBB
11882 // |
11883 // |-----------------|
11884 // (n <= 0) (n > 0)
11885 // | |
11886 // | TP loop Body MBB<--|
11887 // | | |
11888 // \ |___________|
11889 // \ /
11890 // TP exit MBB
11891
11892 MachineFunction *MF = BB->getParent();
11893 MachineFunctionProperties &Properties = MF->getProperties();
11894 MachineRegisterInfo &MRI = MF->getRegInfo();
11895
11896 Register OpDestReg = MI.getOperand(i: 0).getReg();
11897 Register OpSrcReg = MI.getOperand(i: 1).getReg();
11898 Register OpSizeReg = MI.getOperand(i: 2).getReg();
11899
11900 // Allocate the required MBBs and add to parent function.
11901 MachineBasicBlock *TpEntry = BB;
11902 MachineBasicBlock *TpLoopBody = MF->CreateMachineBasicBlock();
11903 MachineBasicBlock *TpExit;
11904
11905 MF->push_back(MBB: TpLoopBody);
11906
11907 // If any instructions are present in the current block after
11908 // MVE_MEMCPYLOOPINST or MVE_MEMSETLOOPINST, split the current block and
11909 // move the instructions into the newly created exit block. If there are no
11910 // instructions add an explicit branch to the FallThrough block and then
11911 // split.
11912 //
11913 // The split is required for two reasons:
11914 // 1) A terminator(t2WhileLoopStart) will be placed at that site.
11915 // 2) Since a TPLoopBody will be added later, any phis in successive blocks
11916 // need to be updated. splitAt() already handles this.
11917 TpExit = BB->splitAt(SplitInst&: MI, UpdateLiveIns: false);
11918 if (TpExit == BB) {
11919 assert(BB->canFallThrough() && "Exit Block must be Fallthrough of the "
11920 "block containing memcpy/memset Pseudo");
11921 TpExit = BB->getFallThrough();
11922 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: ARM::t2B))
11923 .addMBB(MBB: TpExit)
11924 .add(MOs: predOps(Pred: ARMCC::AL));
11925 TpExit = BB->splitAt(SplitInst&: MI, UpdateLiveIns: false);
11926 }
11927
11928 // Add logic for iteration count
11929 Register TotalIterationsReg =
11930 genTPEntry(TpEntry, TpLoopBody, TpExit, OpSizeReg, TII, Dl: dl, MRI);
11931
11932 // Add the vectorized (and predicated) loads/store instructions
11933 bool IsMemcpy = MI.getOpcode() == ARM::MVE_MEMCPYLOOPINST;
11934 genTPLoopBody(TpLoopBody, TpEntry, TpExit, TII, Dl: dl, MRI, OpSrcReg,
11935 OpDestReg, ElementCountReg: OpSizeReg, TotalIterationsReg, IsMemcpy);
11936
11937 // Required to avoid conflict with the MachineVerifier during testing.
11938 Properties.resetNoPHIs();
11939
11940 // Connect the blocks
11941 TpEntry->addSuccessor(Succ: TpLoopBody);
11942 TpLoopBody->addSuccessor(Succ: TpLoopBody);
11943 TpLoopBody->addSuccessor(Succ: TpExit);
11944
11945 // Reorder for a more natural layout
11946 TpLoopBody->moveAfter(NewBefore: TpEntry);
11947 TpExit->moveAfter(NewBefore: TpLoopBody);
11948
11949 // Finally, remove the memcpy Pseudo Instruction
11950 MI.eraseFromParent();
11951
11952 // Return the exit block as it may contain other instructions requiring a
11953 // custom inserter
11954 return TpExit;
11955 }
11956
11957 // The Thumb2 pre-indexed stores have the same MI operands, they just
11958 // define them differently in the .td files from the isel patterns, so
11959 // they need pseudos.
11960 case ARM::t2STR_preidx:
11961 MI.setDesc(TII->get(Opcode: ARM::t2STR_PRE));
11962 return BB;
11963 case ARM::t2STRB_preidx:
11964 MI.setDesc(TII->get(Opcode: ARM::t2STRB_PRE));
11965 return BB;
11966 case ARM::t2STRH_preidx:
11967 MI.setDesc(TII->get(Opcode: ARM::t2STRH_PRE));
11968 return BB;
11969
11970 case ARM::STRi_preidx:
11971 case ARM::STRBi_preidx: {
11972 unsigned NewOpc = MI.getOpcode() == ARM::STRi_preidx ? ARM::STR_PRE_IMM
11973 : ARM::STRB_PRE_IMM;
11974 // Decode the offset.
11975 unsigned Offset = MI.getOperand(i: 4).getImm();
11976 bool isSub = ARM_AM::getAM2Op(AM2Opc: Offset) == ARM_AM::sub;
11977 Offset = ARM_AM::getAM2Offset(AM2Opc: Offset);
11978 if (isSub)
11979 Offset = -Offset;
11980
11981 MachineMemOperand *MMO = *MI.memoperands_begin();
11982 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: NewOpc))
11983 .add(MO: MI.getOperand(i: 0)) // Rn_wb
11984 .add(MO: MI.getOperand(i: 1)) // Rt
11985 .add(MO: MI.getOperand(i: 2)) // Rn
11986 .addImm(Val: Offset) // offset (skip GPR==zero_reg)
11987 .add(MO: MI.getOperand(i: 5)) // pred
11988 .add(MO: MI.getOperand(i: 6))
11989 .addMemOperand(MMO);
11990 MI.eraseFromParent();
11991 return BB;
11992 }
11993 case ARM::STRr_preidx:
11994 case ARM::STRBr_preidx:
11995 case ARM::STRH_preidx: {
11996 unsigned NewOpc;
11997 switch (MI.getOpcode()) {
11998 default: llvm_unreachable("unexpected opcode!");
11999 case ARM::STRr_preidx: NewOpc = ARM::STR_PRE_REG; break;
12000 case ARM::STRBr_preidx: NewOpc = ARM::STRB_PRE_REG; break;
12001 case ARM::STRH_preidx: NewOpc = ARM::STRH_PRE; break;
12002 }
12003 MachineInstrBuilder MIB = BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: NewOpc));
12004 for (const MachineOperand &MO : MI.operands())
12005 MIB.add(MO);
12006 MI.eraseFromParent();
12007 return BB;
12008 }
12009
12010 case ARM::tMOVCCr_pseudo: {
12011 // To "insert" a SELECT_CC instruction, we actually have to insert the
12012 // diamond control-flow pattern. The incoming instruction knows the
12013 // destination vreg to set, the condition code register to branch on, the
12014 // true/false values to select between, and a branch opcode to use.
12015 const BasicBlock *LLVM_BB = BB->getBasicBlock();
12016 MachineFunction::iterator It = ++BB->getIterator();
12017
12018 // thisMBB:
12019 // ...
12020 // TrueVal = ...
12021 // cmpTY ccX, r1, r2
12022 // bCC copy1MBB
12023 // fallthrough --> copy0MBB
12024 MachineBasicBlock *thisMBB = BB;
12025 MachineFunction *F = BB->getParent();
12026 MachineBasicBlock *copy0MBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
12027 MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
12028 F->insert(MBBI: It, MBB: copy0MBB);
12029 F->insert(MBBI: It, MBB: sinkMBB);
12030
12031 // Set the call frame size on entry to the new basic blocks.
12032 unsigned CallFrameSize = TII->getCallFrameSizeAt(MI);
12033 copy0MBB->setCallFrameSize(CallFrameSize);
12034 sinkMBB->setCallFrameSize(CallFrameSize);
12035
12036 // Check whether CPSR is live past the tMOVCCr_pseudo.
12037 const TargetRegisterInfo *TRI = Subtarget->getRegisterInfo();
12038 if (!MI.killsRegister(Reg: ARM::CPSR, /*TRI=*/nullptr) &&
12039 !checkAndUpdateCPSRKill(SelectItr: MI, BB: thisMBB, TRI)) {
12040 copy0MBB->addLiveIn(PhysReg: ARM::CPSR);
12041 sinkMBB->addLiveIn(PhysReg: ARM::CPSR);
12042 }
12043
12044 // Transfer the remainder of BB and its successor edges to sinkMBB.
12045 sinkMBB->splice(Where: sinkMBB->begin(), Other: BB,
12046 From: std::next(x: MachineBasicBlock::iterator(MI)), To: BB->end());
12047 sinkMBB->transferSuccessorsAndUpdatePHIs(FromMBB: BB);
12048
12049 BB->addSuccessor(Succ: copy0MBB);
12050 BB->addSuccessor(Succ: sinkMBB);
12051
12052 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: ARM::tBcc))
12053 .addMBB(MBB: sinkMBB)
12054 .addImm(Val: MI.getOperand(i: 3).getImm())
12055 .addReg(RegNo: MI.getOperand(i: 4).getReg());
12056
12057 // copy0MBB:
12058 // %FalseValue = ...
12059 // # fallthrough to sinkMBB
12060 BB = copy0MBB;
12061
12062 // Update machine-CFG edges
12063 BB->addSuccessor(Succ: sinkMBB);
12064
12065 // sinkMBB:
12066 // %Result = phi [ %FalseValue, copy0MBB ], [ %TrueValue, thisMBB ]
12067 // ...
12068 BB = sinkMBB;
12069 BuildMI(BB&: *BB, I: BB->begin(), MIMD: dl, MCID: TII->get(Opcode: ARM::PHI), DestReg: MI.getOperand(i: 0).getReg())
12070 .addReg(RegNo: MI.getOperand(i: 1).getReg())
12071 .addMBB(MBB: copy0MBB)
12072 .addReg(RegNo: MI.getOperand(i: 2).getReg())
12073 .addMBB(MBB: thisMBB);
12074
12075 MI.eraseFromParent(); // The pseudo instruction is gone now.
12076 return BB;
12077 }
12078
12079 case ARM::BCCi64:
12080 case ARM::BCCZi64: {
12081 // If there is an unconditional branch to the other successor, remove it.
12082 BB->erase(I: std::next(x: MachineBasicBlock::iterator(MI)), E: BB->end());
12083
12084 // Compare both parts that make up the double comparison separately for
12085 // equality.
12086 bool RHSisZero = MI.getOpcode() == ARM::BCCZi64;
12087
12088 Register LHS1 = MI.getOperand(i: 1).getReg();
12089 Register LHS2 = MI.getOperand(i: 2).getReg();
12090 if (RHSisZero) {
12091 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: isThumb2 ? ARM::t2CMPri : ARM::CMPri))
12092 .addReg(RegNo: LHS1)
12093 .addImm(Val: 0)
12094 .add(MOs: predOps(Pred: ARMCC::AL));
12095 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: isThumb2 ? ARM::t2CMPri : ARM::CMPri))
12096 .addReg(RegNo: LHS2).addImm(Val: 0)
12097 .addImm(Val: ARMCC::EQ).addReg(RegNo: ARM::CPSR);
12098 } else {
12099 Register RHS1 = MI.getOperand(i: 3).getReg();
12100 Register RHS2 = MI.getOperand(i: 4).getReg();
12101 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: isThumb2 ? ARM::t2CMPrr : ARM::CMPrr))
12102 .addReg(RegNo: LHS1)
12103 .addReg(RegNo: RHS1)
12104 .add(MOs: predOps(Pred: ARMCC::AL));
12105 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: isThumb2 ? ARM::t2CMPrr : ARM::CMPrr))
12106 .addReg(RegNo: LHS2).addReg(RegNo: RHS2)
12107 .addImm(Val: ARMCC::EQ).addReg(RegNo: ARM::CPSR);
12108 }
12109
12110 MachineBasicBlock *destMBB = MI.getOperand(i: RHSisZero ? 3 : 5).getMBB();
12111 MachineBasicBlock *exitMBB = OtherSucc(MBB: BB, Succ: destMBB);
12112 if (MI.getOperand(i: 0).getImm() == ARMCC::NE)
12113 std::swap(a&: destMBB, b&: exitMBB);
12114
12115 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: isThumb2 ? ARM::t2Bcc : ARM::Bcc))
12116 .addMBB(MBB: destMBB).addImm(Val: ARMCC::EQ).addReg(RegNo: ARM::CPSR);
12117 if (isThumb2)
12118 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: ARM::t2B))
12119 .addMBB(MBB: exitMBB)
12120 .add(MOs: predOps(Pred: ARMCC::AL));
12121 else
12122 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: ARM::B)) .addMBB(MBB: exitMBB);
12123
12124 MI.eraseFromParent(); // The pseudo instruction is gone now.
12125 return BB;
12126 }
12127
12128 case ARM::Int_eh_sjlj_setjmp:
12129 case ARM::Int_eh_sjlj_setjmp_nofp:
12130 case ARM::tInt_eh_sjlj_setjmp:
12131 case ARM::t2Int_eh_sjlj_setjmp:
12132 case ARM::t2Int_eh_sjlj_setjmp_nofp:
12133 return BB;
12134
12135 case ARM::Int_eh_sjlj_setup_dispatch:
12136 EmitSjLjDispatchBlock(MI, MBB: BB);
12137 return BB;
12138 case ARM::COPY_STRUCT_BYVAL_I32:
12139 ++NumLoopByVals;
12140 return EmitStructByval(MI, BB);
12141 case ARM::WIN__CHKSTK:
12142 return EmitLowered__chkstk(MI, MBB: BB);
12143 case ARM::WIN__DBZCHK:
12144 return EmitLowered__dbzchk(MI, MBB: BB);
12145 }
12146}
12147
12148/// Attaches vregs to MEMCPY that it will use as scratch registers
12149/// when it is expanded into LDM/STM. This is done as a post-isel lowering
12150/// instead of as a custom inserter because we need the use list from the SDNode.
12151static void attachMEMCPYScratchRegs(const ARMSubtarget *Subtarget,
12152 MachineInstr &MI, const SDNode *Node) {
12153 bool isThumb1 = Subtarget->isThumb1Only();
12154
12155 MachineFunction *MF = MI.getParent()->getParent();
12156 MachineRegisterInfo &MRI = MF->getRegInfo();
12157 MachineInstrBuilder MIB(*MF, MI);
12158
12159 // If the new dst/src is unused mark it as dead.
12160 if (!Node->hasAnyUseOfValue(Value: 0)) {
12161 MI.getOperand(i: 0).setIsDead(true);
12162 }
12163 if (!Node->hasAnyUseOfValue(Value: 1)) {
12164 MI.getOperand(i: 1).setIsDead(true);
12165 }
12166
12167 // The MEMCPY both defines and kills the scratch registers.
12168 for (unsigned I = 0; I != MI.getOperand(i: 4).getImm(); ++I) {
12169 Register TmpReg = MRI.createVirtualRegister(RegClass: isThumb1 ? &ARM::tGPRRegClass
12170 : &ARM::GPRRegClass);
12171 MIB.addReg(RegNo: TmpReg, Flags: RegState::Define|RegState::Dead);
12172 }
12173}
12174
12175void ARMTargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI,
12176 SDNode *Node) const {
12177 if (MI.getOpcode() == ARM::MEMCPY) {
12178 attachMEMCPYScratchRegs(Subtarget, MI, Node);
12179 return;
12180 }
12181
12182 const MCInstrDesc *MCID = &MI.getDesc();
12183 // Adjust potentially 's' setting instructions after isel, i.e. ADC, SBC, RSB,
12184 // RSC. Coming out of isel, they have an implicit CPSR def, but the optional
12185 // operand is still set to noreg. If needed, set the optional operand's
12186 // register to CPSR, and remove the redundant implicit def.
12187 //
12188 // e.g. ADCS (..., implicit-def CPSR) -> ADC (... opt:def CPSR).
12189
12190 // Rename pseudo opcodes.
12191 unsigned NewOpc = convertAddSubFlagsOpcode(OldOpc: MI.getOpcode());
12192 unsigned ccOutIdx;
12193 if (NewOpc) {
12194 const ARMBaseInstrInfo *TII = Subtarget->getInstrInfo();
12195 MCID = &TII->get(Opcode: NewOpc);
12196
12197 assert(MCID->getNumOperands() ==
12198 MI.getDesc().getNumOperands() + 5 - MI.getDesc().getSize()
12199 && "converted opcode should be the same except for cc_out"
12200 " (and, on Thumb1, pred)");
12201
12202 MI.setDesc(*MCID);
12203
12204 // Add the optional cc_out operand
12205 MI.addOperand(Op: MachineOperand::CreateReg(Reg: 0, /*isDef=*/true));
12206
12207 // On Thumb1, move all input operands to the end, then add the predicate
12208 if (Subtarget->isThumb1Only()) {
12209 for (unsigned c = MCID->getNumOperands() - 4; c--;) {
12210 MI.addOperand(Op: MI.getOperand(i: 1));
12211 MI.removeOperand(OpNo: 1);
12212 }
12213
12214 // Restore the ties
12215 for (unsigned i = MI.getNumOperands(); i--;) {
12216 const MachineOperand& op = MI.getOperand(i);
12217 if (op.isReg() && op.isUse()) {
12218 int DefIdx = MCID->getOperandConstraint(OpNum: i, Constraint: MCOI::TIED_TO);
12219 if (DefIdx != -1)
12220 MI.tieOperands(DefIdx, UseIdx: i);
12221 }
12222 }
12223
12224 MI.addOperand(Op: MachineOperand::CreateImm(Val: ARMCC::AL));
12225 MI.addOperand(Op: MachineOperand::CreateReg(Reg: 0, /*isDef=*/false));
12226 ccOutIdx = 1;
12227 } else
12228 ccOutIdx = MCID->getNumOperands() - 1;
12229 } else
12230 ccOutIdx = MCID->getNumOperands() - 1;
12231
12232 // Any ARM instruction that sets the 's' bit should specify an optional
12233 // "cc_out" operand in the last operand position.
12234 if (!MI.hasOptionalDef() || !MCID->operands()[ccOutIdx].isOptionalDef()) {
12235 assert(!NewOpc && "Optional cc_out operand required");
12236 return;
12237 }
12238 // Look for an implicit def of CPSR added by MachineInstr ctor. Remove it
12239 // since we already have an optional CPSR def.
12240 bool definesCPSR = false;
12241 bool deadCPSR = false;
12242 for (unsigned i = MCID->getNumOperands(), e = MI.getNumOperands(); i != e;
12243 ++i) {
12244 const MachineOperand &MO = MI.getOperand(i);
12245 if (MO.isReg() && MO.isDef() && MO.getReg() == ARM::CPSR) {
12246 definesCPSR = true;
12247 if (MO.isDead())
12248 deadCPSR = true;
12249 MI.removeOperand(OpNo: i);
12250 break;
12251 }
12252 }
12253 if (!definesCPSR) {
12254 assert(!NewOpc && "Optional cc_out operand required");
12255 return;
12256 }
12257 assert(deadCPSR == !Node->hasAnyUseOfValue(1) && "inconsistent dead flag");
12258 if (deadCPSR) {
12259 assert(!MI.getOperand(ccOutIdx).getReg() &&
12260 "expect uninitialized optional cc_out operand");
12261 // Thumb1 instructions must have the S bit even if the CPSR is dead.
12262 if (!Subtarget->isThumb1Only())
12263 return;
12264 }
12265
12266 // If this instruction was defined with an optional CPSR def and its dag node
12267 // had a live implicit CPSR def, then activate the optional CPSR def.
12268 MachineOperand &MO = MI.getOperand(i: ccOutIdx);
12269 MO.setReg(ARM::CPSR);
12270 MO.setIsDef(true);
12271 MO.setIsDead(deadCPSR);
12272}
12273
12274//===----------------------------------------------------------------------===//
12275// ARM Optimization Hooks
12276//===----------------------------------------------------------------------===//
12277
12278// Helper function that checks if N is a null or all ones constant.
12279static inline bool isZeroOrAllOnes(SDValue N, bool AllOnes) {
12280 return AllOnes ? isAllOnesConstant(V: N) : isNullConstant(V: N);
12281}
12282
12283// Return true if N is conditionally 0 or all ones.
12284// Detects these expressions where cc is an i1 value:
12285//
12286// (select cc 0, y) [AllOnes=0]
12287// (select cc y, 0) [AllOnes=0]
12288// (zext cc) [AllOnes=0]
12289// (sext cc) [AllOnes=0/1]
12290// (select cc -1, y) [AllOnes=1]
12291// (select cc y, -1) [AllOnes=1]
12292//
12293// Invert is set when N is the null/all ones constant when CC is false.
12294// OtherOp is set to the alternative value of N.
12295static bool isConditionalZeroOrAllOnes(SDNode *N, bool AllOnes,
12296 SDValue &CC, bool &Invert,
12297 SDValue &OtherOp,
12298 SelectionDAG &DAG) {
12299 switch (N->getOpcode()) {
12300 default: return false;
12301 case ISD::SELECT: {
12302 CC = N->getOperand(Num: 0);
12303 SDValue N1 = N->getOperand(Num: 1);
12304 SDValue N2 = N->getOperand(Num: 2);
12305 if (isZeroOrAllOnes(N: N1, AllOnes)) {
12306 Invert = false;
12307 OtherOp = N2;
12308 return true;
12309 }
12310 if (isZeroOrAllOnes(N: N2, AllOnes)) {
12311 Invert = true;
12312 OtherOp = N1;
12313 return true;
12314 }
12315 return false;
12316 }
12317 case ISD::ZERO_EXTEND:
12318 // (zext cc) can never be the all ones value.
12319 if (AllOnes)
12320 return false;
12321 [[fallthrough]];
12322 case ISD::SIGN_EXTEND: {
12323 SDLoc dl(N);
12324 EVT VT = N->getValueType(ResNo: 0);
12325 CC = N->getOperand(Num: 0);
12326 if (CC.getValueType() != MVT::i1 || CC.getOpcode() != ISD::SETCC)
12327 return false;
12328 Invert = !AllOnes;
12329 if (AllOnes)
12330 // When looking for an AllOnes constant, N is an sext, and the 'other'
12331 // value is 0.
12332 OtherOp = DAG.getConstant(Val: 0, DL: dl, VT);
12333 else if (N->getOpcode() == ISD::ZERO_EXTEND)
12334 // When looking for a 0 constant, N can be zext or sext.
12335 OtherOp = DAG.getConstant(Val: 1, DL: dl, VT);
12336 else
12337 OtherOp = DAG.getAllOnesConstant(DL: dl, VT);
12338 return true;
12339 }
12340 }
12341}
12342
12343// Combine a constant select operand into its use:
12344//
12345// (add (select cc, 0, c), x) -> (select cc, x, (add, x, c))
12346// (sub x, (select cc, 0, c)) -> (select cc, x, (sub, x, c))
12347// (and (select cc, -1, c), x) -> (select cc, x, (and, x, c)) [AllOnes=1]
12348// (or (select cc, 0, c), x) -> (select cc, x, (or, x, c))
12349// (xor (select cc, 0, c), x) -> (select cc, x, (xor, x, c))
12350//
12351// The transform is rejected if the select doesn't have a constant operand that
12352// is null, or all ones when AllOnes is set.
12353//
12354// Also recognize sext/zext from i1:
12355//
12356// (add (zext cc), x) -> (select cc (add x, 1), x)
12357// (add (sext cc), x) -> (select cc (add x, -1), x)
12358//
12359// These transformations eventually create predicated instructions.
12360//
12361// @param N The node to transform.
12362// @param Slct The N operand that is a select.
12363// @param OtherOp The other N operand (x above).
12364// @param DCI Context.
12365// @param AllOnes Require the select constant to be all ones instead of null.
12366// @returns The new node, or SDValue() on failure.
12367static
12368SDValue combineSelectAndUse(SDNode *N, SDValue Slct, SDValue OtherOp,
12369 TargetLowering::DAGCombinerInfo &DCI,
12370 bool AllOnes = false) {
12371 SelectionDAG &DAG = DCI.DAG;
12372 EVT VT = N->getValueType(ResNo: 0);
12373 SDValue NonConstantVal;
12374 SDValue CCOp;
12375 bool SwapSelectOps;
12376 if (!isConditionalZeroOrAllOnes(N: Slct.getNode(), AllOnes, CC&: CCOp, Invert&: SwapSelectOps,
12377 OtherOp&: NonConstantVal, DAG))
12378 return SDValue();
12379
12380 // Slct is now know to be the desired identity constant when CC is true.
12381 SDValue TrueVal = OtherOp;
12382 SDValue FalseVal = DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT,
12383 N1: OtherOp, N2: NonConstantVal);
12384 // Unless SwapSelectOps says CC should be false.
12385 if (SwapSelectOps)
12386 std::swap(a&: TrueVal, b&: FalseVal);
12387
12388 return DAG.getNode(Opcode: ISD::SELECT, DL: SDLoc(N), VT,
12389 N1: CCOp, N2: TrueVal, N3: FalseVal);
12390}
12391
12392// Attempt combineSelectAndUse on each operand of a commutative operator N.
12393static
12394SDValue combineSelectAndUseCommutative(SDNode *N, bool AllOnes,
12395 TargetLowering::DAGCombinerInfo &DCI) {
12396 SDValue N0 = N->getOperand(Num: 0);
12397 SDValue N1 = N->getOperand(Num: 1);
12398 if (N0.getNode()->hasOneUse())
12399 if (SDValue Result = combineSelectAndUse(N, Slct: N0, OtherOp: N1, DCI, AllOnes))
12400 return Result;
12401 if (N1.getNode()->hasOneUse())
12402 if (SDValue Result = combineSelectAndUse(N, Slct: N1, OtherOp: N0, DCI, AllOnes))
12403 return Result;
12404 return SDValue();
12405}
12406
12407static bool IsVUZPShuffleNode(SDNode *N) {
12408 // VUZP shuffle node.
12409 if (N->getOpcode() == ARMISD::VUZP)
12410 return true;
12411
12412 // "VUZP" on i32 is an alias for VTRN.
12413 if (N->getOpcode() == ARMISD::VTRN && N->getValueType(ResNo: 0) == MVT::v2i32)
12414 return true;
12415
12416 return false;
12417}
12418
12419static SDValue AddCombineToVPADD(SDNode *N, SDValue N0, SDValue N1,
12420 TargetLowering::DAGCombinerInfo &DCI,
12421 const ARMSubtarget *Subtarget) {
12422 // Look for ADD(VUZP.0, VUZP.1).
12423 if (!IsVUZPShuffleNode(N: N0.getNode()) || N0.getNode() != N1.getNode() ||
12424 N0 == N1)
12425 return SDValue();
12426
12427 // Make sure the ADD is a 64-bit add; there is no 128-bit VPADD.
12428 if (!N->getValueType(ResNo: 0).is64BitVector())
12429 return SDValue();
12430
12431 // Generate vpadd.
12432 SelectionDAG &DAG = DCI.DAG;
12433 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
12434 SDLoc dl(N);
12435 SDNode *Unzip = N0.getNode();
12436 EVT VT = N->getValueType(ResNo: 0);
12437
12438 SmallVector<SDValue, 8> Ops;
12439 Ops.push_back(Elt: DAG.getConstant(Val: Intrinsic::arm_neon_vpadd, DL: dl,
12440 VT: TLI.getPointerTy(DL: DAG.getDataLayout())));
12441 Ops.push_back(Elt: Unzip->getOperand(Num: 0));
12442 Ops.push_back(Elt: Unzip->getOperand(Num: 1));
12443
12444 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT, Ops);
12445}
12446
12447static SDValue AddCombineVUZPToVPADDL(SDNode *N, SDValue N0, SDValue N1,
12448 TargetLowering::DAGCombinerInfo &DCI,
12449 const ARMSubtarget *Subtarget) {
12450 // Check for two extended operands.
12451 if (!(N0.getOpcode() == ISD::SIGN_EXTEND &&
12452 N1.getOpcode() == ISD::SIGN_EXTEND) &&
12453 !(N0.getOpcode() == ISD::ZERO_EXTEND &&
12454 N1.getOpcode() == ISD::ZERO_EXTEND))
12455 return SDValue();
12456
12457 SDValue N00 = N0.getOperand(i: 0);
12458 SDValue N10 = N1.getOperand(i: 0);
12459
12460 // Look for ADD(SEXT(VUZP.0), SEXT(VUZP.1))
12461 if (!IsVUZPShuffleNode(N: N00.getNode()) || N00.getNode() != N10.getNode() ||
12462 N00 == N10)
12463 return SDValue();
12464
12465 // We only recognize Q register paddl here; this can't be reached until
12466 // after type legalization.
12467 if (!N00.getValueType().is64BitVector() ||
12468 !N0.getValueType().is128BitVector())
12469 return SDValue();
12470
12471 // Generate vpaddl.
12472 SelectionDAG &DAG = DCI.DAG;
12473 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
12474 SDLoc dl(N);
12475 EVT VT = N->getValueType(ResNo: 0);
12476
12477 SmallVector<SDValue, 8> Ops;
12478 // Form vpaddl.sN or vpaddl.uN depending on the kind of extension.
12479 unsigned Opcode;
12480 if (N0.getOpcode() == ISD::SIGN_EXTEND)
12481 Opcode = Intrinsic::arm_neon_vpaddls;
12482 else
12483 Opcode = Intrinsic::arm_neon_vpaddlu;
12484 Ops.push_back(Elt: DAG.getConstant(Val: Opcode, DL: dl,
12485 VT: TLI.getPointerTy(DL: DAG.getDataLayout())));
12486 EVT ElemTy = N00.getValueType().getVectorElementType();
12487 unsigned NumElts = VT.getVectorNumElements();
12488 EVT ConcatVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ElemTy, NumElements: NumElts * 2);
12489 SDValue Concat = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(N), VT: ConcatVT,
12490 N1: N00.getOperand(i: 0), N2: N00.getOperand(i: 1));
12491 Ops.push_back(Elt: Concat);
12492
12493 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT, Ops);
12494}
12495
12496// FIXME: This function shouldn't be necessary; if we lower BUILD_VECTOR in
12497// an appropriate manner, we end up with ADD(VUZP(ZEXT(N))), which is
12498// much easier to match.
12499static SDValue
12500AddCombineBUILD_VECTORToVPADDL(SDNode *N, SDValue N0, SDValue N1,
12501 TargetLowering::DAGCombinerInfo &DCI,
12502 const ARMSubtarget *Subtarget) {
12503 // Only perform optimization if after legalize, and if NEON is available. We
12504 // also expected both operands to be BUILD_VECTORs.
12505 if (DCI.isBeforeLegalize() || !Subtarget->hasNEON()
12506 || N0.getOpcode() != ISD::BUILD_VECTOR
12507 || N1.getOpcode() != ISD::BUILD_VECTOR)
12508 return SDValue();
12509
12510 // Check output type since VPADDL operand elements can only be 8, 16, or 32.
12511 EVT VT = N->getValueType(ResNo: 0);
12512 if (!VT.isInteger() || VT.getVectorElementType() == MVT::i64)
12513 return SDValue();
12514
12515 // Check that the vector operands are of the right form.
12516 // N0 and N1 are BUILD_VECTOR nodes with N number of EXTRACT_VECTOR
12517 // operands, where N is the size of the formed vector.
12518 // Each EXTRACT_VECTOR should have the same input vector and odd or even
12519 // index such that we have a pair wise add pattern.
12520
12521 // Grab the vector that all EXTRACT_VECTOR nodes should be referencing.
12522 if (N0->getOperand(Num: 0)->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
12523 return SDValue();
12524 SDValue Vec = N0->getOperand(Num: 0)->getOperand(Num: 0);
12525 SDNode *V = Vec.getNode();
12526 unsigned nextIndex = 0;
12527
12528 // For each operands to the ADD which are BUILD_VECTORs,
12529 // check to see if each of their operands are an EXTRACT_VECTOR with
12530 // the same vector and appropriate index.
12531 for (unsigned i = 0, e = N0->getNumOperands(); i != e; ++i) {
12532 if (N0->getOperand(Num: i)->getOpcode() == ISD::EXTRACT_VECTOR_ELT
12533 && N1->getOperand(Num: i)->getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
12534
12535 SDValue ExtVec0 = N0->getOperand(Num: i);
12536 SDValue ExtVec1 = N1->getOperand(Num: i);
12537
12538 // First operand is the vector, verify its the same.
12539 if (V != ExtVec0->getOperand(Num: 0).getNode() ||
12540 V != ExtVec1->getOperand(Num: 0).getNode())
12541 return SDValue();
12542
12543 // Second is the constant, verify its correct.
12544 ConstantSDNode *C0 = dyn_cast<ConstantSDNode>(Val: ExtVec0->getOperand(Num: 1));
12545 ConstantSDNode *C1 = dyn_cast<ConstantSDNode>(Val: ExtVec1->getOperand(Num: 1));
12546
12547 // For the constant, we want to see all the even or all the odd.
12548 if (!C0 || !C1 || C0->getZExtValue() != nextIndex
12549 || C1->getZExtValue() != nextIndex+1)
12550 return SDValue();
12551
12552 // Increment index.
12553 nextIndex+=2;
12554 } else
12555 return SDValue();
12556 }
12557
12558 // Don't generate vpaddl+vmovn; we'll match it to vpadd later. Also make sure
12559 // we're using the entire input vector, otherwise there's a size/legality
12560 // mismatch somewhere.
12561 if (nextIndex != Vec.getValueType().getVectorNumElements() ||
12562 Vec.getValueType().getVectorElementType() == VT.getVectorElementType())
12563 return SDValue();
12564
12565 // Create VPADDL node.
12566 SelectionDAG &DAG = DCI.DAG;
12567 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
12568
12569 SDLoc dl(N);
12570
12571 // Build operand list.
12572 SmallVector<SDValue, 8> Ops;
12573 Ops.push_back(Elt: DAG.getConstant(Val: Intrinsic::arm_neon_vpaddls, DL: dl,
12574 VT: TLI.getPointerTy(DL: DAG.getDataLayout())));
12575
12576 // Input is the vector.
12577 Ops.push_back(Elt: Vec);
12578
12579 // Get widened type and narrowed type.
12580 MVT widenType;
12581 unsigned numElem = VT.getVectorNumElements();
12582
12583 EVT inputLaneType = Vec.getValueType().getVectorElementType();
12584 switch (inputLaneType.getSimpleVT().SimpleTy) {
12585 case MVT::i8: widenType = MVT::getVectorVT(VT: MVT::i16, NumElements: numElem); break;
12586 case MVT::i16: widenType = MVT::getVectorVT(VT: MVT::i32, NumElements: numElem); break;
12587 case MVT::i32: widenType = MVT::getVectorVT(VT: MVT::i64, NumElements: numElem); break;
12588 default:
12589 llvm_unreachable("Invalid vector element type for padd optimization.");
12590 }
12591
12592 SDValue tmp = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: widenType, Ops);
12593 unsigned ExtOp = VT.bitsGT(VT: tmp.getValueType()) ? ISD::ANY_EXTEND : ISD::TRUNCATE;
12594 return DAG.getNode(Opcode: ExtOp, DL: dl, VT, Operand: tmp);
12595}
12596
12597static SDValue findMUL_LOHI(SDValue V) {
12598 if (V->getOpcode() == ISD::UMUL_LOHI ||
12599 V->getOpcode() == ISD::SMUL_LOHI)
12600 return V;
12601 return SDValue();
12602}
12603
12604static SDValue AddCombineTo64BitSMLAL16(SDNode *AddcNode, SDNode *AddeNode,
12605 TargetLowering::DAGCombinerInfo &DCI,
12606 const ARMSubtarget *Subtarget) {
12607 if (!Subtarget->hasBaseDSP())
12608 return SDValue();
12609
12610 // SMLALBB, SMLALBT, SMLALTB, SMLALTT multiply two 16-bit values and
12611 // accumulates the product into a 64-bit value. The 16-bit values will
12612 // be sign extended somehow or SRA'd into 32-bit values
12613 // (addc (adde (mul 16bit, 16bit), lo), hi)
12614 SDValue Mul = AddcNode->getOperand(Num: 0);
12615 SDValue Lo = AddcNode->getOperand(Num: 1);
12616 if (Mul.getOpcode() != ISD::MUL) {
12617 Lo = AddcNode->getOperand(Num: 0);
12618 Mul = AddcNode->getOperand(Num: 1);
12619 if (Mul.getOpcode() != ISD::MUL)
12620 return SDValue();
12621 }
12622
12623 SDValue SRA = AddeNode->getOperand(Num: 0);
12624 SDValue Hi = AddeNode->getOperand(Num: 1);
12625 if (SRA.getOpcode() != ISD::SRA) {
12626 SRA = AddeNode->getOperand(Num: 1);
12627 Hi = AddeNode->getOperand(Num: 0);
12628 if (SRA.getOpcode() != ISD::SRA)
12629 return SDValue();
12630 }
12631 if (auto Const = dyn_cast<ConstantSDNode>(Val: SRA.getOperand(i: 1))) {
12632 if (Const->getZExtValue() != 31)
12633 return SDValue();
12634 } else
12635 return SDValue();
12636
12637 if (SRA.getOperand(i: 0) != Mul)
12638 return SDValue();
12639
12640 SelectionDAG &DAG = DCI.DAG;
12641 SDLoc dl(AddcNode);
12642 unsigned Opcode = 0;
12643 SDValue Op0;
12644 SDValue Op1;
12645
12646 if (isS16(Op: Mul.getOperand(i: 0), DAG) && isS16(Op: Mul.getOperand(i: 1), DAG)) {
12647 Opcode = ARMISD::SMLALBB;
12648 Op0 = Mul.getOperand(i: 0);
12649 Op1 = Mul.getOperand(i: 1);
12650 } else if (isS16(Op: Mul.getOperand(i: 0), DAG) && isSRA16(Op: Mul.getOperand(i: 1))) {
12651 Opcode = ARMISD::SMLALBT;
12652 Op0 = Mul.getOperand(i: 0);
12653 Op1 = Mul.getOperand(i: 1).getOperand(i: 0);
12654 } else if (isSRA16(Op: Mul.getOperand(i: 0)) && isS16(Op: Mul.getOperand(i: 1), DAG)) {
12655 Opcode = ARMISD::SMLALTB;
12656 Op0 = Mul.getOperand(i: 0).getOperand(i: 0);
12657 Op1 = Mul.getOperand(i: 1);
12658 } else if (isSRA16(Op: Mul.getOperand(i: 0)) && isSRA16(Op: Mul.getOperand(i: 1))) {
12659 Opcode = ARMISD::SMLALTT;
12660 Op0 = Mul->getOperand(Num: 0).getOperand(i: 0);
12661 Op1 = Mul->getOperand(Num: 1).getOperand(i: 0);
12662 }
12663
12664 if (!Op0 || !Op1)
12665 return SDValue();
12666
12667 SDValue SMLAL = DAG.getNode(Opcode, DL: dl, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
12668 N1: Op0, N2: Op1, N3: Lo, N4: Hi);
12669 // Replace the ADDs' nodes uses by the MLA node's values.
12670 SDValue HiMLALResult(SMLAL.getNode(), 1);
12671 SDValue LoMLALResult(SMLAL.getNode(), 0);
12672
12673 DAG.ReplaceAllUsesOfValueWith(From: SDValue(AddcNode, 0), To: LoMLALResult);
12674 DAG.ReplaceAllUsesOfValueWith(From: SDValue(AddeNode, 0), To: HiMLALResult);
12675 DCI.recursivelyDeleteUnusedNodes(N: AddeNode);
12676
12677 // Return original node to notify the driver to stop replacing.
12678 return SDValue(AddeNode, 0);
12679}
12680
12681static SDValue AddCombineTo64bitMLAL(SDNode *AddeSubeNode,
12682 TargetLowering::DAGCombinerInfo &DCI,
12683 const ARMSubtarget *Subtarget) {
12684 // Look for multiply add opportunities.
12685 // The pattern is a ISD::UMUL_LOHI followed by two add nodes, where
12686 // each add nodes consumes a value from ISD::UMUL_LOHI and there is
12687 // a glue link from the first add to the second add.
12688 // If we find this pattern, we can replace the U/SMUL_LOHI, ADDC, and ADDE by
12689 // a S/UMLAL instruction.
12690 // UMUL_LOHI
12691 // / :lo \ :hi
12692 // V \ [no multiline comment]
12693 // loAdd -> ADDC |
12694 // \ :carry /
12695 // V V
12696 // ADDE <- hiAdd
12697 //
12698 // In the special case where only the higher part of a signed result is used
12699 // and the add to the low part of the result of ISD::UMUL_LOHI adds or subtracts
12700 // a constant with the exact value of 0x80000000, we recognize we are dealing
12701 // with a "rounded multiply and add" (or subtract) and transform it into
12702 // either a ARMISD::SMMLAR or ARMISD::SMMLSR respectively.
12703
12704 assert((AddeSubeNode->getOpcode() == ARMISD::ADDE ||
12705 AddeSubeNode->getOpcode() == ARMISD::SUBE) &&
12706 "Expect an ADDE or SUBE");
12707
12708 assert(AddeSubeNode->getNumOperands() == 3 &&
12709 AddeSubeNode->getOperand(2).getValueType() == MVT::i32 &&
12710 "ADDE node has the wrong inputs");
12711
12712 // Check that we are chained to the right ADDC or SUBC node.
12713 SDNode *AddcSubcNode = AddeSubeNode->getOperand(Num: 2).getNode();
12714 if ((AddeSubeNode->getOpcode() == ARMISD::ADDE &&
12715 AddcSubcNode->getOpcode() != ARMISD::ADDC) ||
12716 (AddeSubeNode->getOpcode() == ARMISD::SUBE &&
12717 AddcSubcNode->getOpcode() != ARMISD::SUBC))
12718 return SDValue();
12719
12720 SDValue AddcSubcOp0 = AddcSubcNode->getOperand(Num: 0);
12721 SDValue AddcSubcOp1 = AddcSubcNode->getOperand(Num: 1);
12722
12723 // Check if the two operands are from the same mul_lohi node.
12724 if (AddcSubcOp0.getNode() == AddcSubcOp1.getNode())
12725 return SDValue();
12726
12727 assert(AddcSubcNode->getNumValues() == 2 &&
12728 AddcSubcNode->getValueType(0) == MVT::i32 &&
12729 "Expect ADDC with two result values. First: i32");
12730
12731 // Check that the ADDC adds the low result of the S/UMUL_LOHI. If not, it
12732 // maybe a SMLAL which multiplies two 16-bit values.
12733 if (AddeSubeNode->getOpcode() == ARMISD::ADDE &&
12734 AddcSubcOp0->getOpcode() != ISD::UMUL_LOHI &&
12735 AddcSubcOp0->getOpcode() != ISD::SMUL_LOHI &&
12736 AddcSubcOp1->getOpcode() != ISD::UMUL_LOHI &&
12737 AddcSubcOp1->getOpcode() != ISD::SMUL_LOHI)
12738 return AddCombineTo64BitSMLAL16(AddcNode: AddcSubcNode, AddeNode: AddeSubeNode, DCI, Subtarget);
12739
12740 // Check for the triangle shape.
12741 SDValue AddeSubeOp0 = AddeSubeNode->getOperand(Num: 0);
12742 SDValue AddeSubeOp1 = AddeSubeNode->getOperand(Num: 1);
12743
12744 // Make sure that the ADDE/SUBE operands are not coming from the same node.
12745 if (AddeSubeOp0.getNode() == AddeSubeOp1.getNode())
12746 return SDValue();
12747
12748 // Find the MUL_LOHI node walking up ADDE/SUBE's operands.
12749 bool IsLeftOperandMUL = false;
12750 SDValue MULOp = findMUL_LOHI(V: AddeSubeOp0);
12751 if (MULOp == SDValue())
12752 MULOp = findMUL_LOHI(V: AddeSubeOp1);
12753 else
12754 IsLeftOperandMUL = true;
12755 if (MULOp == SDValue())
12756 return SDValue();
12757
12758 // Figure out the right opcode.
12759 unsigned Opc = MULOp->getOpcode();
12760 unsigned FinalOpc = (Opc == ISD::SMUL_LOHI) ? ARMISD::SMLAL : ARMISD::UMLAL;
12761
12762 // Figure out the high and low input values to the MLAL node.
12763 SDValue *HiAddSub = nullptr;
12764 SDValue *LoMul = nullptr;
12765 SDValue *LowAddSub = nullptr;
12766
12767 // Ensure that ADDE/SUBE is from high result of ISD::xMUL_LOHI.
12768 if ((AddeSubeOp0 != MULOp.getValue(R: 1)) && (AddeSubeOp1 != MULOp.getValue(R: 1)))
12769 return SDValue();
12770
12771 if (IsLeftOperandMUL)
12772 HiAddSub = &AddeSubeOp1;
12773 else
12774 HiAddSub = &AddeSubeOp0;
12775
12776 // Ensure that LoMul and LowAddSub are taken from correct ISD::SMUL_LOHI node
12777 // whose low result is fed to the ADDC/SUBC we are checking.
12778
12779 if (AddcSubcOp0 == MULOp.getValue(R: 0)) {
12780 LoMul = &AddcSubcOp0;
12781 LowAddSub = &AddcSubcOp1;
12782 }
12783 if (AddcSubcOp1 == MULOp.getValue(R: 0)) {
12784 LoMul = &AddcSubcOp1;
12785 LowAddSub = &AddcSubcOp0;
12786 }
12787
12788 if (!LoMul)
12789 return SDValue();
12790
12791 // If HiAddSub is the same node as ADDC/SUBC or is a predecessor of ADDC/SUBC
12792 // the replacement below will create a cycle.
12793 if (AddcSubcNode == HiAddSub->getNode() ||
12794 AddcSubcNode->isPredecessorOf(N: HiAddSub->getNode()))
12795 return SDValue();
12796
12797 // Create the merged node.
12798 SelectionDAG &DAG = DCI.DAG;
12799
12800 // Start building operand list.
12801 SmallVector<SDValue, 8> Ops;
12802 Ops.push_back(Elt: LoMul->getOperand(i: 0));
12803 Ops.push_back(Elt: LoMul->getOperand(i: 1));
12804
12805 // Check whether we can use SMMLAR, SMMLSR or SMMULR instead. For this to be
12806 // the case, we must be doing signed multiplication and only use the higher
12807 // part of the result of the MLAL, furthermore the LowAddSub must be a constant
12808 // addition or subtraction with the value of 0x800000.
12809 if (Subtarget->hasV6Ops() && Subtarget->hasDSP() && Subtarget->useMulOps() &&
12810 FinalOpc == ARMISD::SMLAL && !AddeSubeNode->hasAnyUseOfValue(Value: 1) &&
12811 LowAddSub->getNode()->getOpcode() == ISD::Constant &&
12812 static_cast<ConstantSDNode *>(LowAddSub->getNode())->getZExtValue() ==
12813 0x80000000) {
12814 Ops.push_back(Elt: *HiAddSub);
12815 if (AddcSubcNode->getOpcode() == ARMISD::SUBC) {
12816 FinalOpc = ARMISD::SMMLSR;
12817 } else {
12818 FinalOpc = ARMISD::SMMLAR;
12819 }
12820 SDValue NewNode = DAG.getNode(Opcode: FinalOpc, DL: SDLoc(AddcSubcNode), VT: MVT::i32, Ops);
12821 DAG.ReplaceAllUsesOfValueWith(From: SDValue(AddeSubeNode, 0), To: NewNode);
12822
12823 return SDValue(AddeSubeNode, 0);
12824 } else if (AddcSubcNode->getOpcode() == ARMISD::SUBC)
12825 // SMMLS is generated during instruction selection and the rest of this
12826 // function can not handle the case where AddcSubcNode is a SUBC.
12827 return SDValue();
12828
12829 // Finish building the operand list for {U/S}MLAL
12830 Ops.push_back(Elt: *LowAddSub);
12831 Ops.push_back(Elt: *HiAddSub);
12832
12833 SDValue MLALNode = DAG.getNode(Opcode: FinalOpc, DL: SDLoc(AddcSubcNode),
12834 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), Ops);
12835
12836 // Replace the ADDs' nodes uses by the MLA node's values.
12837 SDValue HiMLALResult(MLALNode.getNode(), 1);
12838 DAG.ReplaceAllUsesOfValueWith(From: SDValue(AddeSubeNode, 0), To: HiMLALResult);
12839
12840 SDValue LoMLALResult(MLALNode.getNode(), 0);
12841 DAG.ReplaceAllUsesOfValueWith(From: SDValue(AddcSubcNode, 0), To: LoMLALResult);
12842
12843 // Return original node to notify the driver to stop replacing.
12844 return SDValue(AddeSubeNode, 0);
12845}
12846
12847static SDValue AddCombineTo64bitUMAAL(SDNode *AddeNode,
12848 TargetLowering::DAGCombinerInfo &DCI,
12849 const ARMSubtarget *Subtarget) {
12850 // UMAAL is similar to UMLAL except that it adds two unsigned values.
12851 // While trying to combine for the other MLAL nodes, first search for the
12852 // chance to use UMAAL. Check if Addc uses a node which has already
12853 // been combined into a UMLAL. The other pattern is UMLAL using Addc/Adde
12854 // as the addend, and it's handled in PerformUMLALCombine.
12855
12856 if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP())
12857 return AddCombineTo64bitMLAL(AddeSubeNode: AddeNode, DCI, Subtarget);
12858
12859 // Check that we have a glued ADDC node.
12860 SDNode* AddcNode = AddeNode->getOperand(Num: 2).getNode();
12861 if (AddcNode->getOpcode() != ARMISD::ADDC)
12862 return SDValue();
12863
12864 // Find the converted UMAAL or quit if it doesn't exist.
12865 SDNode *UmlalNode = nullptr;
12866 SDValue AddHi;
12867 if (AddcNode->getOperand(Num: 0).getOpcode() == ARMISD::UMLAL) {
12868 UmlalNode = AddcNode->getOperand(Num: 0).getNode();
12869 AddHi = AddcNode->getOperand(Num: 1);
12870 } else if (AddcNode->getOperand(Num: 1).getOpcode() == ARMISD::UMLAL) {
12871 UmlalNode = AddcNode->getOperand(Num: 1).getNode();
12872 AddHi = AddcNode->getOperand(Num: 0);
12873 } else {
12874 return AddCombineTo64bitMLAL(AddeSubeNode: AddeNode, DCI, Subtarget);
12875 }
12876
12877 // The ADDC should be glued to an ADDE node, which uses the same UMLAL as
12878 // the ADDC as well as Zero.
12879 if (!isNullConstant(V: UmlalNode->getOperand(Num: 3)))
12880 return SDValue();
12881
12882 if ((isNullConstant(V: AddeNode->getOperand(Num: 0)) &&
12883 AddeNode->getOperand(Num: 1).getNode() == UmlalNode) ||
12884 (AddeNode->getOperand(Num: 0).getNode() == UmlalNode &&
12885 isNullConstant(V: AddeNode->getOperand(Num: 1)))) {
12886 SelectionDAG &DAG = DCI.DAG;
12887 SDValue Ops[] = { UmlalNode->getOperand(Num: 0), UmlalNode->getOperand(Num: 1),
12888 UmlalNode->getOperand(Num: 2), AddHi };
12889 SDValue UMAAL = DAG.getNode(Opcode: ARMISD::UMAAL, DL: SDLoc(AddcNode),
12890 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), Ops);
12891
12892 // Replace the ADDs' nodes uses by the UMAAL node's values.
12893 DAG.ReplaceAllUsesOfValueWith(From: SDValue(AddeNode, 0), To: SDValue(UMAAL.getNode(), 1));
12894 DAG.ReplaceAllUsesOfValueWith(From: SDValue(AddcNode, 0), To: SDValue(UMAAL.getNode(), 0));
12895
12896 // Return original node to notify the driver to stop replacing.
12897 return SDValue(AddeNode, 0);
12898 }
12899 return SDValue();
12900}
12901
12902static SDValue PerformUMLALCombine(SDNode *N, SelectionDAG &DAG,
12903 const ARMSubtarget *Subtarget) {
12904 if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP())
12905 return SDValue();
12906
12907 // Check that we have a pair of ADDC and ADDE as operands.
12908 // Both addends of the ADDE must be zero.
12909 SDNode* AddcNode = N->getOperand(Num: 2).getNode();
12910 SDNode* AddeNode = N->getOperand(Num: 3).getNode();
12911 if ((AddcNode->getOpcode() == ARMISD::ADDC) &&
12912 (AddeNode->getOpcode() == ARMISD::ADDE) &&
12913 isNullConstant(V: AddeNode->getOperand(Num: 0)) &&
12914 isNullConstant(V: AddeNode->getOperand(Num: 1)) &&
12915 (AddeNode->getOperand(Num: 2).getNode() == AddcNode))
12916 return DAG.getNode(Opcode: ARMISD::UMAAL, DL: SDLoc(N),
12917 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
12918 Ops: {N->getOperand(Num: 0), N->getOperand(Num: 1),
12919 AddcNode->getOperand(Num: 0), AddcNode->getOperand(Num: 1)});
12920 else
12921 return SDValue();
12922}
12923
12924static SDValue PerformAddcSubcCombine(SDNode *N,
12925 TargetLowering::DAGCombinerInfo &DCI,
12926 const ARMSubtarget *Subtarget) {
12927 SelectionDAG &DAG(DCI.DAG);
12928
12929 if (N->getOpcode() == ARMISD::SUBC && N->hasAnyUseOfValue(Value: 1)) {
12930 // (SUBC (ADDE 0, 0, C), 1) -> C
12931 SDValue LHS = N->getOperand(Num: 0);
12932 SDValue RHS = N->getOperand(Num: 1);
12933 if (LHS->getOpcode() == ARMISD::ADDE &&
12934 isNullConstant(V: LHS->getOperand(Num: 0)) &&
12935 isNullConstant(V: LHS->getOperand(Num: 1)) && isOneConstant(V: RHS)) {
12936 return DCI.CombineTo(N, Res0: SDValue(N, 0), Res1: LHS->getOperand(Num: 2));
12937 }
12938 }
12939
12940 if (Subtarget->isThumb1Only()) {
12941 SDValue RHS = N->getOperand(Num: 1);
12942 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: RHS)) {
12943 int32_t imm = C->getSExtValue();
12944 if (imm < 0 && imm > std::numeric_limits<int>::min()) {
12945 SDLoc DL(N);
12946 RHS = DAG.getConstant(Val: -imm, DL, VT: MVT::i32);
12947 unsigned Opcode = (N->getOpcode() == ARMISD::ADDC) ? ARMISD::SUBC
12948 : ARMISD::ADDC;
12949 return DAG.getNode(Opcode, DL, VTList: N->getVTList(), N1: N->getOperand(Num: 0), N2: RHS);
12950 }
12951 }
12952 }
12953
12954 return SDValue();
12955}
12956
12957static SDValue PerformAddeSubeCombine(SDNode *N,
12958 TargetLowering::DAGCombinerInfo &DCI,
12959 const ARMSubtarget *Subtarget) {
12960 if (Subtarget->isThumb1Only()) {
12961 SelectionDAG &DAG = DCI.DAG;
12962 SDValue RHS = N->getOperand(Num: 1);
12963 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: RHS)) {
12964 int64_t imm = C->getSExtValue();
12965 if (imm < 0) {
12966 SDLoc DL(N);
12967
12968 // The with-carry-in form matches bitwise not instead of the negation.
12969 // Effectively, the inverse interpretation of the carry flag already
12970 // accounts for part of the negation.
12971 RHS = DAG.getConstant(Val: ~imm, DL, VT: MVT::i32);
12972
12973 unsigned Opcode = (N->getOpcode() == ARMISD::ADDE) ? ARMISD::SUBE
12974 : ARMISD::ADDE;
12975 return DAG.getNode(Opcode, DL, VTList: N->getVTList(),
12976 N1: N->getOperand(Num: 0), N2: RHS, N3: N->getOperand(Num: 2));
12977 }
12978 }
12979 } else if (N->getOperand(Num: 1)->getOpcode() == ISD::SMUL_LOHI) {
12980 return AddCombineTo64bitMLAL(AddeSubeNode: N, DCI, Subtarget);
12981 }
12982 return SDValue();
12983}
12984
12985static SDValue PerformSELECTCombine(SDNode *N,
12986 TargetLowering::DAGCombinerInfo &DCI,
12987 const ARMSubtarget *Subtarget) {
12988 if (!Subtarget->hasMVEIntegerOps())
12989 return SDValue();
12990
12991 SDLoc dl(N);
12992 SDValue SetCC;
12993 SDValue LHS;
12994 SDValue RHS;
12995 ISD::CondCode CC;
12996 SDValue TrueVal;
12997 SDValue FalseVal;
12998
12999 if (N->getOpcode() == ISD::SELECT &&
13000 N->getOperand(Num: 0)->getOpcode() == ISD::SETCC) {
13001 SetCC = N->getOperand(Num: 0);
13002 LHS = SetCC->getOperand(Num: 0);
13003 RHS = SetCC->getOperand(Num: 1);
13004 CC = cast<CondCodeSDNode>(Val: SetCC->getOperand(Num: 2))->get();
13005 TrueVal = N->getOperand(Num: 1);
13006 FalseVal = N->getOperand(Num: 2);
13007 } else if (N->getOpcode() == ISD::SELECT_CC) {
13008 LHS = N->getOperand(Num: 0);
13009 RHS = N->getOperand(Num: 1);
13010 CC = cast<CondCodeSDNode>(Val: N->getOperand(Num: 4))->get();
13011 TrueVal = N->getOperand(Num: 2);
13012 FalseVal = N->getOperand(Num: 3);
13013 } else {
13014 return SDValue();
13015 }
13016
13017 unsigned int Opcode = 0;
13018 if ((TrueVal->getOpcode() == ISD::VECREDUCE_UMIN ||
13019 FalseVal->getOpcode() == ISD::VECREDUCE_UMIN) &&
13020 (CC == ISD::SETULT || CC == ISD::SETUGT)) {
13021 Opcode = ARMISD::VMINVu;
13022 if (CC == ISD::SETUGT)
13023 std::swap(a&: TrueVal, b&: FalseVal);
13024 } else if ((TrueVal->getOpcode() == ISD::VECREDUCE_SMIN ||
13025 FalseVal->getOpcode() == ISD::VECREDUCE_SMIN) &&
13026 (CC == ISD::SETLT || CC == ISD::SETGT)) {
13027 Opcode = ARMISD::VMINVs;
13028 if (CC == ISD::SETGT)
13029 std::swap(a&: TrueVal, b&: FalseVal);
13030 } else if ((TrueVal->getOpcode() == ISD::VECREDUCE_UMAX ||
13031 FalseVal->getOpcode() == ISD::VECREDUCE_UMAX) &&
13032 (CC == ISD::SETUGT || CC == ISD::SETULT)) {
13033 Opcode = ARMISD::VMAXVu;
13034 if (CC == ISD::SETULT)
13035 std::swap(a&: TrueVal, b&: FalseVal);
13036 } else if ((TrueVal->getOpcode() == ISD::VECREDUCE_SMAX ||
13037 FalseVal->getOpcode() == ISD::VECREDUCE_SMAX) &&
13038 (CC == ISD::SETGT || CC == ISD::SETLT)) {
13039 Opcode = ARMISD::VMAXVs;
13040 if (CC == ISD::SETLT)
13041 std::swap(a&: TrueVal, b&: FalseVal);
13042 } else
13043 return SDValue();
13044
13045 // Normalise to the right hand side being the vector reduction
13046 switch (TrueVal->getOpcode()) {
13047 case ISD::VECREDUCE_UMIN:
13048 case ISD::VECREDUCE_SMIN:
13049 case ISD::VECREDUCE_UMAX:
13050 case ISD::VECREDUCE_SMAX:
13051 std::swap(a&: LHS, b&: RHS);
13052 std::swap(a&: TrueVal, b&: FalseVal);
13053 break;
13054 }
13055
13056 EVT VectorType = FalseVal->getOperand(Num: 0).getValueType();
13057
13058 if (VectorType != MVT::v16i8 && VectorType != MVT::v8i16 &&
13059 VectorType != MVT::v4i32)
13060 return SDValue();
13061
13062 EVT VectorScalarType = VectorType.getVectorElementType();
13063
13064 // The values being selected must also be the ones being compared
13065 if (TrueVal != LHS || FalseVal != RHS)
13066 return SDValue();
13067
13068 EVT LeftType = LHS->getValueType(ResNo: 0);
13069 EVT RightType = RHS->getValueType(ResNo: 0);
13070
13071 // The types must match the reduced type too
13072 if (LeftType != VectorScalarType || RightType != VectorScalarType)
13073 return SDValue();
13074
13075 // Legalise the scalar to an i32
13076 if (VectorScalarType != MVT::i32)
13077 LHS = DCI.DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: MVT::i32, Operand: LHS);
13078
13079 // Generate the reduction as an i32 for legalisation purposes
13080 auto Reduction =
13081 DCI.DAG.getNode(Opcode, DL: dl, VT: MVT::i32, N1: LHS, N2: RHS->getOperand(Num: 0));
13082
13083 // The result isn't actually an i32 so truncate it back to its original type
13084 if (VectorScalarType != MVT::i32)
13085 Reduction = DCI.DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: VectorScalarType, Operand: Reduction);
13086
13087 return Reduction;
13088}
13089
13090// A special combine for the vqdmulh family of instructions. This is one of the
13091// potential set of patterns that could patch this instruction. The base pattern
13092// you would expect to be min(max(ashr(mul(mul(sext(x), 2), sext(y)), 16))).
13093// This matches the different min(max(ashr(mul(mul(sext(x), sext(y)), 2), 16))),
13094// which llvm will have optimized to min(ashr(mul(sext(x), sext(y)), 15))) as
13095// the max is unnecessary.
13096static SDValue PerformVQDMULHCombine(SDNode *N, SelectionDAG &DAG) {
13097 EVT VT = N->getValueType(ResNo: 0);
13098 SDValue Shft;
13099 ConstantSDNode *Clamp;
13100
13101 if (!VT.isVector() || VT.getScalarSizeInBits() > 64)
13102 return SDValue();
13103
13104 if (N->getOpcode() == ISD::SMIN) {
13105 Shft = N->getOperand(Num: 0);
13106 Clamp = isConstOrConstSplat(N: N->getOperand(Num: 1));
13107 } else if (N->getOpcode() == ISD::VSELECT) {
13108 // Detect a SMIN, which for an i64 node will be a vselect/setcc, not a smin.
13109 SDValue Cmp = N->getOperand(Num: 0);
13110 if (Cmp.getOpcode() != ISD::SETCC ||
13111 cast<CondCodeSDNode>(Val: Cmp.getOperand(i: 2))->get() != ISD::SETLT ||
13112 Cmp.getOperand(i: 0) != N->getOperand(Num: 1) ||
13113 Cmp.getOperand(i: 1) != N->getOperand(Num: 2))
13114 return SDValue();
13115 Shft = N->getOperand(Num: 1);
13116 Clamp = isConstOrConstSplat(N: N->getOperand(Num: 2));
13117 } else
13118 return SDValue();
13119
13120 if (!Clamp)
13121 return SDValue();
13122
13123 MVT ScalarType;
13124 int ShftAmt = 0;
13125 switch (Clamp->getSExtValue()) {
13126 case (1 << 7) - 1:
13127 ScalarType = MVT::i8;
13128 ShftAmt = 7;
13129 break;
13130 case (1 << 15) - 1:
13131 ScalarType = MVT::i16;
13132 ShftAmt = 15;
13133 break;
13134 case (1ULL << 31) - 1:
13135 ScalarType = MVT::i32;
13136 ShftAmt = 31;
13137 break;
13138 default:
13139 return SDValue();
13140 }
13141
13142 if (Shft.getOpcode() != ISD::SRA)
13143 return SDValue();
13144 ConstantSDNode *N1 = isConstOrConstSplat(N: Shft.getOperand(i: 1));
13145 if (!N1 || N1->getSExtValue() != ShftAmt)
13146 return SDValue();
13147
13148 SDValue Mul = Shft.getOperand(i: 0);
13149 if (Mul.getOpcode() != ISD::MUL)
13150 return SDValue();
13151
13152 SDValue Ext0 = Mul.getOperand(i: 0);
13153 SDValue Ext1 = Mul.getOperand(i: 1);
13154 if (Ext0.getOpcode() != ISD::SIGN_EXTEND ||
13155 Ext1.getOpcode() != ISD::SIGN_EXTEND)
13156 return SDValue();
13157 EVT VecVT = Ext0.getOperand(i: 0).getValueType();
13158 if (!VecVT.isPow2VectorType() || VecVT.getVectorNumElements() == 1)
13159 return SDValue();
13160 if (Ext1.getOperand(i: 0).getValueType() != VecVT ||
13161 VecVT.getScalarType() != ScalarType ||
13162 VT.getScalarSizeInBits() < ScalarType.getScalarSizeInBits() * 2)
13163 return SDValue();
13164
13165 SDLoc DL(Mul);
13166 unsigned LegalLanes = 128 / (ShftAmt + 1);
13167 EVT LegalVecVT = MVT::getVectorVT(VT: ScalarType, NumElements: LegalLanes);
13168 // For types smaller than legal vectors extend to be legal and only use needed
13169 // lanes.
13170 if (VecVT.getSizeInBits() < 128) {
13171 EVT ExtVecVT =
13172 MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: 128 / VecVT.getVectorNumElements()),
13173 NumElements: VecVT.getVectorNumElements());
13174 SDValue Inp0 =
13175 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: ExtVecVT, Operand: Ext0.getOperand(i: 0));
13176 SDValue Inp1 =
13177 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: ExtVecVT, Operand: Ext1.getOperand(i: 0));
13178 Inp0 = DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL, VT: LegalVecVT, Operand: Inp0);
13179 Inp1 = DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL, VT: LegalVecVT, Operand: Inp1);
13180 SDValue VQDMULH = DAG.getNode(Opcode: ARMISD::VQDMULH, DL, VT: LegalVecVT, N1: Inp0, N2: Inp1);
13181 SDValue Trunc = DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL, VT: ExtVecVT, Operand: VQDMULH);
13182 Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: VecVT, Operand: Trunc);
13183 return DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT, Operand: Trunc);
13184 }
13185
13186 // For larger types, split into legal sized chunks.
13187 assert(VecVT.getSizeInBits() % 128 == 0 && "Expected a power2 type");
13188 unsigned NumParts = VecVT.getSizeInBits() / 128;
13189 SmallVector<SDValue> Parts;
13190 for (unsigned I = 0; I < NumParts; ++I) {
13191 SDValue Inp0 =
13192 DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: LegalVecVT, N1: Ext0.getOperand(i: 0),
13193 N2: DAG.getVectorIdxConstant(Val: I * LegalLanes, DL));
13194 SDValue Inp1 =
13195 DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: LegalVecVT, N1: Ext1.getOperand(i: 0),
13196 N2: DAG.getVectorIdxConstant(Val: I * LegalLanes, DL));
13197 SDValue VQDMULH = DAG.getNode(Opcode: ARMISD::VQDMULH, DL, VT: LegalVecVT, N1: Inp0, N2: Inp1);
13198 Parts.push_back(Elt: VQDMULH);
13199 }
13200 return DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT,
13201 Operand: DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: VecVT, Ops: Parts));
13202}
13203
13204static SDValue PerformVSELECTCombine(SDNode *N,
13205 TargetLowering::DAGCombinerInfo &DCI,
13206 const ARMSubtarget *Subtarget) {
13207 if (!Subtarget->hasMVEIntegerOps())
13208 return SDValue();
13209
13210 // Constant fold vselect 0, A, B -> B
13211 // and vselect 0xffff, A, B -> A
13212 if (N->getOperand(Num: 0).getOpcode() == ARMISD::PREDICATE_CAST &&
13213 isa<ConstantSDNode>(Val: N->getOperand(Num: 0).getOperand(i: 0))) {
13214 unsigned C = N->getOperand(Num: 0).getConstantOperandVal(i: 0);
13215 if (C == 0)
13216 return N->getOperand(Num: 2);
13217 if (C == 0xffff)
13218 return N->getOperand(Num: 1);
13219 }
13220
13221 if (SDValue V = PerformVQDMULHCombine(N, DAG&: DCI.DAG))
13222 return V;
13223
13224 // Transforms vselect(not(cond), lhs, rhs) into vselect(cond, rhs, lhs).
13225 //
13226 // We need to re-implement this optimization here as the implementation in the
13227 // Target-Independent DAGCombiner does not handle the kind of constant we make
13228 // (it calls isConstOrConstSplat with AllowTruncation set to false - and for
13229 // good reason, allowing truncation there would break other targets).
13230 //
13231 // Currently, this is only done for MVE, as it's the only target that benefits
13232 // from this transformation (e.g. VPNOT+VPSEL becomes a single VPSEL).
13233 if (N->getOperand(Num: 0).getOpcode() != ISD::XOR)
13234 return SDValue();
13235 SDValue XOR = N->getOperand(Num: 0);
13236
13237 // Check if the XOR's RHS is either a 1, or a BUILD_VECTOR of 1s.
13238 // It is important to check with truncation allowed as the BUILD_VECTORs we
13239 // generate in those situations will truncate their operands.
13240 ConstantSDNode *Const =
13241 isConstOrConstSplat(N: XOR->getOperand(Num: 1), /*AllowUndefs*/ false,
13242 /*AllowTruncation*/ true);
13243 if (!Const || !Const->isOne())
13244 return SDValue();
13245
13246 // Rewrite into vselect(cond, rhs, lhs).
13247 SDValue Cond = XOR->getOperand(Num: 0);
13248 SDValue LHS = N->getOperand(Num: 1);
13249 SDValue RHS = N->getOperand(Num: 2);
13250 EVT Type = N->getValueType(ResNo: 0);
13251 return DCI.DAG.getNode(Opcode: ISD::VSELECT, DL: SDLoc(N), VT: Type, N1: Cond, N2: RHS, N3: LHS);
13252}
13253
13254// Convert vsetcc([0,1,2,..], splat(n), ult) -> vctp n
13255static SDValue PerformVSetCCToVCTPCombine(SDNode *N,
13256 TargetLowering::DAGCombinerInfo &DCI,
13257 const ARMSubtarget *Subtarget) {
13258 SDValue Op0 = N->getOperand(Num: 0);
13259 SDValue Op1 = N->getOperand(Num: 1);
13260 ISD::CondCode CC = cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get();
13261 EVT VT = N->getValueType(ResNo: 0);
13262
13263 if (!Subtarget->hasMVEIntegerOps() ||
13264 !DCI.DAG.getTargetLoweringInfo().isTypeLegal(VT))
13265 return SDValue();
13266
13267 if (CC == ISD::SETUGT) {
13268 std::swap(a&: Op0, b&: Op1);
13269 CC = ISD::SETULT;
13270 }
13271
13272 if (CC != ISD::SETULT || VT.getScalarSizeInBits() != 1 ||
13273 Op0.getOpcode() != ISD::BUILD_VECTOR)
13274 return SDValue();
13275
13276 // Check first operand is BuildVector of 0,1,2,...
13277 for (unsigned I = 0; I < VT.getVectorNumElements(); I++) {
13278 if (!Op0.getOperand(i: I).isUndef() &&
13279 !(isa<ConstantSDNode>(Val: Op0.getOperand(i: I)) &&
13280 Op0.getConstantOperandVal(i: I) == I))
13281 return SDValue();
13282 }
13283
13284 // The second is a Splat of Op1S
13285 SDValue Op1S = DCI.DAG.getSplatValue(V: Op1);
13286 if (!Op1S)
13287 return SDValue();
13288
13289 unsigned Opc;
13290 switch (VT.getVectorNumElements()) {
13291 case 4:
13292 Opc = Intrinsic::arm_mve_vctp32;
13293 break;
13294 case 8:
13295 Opc = Intrinsic::arm_mve_vctp16;
13296 break;
13297 case 16:
13298 Opc = Intrinsic::arm_mve_vctp8;
13299 break;
13300 default:
13301 return SDValue();
13302 }
13303
13304 SDLoc DL(N);
13305 return DCI.DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT,
13306 N1: DCI.DAG.getConstant(Val: Opc, DL, VT: MVT::i32),
13307 N2: DCI.DAG.getZExtOrTrunc(Op: Op1S, DL, VT: MVT::i32));
13308}
13309
13310/// PerformADDECombine - Target-specific dag combine transform from
13311/// ARMISD::ADDC, ARMISD::ADDE, and ISD::MUL_LOHI to MLAL or
13312/// ARMISD::ADDC, ARMISD::ADDE and ARMISD::UMLAL to ARMISD::UMAAL
13313static SDValue PerformADDECombine(SDNode *N,
13314 TargetLowering::DAGCombinerInfo &DCI,
13315 const ARMSubtarget *Subtarget) {
13316 // Only ARM and Thumb2 support UMLAL/SMLAL.
13317 if (Subtarget->isThumb1Only())
13318 return PerformAddeSubeCombine(N, DCI, Subtarget);
13319
13320 // Only perform the checks after legalize when the pattern is available.
13321 if (DCI.isBeforeLegalize()) return SDValue();
13322
13323 return AddCombineTo64bitUMAAL(AddeNode: N, DCI, Subtarget);
13324}
13325
13326/// PerformADDCombineWithOperands - Try DAG combinations for an ADD with
13327/// operands N0 and N1. This is a helper for PerformADDCombine that is
13328/// called with the default operands, and if that fails, with commuted
13329/// operands.
13330static SDValue PerformADDCombineWithOperands(SDNode *N, SDValue N0, SDValue N1,
13331 TargetLowering::DAGCombinerInfo &DCI,
13332 const ARMSubtarget *Subtarget){
13333 // Attempt to create vpadd for this add.
13334 if (SDValue Result = AddCombineToVPADD(N, N0, N1, DCI, Subtarget))
13335 return Result;
13336
13337 // Attempt to create vpaddl for this add.
13338 if (SDValue Result = AddCombineVUZPToVPADDL(N, N0, N1, DCI, Subtarget))
13339 return Result;
13340 if (SDValue Result = AddCombineBUILD_VECTORToVPADDL(N, N0, N1, DCI,
13341 Subtarget))
13342 return Result;
13343
13344 // fold (add (select cc, 0, c), x) -> (select cc, x, (add, x, c))
13345 if (N0.getNode()->hasOneUse())
13346 if (SDValue Result = combineSelectAndUse(N, Slct: N0, OtherOp: N1, DCI))
13347 return Result;
13348 return SDValue();
13349}
13350
13351static SDValue TryDistrubutionADDVecReduce(SDNode *N, SelectionDAG &DAG) {
13352 EVT VT = N->getValueType(ResNo: 0);
13353 SDValue N0 = N->getOperand(Num: 0);
13354 SDValue N1 = N->getOperand(Num: 1);
13355 SDLoc dl(N);
13356
13357 auto IsVecReduce = [](SDValue Op) {
13358 switch (Op.getOpcode()) {
13359 case ISD::VECREDUCE_ADD:
13360 case ARMISD::VADDVs:
13361 case ARMISD::VADDVu:
13362 case ARMISD::VMLAVs:
13363 case ARMISD::VMLAVu:
13364 return true;
13365 }
13366 return false;
13367 };
13368
13369 auto DistrubuteAddAddVecReduce = [&](SDValue N0, SDValue N1) {
13370 // Distribute add(X, add(vecreduce(Y), vecreduce(Z))) ->
13371 // add(add(X, vecreduce(Y)), vecreduce(Z))
13372 // to make better use of vaddva style instructions.
13373 if (VT == MVT::i32 && N1.getOpcode() == ISD::ADD && !IsVecReduce(N0) &&
13374 IsVecReduce(N1.getOperand(i: 0)) && IsVecReduce(N1.getOperand(i: 1)) &&
13375 !isa<ConstantSDNode>(Val: N0) && N1->hasOneUse()) {
13376 SDValue Add0 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: N0, N2: N1.getOperand(i: 0));
13377 return DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Add0, N2: N1.getOperand(i: 1));
13378 }
13379 // And turn add(add(A, reduce(B)), add(C, reduce(D))) ->
13380 // add(add(add(A, C), reduce(B)), reduce(D))
13381 if (VT == MVT::i32 && N0.getOpcode() == ISD::ADD &&
13382 N1.getOpcode() == ISD::ADD && N0->hasOneUse() && N1->hasOneUse()) {
13383 unsigned N0RedOp = 0;
13384 if (!IsVecReduce(N0.getOperand(i: N0RedOp))) {
13385 N0RedOp = 1;
13386 if (!IsVecReduce(N0.getOperand(i: N0RedOp)))
13387 return SDValue();
13388 }
13389
13390 unsigned N1RedOp = 0;
13391 if (!IsVecReduce(N1.getOperand(i: N1RedOp)))
13392 N1RedOp = 1;
13393 if (!IsVecReduce(N1.getOperand(i: N1RedOp)))
13394 return SDValue();
13395
13396 SDValue Add0 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: N0.getOperand(i: 1 - N0RedOp),
13397 N2: N1.getOperand(i: 1 - N1RedOp));
13398 SDValue Add1 =
13399 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Add0, N2: N0.getOperand(i: N0RedOp));
13400 return DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Add1, N2: N1.getOperand(i: N1RedOp));
13401 }
13402 return SDValue();
13403 };
13404 if (SDValue R = DistrubuteAddAddVecReduce(N0, N1))
13405 return R;
13406 if (SDValue R = DistrubuteAddAddVecReduce(N1, N0))
13407 return R;
13408
13409 // Distribute add(vecreduce(load(Y)), vecreduce(load(Z)))
13410 // Or add(add(X, vecreduce(load(Y))), vecreduce(load(Z)))
13411 // by ascending load offsets. This can help cores prefetch if the order of
13412 // loads is more predictable.
13413 auto DistrubuteVecReduceLoad = [&](SDValue N0, SDValue N1, bool IsForward) {
13414 // Check if two reductions are known to load data where one is before/after
13415 // another. Return negative if N0 loads data before N1, positive if N1 is
13416 // before N0 and 0 otherwise if nothing is known.
13417 auto IsKnownOrderedLoad = [&](SDValue N0, SDValue N1) {
13418 // Look through to the first operand of a MUL, for the VMLA case.
13419 // Currently only looks at the first operand, in the hope they are equal.
13420 if (N0.getOpcode() == ISD::MUL)
13421 N0 = N0.getOperand(i: 0);
13422 if (N1.getOpcode() == ISD::MUL)
13423 N1 = N1.getOperand(i: 0);
13424
13425 // Return true if the two operands are loads to the same object and the
13426 // offset of the first is known to be less than the offset of the second.
13427 LoadSDNode *Load0 = dyn_cast<LoadSDNode>(Val&: N0);
13428 LoadSDNode *Load1 = dyn_cast<LoadSDNode>(Val&: N1);
13429 if (!Load0 || !Load1 || Load0->getChain() != Load1->getChain() ||
13430 !Load0->isSimple() || !Load1->isSimple() || Load0->isIndexed() ||
13431 Load1->isIndexed())
13432 return 0;
13433
13434 auto BaseLocDecomp0 = BaseIndexOffset::match(N: Load0, DAG);
13435 auto BaseLocDecomp1 = BaseIndexOffset::match(N: Load1, DAG);
13436
13437 if (!BaseLocDecomp0.getBase() ||
13438 BaseLocDecomp0.getBase() != BaseLocDecomp1.getBase() ||
13439 !BaseLocDecomp0.hasValidOffset() || !BaseLocDecomp1.hasValidOffset())
13440 return 0;
13441 if (BaseLocDecomp0.getOffset() < BaseLocDecomp1.getOffset())
13442 return -1;
13443 if (BaseLocDecomp0.getOffset() > BaseLocDecomp1.getOffset())
13444 return 1;
13445 return 0;
13446 };
13447
13448 SDValue X;
13449 if (N0.getOpcode() == ISD::ADD && N0->hasOneUse()) {
13450 if (IsVecReduce(N0.getOperand(i: 0)) && IsVecReduce(N0.getOperand(i: 1))) {
13451 int IsBefore = IsKnownOrderedLoad(N0.getOperand(i: 0).getOperand(i: 0),
13452 N0.getOperand(i: 1).getOperand(i: 0));
13453 if (IsBefore < 0) {
13454 X = N0.getOperand(i: 0);
13455 N0 = N0.getOperand(i: 1);
13456 } else if (IsBefore > 0) {
13457 X = N0.getOperand(i: 1);
13458 N0 = N0.getOperand(i: 0);
13459 } else
13460 return SDValue();
13461 } else if (IsVecReduce(N0.getOperand(i: 0))) {
13462 X = N0.getOperand(i: 1);
13463 N0 = N0.getOperand(i: 0);
13464 } else if (IsVecReduce(N0.getOperand(i: 1))) {
13465 X = N0.getOperand(i: 0);
13466 N0 = N0.getOperand(i: 1);
13467 } else
13468 return SDValue();
13469 } else if (IsForward && IsVecReduce(N0) && IsVecReduce(N1) &&
13470 IsKnownOrderedLoad(N0.getOperand(i: 0), N1.getOperand(i: 0)) < 0) {
13471 // Note this is backward to how you would expect. We create
13472 // add(reduce(load + 16), reduce(load + 0)) so that the
13473 // add(reduce(load+16), X) is combined into VADDVA(X, load+16)), leaving
13474 // the X as VADDV(load + 0)
13475 return DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1, N2: N0);
13476 } else
13477 return SDValue();
13478
13479 if (!IsVecReduce(N0) || !IsVecReduce(N1))
13480 return SDValue();
13481
13482 if (IsKnownOrderedLoad(N1.getOperand(i: 0), N0.getOperand(i: 0)) >= 0)
13483 return SDValue();
13484
13485 // Switch from add(add(X, N0), N1) to add(add(X, N1), N0)
13486 SDValue Add0 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: X, N2: N1);
13487 return DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Add0, N2: N0);
13488 };
13489 if (SDValue R = DistrubuteVecReduceLoad(N0, N1, true))
13490 return R;
13491 if (SDValue R = DistrubuteVecReduceLoad(N1, N0, false))
13492 return R;
13493 return SDValue();
13494}
13495
13496static SDValue PerformADDVecReduce(SDNode *N, SelectionDAG &DAG,
13497 const ARMSubtarget *Subtarget) {
13498 if (!Subtarget->hasMVEIntegerOps())
13499 return SDValue();
13500
13501 if (SDValue R = TryDistrubutionADDVecReduce(N, DAG))
13502 return R;
13503
13504 EVT VT = N->getValueType(ResNo: 0);
13505 SDValue N0 = N->getOperand(Num: 0);
13506 SDValue N1 = N->getOperand(Num: 1);
13507 SDLoc dl(N);
13508
13509 if (VT != MVT::i64)
13510 return SDValue();
13511
13512 // We are looking for a i64 add of a VADDLVx. Due to these being i64's, this
13513 // will look like:
13514 // t1: i32,i32 = ARMISD::VADDLVs x
13515 // t2: i64 = build_pair t1, t1:1
13516 // t3: i64 = add t2, y
13517 // Otherwise we try to push the add up above VADDLVAx, to potentially allow
13518 // the add to be simplified separately.
13519 // We also need to check for sext / zext and commutitive adds.
13520 auto MakeVecReduce = [&](unsigned Opcode, unsigned OpcodeA, SDValue NA,
13521 SDValue NB) {
13522 if (NB->getOpcode() != ISD::BUILD_PAIR)
13523 return SDValue();
13524 SDValue VecRed = NB->getOperand(Num: 0);
13525 if ((VecRed->getOpcode() != Opcode && VecRed->getOpcode() != OpcodeA) ||
13526 VecRed.getResNo() != 0 ||
13527 NB->getOperand(Num: 1) != SDValue(VecRed.getNode(), 1))
13528 return SDValue();
13529
13530 if (VecRed->getOpcode() == OpcodeA) {
13531 // add(NA, VADDLVA(Inp), Y) -> VADDLVA(add(NA, Inp), Y)
13532 SDValue Inp = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: MVT::i64,
13533 N1: VecRed.getOperand(i: 0), N2: VecRed.getOperand(i: 1));
13534 NA = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::i64, N1: Inp, N2: NA);
13535 }
13536
13537 SmallVector<SDValue, 4> Ops(2);
13538 std::tie(args&: Ops[0], args&: Ops[1]) = DAG.SplitScalar(N: NA, DL: dl, LoVT: MVT::i32, HiVT: MVT::i32);
13539
13540 unsigned S = VecRed->getOpcode() == OpcodeA ? 2 : 0;
13541 for (unsigned I = S, E = VecRed.getNumOperands(); I < E; I++)
13542 Ops.push_back(Elt: VecRed->getOperand(Num: I));
13543 SDValue Red =
13544 DAG.getNode(Opcode: OpcodeA, DL: dl, VTList: DAG.getVTList(VTs: {MVT::i32, MVT::i32}), Ops);
13545 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: MVT::i64, N1: Red,
13546 N2: SDValue(Red.getNode(), 1));
13547 };
13548
13549 if (SDValue M = MakeVecReduce(ARMISD::VADDLVs, ARMISD::VADDLVAs, N0, N1))
13550 return M;
13551 if (SDValue M = MakeVecReduce(ARMISD::VADDLVu, ARMISD::VADDLVAu, N0, N1))
13552 return M;
13553 if (SDValue M = MakeVecReduce(ARMISD::VADDLVs, ARMISD::VADDLVAs, N1, N0))
13554 return M;
13555 if (SDValue M = MakeVecReduce(ARMISD::VADDLVu, ARMISD::VADDLVAu, N1, N0))
13556 return M;
13557 if (SDValue M = MakeVecReduce(ARMISD::VADDLVps, ARMISD::VADDLVAps, N0, N1))
13558 return M;
13559 if (SDValue M = MakeVecReduce(ARMISD::VADDLVpu, ARMISD::VADDLVApu, N0, N1))
13560 return M;
13561 if (SDValue M = MakeVecReduce(ARMISD::VADDLVps, ARMISD::VADDLVAps, N1, N0))
13562 return M;
13563 if (SDValue M = MakeVecReduce(ARMISD::VADDLVpu, ARMISD::VADDLVApu, N1, N0))
13564 return M;
13565 if (SDValue M = MakeVecReduce(ARMISD::VMLALVs, ARMISD::VMLALVAs, N0, N1))
13566 return M;
13567 if (SDValue M = MakeVecReduce(ARMISD::VMLALVu, ARMISD::VMLALVAu, N0, N1))
13568 return M;
13569 if (SDValue M = MakeVecReduce(ARMISD::VMLALVs, ARMISD::VMLALVAs, N1, N0))
13570 return M;
13571 if (SDValue M = MakeVecReduce(ARMISD::VMLALVu, ARMISD::VMLALVAu, N1, N0))
13572 return M;
13573 if (SDValue M = MakeVecReduce(ARMISD::VMLALVps, ARMISD::VMLALVAps, N0, N1))
13574 return M;
13575 if (SDValue M = MakeVecReduce(ARMISD::VMLALVpu, ARMISD::VMLALVApu, N0, N1))
13576 return M;
13577 if (SDValue M = MakeVecReduce(ARMISD::VMLALVps, ARMISD::VMLALVAps, N1, N0))
13578 return M;
13579 if (SDValue M = MakeVecReduce(ARMISD::VMLALVpu, ARMISD::VMLALVApu, N1, N0))
13580 return M;
13581 return SDValue();
13582}
13583
13584bool
13585ARMTargetLowering::isDesirableToCommuteWithShift(const SDNode *N,
13586 CombineLevel Level) const {
13587 assert((N->getOpcode() == ISD::SHL || N->getOpcode() == ISD::SRA ||
13588 N->getOpcode() == ISD::SRL) &&
13589 "Expected shift op");
13590
13591 SDValue ShiftLHS = N->getOperand(Num: 0);
13592 if (!ShiftLHS->hasOneUse())
13593 return false;
13594
13595 if (ShiftLHS.getOpcode() == ISD::SIGN_EXTEND &&
13596 !ShiftLHS.getOperand(i: 0)->hasOneUse())
13597 return false;
13598
13599 if (Level == BeforeLegalizeTypes)
13600 return true;
13601
13602 if (N->getOpcode() != ISD::SHL)
13603 return true;
13604
13605 if (Subtarget->isThumb1Only()) {
13606 // Avoid making expensive immediates by commuting shifts. (This logic
13607 // only applies to Thumb1 because ARM and Thumb2 immediates can be shifted
13608 // for free.)
13609 if (N->getOpcode() != ISD::SHL)
13610 return true;
13611 SDValue N1 = N->getOperand(Num: 0);
13612 if (N1->getOpcode() != ISD::ADD && N1->getOpcode() != ISD::AND &&
13613 N1->getOpcode() != ISD::OR && N1->getOpcode() != ISD::XOR)
13614 return true;
13615 if (auto *Const = dyn_cast<ConstantSDNode>(Val: N1->getOperand(Num: 1))) {
13616 if (Const->getAPIntValue().ult(RHS: 256))
13617 return false;
13618 if (N1->getOpcode() == ISD::ADD && Const->getAPIntValue().slt(RHS: 0) &&
13619 Const->getAPIntValue().sgt(RHS: -256))
13620 return false;
13621 }
13622 return true;
13623 }
13624
13625 // Turn off commute-with-shift transform after legalization, so it doesn't
13626 // conflict with PerformSHLSimplify. (We could try to detect when
13627 // PerformSHLSimplify would trigger more precisely, but it isn't
13628 // really necessary.)
13629 return false;
13630}
13631
13632bool ARMTargetLowering::isDesirableToCommuteXorWithShift(
13633 const SDNode *N) const {
13634 assert(N->getOpcode() == ISD::XOR &&
13635 (N->getOperand(0).getOpcode() == ISD::SHL ||
13636 N->getOperand(0).getOpcode() == ISD::SRL) &&
13637 "Expected XOR(SHIFT) pattern");
13638
13639 // Only commute if the entire NOT mask is a hidden shifted mask.
13640 auto *XorC = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
13641 auto *ShiftC = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 0).getOperand(i: 1));
13642 if (XorC && ShiftC) {
13643 unsigned MaskIdx, MaskLen;
13644 if (XorC->getAPIntValue().isShiftedMask(MaskIdx, MaskLen)) {
13645 unsigned ShiftAmt = ShiftC->getZExtValue();
13646 unsigned BitWidth = N->getValueType(ResNo: 0).getScalarSizeInBits();
13647 if (N->getOperand(Num: 0).getOpcode() == ISD::SHL)
13648 return MaskIdx == ShiftAmt && MaskLen == (BitWidth - ShiftAmt);
13649 return MaskIdx == 0 && MaskLen == (BitWidth - ShiftAmt);
13650 }
13651 }
13652
13653 return false;
13654}
13655
13656bool ARMTargetLowering::shouldFoldConstantShiftPairToMask(
13657 const SDNode *N) const {
13658 assert(((N->getOpcode() == ISD::SHL &&
13659 N->getOperand(0).getOpcode() == ISD::SRL) ||
13660 (N->getOpcode() == ISD::SRL &&
13661 N->getOperand(0).getOpcode() == ISD::SHL)) &&
13662 "Expected shift-shift mask");
13663
13664 if (!Subtarget->isThumb1Only())
13665 return true;
13666
13667 EVT VT = N->getValueType(ResNo: 0);
13668 if (VT.getScalarSizeInBits() > 32)
13669 return true;
13670
13671 return false;
13672}
13673
13674bool ARMTargetLowering::shouldFoldSelectWithIdentityConstant(
13675 unsigned BinOpcode, EVT VT, unsigned SelectOpcode, SDValue X,
13676 SDValue Y) const {
13677 return Subtarget->hasMVEIntegerOps() && isTypeLegal(VT) &&
13678 SelectOpcode == ISD::VSELECT;
13679}
13680
13681bool ARMTargetLowering::preferIncOfAddToSubOfNot(EVT VT) const {
13682 if (!Subtarget->hasNEON() && !Subtarget->hasMVEIntegerOps()) {
13683 if (Subtarget->isThumb1Only())
13684 return VT.getScalarSizeInBits() <= 32;
13685 return true;
13686 }
13687 return VT.isScalarInteger();
13688}
13689
13690bool ARMTargetLowering::shouldConvertFpToSat(unsigned Op, EVT FPVT,
13691 EVT VT) const {
13692 if (!isOperationLegalOrCustom(Op, VT) || !FPVT.isSimple())
13693 return false;
13694
13695 switch (FPVT.getSimpleVT().SimpleTy) {
13696 case MVT::f16:
13697 return Subtarget->hasVFP2Base();
13698 case MVT::f32:
13699 return Subtarget->hasVFP2Base();
13700 case MVT::f64:
13701 return Subtarget->hasFP64();
13702 case MVT::v4f32:
13703 case MVT::v8f16:
13704 return Subtarget->hasMVEFloatOps();
13705 default:
13706 return false;
13707 }
13708}
13709
13710static SDValue PerformSHLSimplify(SDNode *N,
13711 TargetLowering::DAGCombinerInfo &DCI,
13712 const ARMSubtarget *ST) {
13713 // Allow the generic combiner to identify potential bswaps.
13714 if (DCI.isBeforeLegalize())
13715 return SDValue();
13716
13717 // DAG combiner will fold:
13718 // (shl (add x, c1), c2) -> (add (shl x, c2), c1 << c2)
13719 // (shl (or x, c1), c2) -> (or (shl x, c2), c1 << c2
13720 // Other code patterns that can be also be modified have the following form:
13721 // b + ((a << 1) | 510)
13722 // b + ((a << 1) & 510)
13723 // b + ((a << 1) ^ 510)
13724 // b + ((a << 1) + 510)
13725
13726 // Many instructions can perform the shift for free, but it requires both
13727 // the operands to be registers. If c1 << c2 is too large, a mov immediate
13728 // instruction will needed. So, unfold back to the original pattern if:
13729 // - if c1 and c2 are small enough that they don't require mov imms.
13730 // - the user(s) of the node can perform an shl
13731
13732 // No shifted operands for 16-bit instructions.
13733 if (ST->isThumb1Only())
13734 return SDValue();
13735
13736 // Check that all the users could perform the shl themselves.
13737 for (auto *U : N->users()) {
13738 switch(U->getOpcode()) {
13739 default:
13740 return SDValue();
13741 case ISD::SUB:
13742 case ISD::ADD:
13743 case ISD::AND:
13744 case ISD::OR:
13745 case ISD::XOR:
13746 case ISD::SETCC:
13747 case ARMISD::CMP:
13748 // Check that the user isn't already using a constant because there
13749 // aren't any instructions that support an immediate operand and a
13750 // shifted operand.
13751 if (isa<ConstantSDNode>(Val: U->getOperand(Num: 0)) ||
13752 isa<ConstantSDNode>(Val: U->getOperand(Num: 1)))
13753 return SDValue();
13754
13755 // Check that it's not already using a shift.
13756 if (U->getOperand(Num: 0).getOpcode() == ISD::SHL ||
13757 U->getOperand(Num: 1).getOpcode() == ISD::SHL)
13758 return SDValue();
13759 break;
13760 }
13761 }
13762
13763 if (N->getOpcode() != ISD::ADD && N->getOpcode() != ISD::OR &&
13764 N->getOpcode() != ISD::XOR && N->getOpcode() != ISD::AND)
13765 return SDValue();
13766
13767 if (N->getOperand(Num: 0).getOpcode() != ISD::SHL)
13768 return SDValue();
13769
13770 SDValue SHL = N->getOperand(Num: 0);
13771
13772 auto *C1ShlC2 = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
13773 auto *C2 = dyn_cast<ConstantSDNode>(Val: SHL.getOperand(i: 1));
13774 if (!C1ShlC2 || !C2)
13775 return SDValue();
13776
13777 APInt C2Int = C2->getAPIntValue();
13778 APInt C1Int = C1ShlC2->getAPIntValue();
13779 unsigned C2Width = C2Int.getBitWidth();
13780 if (C2Int.uge(RHS: C2Width))
13781 return SDValue();
13782 uint64_t C2Value = C2Int.getZExtValue();
13783
13784 // Check that performing a lshr will not lose any information.
13785 APInt Mask = APInt::getHighBitsSet(numBits: C2Width, hiBitsSet: C2Width - C2Value);
13786 if ((C1Int & Mask) != C1Int)
13787 return SDValue();
13788
13789 // Shift the first constant.
13790 C1Int.lshrInPlace(ShiftAmt: C2Int);
13791
13792 // The immediates are encoded as an 8-bit value that can be rotated.
13793 auto LargeImm = [](const APInt &Imm) {
13794 unsigned Zeros = Imm.countl_zero() + Imm.countr_zero();
13795 return Imm.getBitWidth() - Zeros > 8;
13796 };
13797
13798 if (LargeImm(C1Int) || LargeImm(C2Int))
13799 return SDValue();
13800
13801 SelectionDAG &DAG = DCI.DAG;
13802 SDLoc dl(N);
13803 SDValue X = SHL.getOperand(i: 0);
13804 SDValue BinOp = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: MVT::i32, N1: X,
13805 N2: DAG.getConstant(Val: C1Int, DL: dl, VT: MVT::i32));
13806 // Shift left to compensate for the lshr of C1Int.
13807 SDValue Res = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: MVT::i32, N1: BinOp, N2: SHL.getOperand(i: 1));
13808
13809 LLVM_DEBUG(dbgs() << "Simplify shl use:\n"; SHL.getOperand(0).dump();
13810 SHL.dump(); N->dump());
13811 LLVM_DEBUG(dbgs() << "Into:\n"; X.dump(); BinOp.dump(); Res.dump());
13812 return Res;
13813}
13814
13815
13816/// PerformADDCombine - Target-specific dag combine xforms for ISD::ADD.
13817///
13818static SDValue PerformADDCombine(SDNode *N,
13819 TargetLowering::DAGCombinerInfo &DCI,
13820 const ARMSubtarget *Subtarget) {
13821 SDValue N0 = N->getOperand(Num: 0);
13822 SDValue N1 = N->getOperand(Num: 1);
13823
13824 // Only works one way, because it needs an immediate operand.
13825 if (SDValue Result = PerformSHLSimplify(N, DCI, ST: Subtarget))
13826 return Result;
13827
13828 if (SDValue Result = PerformADDVecReduce(N, DAG&: DCI.DAG, Subtarget))
13829 return Result;
13830
13831 // First try with the default operand order.
13832 if (SDValue Result = PerformADDCombineWithOperands(N, N0, N1, DCI, Subtarget))
13833 return Result;
13834
13835 // If that didn't work, try again with the operands commuted.
13836 return PerformADDCombineWithOperands(N, N0: N1, N1: N0, DCI, Subtarget);
13837}
13838
13839// Combine (sub 0, (csinc X, Y, CC)) -> (csinv -X, Y, CC)
13840// providing -X is as cheap as X (currently, just a constant).
13841static SDValue PerformSubCSINCCombine(SDNode *N, SelectionDAG &DAG) {
13842 if (N->getValueType(ResNo: 0) != MVT::i32 || !isNullConstant(V: N->getOperand(Num: 0)))
13843 return SDValue();
13844 SDValue CSINC = N->getOperand(Num: 1);
13845 if (CSINC.getOpcode() != ARMISD::CSINC || !CSINC.hasOneUse())
13846 return SDValue();
13847
13848 ConstantSDNode *X = dyn_cast<ConstantSDNode>(Val: CSINC.getOperand(i: 0));
13849 if (!X)
13850 return SDValue();
13851
13852 return DAG.getNode(Opcode: ARMISD::CSINV, DL: SDLoc(N), VT: MVT::i32,
13853 N1: DAG.getNode(Opcode: ISD::SUB, DL: SDLoc(N), VT: MVT::i32, N1: N->getOperand(Num: 0),
13854 N2: CSINC.getOperand(i: 0)),
13855 N2: CSINC.getOperand(i: 1), N3: CSINC.getOperand(i: 2),
13856 N4: CSINC.getOperand(i: 3));
13857}
13858
13859static int getNegationCost(SDValue Op) {
13860 // Free to negate.
13861 if (isa<ConstantSDNode>(Val: Op))
13862 return 0;
13863
13864 // Will save one instruction.
13865 if (Op.getOpcode() == ISD::SUB && isNullConstant(V: Op.getOperand(i: 0)))
13866 return -1;
13867
13868 // Can freely negate by converting sra <-> srl.
13869 if (Op.getOpcode() == ISD::SRA || Op.getOpcode() == ISD::SRL) {
13870 ConstantSDNode *ShiftAmt = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1));
13871 if (Op.hasOneUse() && ShiftAmt &&
13872 ShiftAmt->getZExtValue() == Op.getValueType().getScalarSizeInBits() - 1)
13873 return 0;
13874 }
13875
13876 // Will have to create sub.
13877 return 1;
13878}
13879
13880// Try to fold
13881//
13882// (neg (cmov X, Y)) -> (cmov (neg X), (neg Y))
13883//
13884// The folding helps cmov to be matched with csneg without generating
13885// redundant neg instruction.
13886static SDValue performNegCMovCombine(SDNode *N, SelectionDAG &DAG) {
13887 assert(N->getOpcode() == ISD::SUB);
13888 if (!isNullConstant(V: N->getOperand(Num: 0)))
13889 return SDValue();
13890
13891 SDValue CMov = N->getOperand(Num: 1);
13892 if (CMov.getOpcode() != ARMISD::CMOV || !CMov->hasOneUse())
13893 return SDValue();
13894
13895 SDValue N0 = CMov.getOperand(i: 0);
13896 SDValue N1 = CMov.getOperand(i: 1);
13897
13898 // Only perform the fold if we actually save something.
13899 if (getNegationCost(Op: N0) + getNegationCost(Op: N1) > 0)
13900 return SDValue();
13901
13902 SDLoc DL(N);
13903 EVT VT = CMov.getValueType();
13904
13905 SDValue N0N = DAG.getNegative(Val: N0, DL, VT);
13906 SDValue N1N = DAG.getNegative(Val: N1, DL, VT);
13907 return DAG.getNode(Opcode: ARMISD::CMOV, DL, VT, N1: N0N, N2: N1N, N3: CMov.getOperand(i: 2),
13908 N4: CMov.getOperand(i: 3));
13909}
13910
13911/// PerformSUBCombine - Target-specific dag combine xforms for ISD::SUB.
13912///
13913static SDValue PerformSUBCombine(SDNode *N,
13914 TargetLowering::DAGCombinerInfo &DCI,
13915 const ARMSubtarget *Subtarget) {
13916 SDValue N0 = N->getOperand(Num: 0);
13917 SDValue N1 = N->getOperand(Num: 1);
13918
13919 // fold (sub x, (select cc, 0, c)) -> (select cc, x, (sub, x, c))
13920 if (N1.getNode()->hasOneUse())
13921 if (SDValue Result = combineSelectAndUse(N, Slct: N1, OtherOp: N0, DCI))
13922 return Result;
13923
13924 if (SDValue R = PerformSubCSINCCombine(N, DAG&: DCI.DAG))
13925 return R;
13926
13927 if (SDValue Val = performNegCMovCombine(N, DAG&: DCI.DAG))
13928 return Val;
13929
13930 if (!Subtarget->hasMVEIntegerOps() || !N->getValueType(ResNo: 0).isVector())
13931 return SDValue();
13932
13933 // Fold (sub (ARMvmovImm 0), (ARMvdup x)) -> (ARMvdup (sub 0, x))
13934 // so that we can readily pattern match more mve instructions which can use
13935 // a scalar operand.
13936 SDValue VDup = N->getOperand(Num: 1);
13937 if (VDup->getOpcode() != ARMISD::VDUP)
13938 return SDValue();
13939
13940 SDValue VMov = N->getOperand(Num: 0);
13941 if (VMov->getOpcode() == ISD::BITCAST)
13942 VMov = VMov->getOperand(Num: 0);
13943
13944 if (VMov->getOpcode() != ARMISD::VMOVIMM || !isZeroVector(N: VMov))
13945 return SDValue();
13946
13947 SDLoc dl(N);
13948 SDValue Negate = DCI.DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32,
13949 N1: DCI.DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32),
13950 N2: VDup->getOperand(Num: 0));
13951 return DCI.DAG.getNode(Opcode: ARMISD::VDUP, DL: dl, VT: N->getValueType(ResNo: 0), Operand: Negate);
13952}
13953
13954/// PerformVMULCombine
13955/// Distribute (A + B) * C to (A * C) + (B * C) to take advantage of the
13956/// special multiplier accumulator forwarding.
13957/// vmul d3, d0, d2
13958/// vmla d3, d1, d2
13959/// is faster than
13960/// vadd d3, d0, d1
13961/// vmul d3, d3, d2
13962// However, for (A + B) * (A + B),
13963// vadd d2, d0, d1
13964// vmul d3, d0, d2
13965// vmla d3, d1, d2
13966// is slower than
13967// vadd d2, d0, d1
13968// vmul d3, d2, d2
13969static SDValue PerformVMULCombine(SDNode *N,
13970 TargetLowering::DAGCombinerInfo &DCI,
13971 const ARMSubtarget *Subtarget) {
13972 if (!Subtarget->hasVMLxForwarding())
13973 return SDValue();
13974
13975 SelectionDAG &DAG = DCI.DAG;
13976 SDValue N0 = N->getOperand(Num: 0);
13977 SDValue N1 = N->getOperand(Num: 1);
13978 unsigned Opcode = N0.getOpcode();
13979 if (Opcode != ISD::ADD && Opcode != ISD::SUB &&
13980 Opcode != ISD::FADD && Opcode != ISD::FSUB) {
13981 Opcode = N1.getOpcode();
13982 if (Opcode != ISD::ADD && Opcode != ISD::SUB &&
13983 Opcode != ISD::FADD && Opcode != ISD::FSUB)
13984 return SDValue();
13985 std::swap(a&: N0, b&: N1);
13986 }
13987
13988 if (N0 == N1)
13989 return SDValue();
13990
13991 EVT VT = N->getValueType(ResNo: 0);
13992 SDLoc DL(N);
13993 SDValue N00 = N0->getOperand(Num: 0);
13994 SDValue N01 = N0->getOperand(Num: 1);
13995 return DAG.getNode(Opcode, DL, VT,
13996 N1: DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: N00, N2: N1),
13997 N2: DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: N01, N2: N1));
13998}
13999
14000static SDValue PerformMVEVMULLCombine(SDNode *N, SelectionDAG &DAG,
14001 const ARMSubtarget *Subtarget) {
14002 EVT VT = N->getValueType(ResNo: 0);
14003 if (VT != MVT::v2i64)
14004 return SDValue();
14005
14006 SDValue N0 = N->getOperand(Num: 0);
14007 SDValue N1 = N->getOperand(Num: 1);
14008
14009 auto IsSignExt = [&](SDValue Op) {
14010 if (Op->getOpcode() != ISD::SIGN_EXTEND_INREG)
14011 return SDValue();
14012 EVT VT = cast<VTSDNode>(Val: Op->getOperand(Num: 1))->getVT();
14013 if (VT.getScalarSizeInBits() == 32)
14014 return Op->getOperand(Num: 0);
14015 return SDValue();
14016 };
14017 auto IsZeroExt = [&](SDValue Op) {
14018 // Zero extends are a little more awkward. At the point we are matching
14019 // this, we are looking for an AND with a (-1, 0, -1, 0) buildvector mask.
14020 // That might be before of after a bitcast depending on how the and is
14021 // placed. Because this has to look through bitcasts, it is currently only
14022 // supported on LE.
14023 if (!Subtarget->isLittle())
14024 return SDValue();
14025
14026 SDValue And = Op;
14027 if (And->getOpcode() == ISD::BITCAST)
14028 And = And->getOperand(Num: 0);
14029 if (And->getOpcode() != ISD::AND)
14030 return SDValue();
14031 SDValue Mask = And->getOperand(Num: 1);
14032 if (Mask->getOpcode() == ISD::BITCAST)
14033 Mask = Mask->getOperand(Num: 0);
14034
14035 if (Mask->getOpcode() != ISD::BUILD_VECTOR ||
14036 Mask.getValueType() != MVT::v4i32)
14037 return SDValue();
14038 if (isAllOnesConstant(V: Mask->getOperand(Num: 0)) &&
14039 isNullConstant(V: Mask->getOperand(Num: 1)) &&
14040 isAllOnesConstant(V: Mask->getOperand(Num: 2)) &&
14041 isNullConstant(V: Mask->getOperand(Num: 3)))
14042 return And->getOperand(Num: 0);
14043 return SDValue();
14044 };
14045
14046 SDLoc dl(N);
14047 if (SDValue Op0 = IsSignExt(N0)) {
14048 if (SDValue Op1 = IsSignExt(N1)) {
14049 SDValue New0a = DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT: MVT::v4i32, Operand: Op0);
14050 SDValue New1a = DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT: MVT::v4i32, Operand: Op1);
14051 return DAG.getNode(Opcode: ARMISD::VMULLs, DL: dl, VT, N1: New0a, N2: New1a);
14052 }
14053 }
14054 if (SDValue Op0 = IsZeroExt(N0)) {
14055 if (SDValue Op1 = IsZeroExt(N1)) {
14056 SDValue New0a = DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT: MVT::v4i32, Operand: Op0);
14057 SDValue New1a = DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT: MVT::v4i32, Operand: Op1);
14058 return DAG.getNode(Opcode: ARMISD::VMULLu, DL: dl, VT, N1: New0a, N2: New1a);
14059 }
14060 }
14061
14062 return SDValue();
14063}
14064
14065static SDValue PerformMULCombine(SDNode *N,
14066 TargetLowering::DAGCombinerInfo &DCI,
14067 const ARMSubtarget *Subtarget) {
14068 SelectionDAG &DAG = DCI.DAG;
14069
14070 EVT VT = N->getValueType(ResNo: 0);
14071 if (Subtarget->hasMVEIntegerOps() && VT == MVT::v2i64)
14072 return PerformMVEVMULLCombine(N, DAG, Subtarget);
14073
14074 if (Subtarget->isThumb1Only())
14075 return SDValue();
14076
14077 if (DCI.isBeforeLegalize() || DCI.isCalledByLegalizer())
14078 return SDValue();
14079
14080 if (VT.is64BitVector() || VT.is128BitVector())
14081 return PerformVMULCombine(N, DCI, Subtarget);
14082 if (VT != MVT::i32)
14083 return SDValue();
14084
14085 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
14086 if (!C)
14087 return SDValue();
14088
14089 int64_t MulAmt = C->getSExtValue();
14090 unsigned ShiftAmt = llvm::countr_zero<uint64_t>(Val: MulAmt);
14091
14092 ShiftAmt = ShiftAmt & (32 - 1);
14093 SDValue V = N->getOperand(Num: 0);
14094 SDLoc DL(N);
14095
14096 SDValue Res;
14097 MulAmt >>= ShiftAmt;
14098
14099 if (MulAmt >= 0) {
14100 if (llvm::has_single_bit<uint32_t>(Value: MulAmt - 1)) {
14101 // (mul x, 2^N + 1) => (add (shl x, N), x)
14102 Res = DAG.getNode(Opcode: ISD::ADD, DL, VT,
14103 N1: V,
14104 N2: DAG.getNode(Opcode: ISD::SHL, DL, VT,
14105 N1: V,
14106 N2: DAG.getConstant(Val: Log2_32(Value: MulAmt - 1), DL,
14107 VT: MVT::i32)));
14108 } else if (llvm::has_single_bit<uint32_t>(Value: MulAmt + 1)) {
14109 // (mul x, 2^N - 1) => (sub (shl x, N), x)
14110 Res = DAG.getNode(Opcode: ISD::SUB, DL, VT,
14111 N1: DAG.getNode(Opcode: ISD::SHL, DL, VT,
14112 N1: V,
14113 N2: DAG.getConstant(Val: Log2_32(Value: MulAmt + 1), DL,
14114 VT: MVT::i32)),
14115 N2: V);
14116 } else
14117 return SDValue();
14118 } else {
14119 uint64_t MulAmtAbs = -MulAmt;
14120 if (llvm::has_single_bit<uint32_t>(Value: MulAmtAbs + 1)) {
14121 // (mul x, -(2^N - 1)) => (sub x, (shl x, N))
14122 Res = DAG.getNode(Opcode: ISD::SUB, DL, VT,
14123 N1: V,
14124 N2: DAG.getNode(Opcode: ISD::SHL, DL, VT,
14125 N1: V,
14126 N2: DAG.getConstant(Val: Log2_32(Value: MulAmtAbs + 1), DL,
14127 VT: MVT::i32)));
14128 } else if (llvm::has_single_bit<uint32_t>(Value: MulAmtAbs - 1)) {
14129 // (mul x, -(2^N + 1)) => - (add (shl x, N), x)
14130 Res = DAG.getNode(Opcode: ISD::ADD, DL, VT,
14131 N1: V,
14132 N2: DAG.getNode(Opcode: ISD::SHL, DL, VT,
14133 N1: V,
14134 N2: DAG.getConstant(Val: Log2_32(Value: MulAmtAbs - 1), DL,
14135 VT: MVT::i32)));
14136 Res = DAG.getNode(Opcode: ISD::SUB, DL, VT,
14137 N1: DAG.getConstant(Val: 0, DL, VT: MVT::i32), N2: Res);
14138 } else
14139 return SDValue();
14140 }
14141
14142 if (ShiftAmt != 0)
14143 Res = DAG.getNode(Opcode: ISD::SHL, DL, VT,
14144 N1: Res, N2: DAG.getConstant(Val: ShiftAmt, DL, VT: MVT::i32));
14145
14146 // Do not add new nodes to DAG combiner worklist.
14147 DCI.CombineTo(N, Res, AddTo: false);
14148 return SDValue();
14149}
14150
14151static SDValue CombineANDShift(SDNode *N,
14152 TargetLowering::DAGCombinerInfo &DCI,
14153 const ARMSubtarget *Subtarget) {
14154 // Allow DAGCombine to pattern-match before we touch the canonical form.
14155 if (DCI.isBeforeLegalize() || DCI.isCalledByLegalizer())
14156 return SDValue();
14157
14158 if (N->getValueType(ResNo: 0) != MVT::i32)
14159 return SDValue();
14160
14161 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
14162 if (!N1C)
14163 return SDValue();
14164
14165 uint32_t C1 = (uint32_t)N1C->getZExtValue();
14166 // Don't transform uxtb/uxth.
14167 if (C1 == 255 || C1 == 65535)
14168 return SDValue();
14169
14170 SDNode *N0 = N->getOperand(Num: 0).getNode();
14171 if (!N0->hasOneUse())
14172 return SDValue();
14173
14174 if (N0->getOpcode() != ISD::SHL && N0->getOpcode() != ISD::SRL)
14175 return SDValue();
14176
14177 bool LeftShift = N0->getOpcode() == ISD::SHL;
14178
14179 ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(Val: N0->getOperand(Num: 1));
14180 if (!N01C)
14181 return SDValue();
14182
14183 uint32_t C2 = (uint32_t)N01C->getZExtValue();
14184 if (!C2 || C2 >= 32)
14185 return SDValue();
14186
14187 // Clear irrelevant bits in the mask.
14188 if (LeftShift)
14189 C1 &= (-1U << C2);
14190 else
14191 C1 &= (-1U >> C2);
14192
14193 SelectionDAG &DAG = DCI.DAG;
14194 SDLoc DL(N);
14195
14196 // We have a pattern of the form "(and (shl x, c2) c1)" or
14197 // "(and (srl x, c2) c1)", where c1 is a shifted mask. Try to
14198 // transform to a pair of shifts, to save materializing c1.
14199
14200 // First pattern: right shift, then mask off leading bits.
14201 // FIXME: Use demanded bits?
14202 if (!LeftShift && isMask_32(Value: C1)) {
14203 uint32_t C3 = llvm::countl_zero(Val: C1);
14204 if (C2 < C3) {
14205 SDValue SHL = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i32, N1: N0->getOperand(Num: 0),
14206 N2: DAG.getConstant(Val: C3 - C2, DL, VT: MVT::i32));
14207 return DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: SHL,
14208 N2: DAG.getConstant(Val: C3, DL, VT: MVT::i32));
14209 }
14210 }
14211
14212 // First pattern, reversed: left shift, then mask off trailing bits.
14213 if (LeftShift && isMask_32(Value: ~C1)) {
14214 uint32_t C3 = llvm::countr_zero(Val: C1);
14215 if (C2 < C3) {
14216 SDValue SHL = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: N0->getOperand(Num: 0),
14217 N2: DAG.getConstant(Val: C3 - C2, DL, VT: MVT::i32));
14218 return DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i32, N1: SHL,
14219 N2: DAG.getConstant(Val: C3, DL, VT: MVT::i32));
14220 }
14221 }
14222
14223 // Second pattern: left shift, then mask off leading bits.
14224 // FIXME: Use demanded bits?
14225 if (LeftShift && isShiftedMask_32(Value: C1)) {
14226 uint32_t Trailing = llvm::countr_zero(Val: C1);
14227 uint32_t C3 = llvm::countl_zero(Val: C1);
14228 if (Trailing == C2 && C2 + C3 < 32) {
14229 SDValue SHL = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i32, N1: N0->getOperand(Num: 0),
14230 N2: DAG.getConstant(Val: C2 + C3, DL, VT: MVT::i32));
14231 return DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: SHL,
14232 N2: DAG.getConstant(Val: C3, DL, VT: MVT::i32));
14233 }
14234 }
14235
14236 // Second pattern, reversed: right shift, then mask off trailing bits.
14237 // FIXME: Handle other patterns of known/demanded bits.
14238 if (!LeftShift && isShiftedMask_32(Value: C1)) {
14239 uint32_t Leading = llvm::countl_zero(Val: C1);
14240 uint32_t C3 = llvm::countr_zero(Val: C1);
14241 if (Leading == C2 && C2 + C3 < 32) {
14242 SDValue SHL = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: N0->getOperand(Num: 0),
14243 N2: DAG.getConstant(Val: C2 + C3, DL, VT: MVT::i32));
14244 return DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i32, N1: SHL,
14245 N2: DAG.getConstant(Val: C3, DL, VT: MVT::i32));
14246 }
14247 }
14248
14249 // Transform "(and (shl x, c2) c1)" into "(shl (and x, c1>>c2), c2)"
14250 // if "c1 >> c2" is a cheaper immediate than "c1"
14251 if (LeftShift &&
14252 HasLowerConstantMaterializationCost(Val1: C1 >> C2, Val2: C1, Subtarget)) {
14253
14254 SDValue And = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: N0->getOperand(Num: 0),
14255 N2: DAG.getConstant(Val: C1 >> C2, DL, VT: MVT::i32));
14256 return DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i32, N1: And,
14257 N2: DAG.getConstant(Val: C2, DL, VT: MVT::i32));
14258 }
14259
14260 return SDValue();
14261}
14262
14263static SDValue PerformANDCombine(SDNode *N,
14264 TargetLowering::DAGCombinerInfo &DCI,
14265 const ARMSubtarget *Subtarget) {
14266 // Attempt to use immediate-form VBIC
14267 BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(Val: N->getOperand(Num: 1));
14268 SDLoc dl(N);
14269 EVT VT = N->getValueType(ResNo: 0);
14270 SelectionDAG &DAG = DCI.DAG;
14271
14272 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT) || VT == MVT::v2i1 ||
14273 VT == MVT::v4i1 || VT == MVT::v8i1 || VT == MVT::v16i1)
14274 return SDValue();
14275
14276 APInt SplatBits, SplatUndef;
14277 unsigned SplatBitSize;
14278 bool HasAnyUndefs;
14279 if (BVN && (Subtarget->hasNEON() || Subtarget->hasMVEIntegerOps()) &&
14280 BVN->isConstantSplat(SplatValue&: SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
14281 if (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32 ||
14282 SplatBitSize == 64) {
14283 EVT VbicVT;
14284 SDValue Val = isVMOVModifiedImm(SplatBits: (~SplatBits).getZExtValue(),
14285 SplatUndef: SplatUndef.getZExtValue(), SplatBitSize,
14286 DAG, dl, VT&: VbicVT, VectorVT: VT, type: OtherModImm);
14287 if (Val.getNode()) {
14288 SDValue Input =
14289 DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT: VbicVT, Operand: N->getOperand(Num: 0));
14290 SDValue Vbic = DAG.getNode(Opcode: ARMISD::VBICIMM, DL: dl, VT: VbicVT, N1: Input, N2: Val);
14291 return DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT, Operand: Vbic);
14292 }
14293 }
14294 }
14295
14296 if (!Subtarget->isThumb1Only()) {
14297 // fold (and (select cc, -1, c), x) -> (select cc, x, (and, x, c))
14298 if (SDValue Result = combineSelectAndUseCommutative(N, AllOnes: true, DCI))
14299 return Result;
14300
14301 if (SDValue Result = PerformSHLSimplify(N, DCI, ST: Subtarget))
14302 return Result;
14303 }
14304
14305 if (Subtarget->isThumb1Only())
14306 if (SDValue Result = CombineANDShift(N, DCI, Subtarget))
14307 return Result;
14308
14309 return SDValue();
14310}
14311
14312// Try combining OR nodes to SMULWB, SMULWT.
14313static SDValue PerformORCombineToSMULWBT(SDNode *OR,
14314 TargetLowering::DAGCombinerInfo &DCI,
14315 const ARMSubtarget *Subtarget) {
14316 if (!Subtarget->hasV6Ops() ||
14317 (Subtarget->isThumb() &&
14318 (!Subtarget->hasThumb2() || !Subtarget->hasDSP())))
14319 return SDValue();
14320
14321 SDValue SRL = OR->getOperand(Num: 0);
14322 SDValue SHL = OR->getOperand(Num: 1);
14323
14324 if (SRL.getOpcode() != ISD::SRL || SHL.getOpcode() != ISD::SHL) {
14325 SRL = OR->getOperand(Num: 1);
14326 SHL = OR->getOperand(Num: 0);
14327 }
14328 if (!isSRL16(Op: SRL) || !isSHL16(Op: SHL))
14329 return SDValue();
14330
14331 // The first operands to the shifts need to be the two results from the
14332 // same smul_lohi node.
14333 if ((SRL.getOperand(i: 0).getNode() != SHL.getOperand(i: 0).getNode()) ||
14334 SRL.getOperand(i: 0).getOpcode() != ISD::SMUL_LOHI)
14335 return SDValue();
14336
14337 SDNode *SMULLOHI = SRL.getOperand(i: 0).getNode();
14338 if (SRL.getOperand(i: 0) != SDValue(SMULLOHI, 0) ||
14339 SHL.getOperand(i: 0) != SDValue(SMULLOHI, 1))
14340 return SDValue();
14341
14342 // Now we have:
14343 // (or (srl (smul_lohi ?, ?), 16), (shl (smul_lohi ?, ?), 16)))
14344 // For SMUL[B|T] smul_lohi will take a 32-bit and a 16-bit arguments.
14345 // For SMUWB the 16-bit value will signed extended somehow.
14346 // For SMULWT only the SRA is required.
14347 // Check both sides of SMUL_LOHI
14348 SDValue OpS16 = SMULLOHI->getOperand(Num: 0);
14349 SDValue OpS32 = SMULLOHI->getOperand(Num: 1);
14350
14351 SelectionDAG &DAG = DCI.DAG;
14352 if (!isS16(Op: OpS16, DAG) && !isSRA16(Op: OpS16)) {
14353 OpS16 = OpS32;
14354 OpS32 = SMULLOHI->getOperand(Num: 0);
14355 }
14356
14357 SDLoc dl(OR);
14358 unsigned Opcode = 0;
14359 if (isS16(Op: OpS16, DAG))
14360 Opcode = ARMISD::SMULWB;
14361 else if (isSRA16(Op: OpS16)) {
14362 Opcode = ARMISD::SMULWT;
14363 OpS16 = OpS16->getOperand(Num: 0);
14364 }
14365 else
14366 return SDValue();
14367
14368 SDValue Res = DAG.getNode(Opcode, DL: dl, VT: MVT::i32, N1: OpS32, N2: OpS16);
14369 DAG.ReplaceAllUsesOfValueWith(From: SDValue(OR, 0), To: Res);
14370 return SDValue(OR, 0);
14371}
14372
14373static SDValue PerformORCombineToBFI(SDNode *N,
14374 TargetLowering::DAGCombinerInfo &DCI,
14375 const ARMSubtarget *Subtarget) {
14376 // BFI is only available on V6T2+
14377 if (Subtarget->isThumb1Only() || !Subtarget->hasV6T2Ops())
14378 return SDValue();
14379
14380 EVT VT = N->getValueType(ResNo: 0);
14381 SDValue N0 = N->getOperand(Num: 0);
14382 SDValue N1 = N->getOperand(Num: 1);
14383 SelectionDAG &DAG = DCI.DAG;
14384 SDLoc DL(N);
14385 // 1) or (and A, mask), val => ARMbfi A, val, mask
14386 // iff (val & mask) == val
14387 //
14388 // 2) or (and A, mask), (and B, mask2) => ARMbfi A, (lsr B, amt), mask
14389 // 2a) iff isBitFieldInvertedMask(mask) && isBitFieldInvertedMask(~mask2)
14390 // && mask == ~mask2
14391 // 2b) iff isBitFieldInvertedMask(~mask) && isBitFieldInvertedMask(mask2)
14392 // && ~mask == mask2
14393 // (i.e., copy a bitfield value into another bitfield of the same width)
14394
14395 if (VT != MVT::i32)
14396 return SDValue();
14397
14398 SDValue N00 = N0.getOperand(i: 0);
14399
14400 // The value and the mask need to be constants so we can verify this is
14401 // actually a bitfield set. If the mask is 0xffff, we can do better
14402 // via a movt instruction, so don't use BFI in that case.
14403 SDValue MaskOp = N0.getOperand(i: 1);
14404 ConstantSDNode *MaskC = dyn_cast<ConstantSDNode>(Val&: MaskOp);
14405 if (!MaskC)
14406 return SDValue();
14407 unsigned Mask = MaskC->getZExtValue();
14408 if (Mask == 0xffff)
14409 return SDValue();
14410 SDValue Res;
14411 // Case (1): or (and A, mask), val => ARMbfi A, val, mask
14412 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(Val&: N1);
14413 if (N1C) {
14414 unsigned Val = N1C->getZExtValue();
14415 if ((Val & ~Mask) != Val)
14416 return SDValue();
14417
14418 if (ARM::isBitFieldInvertedMask(v: Mask)) {
14419 Val >>= llvm::countr_zero(Val: ~Mask);
14420
14421 Res = DAG.getNode(Opcode: ARMISD::BFI, DL, VT, N1: N00,
14422 N2: DAG.getConstant(Val, DL, VT: MVT::i32),
14423 N3: DAG.getConstant(Val: Mask, DL, VT: MVT::i32));
14424
14425 DCI.CombineTo(N, Res, AddTo: false);
14426 // Return value from the original node to inform the combiner than N is
14427 // now dead.
14428 return SDValue(N, 0);
14429 }
14430 } else if (N1.getOpcode() == ISD::AND) {
14431 // case (2) or (and A, mask), (and B, mask2) => ARMbfi A, (lsr B, amt), mask
14432 ConstantSDNode *N11C = dyn_cast<ConstantSDNode>(Val: N1.getOperand(i: 1));
14433 if (!N11C)
14434 return SDValue();
14435 unsigned Mask2 = N11C->getZExtValue();
14436
14437 // Mask and ~Mask2 (or reverse) must be equivalent for the BFI pattern
14438 // as is to match.
14439 if (ARM::isBitFieldInvertedMask(v: Mask) &&
14440 (Mask == ~Mask2)) {
14441 // The pack halfword instruction works better for masks that fit it,
14442 // so use that when it's available.
14443 if (Subtarget->hasDSP() &&
14444 (Mask == 0xffff || Mask == 0xffff0000))
14445 return SDValue();
14446 // 2a
14447 unsigned amt = llvm::countr_zero(Val: Mask2);
14448 Res = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: N1.getOperand(i: 0),
14449 N2: DAG.getConstant(Val: amt, DL, VT: MVT::i32));
14450 Res = DAG.getNode(Opcode: ARMISD::BFI, DL, VT, N1: N00, N2: Res,
14451 N3: DAG.getConstant(Val: Mask, DL, VT: MVT::i32));
14452 DCI.CombineTo(N, Res, AddTo: false);
14453 // Return value from the original node to inform the combiner than N is
14454 // now dead.
14455 return SDValue(N, 0);
14456 } else if (ARM::isBitFieldInvertedMask(v: ~Mask) &&
14457 (~Mask == Mask2)) {
14458 // The pack halfword instruction works better for masks that fit it,
14459 // so use that when it's available.
14460 if (Subtarget->hasDSP() &&
14461 (Mask2 == 0xffff || Mask2 == 0xffff0000))
14462 return SDValue();
14463 // 2b
14464 unsigned lsb = llvm::countr_zero(Val: Mask);
14465 Res = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: N00,
14466 N2: DAG.getConstant(Val: lsb, DL, VT: MVT::i32));
14467 Res = DAG.getNode(Opcode: ARMISD::BFI, DL, VT, N1: N1.getOperand(i: 0), N2: Res,
14468 N3: DAG.getConstant(Val: Mask2, DL, VT: MVT::i32));
14469 DCI.CombineTo(N, Res, AddTo: false);
14470 // Return value from the original node to inform the combiner than N is
14471 // now dead.
14472 return SDValue(N, 0);
14473 }
14474 }
14475
14476 if (DAG.MaskedValueIsZero(Op: N1, Mask: MaskC->getAPIntValue()) &&
14477 N00.getOpcode() == ISD::SHL && isa<ConstantSDNode>(Val: N00.getOperand(i: 1)) &&
14478 ARM::isBitFieldInvertedMask(v: ~Mask)) {
14479 // Case (3): or (and (shl A, #shamt), mask), B => ARMbfi B, A, ~mask
14480 // where lsb(mask) == #shamt and masked bits of B are known zero.
14481 SDValue ShAmt = N00.getOperand(i: 1);
14482 unsigned ShAmtC = ShAmt->getAsZExtVal();
14483 unsigned LSB = llvm::countr_zero(Val: Mask);
14484 if (ShAmtC != LSB)
14485 return SDValue();
14486
14487 Res = DAG.getNode(Opcode: ARMISD::BFI, DL, VT, N1, N2: N00.getOperand(i: 0),
14488 N3: DAG.getConstant(Val: ~Mask, DL, VT: MVT::i32));
14489
14490 DCI.CombineTo(N, Res, AddTo: false);
14491 // Return value from the original node to inform the combiner than N is
14492 // now dead.
14493 return SDValue(N, 0);
14494 }
14495
14496 return SDValue();
14497}
14498
14499static bool isValidMVECond(unsigned CC, bool IsFloat) {
14500 switch (CC) {
14501 case ARMCC::EQ:
14502 case ARMCC::NE:
14503 case ARMCC::LE:
14504 case ARMCC::GT:
14505 case ARMCC::GE:
14506 case ARMCC::LT:
14507 return true;
14508 case ARMCC::HS:
14509 case ARMCC::HI:
14510 return !IsFloat;
14511 default:
14512 return false;
14513 };
14514}
14515
14516static ARMCC::CondCodes getVCMPCondCode(SDValue N) {
14517 if (N->getOpcode() == ARMISD::VCMP)
14518 return (ARMCC::CondCodes)N->getConstantOperandVal(Num: 2);
14519 else if (N->getOpcode() == ARMISD::VCMPZ)
14520 return (ARMCC::CondCodes)N->getConstantOperandVal(Num: 1);
14521 else
14522 llvm_unreachable("Not a VCMP/VCMPZ!");
14523}
14524
14525static bool CanInvertMVEVCMP(SDValue N) {
14526 ARMCC::CondCodes CC = ARMCC::getOppositeCondition(CC: getVCMPCondCode(N));
14527 return isValidMVECond(CC, IsFloat: N->getOperand(Num: 0).getValueType().isFloatingPoint());
14528}
14529
14530static SDValue PerformORCombine_i1(SDNode *N, SelectionDAG &DAG,
14531 const ARMSubtarget *Subtarget) {
14532 // Try to invert "or A, B" -> "and ~A, ~B", as the "and" is easier to chain
14533 // together with predicates
14534 EVT VT = N->getValueType(ResNo: 0);
14535 SDLoc DL(N);
14536 SDValue N0 = N->getOperand(Num: 0);
14537 SDValue N1 = N->getOperand(Num: 1);
14538
14539 auto IsFreelyInvertable = [&](SDValue V) {
14540 if (V->getOpcode() == ARMISD::VCMP || V->getOpcode() == ARMISD::VCMPZ)
14541 return CanInvertMVEVCMP(N: V);
14542 return false;
14543 };
14544
14545 // At least one operand must be freely invertable.
14546 if (!(IsFreelyInvertable(N0) || IsFreelyInvertable(N1)))
14547 return SDValue();
14548
14549 SDValue NewN0 = DAG.getLogicalNOT(DL, Val: N0, VT);
14550 SDValue NewN1 = DAG.getLogicalNOT(DL, Val: N1, VT);
14551 SDValue And = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: NewN0, N2: NewN1);
14552 return DAG.getLogicalNOT(DL, Val: And, VT);
14553}
14554
14555// Try to form a NEON shift-{right, left}-and-insert (VSRI/VSLI) from:
14556// (or (and X, splat (i32 C1)), (srl Y, splat (i32 C2))) -> VSRI X, Y, #C2
14557// (or (and X, splat (i32 C1)), (shl Y, splat (i32 C2))) -> VSLI X, Y, #C2
14558// where C1 is a mask that preserves the bits not written by the shift/insert,
14559// i.e. `C1 == (1 << C2) - 1`.
14560static SDValue PerformORCombineToShiftInsert(SelectionDAG &DAG, SDValue AndOp,
14561 SDValue ShiftOp, EVT VT,
14562 SDLoc dl) {
14563 // Match (and X, Mask)
14564 if (AndOp.getOpcode() != ISD::AND)
14565 return SDValue();
14566
14567 SDValue X = AndOp.getOperand(i: 0);
14568 SDValue Mask = AndOp.getOperand(i: 1);
14569
14570 ConstantSDNode *MaskC = isConstOrConstSplat(N: Mask, AllowUndefs: false, AllowTruncation: true);
14571 if (!MaskC)
14572 return SDValue();
14573 APInt MaskBits =
14574 MaskC->getAPIntValue().trunc(width: Mask.getScalarValueSizeInBits());
14575
14576 // Match shift (srl/shl Y, CntVec)
14577 int64_t Cnt = 0;
14578 bool IsShiftRight = false;
14579 SDValue Y;
14580
14581 if (ShiftOp.getOpcode() == ARMISD::VSHRuIMM) {
14582 IsShiftRight = true;
14583 Y = ShiftOp.getOperand(i: 0);
14584 Cnt = ShiftOp.getConstantOperandVal(i: 1);
14585 } else if (ShiftOp.getOpcode() == ARMISD::VSHLIMM) {
14586 Y = ShiftOp.getOperand(i: 0);
14587 Cnt = ShiftOp.getConstantOperandVal(i: 1);
14588 } else {
14589 return SDValue();
14590 }
14591
14592 unsigned ElemBits = VT.getScalarSizeInBits();
14593 APInt RequiredMask = IsShiftRight
14594 ? APInt::getHighBitsSet(numBits: ElemBits, hiBitsSet: (unsigned)Cnt)
14595 : APInt::getLowBitsSet(numBits: ElemBits, loBitsSet: (unsigned)Cnt);
14596 if (MaskBits != RequiredMask)
14597 return SDValue();
14598
14599 unsigned Opc = IsShiftRight ? ARMISD::VSRIIMM : ARMISD::VSLIIMM;
14600 return DAG.getNode(Opcode: Opc, DL: dl, VT, N1: X, N2: Y, N3: DAG.getConstant(Val: Cnt, DL: dl, VT: MVT::i32));
14601}
14602
14603/// PerformORCombine - Target-specific dag combine xforms for ISD::OR
14604static SDValue PerformORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
14605 const ARMSubtarget *Subtarget) {
14606 // Attempt to use immediate-form VORR
14607 BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(Val: N->getOperand(Num: 1));
14608 SDLoc dl(N);
14609 EVT VT = N->getValueType(ResNo: 0);
14610 SelectionDAG &DAG = DCI.DAG;
14611
14612 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
14613 return SDValue();
14614
14615 if (Subtarget->hasMVEIntegerOps() && (VT == MVT::v2i1 || VT == MVT::v4i1 ||
14616 VT == MVT::v8i1 || VT == MVT::v16i1))
14617 return PerformORCombine_i1(N, DAG, Subtarget);
14618
14619 APInt SplatBits, SplatUndef;
14620 unsigned SplatBitSize;
14621 bool HasAnyUndefs;
14622 if (BVN && (Subtarget->hasNEON() || Subtarget->hasMVEIntegerOps()) &&
14623 BVN->isConstantSplat(SplatValue&: SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
14624 if (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32 ||
14625 SplatBitSize == 64) {
14626 EVT VorrVT;
14627 SDValue Val =
14628 isVMOVModifiedImm(SplatBits: SplatBits.getZExtValue(), SplatUndef: SplatUndef.getZExtValue(),
14629 SplatBitSize, DAG, dl, VT&: VorrVT, VectorVT: VT, type: OtherModImm);
14630 if (Val.getNode()) {
14631 SDValue Input =
14632 DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT: VorrVT, Operand: N->getOperand(Num: 0));
14633 SDValue Vorr = DAG.getNode(Opcode: ARMISD::VORRIMM, DL: dl, VT: VorrVT, N1: Input, N2: Val);
14634 return DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT, Operand: Vorr);
14635 }
14636 }
14637 }
14638
14639 if (!Subtarget->isThumb1Only()) {
14640 // fold (or (select cc, 0, c), x) -> (select cc, x, (or, x, c))
14641 if (SDValue Result = combineSelectAndUseCommutative(N, AllOnes: false, DCI))
14642 return Result;
14643 if (SDValue Result = PerformORCombineToSMULWBT(OR: N, DCI, Subtarget))
14644 return Result;
14645 }
14646
14647 SDValue N0 = N->getOperand(Num: 0);
14648 SDValue N1 = N->getOperand(Num: 1);
14649
14650 // (or (and X, C1), (srl Y, C2)) -> VSRI X, Y, #C2
14651 // (or (and X, C1), (shl Y, C2)) -> VSLI X, Y, #C2
14652 if (VT.isVector() &&
14653 ((Subtarget->hasNEON() && DAG.getTargetLoweringInfo().isTypeLegal(VT)) ||
14654 (Subtarget->hasMVEIntegerOps() &&
14655 (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32)))) {
14656 if (SDValue ShiftInsert =
14657 PerformORCombineToShiftInsert(DAG, AndOp: N0, ShiftOp: N1, VT, dl))
14658 return ShiftInsert;
14659
14660 if (SDValue ShiftInsert =
14661 PerformORCombineToShiftInsert(DAG, AndOp: N1, ShiftOp: N0, VT, dl))
14662 return ShiftInsert;
14663 }
14664
14665 // (or (and B, A), (and C, ~A)) => (VBSL A, B, C) when A is a constant.
14666 if (Subtarget->hasNEON() && N1.getOpcode() == ISD::AND && VT.isVector() &&
14667 DAG.getTargetLoweringInfo().isTypeLegal(VT)) {
14668
14669 // The code below optimizes (or (and X, Y), Z).
14670 // The AND operand needs to have a single user to make these optimizations
14671 // profitable.
14672 if (N0.getOpcode() != ISD::AND || !N0.hasOneUse())
14673 return SDValue();
14674
14675 APInt SplatUndef;
14676 unsigned SplatBitSize;
14677 bool HasAnyUndefs;
14678
14679 APInt SplatBits0, SplatBits1;
14680 BuildVectorSDNode *BVN0 = dyn_cast<BuildVectorSDNode>(Val: N0->getOperand(Num: 1));
14681 BuildVectorSDNode *BVN1 = dyn_cast<BuildVectorSDNode>(Val: N1->getOperand(Num: 1));
14682 // Ensure that the second operand of both ands are constants
14683 if (BVN0 && BVN0->isConstantSplat(SplatValue&: SplatBits0, SplatUndef, SplatBitSize,
14684 HasAnyUndefs) && !HasAnyUndefs) {
14685 if (BVN1 && BVN1->isConstantSplat(SplatValue&: SplatBits1, SplatUndef, SplatBitSize,
14686 HasAnyUndefs) && !HasAnyUndefs) {
14687 // Ensure that the bit width of the constants are the same and that
14688 // the splat arguments are logical inverses as per the pattern we
14689 // are trying to simplify.
14690 if (SplatBits0.getBitWidth() == SplatBits1.getBitWidth() &&
14691 SplatBits0 == ~SplatBits1) {
14692 // Canonicalize the vector type to make instruction selection
14693 // simpler.
14694 EVT CanonicalVT = VT.is128BitVector() ? MVT::v4i32 : MVT::v2i32;
14695 SDValue Mask = DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl,
14696 VT: CanonicalVT, Operand: N0->getOperand(Num: 1));
14697 SDValue LHS = DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl,
14698 VT: CanonicalVT, Operand: N0->getOperand(Num: 0));
14699 SDValue RHS = DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl,
14700 VT: CanonicalVT, Operand: N1->getOperand(Num: 0));
14701 SDValue Result =
14702 DAG.getNode(Opcode: ARMISD::VBSP, DL: dl, VT: CanonicalVT, N1: Mask, N2: LHS, N3: RHS);
14703 return DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT, Operand: Result);
14704 }
14705 }
14706 }
14707 }
14708
14709 // Try to use the ARM/Thumb2 BFI (bitfield insert) instruction when
14710 // reasonable.
14711 if (N0.getOpcode() == ISD::AND && N0.hasOneUse()) {
14712 if (SDValue Res = PerformORCombineToBFI(N, DCI, Subtarget))
14713 return Res;
14714 }
14715
14716 if (SDValue Result = PerformSHLSimplify(N, DCI, ST: Subtarget))
14717 return Result;
14718
14719 // (or x, (csinc 0, 0, cc)) -> (csinc x, 0, cc)
14720 // providing that the x is 0 or 1.
14721 SDValue CSINC = N1;
14722 SDValue Other = N0;
14723 if (CSINC.getOpcode() != ARMISD::CSINC)
14724 std::swap(a&: CSINC, b&: Other);
14725 if (CSINC.getOpcode() == ARMISD::CSINC &&
14726 isNullConstant(V: CSINC.getOperand(i: 0)) &&
14727 isNullConstant(V: CSINC.getOperand(i: 1)) &&
14728 DAG.MaskedValueIsZero(Op: Other, Mask: APInt::getHighBitsSet(numBits: 32, hiBitsSet: 31)))
14729 return DAG.getNode(Opcode: ARMISD::CSINC, DL: dl, VT, N1: Other, N2: CSINC.getOperand(i: 1),
14730 N3: CSINC.getOperand(i: 2), N4: CSINC.getOperand(i: 3));
14731
14732 return SDValue();
14733}
14734
14735static SDValue PerformXORCombine(SDNode *N,
14736 TargetLowering::DAGCombinerInfo &DCI,
14737 const ARMSubtarget *Subtarget) {
14738 EVT VT = N->getValueType(ResNo: 0);
14739 SelectionDAG &DAG = DCI.DAG;
14740
14741 if(!DAG.getTargetLoweringInfo().isTypeLegal(VT))
14742 return SDValue();
14743
14744 if (!Subtarget->isThumb1Only()) {
14745 // fold (xor (select cc, 0, c), x) -> (select cc, x, (xor, x, c))
14746 if (SDValue Result = combineSelectAndUseCommutative(N, AllOnes: false, DCI))
14747 return Result;
14748
14749 if (SDValue Result = PerformSHLSimplify(N, DCI, ST: Subtarget))
14750 return Result;
14751 }
14752
14753 if (Subtarget->hasMVEIntegerOps()) {
14754 // fold (xor(vcmp/z, 1)) into a vcmp with the opposite condition.
14755 SDValue N0 = N->getOperand(Num: 0);
14756 SDValue N1 = N->getOperand(Num: 1);
14757 const TargetLowering *TLI = Subtarget->getTargetLowering();
14758 if (TLI->isConstTrueVal(N: N1) &&
14759 (N0->getOpcode() == ARMISD::VCMP || N0->getOpcode() == ARMISD::VCMPZ)) {
14760 if (CanInvertMVEVCMP(N: N0)) {
14761 SDLoc DL(N0);
14762 ARMCC::CondCodes CC = ARMCC::getOppositeCondition(CC: getVCMPCondCode(N: N0));
14763
14764 SmallVector<SDValue, 4> Ops;
14765 Ops.push_back(Elt: N0->getOperand(Num: 0));
14766 if (N0->getOpcode() == ARMISD::VCMP)
14767 Ops.push_back(Elt: N0->getOperand(Num: 1));
14768 Ops.push_back(Elt: DAG.getConstant(Val: CC, DL, VT: MVT::i32));
14769 return DAG.getNode(Opcode: N0->getOpcode(), DL, VT: N0->getValueType(ResNo: 0), Ops);
14770 }
14771 }
14772 }
14773
14774 return SDValue();
14775}
14776
14777// ParseBFI - given a BFI instruction in N, extract the "from" value (Rn) and return it,
14778// and fill in FromMask and ToMask with (consecutive) bits in "from" to be extracted and
14779// their position in "to" (Rd).
14780static SDValue ParseBFI(SDNode *N, APInt &ToMask, APInt &FromMask) {
14781 assert(N->getOpcode() == ARMISD::BFI);
14782
14783 SDValue From = N->getOperand(Num: 1);
14784 ToMask = ~N->getConstantOperandAPInt(Num: 2);
14785 FromMask = APInt::getLowBitsSet(numBits: ToMask.getBitWidth(), loBitsSet: ToMask.popcount());
14786
14787 // If the Base came from a SHR #C, we can deduce that it is really testing bit
14788 // #C in the base of the SHR.
14789 if (From->getOpcode() == ISD::SRL &&
14790 isa<ConstantSDNode>(Val: From->getOperand(Num: 1))) {
14791 APInt Shift = From->getConstantOperandAPInt(Num: 1);
14792 assert(Shift.getLimitedValue() < 32 && "Shift too large!");
14793 FromMask <<= Shift.getLimitedValue(Limit: 31);
14794 From = From->getOperand(Num: 0);
14795 }
14796
14797 return From;
14798}
14799
14800// If A and B contain one contiguous set of bits, does A | B == A . B?
14801//
14802// Neither A nor B must be zero.
14803static bool BitsProperlyConcatenate(const APInt &A, const APInt &B) {
14804 unsigned LastActiveBitInA = A.countr_zero();
14805 unsigned FirstActiveBitInB = B.getBitWidth() - B.countl_zero() - 1;
14806 return LastActiveBitInA - 1 == FirstActiveBitInB;
14807}
14808
14809static SDValue FindBFIToCombineWith(SDNode *N) {
14810 // We have a BFI in N. Find a BFI it can combine with, if one exists.
14811 APInt ToMask, FromMask;
14812 SDValue From = ParseBFI(N, ToMask, FromMask);
14813 SDValue To = N->getOperand(Num: 0);
14814
14815 SDValue V = To;
14816 if (V.getOpcode() != ARMISD::BFI)
14817 return SDValue();
14818
14819 APInt NewToMask, NewFromMask;
14820 SDValue NewFrom = ParseBFI(N: V.getNode(), ToMask&: NewToMask, FromMask&: NewFromMask);
14821 if (NewFrom != From)
14822 return SDValue();
14823
14824 // Do the written bits conflict with any we've seen so far?
14825 if ((NewToMask & ToMask).getBoolValue())
14826 // Conflicting bits.
14827 return SDValue();
14828
14829 // Are the new bits contiguous when combined with the old bits?
14830 if (BitsProperlyConcatenate(A: ToMask, B: NewToMask) &&
14831 BitsProperlyConcatenate(A: FromMask, B: NewFromMask))
14832 return V;
14833 if (BitsProperlyConcatenate(A: NewToMask, B: ToMask) &&
14834 BitsProperlyConcatenate(A: NewFromMask, B: FromMask))
14835 return V;
14836
14837 return SDValue();
14838}
14839
14840static SDValue PerformBFICombine(SDNode *N, SelectionDAG &DAG) {
14841 SDValue N0 = N->getOperand(Num: 0);
14842 SDValue N1 = N->getOperand(Num: 1);
14843
14844 if (N1.getOpcode() == ISD::AND) {
14845 // (bfi A, (and B, Mask1), Mask2) -> (bfi A, B, Mask2) iff
14846 // the bits being cleared by the AND are not demanded by the BFI.
14847 ConstantSDNode *N11C = dyn_cast<ConstantSDNode>(Val: N1.getOperand(i: 1));
14848 if (!N11C)
14849 return SDValue();
14850 unsigned InvMask = N->getConstantOperandVal(Num: 2);
14851 unsigned LSB = llvm::countr_zero(Val: ~InvMask);
14852 unsigned Width = llvm::bit_width<unsigned>(Value: ~InvMask) - LSB;
14853 assert(Width <
14854 static_cast<unsigned>(std::numeric_limits<unsigned>::digits) &&
14855 "undefined behavior");
14856 unsigned Mask = (1u << Width) - 1;
14857 unsigned Mask2 = N11C->getZExtValue();
14858 if ((Mask & (~Mask2)) == 0)
14859 return DAG.getNode(Opcode: ARMISD::BFI, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
14860 N1: N->getOperand(Num: 0), N2: N1.getOperand(i: 0), N3: N->getOperand(Num: 2));
14861 return SDValue();
14862 }
14863
14864 // Look for another BFI to combine with.
14865 if (SDValue CombineBFI = FindBFIToCombineWith(N)) {
14866 // We've found a BFI.
14867 APInt ToMask1, FromMask1;
14868 SDValue From1 = ParseBFI(N, ToMask&: ToMask1, FromMask&: FromMask1);
14869
14870 APInt ToMask2, FromMask2;
14871 SDValue From2 = ParseBFI(N: CombineBFI.getNode(), ToMask&: ToMask2, FromMask&: FromMask2);
14872 assert(From1 == From2);
14873 (void)From2;
14874
14875 // Create a new BFI, combining the two together.
14876 APInt NewFromMask = FromMask1 | FromMask2;
14877 APInt NewToMask = ToMask1 | ToMask2;
14878
14879 EVT VT = N->getValueType(ResNo: 0);
14880 SDLoc dl(N);
14881
14882 if (NewFromMask[0] == 0)
14883 From1 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: From1,
14884 N2: DAG.getConstant(Val: NewFromMask.countr_zero(), DL: dl, VT));
14885 return DAG.getNode(Opcode: ARMISD::BFI, DL: dl, VT, N1: CombineBFI.getOperand(i: 0), N2: From1,
14886 N3: DAG.getConstant(Val: ~NewToMask, DL: dl, VT));
14887 }
14888
14889 // Reassociate BFI(BFI (A, B, M1), C, M2) to BFI(BFI (A, C, M2), B, M1) so
14890 // that lower bit insertions are performed first, providing that M1 and M2
14891 // do no overlap. This can allow multiple BFI instructions to be combined
14892 // together by the other folds above.
14893 if (N->getOperand(Num: 0).getOpcode() == ARMISD::BFI) {
14894 APInt ToMask1 = ~N->getConstantOperandAPInt(Num: 2);
14895 APInt ToMask2 = ~N0.getConstantOperandAPInt(i: 2);
14896
14897 if (!N0.hasOneUse() || (ToMask1 & ToMask2) != 0 ||
14898 ToMask1.countl_zero() < ToMask2.countl_zero())
14899 return SDValue();
14900
14901 EVT VT = N->getValueType(ResNo: 0);
14902 SDLoc dl(N);
14903 SDValue BFI1 = DAG.getNode(Opcode: ARMISD::BFI, DL: dl, VT, N1: N0.getOperand(i: 0),
14904 N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2));
14905 return DAG.getNode(Opcode: ARMISD::BFI, DL: dl, VT, N1: BFI1, N2: N0.getOperand(i: 1),
14906 N3: N0.getOperand(i: 2));
14907 }
14908
14909 return SDValue();
14910}
14911
14912// Check that N is CMPZ(CSINC(0, 0, CC, X)),
14913// or CMPZ(CMOV(1, 0, CC, X))
14914// return X if valid.
14915static SDValue IsCMPZCSINC(SDNode *Cmp, ARMCC::CondCodes &CC) {
14916 if (Cmp->getOpcode() != ARMISD::CMPZ || !isNullConstant(V: Cmp->getOperand(Num: 1)))
14917 return SDValue();
14918 SDValue CSInc = Cmp->getOperand(Num: 0);
14919
14920 // Ignore any `And 1` nodes that may not yet have been removed. We are
14921 // looking for a value that produces 1/0, so these have no effect on the
14922 // code.
14923 while (CSInc.getOpcode() == ISD::AND &&
14924 isa<ConstantSDNode>(Val: CSInc.getOperand(i: 1)) &&
14925 CSInc.getConstantOperandVal(i: 1) == 1 && CSInc->hasOneUse())
14926 CSInc = CSInc.getOperand(i: 0);
14927
14928 if (CSInc.getOpcode() == ARMISD::CSINC &&
14929 isNullConstant(V: CSInc.getOperand(i: 0)) &&
14930 isNullConstant(V: CSInc.getOperand(i: 1)) && CSInc->hasOneUse()) {
14931 CC = (ARMCC::CondCodes)CSInc.getConstantOperandVal(i: 2);
14932 return CSInc.getOperand(i: 3);
14933 }
14934 if (CSInc.getOpcode() == ARMISD::CMOV && isOneConstant(V: CSInc.getOperand(i: 0)) &&
14935 isNullConstant(V: CSInc.getOperand(i: 1)) && CSInc->hasOneUse()) {
14936 CC = (ARMCC::CondCodes)CSInc.getConstantOperandVal(i: 2);
14937 return CSInc.getOperand(i: 3);
14938 }
14939 if (CSInc.getOpcode() == ARMISD::CMOV && isOneConstant(V: CSInc.getOperand(i: 1)) &&
14940 isNullConstant(V: CSInc.getOperand(i: 0)) && CSInc->hasOneUse()) {
14941 CC = ARMCC::getOppositeCondition(
14942 CC: (ARMCC::CondCodes)CSInc.getConstantOperandVal(i: 2));
14943 return CSInc.getOperand(i: 3);
14944 }
14945 return SDValue();
14946}
14947
14948static SDValue PerformCMPZCombine(SDNode *N, SelectionDAG &DAG) {
14949 // Given CMPZ(CSINC(C, 0, 0, EQ), 0), we can just use C directly. As in
14950 // t92: flags = ARMISD::CMPZ t74, 0
14951 // t93: i32 = ARMISD::CSINC 0, 0, 1, t92
14952 // t96: flags = ARMISD::CMPZ t93, 0
14953 // t114: i32 = ARMISD::CSINV 0, 0, 0, t96
14954 ARMCC::CondCodes Cond;
14955 if (SDValue C = IsCMPZCSINC(Cmp: N, CC&: Cond))
14956 if (Cond == ARMCC::EQ)
14957 return C;
14958 return SDValue();
14959}
14960
14961static SDValue PerformCSETCombine(SDNode *N, SelectionDAG &DAG) {
14962 // Fold away an unnecessary CMPZ/CSINC
14963 // CSXYZ A, B, C1 (CMPZ (CSINC 0, 0, C2, D), 0) ->
14964 // if C1==EQ -> CSXYZ A, B, C2, D
14965 // if C1==NE -> CSXYZ A, B, NOT(C2), D
14966 ARMCC::CondCodes Cond;
14967 if (SDValue C = IsCMPZCSINC(Cmp: N->getOperand(Num: 3).getNode(), CC&: Cond)) {
14968 if (N->getConstantOperandVal(Num: 2) == ARMCC::EQ)
14969 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: MVT::i32, N1: N->getOperand(Num: 0),
14970 N2: N->getOperand(Num: 1),
14971 N3: DAG.getConstant(Val: Cond, DL: SDLoc(N), VT: MVT::i32), N4: C);
14972 if (N->getConstantOperandVal(Num: 2) == ARMCC::NE)
14973 return DAG.getNode(
14974 Opcode: N->getOpcode(), DL: SDLoc(N), VT: MVT::i32, N1: N->getOperand(Num: 0),
14975 N2: N->getOperand(Num: 1),
14976 N3: DAG.getConstant(Val: ARMCC::getOppositeCondition(CC: Cond), DL: SDLoc(N), VT: MVT::i32), N4: C);
14977 }
14978 return SDValue();
14979}
14980
14981/// PerformVMOVRRDCombine - Target-specific dag combine xforms for
14982/// ARMISD::VMOVRRD.
14983static SDValue PerformVMOVRRDCombine(SDNode *N,
14984 TargetLowering::DAGCombinerInfo &DCI,
14985 const ARMSubtarget *Subtarget) {
14986 // vmovrrd(vmovdrr x, y) -> x,y
14987 SDValue InDouble = N->getOperand(Num: 0);
14988 if (InDouble.getOpcode() == ARMISD::VMOVDRR && Subtarget->hasFP64())
14989 return DCI.CombineTo(N, Res0: InDouble.getOperand(i: 0), Res1: InDouble.getOperand(i: 1));
14990
14991 // vmovrrd(load f64) -> (load i32), (load i32)
14992 SDNode *InNode = InDouble.getNode();
14993 if (ISD::isNormalLoad(N: InNode) && InNode->hasOneUse() &&
14994 InNode->getValueType(ResNo: 0) == MVT::f64 &&
14995 InNode->getOperand(Num: 1).getOpcode() == ISD::FrameIndex &&
14996 !cast<LoadSDNode>(Val: InNode)->isVolatile()) {
14997 // TODO: Should this be done for non-FrameIndex operands?
14998 LoadSDNode *LD = cast<LoadSDNode>(Val: InNode);
14999
15000 SelectionDAG &DAG = DCI.DAG;
15001 SDLoc DL(LD);
15002 SDValue BasePtr = LD->getBasePtr();
15003 SDValue NewLD1 =
15004 DAG.getLoad(VT: MVT::i32, dl: DL, Chain: LD->getChain(), Ptr: BasePtr, PtrInfo: LD->getPointerInfo(),
15005 Alignment: LD->getAlign(), MMOFlags: LD->getMemOperand()->getFlags());
15006
15007 SDValue OffsetPtr = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i32, N1: BasePtr,
15008 N2: DAG.getConstant(Val: 4, DL, VT: MVT::i32));
15009
15010 SDValue NewLD2 = DAG.getLoad(VT: MVT::i32, dl: DL, Chain: LD->getChain(), Ptr: OffsetPtr,
15011 PtrInfo: LD->getPointerInfo().getWithOffset(O: 4),
15012 Alignment: commonAlignment(A: LD->getAlign(), Offset: 4),
15013 MMOFlags: LD->getMemOperand()->getFlags());
15014
15015 DAG.ReplaceAllUsesOfValueWith(From: SDValue(LD, 1), To: NewLD2.getValue(R: 1));
15016 if (DCI.DAG.getDataLayout().isBigEndian())
15017 std::swap (a&: NewLD1, b&: NewLD2);
15018 SDValue Result = DCI.CombineTo(N, Res0: NewLD1, Res1: NewLD2);
15019 return Result;
15020 }
15021
15022 // VMOVRRD(extract(..(build_vector(a, b, c, d)))) -> a,b or c,d
15023 // VMOVRRD(extract(insert_vector(insert_vector(.., a, l1), b, l2))) -> a,b
15024 if (InDouble.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
15025 isa<ConstantSDNode>(Val: InDouble.getOperand(i: 1))) {
15026 SDValue BV = InDouble.getOperand(i: 0);
15027 // Look up through any nop bitcasts and vector_reg_casts. bitcasts may
15028 // change lane order under big endian.
15029 bool BVSwap = BV.getOpcode() == ISD::BITCAST;
15030 while (
15031 (BV.getOpcode() == ISD::BITCAST ||
15032 BV.getOpcode() == ARMISD::VECTOR_REG_CAST) &&
15033 (BV.getValueType() == MVT::v2f64 || BV.getValueType() == MVT::v2i64)) {
15034 BVSwap = BV.getOpcode() == ISD::BITCAST;
15035 BV = BV.getOperand(i: 0);
15036 }
15037 if (BV.getValueType() != MVT::v4i32)
15038 return SDValue();
15039
15040 // Handle buildvectors, pulling out the correct lane depending on
15041 // endianness.
15042 unsigned Offset = InDouble.getConstantOperandVal(i: 1) == 1 ? 2 : 0;
15043 if (BV.getOpcode() == ISD::BUILD_VECTOR) {
15044 SDValue Op0 = BV.getOperand(i: Offset);
15045 SDValue Op1 = BV.getOperand(i: Offset + 1);
15046 if (!Subtarget->isLittle() && BVSwap)
15047 std::swap(a&: Op0, b&: Op1);
15048
15049 return DCI.DAG.getMergeValues(Ops: {Op0, Op1}, dl: SDLoc(N));
15050 }
15051
15052 // A chain of insert_vectors, grabbing the correct value of the chain of
15053 // inserts.
15054 SDValue Op0, Op1;
15055 while (BV.getOpcode() == ISD::INSERT_VECTOR_ELT) {
15056 if (isa<ConstantSDNode>(Val: BV.getOperand(i: 2))) {
15057 if (BV.getConstantOperandVal(i: 2) == Offset && !Op0)
15058 Op0 = BV.getOperand(i: 1);
15059 if (BV.getConstantOperandVal(i: 2) == Offset + 1 && !Op1)
15060 Op1 = BV.getOperand(i: 1);
15061 }
15062 BV = BV.getOperand(i: 0);
15063 }
15064 if (!Subtarget->isLittle() && BVSwap)
15065 std::swap(a&: Op0, b&: Op1);
15066 if (Op0 && Op1)
15067 return DCI.DAG.getMergeValues(Ops: {Op0, Op1}, dl: SDLoc(N));
15068 }
15069
15070 return SDValue();
15071}
15072
15073/// PerformVMOVDRRCombine - Target-specific dag combine xforms for
15074/// ARMISD::VMOVDRR. This is also used for BUILD_VECTORs with 2 operands.
15075static SDValue PerformVMOVDRRCombine(SDNode *N, SelectionDAG &DAG) {
15076 // N=vmovrrd(X); vmovdrr(N:0, N:1) -> bit_convert(X)
15077 SDValue Op0 = N->getOperand(Num: 0);
15078 SDValue Op1 = N->getOperand(Num: 1);
15079 if (Op0.getOpcode() == ISD::BITCAST)
15080 Op0 = Op0.getOperand(i: 0);
15081 if (Op1.getOpcode() == ISD::BITCAST)
15082 Op1 = Op1.getOperand(i: 0);
15083 if (Op0.getOpcode() == ARMISD::VMOVRRD &&
15084 Op0.getNode() == Op1.getNode() &&
15085 Op0.getResNo() == 0 && Op1.getResNo() == 1)
15086 return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N),
15087 VT: N->getValueType(ResNo: 0), Operand: Op0.getOperand(i: 0));
15088 return SDValue();
15089}
15090
15091static SDValue PerformVMOVhrCombine(SDNode *N,
15092 TargetLowering::DAGCombinerInfo &DCI) {
15093 SDValue Op0 = N->getOperand(Num: 0);
15094
15095 // VMOVhr (VMOVrh (X)) -> X
15096 if (Op0->getOpcode() == ARMISD::VMOVrh)
15097 return Op0->getOperand(Num: 0);
15098
15099 // FullFP16: half values are passed in S-registers, and we don't
15100 // need any of the bitcast and moves:
15101 //
15102 // t2: f32,ch1,gl1? = CopyFromReg ch, Register:f32 %0, gl?
15103 // t5: i32 = bitcast t2
15104 // t18: f16 = ARMISD::VMOVhr t5
15105 // =>
15106 // tN: f16,ch2,gl2? = CopyFromReg ch, Register::f32 %0, gl?
15107 if (Op0->getOpcode() == ISD::BITCAST) {
15108 SDValue Copy = Op0->getOperand(Num: 0);
15109 if (Copy.getValueType() == MVT::f32 &&
15110 Copy->getOpcode() == ISD::CopyFromReg) {
15111 bool HasGlue = Copy->getNumOperands() == 3;
15112 SDValue Ops[] = {Copy->getOperand(Num: 0), Copy->getOperand(Num: 1),
15113 HasGlue ? Copy->getOperand(Num: 2) : SDValue()};
15114 EVT OutTys[] = {N->getValueType(ResNo: 0), MVT::Other, MVT::Glue};
15115 SDValue NewCopy =
15116 DCI.DAG.getNode(Opcode: ISD::CopyFromReg, DL: SDLoc(N),
15117 VTList: DCI.DAG.getVTList(VTs: ArrayRef(OutTys, HasGlue ? 3 : 2)),
15118 Ops: ArrayRef(Ops, HasGlue ? 3 : 2));
15119
15120 // Update Users, Chains, and Potential Glue.
15121 DCI.DAG.ReplaceAllUsesOfValueWith(From: SDValue(N, 0), To: NewCopy.getValue(R: 0));
15122 DCI.DAG.ReplaceAllUsesOfValueWith(From: Copy.getValue(R: 1), To: NewCopy.getValue(R: 1));
15123 if (HasGlue)
15124 DCI.DAG.ReplaceAllUsesOfValueWith(From: Copy.getValue(R: 2),
15125 To: NewCopy.getValue(R: 2));
15126
15127 return NewCopy;
15128 }
15129 }
15130
15131 // fold (VMOVhr (load x)) -> (load (f16*)x)
15132 if (LoadSDNode *LN0 = dyn_cast<LoadSDNode>(Val&: Op0)) {
15133 if (LN0->hasOneUse() && LN0->isUnindexed() &&
15134 LN0->getMemoryVT() == MVT::i16) {
15135 SDValue Load =
15136 DCI.DAG.getLoad(VT: N->getValueType(ResNo: 0), dl: SDLoc(N), Chain: LN0->getChain(),
15137 Ptr: LN0->getBasePtr(), MMO: LN0->getMemOperand());
15138 DCI.DAG.ReplaceAllUsesOfValueWith(From: SDValue(N, 0), To: Load.getValue(R: 0));
15139 DCI.DAG.ReplaceAllUsesOfValueWith(From: Op0.getValue(R: 1), To: Load.getValue(R: 1));
15140 return Load;
15141 }
15142 }
15143
15144 // Only the bottom 16 bits of the source register are used.
15145 APInt DemandedMask = APInt::getLowBitsSet(numBits: 32, loBitsSet: 16);
15146 const TargetLowering &TLI = DCI.DAG.getTargetLoweringInfo();
15147 if (TLI.SimplifyDemandedBits(Op: Op0, DemandedBits: DemandedMask, DCI))
15148 return SDValue(N, 0);
15149
15150 return SDValue();
15151}
15152
15153static SDValue PerformVMOVrhCombine(SDNode *N, SelectionDAG &DAG) {
15154 SDValue N0 = N->getOperand(Num: 0);
15155 EVT VT = N->getValueType(ResNo: 0);
15156
15157 // fold (VMOVrh (fpconst x)) -> const x
15158 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val&: N0)) {
15159 APFloat V = C->getValueAPF();
15160 return DAG.getConstant(Val: V.bitcastToAPInt().getZExtValue(), DL: SDLoc(N), VT);
15161 }
15162
15163 // fold (VMOVrh (load x)) -> (zextload (i16*)x)
15164 if (ISD::isNormalLoad(N: N0.getNode()) && N0.hasOneUse()) {
15165 LoadSDNode *LN0 = cast<LoadSDNode>(Val&: N0);
15166
15167 SDValue Load =
15168 DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl: SDLoc(N), VT, Chain: LN0->getChain(),
15169 Ptr: LN0->getBasePtr(), MemVT: MVT::i16, MMO: LN0->getMemOperand());
15170 DAG.ReplaceAllUsesOfValueWith(From: SDValue(N, 0), To: Load.getValue(R: 0));
15171 DAG.ReplaceAllUsesOfValueWith(From: N0.getValue(R: 1), To: Load.getValue(R: 1));
15172 return Load;
15173 }
15174
15175 // Fold VMOVrh(extract(x, n)) -> vgetlaneu(x, n)
15176 if (N0->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
15177 isa<ConstantSDNode>(Val: N0->getOperand(Num: 1)))
15178 return DAG.getNode(Opcode: ARMISD::VGETLANEu, DL: SDLoc(N), VT, N1: N0->getOperand(Num: 0),
15179 N2: N0->getOperand(Num: 1));
15180
15181 return SDValue();
15182}
15183
15184/// hasNormalLoadOperand - Check if any of the operands of a BUILD_VECTOR node
15185/// are normal, non-volatile loads. If so, it is profitable to bitcast an
15186/// i64 vector to have f64 elements, since the value can then be loaded
15187/// directly into a VFP register.
15188static bool hasNormalLoadOperand(SDNode *N) {
15189 unsigned NumElts = N->getValueType(ResNo: 0).getVectorNumElements();
15190 for (unsigned i = 0; i < NumElts; ++i) {
15191 SDNode *Elt = N->getOperand(Num: i).getNode();
15192 if (ISD::isNormalLoad(N: Elt) && !cast<LoadSDNode>(Val: Elt)->isVolatile())
15193 return true;
15194 }
15195 return false;
15196}
15197
15198/// PerformBUILD_VECTORCombine - Target-specific dag combine xforms for
15199/// ISD::BUILD_VECTOR.
15200static SDValue PerformBUILD_VECTORCombine(SDNode *N,
15201 TargetLowering::DAGCombinerInfo &DCI,
15202 const ARMSubtarget *Subtarget) {
15203 // build_vector(N=ARMISD::VMOVRRD(X), N:1) -> bit_convert(X):
15204 // VMOVRRD is introduced when legalizing i64 types. It forces the i64 value
15205 // into a pair of GPRs, which is fine when the value is used as a scalar,
15206 // but if the i64 value is converted to a vector, we need to undo the VMOVRRD.
15207 SelectionDAG &DAG = DCI.DAG;
15208 if (N->getNumOperands() == 2)
15209 if (SDValue RV = PerformVMOVDRRCombine(N, DAG))
15210 return RV;
15211
15212 // Load i64 elements as f64 values so that type legalization does not split
15213 // them up into i32 values.
15214 EVT VT = N->getValueType(ResNo: 0);
15215 if (VT.getVectorElementType() != MVT::i64 || !hasNormalLoadOperand(N))
15216 return SDValue();
15217 SDLoc dl(N);
15218 SmallVector<SDValue, 8> Ops;
15219 unsigned NumElts = VT.getVectorNumElements();
15220 for (unsigned i = 0; i < NumElts; ++i) {
15221 SDValue V = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::f64, Operand: N->getOperand(Num: i));
15222 Ops.push_back(Elt: V);
15223 // Make the DAGCombiner fold the bitcast.
15224 DCI.AddToWorklist(N: V.getNode());
15225 }
15226 EVT FloatVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::f64, NumElements: NumElts);
15227 SDValue BV = DAG.getBuildVector(VT: FloatVT, DL: dl, Ops);
15228 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: BV);
15229}
15230
15231/// Target-specific dag combine xforms for ARMISD::BUILD_VECTOR.
15232static SDValue
15233PerformARMBUILD_VECTORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
15234 // ARMISD::BUILD_VECTOR is introduced when legalizing ISD::BUILD_VECTOR.
15235 // At that time, we may have inserted bitcasts from integer to float.
15236 // If these bitcasts have survived DAGCombine, change the lowering of this
15237 // BUILD_VECTOR in something more vector friendly, i.e., that does not
15238 // force to use floating point types.
15239
15240 // Make sure we can change the type of the vector.
15241 // This is possible iff:
15242 // 1. The vector is only used in a bitcast to a integer type. I.e.,
15243 // 1.1. Vector is used only once.
15244 // 1.2. Use is a bit convert to an integer type.
15245 // 2. The size of its operands are 32-bits (64-bits are not legal).
15246 EVT VT = N->getValueType(ResNo: 0);
15247 EVT EltVT = VT.getVectorElementType();
15248
15249 // Check 1.1. and 2.
15250 if (EltVT.getSizeInBits() != 32 || !N->hasOneUse())
15251 return SDValue();
15252
15253 // By construction, the input type must be float.
15254 assert(EltVT == MVT::f32 && "Unexpected type!");
15255
15256 // Check 1.2.
15257 SDNode *Use = *N->user_begin();
15258 if (Use->getOpcode() != ISD::BITCAST ||
15259 Use->getValueType(ResNo: 0).isFloatingPoint())
15260 return SDValue();
15261
15262 // Check profitability.
15263 // Model is, if more than half of the relevant operands are bitcast from
15264 // i32, turn the build_vector into a sequence of insert_vector_elt.
15265 // Relevant operands are everything that is not statically
15266 // (i.e., at compile time) bitcasted.
15267 unsigned NumOfBitCastedElts = 0;
15268 unsigned NumElts = VT.getVectorNumElements();
15269 unsigned NumOfRelevantElts = NumElts;
15270 for (unsigned Idx = 0; Idx < NumElts; ++Idx) {
15271 SDValue Elt = N->getOperand(Num: Idx);
15272 if (Elt->getOpcode() == ISD::BITCAST) {
15273 // Assume only bit cast to i32 will go away.
15274 if (Elt->getOperand(Num: 0).getValueType() == MVT::i32)
15275 ++NumOfBitCastedElts;
15276 } else if (Elt.isUndef() || isa<ConstantSDNode>(Val: Elt))
15277 // Constants are statically casted, thus do not count them as
15278 // relevant operands.
15279 --NumOfRelevantElts;
15280 }
15281
15282 // Check if more than half of the elements require a non-free bitcast.
15283 if (NumOfBitCastedElts <= NumOfRelevantElts / 2)
15284 return SDValue();
15285
15286 SelectionDAG &DAG = DCI.DAG;
15287 // Create the new vector type.
15288 EVT VecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32, NumElements: NumElts);
15289 // Check if the type is legal.
15290 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
15291 if (!TLI.isTypeLegal(VT: VecVT))
15292 return SDValue();
15293
15294 // Combine:
15295 // ARMISD::BUILD_VECTOR E1, E2, ..., EN.
15296 // => BITCAST INSERT_VECTOR_ELT
15297 // (INSERT_VECTOR_ELT (...), (BITCAST EN-1), N-1),
15298 // (BITCAST EN), N.
15299 SDValue Vec = DAG.getUNDEF(VT: VecVT);
15300 SDLoc dl(N);
15301 for (unsigned Idx = 0 ; Idx < NumElts; ++Idx) {
15302 SDValue V = N->getOperand(Num: Idx);
15303 if (V.isUndef())
15304 continue;
15305 if (V.getOpcode() == ISD::BITCAST &&
15306 V->getOperand(Num: 0).getValueType() == MVT::i32)
15307 // Fold obvious case.
15308 V = V.getOperand(i: 0);
15309 else {
15310 V = DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(V), VT: MVT::i32, Operand: V);
15311 // Make the DAGCombiner fold the bitcasts.
15312 DCI.AddToWorklist(N: V.getNode());
15313 }
15314 SDValue LaneIdx = DAG.getConstant(Val: Idx, DL: dl, VT: MVT::i32);
15315 Vec = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: VecVT, N1: Vec, N2: V, N3: LaneIdx);
15316 }
15317 Vec = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: Vec);
15318 // Make the DAGCombiner fold the bitcasts.
15319 DCI.AddToWorklist(N: Vec.getNode());
15320 return Vec;
15321}
15322
15323static SDValue
15324PerformPREDICATE_CASTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
15325 EVT VT = N->getValueType(ResNo: 0);
15326 SDValue Op = N->getOperand(Num: 0);
15327 SDLoc dl(N);
15328
15329 // PREDICATE_CAST(PREDICATE_CAST(x)) == PREDICATE_CAST(x)
15330 if (Op->getOpcode() == ARMISD::PREDICATE_CAST) {
15331 // If the valuetypes are the same, we can remove the cast entirely.
15332 if (Op->getOperand(Num: 0).getValueType() == VT)
15333 return Op->getOperand(Num: 0);
15334 return DCI.DAG.getNode(Opcode: ARMISD::PREDICATE_CAST, DL: dl, VT, Operand: Op->getOperand(Num: 0));
15335 }
15336
15337 // Turn pred_cast(xor x, -1) into xor(pred_cast x, -1), in order to produce
15338 // more VPNOT which might get folded as else predicates.
15339 if (Op.getValueType() == MVT::i32 && isBitwiseNot(V: Op)) {
15340 SDValue X =
15341 DCI.DAG.getNode(Opcode: ARMISD::PREDICATE_CAST, DL: dl, VT, Operand: Op->getOperand(Num: 0));
15342 SDValue C = DCI.DAG.getNode(Opcode: ARMISD::PREDICATE_CAST, DL: dl, VT,
15343 Operand: DCI.DAG.getConstant(Val: 65535, DL: dl, VT: MVT::i32));
15344 return DCI.DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: X, N2: C);
15345 }
15346
15347 // Only the bottom 16 bits of the source register are used.
15348 if (Op.getValueType() == MVT::i32) {
15349 APInt DemandedMask = APInt::getLowBitsSet(numBits: 32, loBitsSet: 16);
15350 const TargetLowering &TLI = DCI.DAG.getTargetLoweringInfo();
15351 if (TLI.SimplifyDemandedBits(Op, DemandedBits: DemandedMask, DCI))
15352 return SDValue(N, 0);
15353 }
15354 return SDValue();
15355}
15356
15357static SDValue PerformVECTOR_REG_CASTCombine(SDNode *N, SelectionDAG &DAG,
15358 const ARMSubtarget *ST) {
15359 EVT VT = N->getValueType(ResNo: 0);
15360 SDValue Op = N->getOperand(Num: 0);
15361 SDLoc dl(N);
15362
15363 // Under Little endian, a VECTOR_REG_CAST is equivalent to a BITCAST
15364 if (ST->isLittle())
15365 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: Op);
15366
15367 // VT VECTOR_REG_CAST (VT Op) -> Op
15368 if (Op.getValueType() == VT)
15369 return Op;
15370 // VECTOR_REG_CAST undef -> undef
15371 if (Op.isUndef())
15372 return DAG.getUNDEF(VT);
15373
15374 // VECTOR_REG_CAST(VECTOR_REG_CAST(x)) == VECTOR_REG_CAST(x)
15375 if (Op->getOpcode() == ARMISD::VECTOR_REG_CAST) {
15376 // If the valuetypes are the same, we can remove the cast entirely.
15377 if (Op->getOperand(Num: 0).getValueType() == VT)
15378 return Op->getOperand(Num: 0);
15379 return DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT, Operand: Op->getOperand(Num: 0));
15380 }
15381
15382 return SDValue();
15383}
15384
15385static SDValue PerformVCMPCombine(SDNode *N, SelectionDAG &DAG,
15386 const ARMSubtarget *Subtarget) {
15387 if (!Subtarget->hasMVEIntegerOps())
15388 return SDValue();
15389
15390 EVT VT = N->getValueType(ResNo: 0);
15391 SDValue Op0 = N->getOperand(Num: 0);
15392 SDValue Op1 = N->getOperand(Num: 1);
15393 ARMCC::CondCodes Cond = (ARMCC::CondCodes)N->getConstantOperandVal(Num: 2);
15394 SDLoc dl(N);
15395
15396 // vcmp X, 0, cc -> vcmpz X, cc
15397 if (isZeroVector(N: Op1))
15398 return DAG.getNode(Opcode: ARMISD::VCMPZ, DL: dl, VT, N1: Op0, N2: N->getOperand(Num: 2));
15399
15400 unsigned SwappedCond = getSwappedCondition(CC: Cond);
15401 if (isValidMVECond(CC: SwappedCond, IsFloat: VT.isFloatingPoint())) {
15402 // vcmp 0, X, cc -> vcmpz X, reversed(cc)
15403 if (isZeroVector(N: Op0))
15404 return DAG.getNode(Opcode: ARMISD::VCMPZ, DL: dl, VT, N1: Op1,
15405 N2: DAG.getConstant(Val: SwappedCond, DL: dl, VT: MVT::i32));
15406 // vcmp vdup(Y), X, cc -> vcmp X, vdup(Y), reversed(cc)
15407 if (Op0->getOpcode() == ARMISD::VDUP && Op1->getOpcode() != ARMISD::VDUP)
15408 return DAG.getNode(Opcode: ARMISD::VCMP, DL: dl, VT, N1: Op1, N2: Op0,
15409 N3: DAG.getConstant(Val: SwappedCond, DL: dl, VT: MVT::i32));
15410 }
15411
15412 return SDValue();
15413}
15414
15415/// PerformInsertEltCombine - Target-specific dag combine xforms for
15416/// ISD::INSERT_VECTOR_ELT.
15417static SDValue PerformInsertEltCombine(SDNode *N,
15418 TargetLowering::DAGCombinerInfo &DCI) {
15419 // Bitcast an i64 load inserted into a vector to f64.
15420 // Otherwise, the i64 value will be legalized to a pair of i32 values.
15421 EVT VT = N->getValueType(ResNo: 0);
15422 SDNode *Elt = N->getOperand(Num: 1).getNode();
15423 if (VT.getVectorElementType() != MVT::i64 ||
15424 !ISD::isNormalLoad(N: Elt) || cast<LoadSDNode>(Val: Elt)->isVolatile())
15425 return SDValue();
15426
15427 SelectionDAG &DAG = DCI.DAG;
15428 SDLoc dl(N);
15429 EVT FloatVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::f64,
15430 NumElements: VT.getVectorNumElements());
15431 SDValue Vec = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: FloatVT, Operand: N->getOperand(Num: 0));
15432 SDValue V = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::f64, Operand: N->getOperand(Num: 1));
15433 // Make the DAGCombiner fold the bitcasts.
15434 DCI.AddToWorklist(N: Vec.getNode());
15435 DCI.AddToWorklist(N: V.getNode());
15436 SDValue InsElt = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: FloatVT,
15437 N1: Vec, N2: V, N3: N->getOperand(Num: 2));
15438 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: InsElt);
15439}
15440
15441// Convert a pair of extracts from the same base vector to a VMOVRRD. Either
15442// directly or bitcast to an integer if the original is a float vector.
15443// extract(x, n); extract(x, n+1) -> VMOVRRD(extract v2f64 x, n/2)
15444// bitcast(extract(x, n)); bitcast(extract(x, n+1)) -> VMOVRRD(extract x, n/2)
15445static SDValue
15446PerformExtractEltToVMOVRRD(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
15447 EVT VT = N->getValueType(ResNo: 0);
15448 SDLoc dl(N);
15449
15450 if (!DCI.isAfterLegalizeDAG() || VT != MVT::i32 ||
15451 !DCI.DAG.getTargetLoweringInfo().isTypeLegal(VT: MVT::f64))
15452 return SDValue();
15453
15454 SDValue Ext = SDValue(N, 0);
15455 if (Ext.getOpcode() == ISD::BITCAST &&
15456 Ext.getOperand(i: 0).getValueType() == MVT::f32)
15457 Ext = Ext.getOperand(i: 0);
15458 if (Ext.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
15459 !isa<ConstantSDNode>(Val: Ext.getOperand(i: 1)) ||
15460 Ext.getConstantOperandVal(i: 1) % 2 != 0)
15461 return SDValue();
15462 if (Ext->hasOneUse() && (Ext->user_begin()->getOpcode() == ISD::SINT_TO_FP ||
15463 Ext->user_begin()->getOpcode() == ISD::UINT_TO_FP))
15464 return SDValue();
15465
15466 SDValue Op0 = Ext.getOperand(i: 0);
15467 EVT VecVT = Op0.getValueType();
15468 unsigned ResNo = Op0.getResNo();
15469 unsigned Lane = Ext.getConstantOperandVal(i: 1);
15470 if (VecVT.getVectorNumElements() != 4)
15471 return SDValue();
15472
15473 // Find another extract, of Lane + 1
15474 auto OtherIt = find_if(Range: Op0->users(), P: [&](SDNode *V) {
15475 return V->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
15476 isa<ConstantSDNode>(Val: V->getOperand(Num: 1)) &&
15477 V->getConstantOperandVal(Num: 1) == Lane + 1 &&
15478 V->getOperand(Num: 0).getResNo() == ResNo;
15479 });
15480 if (OtherIt == Op0->users().end())
15481 return SDValue();
15482
15483 // For float extracts, we need to be converting to a i32 for both vector
15484 // lanes.
15485 SDValue OtherExt(*OtherIt, 0);
15486 if (OtherExt.getValueType() != MVT::i32) {
15487 if (!OtherExt->hasOneUse() ||
15488 OtherExt->user_begin()->getOpcode() != ISD::BITCAST ||
15489 OtherExt->user_begin()->getValueType(ResNo: 0) != MVT::i32)
15490 return SDValue();
15491 OtherExt = SDValue(*OtherExt->user_begin(), 0);
15492 }
15493
15494 // Convert the type to a f64 and extract with a VMOVRRD.
15495 SDValue F64 = DCI.DAG.getNode(
15496 Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: MVT::f64,
15497 N1: DCI.DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT: MVT::v2f64, Operand: Op0),
15498 N2: DCI.DAG.getConstant(Val: Ext.getConstantOperandVal(i: 1) / 2, DL: dl, VT: MVT::i32));
15499 SDValue VMOVRRD =
15500 DCI.DAG.getNode(Opcode: ARMISD::VMOVRRD, DL: dl, ResultTys: {MVT::i32, MVT::i32}, Ops: F64);
15501
15502 DCI.CombineTo(N: OtherExt.getNode(), Res: SDValue(VMOVRRD.getNode(), 1));
15503 return VMOVRRD;
15504}
15505
15506static SDValue PerformExtractEltCombine(SDNode *N,
15507 TargetLowering::DAGCombinerInfo &DCI,
15508 const ARMSubtarget *ST) {
15509 SDValue Op0 = N->getOperand(Num: 0);
15510 EVT VT = N->getValueType(ResNo: 0);
15511 SDLoc dl(N);
15512
15513 // extract (vdup x) -> x
15514 if (Op0->getOpcode() == ARMISD::VDUP) {
15515 SDValue X = Op0->getOperand(Num: 0);
15516 if (VT == MVT::f16 && X.getValueType() == MVT::i32)
15517 return DCI.DAG.getNode(Opcode: ARMISD::VMOVhr, DL: dl, VT, Operand: X);
15518 if (VT == MVT::i32 && X.getValueType() == MVT::f16)
15519 return DCI.DAG.getNode(Opcode: ARMISD::VMOVrh, DL: dl, VT, Operand: X);
15520 if (VT == MVT::f32 && X.getValueType() == MVT::i32)
15521 return DCI.DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: X);
15522
15523 while (X.getValueType() != VT && X->getOpcode() == ISD::BITCAST)
15524 X = X->getOperand(Num: 0);
15525 if (X.getValueType() == VT)
15526 return X;
15527 }
15528
15529 // extract ARM_BUILD_VECTOR -> x
15530 if (Op0->getOpcode() == ARMISD::BUILD_VECTOR &&
15531 isa<ConstantSDNode>(Val: N->getOperand(Num: 1)) &&
15532 N->getConstantOperandVal(Num: 1) < Op0.getNumOperands()) {
15533 return Op0.getOperand(i: N->getConstantOperandVal(Num: 1));
15534 }
15535
15536 // extract(bitcast(BUILD_VECTOR(VMOVDRR(a, b), ..))) -> a or b
15537 if (Op0.getValueType() == MVT::v4i32 &&
15538 isa<ConstantSDNode>(Val: N->getOperand(Num: 1)) &&
15539 Op0.getOpcode() == ISD::BITCAST &&
15540 Op0.getOperand(i: 0).getOpcode() == ISD::BUILD_VECTOR &&
15541 Op0.getOperand(i: 0).getValueType() == MVT::v2f64) {
15542 SDValue BV = Op0.getOperand(i: 0);
15543 unsigned Offset = N->getConstantOperandVal(Num: 1);
15544 SDValue MOV = BV.getOperand(i: Offset < 2 ? 0 : 1);
15545 if (MOV.getOpcode() == ARMISD::VMOVDRR)
15546 return MOV.getOperand(i: ST->isLittle() ? Offset % 2 : 1 - Offset % 2);
15547 }
15548
15549 // extract x, n; extract x, n+1 -> VMOVRRD x
15550 if (SDValue R = PerformExtractEltToVMOVRRD(N, DCI))
15551 return R;
15552
15553 // extract (MVETrunc(x)) -> extract x
15554 if (Op0->getOpcode() == ARMISD::MVETRUNC) {
15555 unsigned Idx = N->getConstantOperandVal(Num: 1);
15556 unsigned Vec =
15557 Idx / Op0->getOperand(Num: 0).getValueType().getVectorNumElements();
15558 unsigned SubIdx =
15559 Idx % Op0->getOperand(Num: 0).getValueType().getVectorNumElements();
15560 return DCI.DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT, N1: Op0.getOperand(i: Vec),
15561 N2: DCI.DAG.getConstant(Val: SubIdx, DL: dl, VT: MVT::i32));
15562 }
15563
15564 // extract(bitcast(BUILD_VECTOR(extract(bitcast(a)), ..))) -> extract(a)
15565 if (ST->isLittle() && Op0.getOpcode() == ISD::BITCAST &&
15566 Op0.getOperand(i: 0).getOpcode() == ARMISD::BUILD_VECTOR &&
15567 isa<ConstantSDNode>(Val: N->getOperand(Num: 1)) &&
15568 Op0.getScalarValueSizeInBits() <=
15569 Op0.getOperand(i: 0).getScalarValueSizeInBits()) {
15570 unsigned Lane = N->getConstantOperandVal(Num: 1);
15571 EVT ExtVT = Op0.getValueType();
15572 EVT BVVT = Op0.getOperand(i: 0).getValueType();
15573 unsigned BVLane =
15574 (Lane * BVVT.getVectorNumElements()) / ExtVT.getVectorNumElements();
15575 assert(BVLane < Op0.getOperand(0).getNumOperands());
15576 SDValue Ext = Op0.getOperand(i: 0).getOperand(i: BVLane);
15577 if (Ext.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
15578 Ext.getOperand(i: 0).getOpcode() == ISD::BITCAST &&
15579 isa<ConstantSDNode>(Val: Ext.getOperand(i: 1)) &&
15580 Ext.getOperand(i: 0).getOperand(i: 0).getValueType() == ExtVT) {
15581 unsigned InnerLane = Ext.getConstantOperandVal(i: 1);
15582 unsigned BVSubLane = Lane - (BVLane * ExtVT.getVectorNumElements()) /
15583 BVVT.getVectorNumElements();
15584 unsigned FinalLane = (InnerLane * ExtVT.getVectorNumElements()) /
15585 BVVT.getVectorNumElements() +
15586 BVSubLane;
15587 return DCI.DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT,
15588 N1: Ext.getOperand(i: 0).getOperand(i: 0),
15589 N2: DCI.DAG.getConstant(Val: FinalLane, DL: dl, VT: MVT::i32));
15590 }
15591 }
15592
15593 return SDValue();
15594}
15595
15596static SDValue PerformSignExtendInregCombine(SDNode *N, SelectionDAG &DAG) {
15597 SDValue Op = N->getOperand(Num: 0);
15598 EVT VT = N->getValueType(ResNo: 0);
15599
15600 // sext_inreg(VGETLANEu) -> VGETLANEs
15601 if (Op.getOpcode() == ARMISD::VGETLANEu &&
15602 cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT() ==
15603 Op.getOperand(i: 0).getValueType().getScalarType())
15604 return DAG.getNode(Opcode: ARMISD::VGETLANEs, DL: SDLoc(N), VT, N1: Op.getOperand(i: 0),
15605 N2: Op.getOperand(i: 1));
15606
15607 return SDValue();
15608}
15609
15610static SDValue
15611PerformInsertSubvectorCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
15612 SDValue Vec = N->getOperand(Num: 0);
15613 SDValue SubVec = N->getOperand(Num: 1);
15614 uint64_t IdxVal = N->getConstantOperandVal(Num: 2);
15615 EVT VecVT = Vec.getValueType();
15616 EVT SubVT = SubVec.getValueType();
15617
15618 // Only do this for legal fixed vector types.
15619 if (!VecVT.isFixedLengthVector() ||
15620 !DCI.DAG.getTargetLoweringInfo().isTypeLegal(VT: VecVT) ||
15621 !DCI.DAG.getTargetLoweringInfo().isTypeLegal(VT: SubVT))
15622 return SDValue();
15623
15624 // Ignore widening patterns.
15625 if (IdxVal == 0 && Vec.isUndef())
15626 return SDValue();
15627
15628 // Subvector must be half the width and an "aligned" insertion.
15629 unsigned NumSubElts = SubVT.getVectorNumElements();
15630 if ((SubVT.getSizeInBits() * 2) != VecVT.getSizeInBits() ||
15631 (IdxVal != 0 && IdxVal != NumSubElts))
15632 return SDValue();
15633
15634 // Fold insert_subvector -> concat_vectors
15635 // insert_subvector(Vec,Sub,lo) -> concat_vectors(Sub,extract(Vec,hi))
15636 // insert_subvector(Vec,Sub,hi) -> concat_vectors(extract(Vec,lo),Sub)
15637 SDLoc DL(N);
15638 SDValue Lo, Hi;
15639 if (IdxVal == 0) {
15640 Lo = SubVec;
15641 Hi = DCI.DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: SubVT, N1: Vec,
15642 N2: DCI.DAG.getVectorIdxConstant(Val: NumSubElts, DL));
15643 } else {
15644 Lo = DCI.DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: SubVT, N1: Vec,
15645 N2: DCI.DAG.getVectorIdxConstant(Val: 0, DL));
15646 Hi = SubVec;
15647 }
15648 return DCI.DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: VecVT, N1: Lo, N2: Hi);
15649}
15650
15651// shuffle(MVETrunc(x, y)) -> VMOVN(x, y)
15652static SDValue PerformShuffleVMOVNCombine(ShuffleVectorSDNode *N,
15653 SelectionDAG &DAG) {
15654 SDValue Trunc = N->getOperand(Num: 0);
15655 EVT VT = Trunc.getValueType();
15656 if (Trunc.getOpcode() != ARMISD::MVETRUNC || !N->getOperand(Num: 1).isUndef())
15657 return SDValue();
15658
15659 SDLoc DL(Trunc);
15660 if (isVMOVNTruncMask(M: N->getMask(), ToVT: VT, rev: false))
15661 return DAG.getNode(
15662 Opcode: ARMISD::VMOVN, DL, VT,
15663 N1: DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL, VT, Operand: Trunc.getOperand(i: 0)),
15664 N2: DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL, VT, Operand: Trunc.getOperand(i: 1)),
15665 N3: DAG.getConstant(Val: 1, DL, VT: MVT::i32));
15666 else if (isVMOVNTruncMask(M: N->getMask(), ToVT: VT, rev: true))
15667 return DAG.getNode(
15668 Opcode: ARMISD::VMOVN, DL, VT,
15669 N1: DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL, VT, Operand: Trunc.getOperand(i: 1)),
15670 N2: DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL, VT, Operand: Trunc.getOperand(i: 0)),
15671 N3: DAG.getConstant(Val: 1, DL, VT: MVT::i32));
15672 return SDValue();
15673}
15674
15675/// PerformVECTOR_SHUFFLECombine - Target-specific dag combine xforms for
15676/// ISD::VECTOR_SHUFFLE.
15677static SDValue PerformVECTOR_SHUFFLECombine(SDNode *N, SelectionDAG &DAG) {
15678 if (SDValue R = PerformShuffleVMOVNCombine(N: cast<ShuffleVectorSDNode>(Val: N), DAG))
15679 return R;
15680
15681 // The LLVM shufflevector instruction does not require the shuffle mask
15682 // length to match the operand vector length, but ISD::VECTOR_SHUFFLE does
15683 // have that requirement. When translating to ISD::VECTOR_SHUFFLE, if the
15684 // operands do not match the mask length, they are extended by concatenating
15685 // them with undef vectors. That is probably the right thing for other
15686 // targets, but for NEON it is better to concatenate two double-register
15687 // size vector operands into a single quad-register size vector. Do that
15688 // transformation here:
15689 // shuffle(concat(v1, undef), concat(v2, undef)) ->
15690 // shuffle(concat(v1, v2), undef)
15691 SDValue Op0 = N->getOperand(Num: 0);
15692 SDValue Op1 = N->getOperand(Num: 1);
15693 if (Op0.getOpcode() != ISD::CONCAT_VECTORS ||
15694 Op1.getOpcode() != ISD::CONCAT_VECTORS ||
15695 Op0.getNumOperands() != 2 ||
15696 Op1.getNumOperands() != 2)
15697 return SDValue();
15698 SDValue Concat0Op1 = Op0.getOperand(i: 1);
15699 SDValue Concat1Op1 = Op1.getOperand(i: 1);
15700 if (!Concat0Op1.isUndef() || !Concat1Op1.isUndef())
15701 return SDValue();
15702 // Skip the transformation if any of the types are illegal.
15703 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
15704 EVT VT = N->getValueType(ResNo: 0);
15705 if (!TLI.isTypeLegal(VT) ||
15706 !TLI.isTypeLegal(VT: Concat0Op1.getValueType()) ||
15707 !TLI.isTypeLegal(VT: Concat1Op1.getValueType()))
15708 return SDValue();
15709
15710 SDValue NewConcat = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(N), VT,
15711 N1: Op0.getOperand(i: 0), N2: Op1.getOperand(i: 0));
15712 // Translate the shuffle mask.
15713 SmallVector<int, 16> NewMask;
15714 unsigned NumElts = VT.getVectorNumElements();
15715 unsigned HalfElts = NumElts/2;
15716 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val: N);
15717 for (unsigned n = 0; n < NumElts; ++n) {
15718 int MaskElt = SVN->getMaskElt(Idx: n);
15719 int NewElt = -1;
15720 if (MaskElt < (int)HalfElts)
15721 NewElt = MaskElt;
15722 else if (MaskElt >= (int)NumElts && MaskElt < (int)(NumElts + HalfElts))
15723 NewElt = HalfElts + MaskElt - NumElts;
15724 NewMask.push_back(Elt: NewElt);
15725 }
15726 return DAG.getVectorShuffle(VT, dl: SDLoc(N), N1: NewConcat,
15727 N2: DAG.getUNDEF(VT), Mask: NewMask);
15728}
15729
15730/// Load/store instruction that can be merged with a base address
15731/// update
15732struct BaseUpdateTarget {
15733 SDNode *N;
15734 bool isIntrinsic;
15735 bool isStore;
15736 unsigned AddrOpIdx;
15737};
15738
15739struct BaseUpdateUser {
15740 /// Instruction that updates a pointer
15741 SDNode *N;
15742 /// Pointer increment operand
15743 SDValue Inc;
15744 /// Pointer increment value if it is a constant, or 0 otherwise
15745 unsigned ConstInc;
15746};
15747
15748static bool isValidBaseUpdate(SDNode *N, SDNode *User) {
15749 // Check that the add is independent of the load/store.
15750 // Otherwise, folding it would create a cycle. Search through Addr
15751 // as well, since the User may not be a direct user of Addr and
15752 // only share a base pointer.
15753 SmallPtrSet<const SDNode *, 32> Visited;
15754 SmallVector<const SDNode *, 16> Worklist;
15755 Worklist.push_back(Elt: N);
15756 Worklist.push_back(Elt: User);
15757 const unsigned MaxSteps = 1024;
15758 if (SDNode::hasPredecessorHelper(N, Visited, Worklist, MaxSteps) ||
15759 SDNode::hasPredecessorHelper(N: User, Visited, Worklist, MaxSteps))
15760 return false;
15761 return true;
15762}
15763
15764static bool TryCombineBaseUpdate(struct BaseUpdateTarget &Target,
15765 struct BaseUpdateUser &User,
15766 bool SimpleConstIncOnly,
15767 TargetLowering::DAGCombinerInfo &DCI) {
15768 SelectionDAG &DAG = DCI.DAG;
15769 SDNode *N = Target.N;
15770 MemSDNode *MemN = cast<MemSDNode>(Val: N);
15771 SDLoc dl(N);
15772
15773 // Find the new opcode for the updating load/store.
15774 bool isLoadOp = true;
15775 bool isLaneOp = false;
15776 // Workaround for vst1x and vld1x intrinsics which do not have alignment
15777 // as an operand.
15778 bool hasAlignment = true;
15779 unsigned NewOpc = 0;
15780 unsigned NumVecs = 0;
15781 if (Target.isIntrinsic) {
15782 unsigned IntNo = N->getConstantOperandVal(Num: 1);
15783 switch (IntNo) {
15784 default:
15785 llvm_unreachable("unexpected intrinsic for Neon base update");
15786 case Intrinsic::arm_neon_vld1:
15787 NewOpc = ARMISD::VLD1_UPD;
15788 NumVecs = 1;
15789 break;
15790 case Intrinsic::arm_neon_vld2:
15791 NewOpc = ARMISD::VLD2_UPD;
15792 NumVecs = 2;
15793 break;
15794 case Intrinsic::arm_neon_vld3:
15795 NewOpc = ARMISD::VLD3_UPD;
15796 NumVecs = 3;
15797 break;
15798 case Intrinsic::arm_neon_vld4:
15799 NewOpc = ARMISD::VLD4_UPD;
15800 NumVecs = 4;
15801 break;
15802 case Intrinsic::arm_neon_vld1x2:
15803 NewOpc = ARMISD::VLD1x2_UPD;
15804 NumVecs = 2;
15805 hasAlignment = false;
15806 break;
15807 case Intrinsic::arm_neon_vld1x3:
15808 NewOpc = ARMISD::VLD1x3_UPD;
15809 NumVecs = 3;
15810 hasAlignment = false;
15811 break;
15812 case Intrinsic::arm_neon_vld1x4:
15813 NewOpc = ARMISD::VLD1x4_UPD;
15814 NumVecs = 4;
15815 hasAlignment = false;
15816 break;
15817 case Intrinsic::arm_neon_vld2dup:
15818 NewOpc = ARMISD::VLD2DUP_UPD;
15819 NumVecs = 2;
15820 break;
15821 case Intrinsic::arm_neon_vld3dup:
15822 NewOpc = ARMISD::VLD3DUP_UPD;
15823 NumVecs = 3;
15824 break;
15825 case Intrinsic::arm_neon_vld4dup:
15826 NewOpc = ARMISD::VLD4DUP_UPD;
15827 NumVecs = 4;
15828 break;
15829 case Intrinsic::arm_neon_vld2lane:
15830 NewOpc = ARMISD::VLD2LN_UPD;
15831 NumVecs = 2;
15832 isLaneOp = true;
15833 break;
15834 case Intrinsic::arm_neon_vld3lane:
15835 NewOpc = ARMISD::VLD3LN_UPD;
15836 NumVecs = 3;
15837 isLaneOp = true;
15838 break;
15839 case Intrinsic::arm_neon_vld4lane:
15840 NewOpc = ARMISD::VLD4LN_UPD;
15841 NumVecs = 4;
15842 isLaneOp = true;
15843 break;
15844 case Intrinsic::arm_neon_vst1:
15845 NewOpc = ARMISD::VST1_UPD;
15846 NumVecs = 1;
15847 isLoadOp = false;
15848 break;
15849 case Intrinsic::arm_neon_vst2:
15850 NewOpc = ARMISD::VST2_UPD;
15851 NumVecs = 2;
15852 isLoadOp = false;
15853 break;
15854 case Intrinsic::arm_neon_vst3:
15855 NewOpc = ARMISD::VST3_UPD;
15856 NumVecs = 3;
15857 isLoadOp = false;
15858 break;
15859 case Intrinsic::arm_neon_vst4:
15860 NewOpc = ARMISD::VST4_UPD;
15861 NumVecs = 4;
15862 isLoadOp = false;
15863 break;
15864 case Intrinsic::arm_neon_vst2lane:
15865 NewOpc = ARMISD::VST2LN_UPD;
15866 NumVecs = 2;
15867 isLoadOp = false;
15868 isLaneOp = true;
15869 break;
15870 case Intrinsic::arm_neon_vst3lane:
15871 NewOpc = ARMISD::VST3LN_UPD;
15872 NumVecs = 3;
15873 isLoadOp = false;
15874 isLaneOp = true;
15875 break;
15876 case Intrinsic::arm_neon_vst4lane:
15877 NewOpc = ARMISD::VST4LN_UPD;
15878 NumVecs = 4;
15879 isLoadOp = false;
15880 isLaneOp = true;
15881 break;
15882 case Intrinsic::arm_neon_vst1x2:
15883 NewOpc = ARMISD::VST1x2_UPD;
15884 NumVecs = 2;
15885 isLoadOp = false;
15886 hasAlignment = false;
15887 break;
15888 case Intrinsic::arm_neon_vst1x3:
15889 NewOpc = ARMISD::VST1x3_UPD;
15890 NumVecs = 3;
15891 isLoadOp = false;
15892 hasAlignment = false;
15893 break;
15894 case Intrinsic::arm_neon_vst1x4:
15895 NewOpc = ARMISD::VST1x4_UPD;
15896 NumVecs = 4;
15897 isLoadOp = false;
15898 hasAlignment = false;
15899 break;
15900 }
15901 } else {
15902 isLaneOp = true;
15903 switch (N->getOpcode()) {
15904 default:
15905 llvm_unreachable("unexpected opcode for Neon base update");
15906 case ARMISD::VLD1DUP:
15907 NewOpc = ARMISD::VLD1DUP_UPD;
15908 NumVecs = 1;
15909 break;
15910 case ARMISD::VLD2DUP:
15911 NewOpc = ARMISD::VLD2DUP_UPD;
15912 NumVecs = 2;
15913 break;
15914 case ARMISD::VLD3DUP:
15915 NewOpc = ARMISD::VLD3DUP_UPD;
15916 NumVecs = 3;
15917 break;
15918 case ARMISD::VLD4DUP:
15919 NewOpc = ARMISD::VLD4DUP_UPD;
15920 NumVecs = 4;
15921 break;
15922 case ISD::LOAD:
15923 NewOpc = ARMISD::VLD1_UPD;
15924 NumVecs = 1;
15925 isLaneOp = false;
15926 break;
15927 case ISD::STORE:
15928 NewOpc = ARMISD::VST1_UPD;
15929 NumVecs = 1;
15930 isLaneOp = false;
15931 isLoadOp = false;
15932 break;
15933 }
15934 }
15935
15936 // Find the size of memory referenced by the load/store.
15937 EVT VecTy;
15938 if (isLoadOp) {
15939 VecTy = N->getValueType(ResNo: 0);
15940 } else if (Target.isIntrinsic) {
15941 VecTy = N->getOperand(Num: Target.AddrOpIdx + 1).getValueType();
15942 } else {
15943 assert(Target.isStore &&
15944 "Node has to be a load, a store, or an intrinsic!");
15945 VecTy = N->getOperand(Num: 1).getValueType();
15946 }
15947
15948 bool isVLDDUPOp =
15949 NewOpc == ARMISD::VLD1DUP_UPD || NewOpc == ARMISD::VLD2DUP_UPD ||
15950 NewOpc == ARMISD::VLD3DUP_UPD || NewOpc == ARMISD::VLD4DUP_UPD;
15951
15952 unsigned NumBytes = NumVecs * VecTy.getSizeInBits() / 8;
15953 if (isLaneOp || isVLDDUPOp)
15954 NumBytes /= VecTy.getVectorNumElements();
15955
15956 if (NumBytes >= 3 * 16 && User.ConstInc != NumBytes) {
15957 // VLD3/4 and VST3/4 for 128-bit vectors are implemented with two
15958 // separate instructions that make it harder to use a non-constant update.
15959 return false;
15960 }
15961
15962 if (SimpleConstIncOnly && User.ConstInc != NumBytes)
15963 return false;
15964
15965 if (!isValidBaseUpdate(N, User: User.N))
15966 return false;
15967
15968 // OK, we found an ADD we can fold into the base update.
15969 // Now, create a _UPD node, taking care of not breaking alignment.
15970
15971 EVT AlignedVecTy = VecTy;
15972 Align Alignment = MemN->getAlign();
15973
15974 // If this is a less-than-standard-aligned load/store, change the type to
15975 // match the standard alignment.
15976 // The alignment is overlooked when selecting _UPD variants; and it's
15977 // easier to introduce bitcasts here than fix that.
15978 // There are 3 ways to get to this base-update combine:
15979 // - intrinsics: they are assumed to be properly aligned (to the standard
15980 // alignment of the memory type), so we don't need to do anything.
15981 // - ARMISD::VLDx nodes: they are only generated from the aforementioned
15982 // intrinsics, so, likewise, there's nothing to do.
15983 // - generic load/store instructions: the alignment is specified as an
15984 // explicit operand, rather than implicitly as the standard alignment
15985 // of the memory type (like the intrinsics). We need to change the
15986 // memory type to match the explicit alignment. That way, we don't
15987 // generate non-standard-aligned ARMISD::VLDx nodes.
15988 if (isa<LSBaseSDNode>(Val: N)) {
15989 if (Alignment.value() < VecTy.getScalarSizeInBits() / 8) {
15990 MVT EltTy = MVT::getIntegerVT(BitWidth: Alignment.value() * 8);
15991 assert(NumVecs == 1 && "Unexpected multi-element generic load/store.");
15992 assert(!isLaneOp && "Unexpected generic load/store lane.");
15993 unsigned NumElts = NumBytes / (EltTy.getSizeInBits() / 8);
15994 AlignedVecTy = MVT::getVectorVT(VT: EltTy, NumElements: NumElts);
15995 }
15996 // Don't set an explicit alignment on regular load/stores that we want
15997 // to transform to VLD/VST 1_UPD nodes.
15998 // This matches the behavior of regular load/stores, which only get an
15999 // explicit alignment if the MMO alignment is larger than the standard
16000 // alignment of the memory type.
16001 // Intrinsics, however, always get an explicit alignment, set to the
16002 // alignment of the MMO.
16003 Alignment = Align(1);
16004 }
16005
16006 // Create the new updating load/store node.
16007 // First, create an SDVTList for the new updating node's results.
16008 EVT Tys[6];
16009 unsigned NumResultVecs = (isLoadOp ? NumVecs : 0);
16010 unsigned n;
16011 for (n = 0; n < NumResultVecs; ++n)
16012 Tys[n] = AlignedVecTy;
16013 Tys[n++] = MVT::i32;
16014 Tys[n] = MVT::Other;
16015 SDVTList SDTys = DAG.getVTList(VTs: ArrayRef(Tys, NumResultVecs + 2));
16016
16017 // Then, gather the new node's operands.
16018 SmallVector<SDValue, 8> Ops;
16019 Ops.push_back(Elt: N->getOperand(Num: 0)); // incoming chain
16020 Ops.push_back(Elt: N->getOperand(Num: Target.AddrOpIdx));
16021 Ops.push_back(Elt: User.Inc);
16022
16023 if (StoreSDNode *StN = dyn_cast<StoreSDNode>(Val: N)) {
16024 // Try to match the intrinsic's signature
16025 Ops.push_back(Elt: StN->getValue());
16026 } else {
16027 // Loads (and of course intrinsics) match the intrinsics' signature,
16028 // so just add all but the alignment operand.
16029 unsigned LastOperand =
16030 hasAlignment ? N->getNumOperands() - 1 : N->getNumOperands();
16031 for (unsigned i = Target.AddrOpIdx + 1; i < LastOperand; ++i)
16032 Ops.push_back(Elt: N->getOperand(Num: i));
16033 }
16034
16035 // For all node types, the alignment operand is always the last one.
16036 Ops.push_back(Elt: DAG.getConstant(Val: Alignment.value(), DL: dl, VT: MVT::i32));
16037
16038 // If this is a non-standard-aligned STORE, the penultimate operand is the
16039 // stored value. Bitcast it to the aligned type.
16040 if (AlignedVecTy != VecTy && N->getOpcode() == ISD::STORE) {
16041 SDValue &StVal = Ops[Ops.size() - 2];
16042 StVal = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: AlignedVecTy, Operand: StVal);
16043 }
16044
16045 EVT LoadVT = isLaneOp ? VecTy.getVectorElementType() : AlignedVecTy;
16046 SDValue UpdN = DAG.getMemIntrinsicNode(Opcode: NewOpc, dl, VTList: SDTys, Ops, MemVT: LoadVT,
16047 MMO: MemN->getMemOperand());
16048
16049 // Update the uses.
16050 SmallVector<SDValue, 5> NewResults;
16051 for (unsigned i = 0; i < NumResultVecs; ++i)
16052 NewResults.push_back(Elt: SDValue(UpdN.getNode(), i));
16053
16054 // If this is an non-standard-aligned LOAD, the first result is the loaded
16055 // value. Bitcast it to the expected result type.
16056 if (AlignedVecTy != VecTy && N->getOpcode() == ISD::LOAD) {
16057 SDValue &LdVal = NewResults[0];
16058 LdVal = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: VecTy, Operand: LdVal);
16059 }
16060
16061 NewResults.push_back(Elt: SDValue(UpdN.getNode(), NumResultVecs + 1)); // chain
16062 DCI.CombineTo(N, To: NewResults);
16063 DCI.CombineTo(N: User.N, Res: SDValue(UpdN.getNode(), NumResultVecs));
16064
16065 return true;
16066}
16067
16068// If (opcode ptr inc) is and ADD-like instruction, return the
16069// increment value. Otherwise return 0.
16070static unsigned getPointerConstIncrement(unsigned Opcode, SDValue Ptr,
16071 SDValue Inc, const SelectionDAG &DAG) {
16072 ConstantSDNode *CInc = dyn_cast<ConstantSDNode>(Val: Inc.getNode());
16073 if (!CInc)
16074 return 0;
16075
16076 switch (Opcode) {
16077 case ARMISD::VLD1_UPD:
16078 case ISD::ADD:
16079 return CInc->getZExtValue();
16080 case ISD::OR: {
16081 if (DAG.haveNoCommonBitsSet(A: Ptr, B: Inc)) {
16082 // (OR ptr inc) is the same as (ADD ptr inc)
16083 return CInc->getZExtValue();
16084 }
16085 return 0;
16086 }
16087 default:
16088 return 0;
16089 }
16090}
16091
16092static bool findPointerConstIncrement(SDNode *N, SDValue *Ptr, SDValue *CInc) {
16093 switch (N->getOpcode()) {
16094 case ISD::ADD:
16095 case ISD::OR: {
16096 if (isa<ConstantSDNode>(Val: N->getOperand(Num: 1))) {
16097 *Ptr = N->getOperand(Num: 0);
16098 *CInc = N->getOperand(Num: 1);
16099 return true;
16100 }
16101 return false;
16102 }
16103 case ARMISD::VLD1_UPD: {
16104 if (isa<ConstantSDNode>(Val: N->getOperand(Num: 2))) {
16105 *Ptr = N->getOperand(Num: 1);
16106 *CInc = N->getOperand(Num: 2);
16107 return true;
16108 }
16109 return false;
16110 }
16111 default:
16112 return false;
16113 }
16114}
16115
16116/// CombineBaseUpdate - Target-specific DAG combine function for VLDDUP,
16117/// NEON load/store intrinsics, and generic vector load/stores, to merge
16118/// base address updates.
16119/// For generic load/stores, the memory type is assumed to be a vector.
16120/// The caller is assumed to have checked legality.
16121static SDValue CombineBaseUpdate(SDNode *N,
16122 TargetLowering::DAGCombinerInfo &DCI) {
16123 const bool isIntrinsic = (N->getOpcode() == ISD::INTRINSIC_VOID ||
16124 N->getOpcode() == ISD::INTRINSIC_W_CHAIN);
16125 const bool isStore = N->getOpcode() == ISD::STORE;
16126 const unsigned AddrOpIdx = ((isIntrinsic || isStore) ? 2 : 1);
16127 BaseUpdateTarget Target = {.N: N, .isIntrinsic: isIntrinsic, .isStore: isStore, .AddrOpIdx: AddrOpIdx};
16128
16129 // Limit the number of possible base-updates we look at to prevent degenerate
16130 // cases.
16131 unsigned MaxBaseUpdates = ArmMaxBaseUpdatesToCheck;
16132
16133 SDValue Addr = N->getOperand(Num: AddrOpIdx);
16134
16135 SmallVector<BaseUpdateUser, 8> BaseUpdates;
16136
16137 // Search for a use of the address operand that is an increment.
16138 for (SDUse &Use : Addr->uses()) {
16139 SDNode *User = Use.getUser();
16140 if (Use.getResNo() != Addr.getResNo() || User->getNumOperands() != 2)
16141 continue;
16142
16143 SDValue Inc = User->getOperand(Num: Use.getOperandNo() == 1 ? 0 : 1);
16144 unsigned ConstInc =
16145 getPointerConstIncrement(Opcode: User->getOpcode(), Ptr: Addr, Inc, DAG: DCI.DAG);
16146
16147 if (ConstInc || User->getOpcode() == ISD::ADD) {
16148 BaseUpdates.push_back(Elt: {.N: User, .Inc: Inc, .ConstInc: ConstInc});
16149 if (BaseUpdates.size() >= MaxBaseUpdates)
16150 break;
16151 }
16152 }
16153
16154 // If the address is a constant pointer increment itself, find
16155 // another constant increment that has the same base operand
16156 SDValue Base;
16157 SDValue CInc;
16158 if (findPointerConstIncrement(N: Addr.getNode(), Ptr: &Base, CInc: &CInc)) {
16159 unsigned Offset =
16160 getPointerConstIncrement(Opcode: Addr->getOpcode(), Ptr: Base, Inc: CInc, DAG: DCI.DAG);
16161 if (Offset) {
16162 for (SDUse &Use : Base->uses()) {
16163
16164 SDNode *User = Use.getUser();
16165 if (Use.getResNo() != Base.getResNo() || User == Addr.getNode() ||
16166 User->getNumOperands() != 2)
16167 continue;
16168
16169 SDValue UserInc = User->getOperand(Num: Use.getOperandNo() == 0 ? 1 : 0);
16170 unsigned UserOffset =
16171 getPointerConstIncrement(Opcode: User->getOpcode(), Ptr: Base, Inc: UserInc, DAG: DCI.DAG);
16172
16173 if (!UserOffset || UserOffset <= Offset)
16174 continue;
16175
16176 unsigned NewConstInc = UserOffset - Offset;
16177 SDValue NewInc = DCI.DAG.getConstant(Val: NewConstInc, DL: SDLoc(N), VT: MVT::i32);
16178 BaseUpdates.push_back(Elt: {.N: User, .Inc: NewInc, .ConstInc: NewConstInc});
16179 if (BaseUpdates.size() >= MaxBaseUpdates)
16180 break;
16181 }
16182 }
16183 }
16184
16185 // Try to fold the load/store with an update that matches memory
16186 // access size. This should work well for sequential loads.
16187 unsigned NumValidUpd = BaseUpdates.size();
16188 for (unsigned I = 0; I < NumValidUpd; I++) {
16189 BaseUpdateUser &User = BaseUpdates[I];
16190 if (TryCombineBaseUpdate(Target, User, /*SimpleConstIncOnly=*/true, DCI))
16191 return SDValue();
16192 }
16193
16194 // Try to fold with other users. Non-constant updates are considered
16195 // first, and constant updates are sorted to not break a sequence of
16196 // strided accesses (if there is any).
16197 llvm::stable_sort(Range&: BaseUpdates,
16198 C: [](const BaseUpdateUser &LHS, const BaseUpdateUser &RHS) {
16199 return LHS.ConstInc < RHS.ConstInc;
16200 });
16201 for (BaseUpdateUser &User : BaseUpdates) {
16202 if (TryCombineBaseUpdate(Target, User, /*SimpleConstIncOnly=*/false, DCI))
16203 return SDValue();
16204 }
16205 return SDValue();
16206}
16207
16208static SDValue PerformVLDCombine(SDNode *N,
16209 TargetLowering::DAGCombinerInfo &DCI) {
16210 if (DCI.isBeforeLegalize() || DCI.isCalledByLegalizer())
16211 return SDValue();
16212
16213 return CombineBaseUpdate(N, DCI);
16214}
16215
16216static SDValue PerformMVEVLDCombine(SDNode *N,
16217 TargetLowering::DAGCombinerInfo &DCI) {
16218 if (DCI.isBeforeLegalize() || DCI.isCalledByLegalizer())
16219 return SDValue();
16220
16221 SelectionDAG &DAG = DCI.DAG;
16222 SDValue Addr = N->getOperand(Num: 2);
16223 MemSDNode *MemN = cast<MemSDNode>(Val: N);
16224 SDLoc dl(N);
16225
16226 // For the stores, where there are multiple intrinsics we only actually want
16227 // to post-inc the last of the them.
16228 unsigned IntNo = N->getConstantOperandVal(Num: 1);
16229 if (IntNo == Intrinsic::arm_mve_vst2q && N->getConstantOperandVal(Num: 5) != 1)
16230 return SDValue();
16231 if (IntNo == Intrinsic::arm_mve_vst4q && N->getConstantOperandVal(Num: 7) != 3)
16232 return SDValue();
16233
16234 // Search for a use of the address operand that is an increment.
16235 for (SDUse &Use : Addr->uses()) {
16236 SDNode *User = Use.getUser();
16237 if (User->getOpcode() != ISD::ADD || Use.getResNo() != Addr.getResNo())
16238 continue;
16239
16240 // Check that the add is independent of the load/store. Otherwise, folding
16241 // it would create a cycle. We can avoid searching through Addr as it's a
16242 // predecessor to both.
16243 SmallPtrSet<const SDNode *, 32> Visited;
16244 SmallVector<const SDNode *, 16> Worklist;
16245 Visited.insert(Ptr: Addr.getNode());
16246 Worklist.push_back(Elt: N);
16247 Worklist.push_back(Elt: User);
16248 const unsigned MaxSteps = 1024;
16249 if (SDNode::hasPredecessorHelper(N, Visited, Worklist, MaxSteps) ||
16250 SDNode::hasPredecessorHelper(N: User, Visited, Worklist, MaxSteps))
16251 continue;
16252
16253 // Find the new opcode for the updating load/store.
16254 bool isLoadOp = true;
16255 unsigned NewOpc = 0;
16256 unsigned NumVecs = 0;
16257 switch (IntNo) {
16258 default:
16259 llvm_unreachable("unexpected intrinsic for MVE VLDn combine");
16260 case Intrinsic::arm_mve_vld2q:
16261 NewOpc = ARMISD::VLD2_UPD;
16262 NumVecs = 2;
16263 break;
16264 case Intrinsic::arm_mve_vld4q:
16265 NewOpc = ARMISD::VLD4_UPD;
16266 NumVecs = 4;
16267 break;
16268 case Intrinsic::arm_mve_vst2q:
16269 NewOpc = ARMISD::VST2_UPD;
16270 NumVecs = 2;
16271 isLoadOp = false;
16272 break;
16273 case Intrinsic::arm_mve_vst4q:
16274 NewOpc = ARMISD::VST4_UPD;
16275 NumVecs = 4;
16276 isLoadOp = false;
16277 break;
16278 }
16279
16280 // Find the size of memory referenced by the load/store.
16281 EVT VecTy;
16282 if (isLoadOp) {
16283 VecTy = N->getValueType(ResNo: 0);
16284 } else {
16285 VecTy = N->getOperand(Num: 3).getValueType();
16286 }
16287
16288 unsigned NumBytes = NumVecs * VecTy.getSizeInBits() / 8;
16289
16290 // If the increment is a constant, it must match the memory ref size.
16291 SDValue Inc = User->getOperand(Num: User->getOperand(Num: 0) == Addr ? 1 : 0);
16292 ConstantSDNode *CInc = dyn_cast<ConstantSDNode>(Val: Inc.getNode());
16293 if (!CInc || CInc->getZExtValue() != NumBytes)
16294 continue;
16295
16296 // Create the new updating load/store node.
16297 // First, create an SDVTList for the new updating node's results.
16298 EVT Tys[6];
16299 unsigned NumResultVecs = (isLoadOp ? NumVecs : 0);
16300 unsigned n;
16301 for (n = 0; n < NumResultVecs; ++n)
16302 Tys[n] = VecTy;
16303 Tys[n++] = MVT::i32;
16304 Tys[n] = MVT::Other;
16305 SDVTList SDTys = DAG.getVTList(VTs: ArrayRef(Tys, NumResultVecs + 2));
16306
16307 // Then, gather the new node's operands.
16308 SmallVector<SDValue, 8> Ops;
16309 Ops.push_back(Elt: N->getOperand(Num: 0)); // incoming chain
16310 Ops.push_back(Elt: N->getOperand(Num: 2)); // ptr
16311 Ops.push_back(Elt: Inc);
16312
16313 for (unsigned i = 3; i < N->getNumOperands(); ++i)
16314 Ops.push_back(Elt: N->getOperand(Num: i));
16315
16316 SDValue UpdN = DAG.getMemIntrinsicNode(Opcode: NewOpc, dl, VTList: SDTys, Ops, MemVT: VecTy,
16317 MMO: MemN->getMemOperand());
16318
16319 // Update the uses.
16320 SmallVector<SDValue, 5> NewResults;
16321 for (unsigned i = 0; i < NumResultVecs; ++i)
16322 NewResults.push_back(Elt: SDValue(UpdN.getNode(), i));
16323
16324 NewResults.push_back(Elt: SDValue(UpdN.getNode(), NumResultVecs + 1)); // chain
16325 DCI.CombineTo(N, To: NewResults);
16326 DCI.CombineTo(N: User, Res: SDValue(UpdN.getNode(), NumResultVecs));
16327
16328 break;
16329 }
16330
16331 return SDValue();
16332}
16333
16334/// CombineVLDDUP - For a VDUPLANE node N, check if its source operand is a
16335/// vldN-lane (N > 1) intrinsic, and if all the other uses of that intrinsic
16336/// are also VDUPLANEs. If so, combine them to a vldN-dup operation and
16337/// return true.
16338static bool CombineVLDDUP(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
16339 SelectionDAG &DAG = DCI.DAG;
16340 EVT VT = N->getValueType(ResNo: 0);
16341 // vldN-dup instructions only support 64-bit vectors for N > 1.
16342 if (!VT.is64BitVector())
16343 return false;
16344
16345 // Check if the VDUPLANE operand is a vldN-dup intrinsic.
16346 SDNode *VLD = N->getOperand(Num: 0).getNode();
16347 if (VLD->getOpcode() != ISD::INTRINSIC_W_CHAIN)
16348 return false;
16349 unsigned NumVecs = 0;
16350 unsigned NewOpc = 0;
16351 unsigned IntNo = VLD->getConstantOperandVal(Num: 1);
16352 if (IntNo == Intrinsic::arm_neon_vld2lane) {
16353 NumVecs = 2;
16354 NewOpc = ARMISD::VLD2DUP;
16355 } else if (IntNo == Intrinsic::arm_neon_vld3lane) {
16356 NumVecs = 3;
16357 NewOpc = ARMISD::VLD3DUP;
16358 } else if (IntNo == Intrinsic::arm_neon_vld4lane) {
16359 NumVecs = 4;
16360 NewOpc = ARMISD::VLD4DUP;
16361 } else {
16362 return false;
16363 }
16364
16365 // First check that all the vldN-lane uses are VDUPLANEs and that the lane
16366 // numbers match the load.
16367 unsigned VLDLaneNo = VLD->getConstantOperandVal(Num: NumVecs + 3);
16368 for (SDUse &Use : VLD->uses()) {
16369 // Ignore uses of the chain result.
16370 if (Use.getResNo() == NumVecs)
16371 continue;
16372 SDNode *User = Use.getUser();
16373 if (User->getOpcode() != ARMISD::VDUPLANE ||
16374 VLDLaneNo != User->getConstantOperandVal(Num: 1))
16375 return false;
16376 }
16377
16378 // Create the vldN-dup node.
16379 EVT Tys[5];
16380 unsigned n;
16381 for (n = 0; n < NumVecs; ++n)
16382 Tys[n] = VT;
16383 Tys[n] = MVT::Other;
16384 SDVTList SDTys = DAG.getVTList(VTs: ArrayRef(Tys, NumVecs + 1));
16385 SDValue Ops[] = { VLD->getOperand(Num: 0), VLD->getOperand(Num: 2) };
16386 MemIntrinsicSDNode *VLDMemInt = cast<MemIntrinsicSDNode>(Val: VLD);
16387 SDValue VLDDup = DAG.getMemIntrinsicNode(Opcode: NewOpc, dl: SDLoc(VLD), VTList: SDTys,
16388 Ops, MemVT: VLDMemInt->getMemoryVT(),
16389 MMO: VLDMemInt->getMemOperand());
16390
16391 // Update the uses.
16392 for (SDUse &Use : VLD->uses()) {
16393 unsigned ResNo = Use.getResNo();
16394 // Ignore uses of the chain result.
16395 if (ResNo == NumVecs)
16396 continue;
16397 DCI.CombineTo(N: Use.getUser(), Res: SDValue(VLDDup.getNode(), ResNo));
16398 }
16399
16400 // Now the vldN-lane intrinsic is dead except for its chain result.
16401 // Update uses of the chain.
16402 std::vector<SDValue> VLDDupResults;
16403 for (unsigned n = 0; n < NumVecs; ++n)
16404 VLDDupResults.push_back(x: SDValue(VLDDup.getNode(), n));
16405 VLDDupResults.push_back(x: SDValue(VLDDup.getNode(), NumVecs));
16406 DCI.CombineTo(N: VLD, To: VLDDupResults);
16407
16408 return true;
16409}
16410
16411/// PerformVDUPLANECombine - Target-specific dag combine xforms for
16412/// ARMISD::VDUPLANE.
16413static SDValue PerformVDUPLANECombine(SDNode *N,
16414 TargetLowering::DAGCombinerInfo &DCI,
16415 const ARMSubtarget *Subtarget) {
16416 SDValue Op = N->getOperand(Num: 0);
16417 EVT VT = N->getValueType(ResNo: 0);
16418
16419 // On MVE, we just convert the VDUPLANE to a VDUP with an extract.
16420 if (Subtarget->hasMVEIntegerOps()) {
16421 EVT ExtractVT = VT.getVectorElementType();
16422 // We need to ensure we are creating a legal type.
16423 if (!DCI.DAG.getTargetLoweringInfo().isTypeLegal(VT: ExtractVT))
16424 ExtractVT = MVT::i32;
16425 SDValue Extract = DCI.DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N), VT: ExtractVT,
16426 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1));
16427 return DCI.DAG.getNode(Opcode: ARMISD::VDUP, DL: SDLoc(N), VT, Operand: Extract);
16428 }
16429
16430 // If the source is a vldN-lane (N > 1) intrinsic, and all the other uses
16431 // of that intrinsic are also VDUPLANEs, combine them to a vldN-dup operation.
16432 if (CombineVLDDUP(N, DCI))
16433 return SDValue(N, 0);
16434
16435 // If the source is already a VMOVIMM or VMVNIMM splat, the VDUPLANE is
16436 // redundant. Ignore bit_converts for now; element sizes are checked below.
16437 while (Op.getOpcode() == ISD::BITCAST)
16438 Op = Op.getOperand(i: 0);
16439 if (Op.getOpcode() != ARMISD::VMOVIMM && Op.getOpcode() != ARMISD::VMVNIMM)
16440 return SDValue();
16441
16442 // Make sure the VMOV element size is not bigger than the VDUPLANE elements.
16443 unsigned EltSize = Op.getScalarValueSizeInBits();
16444 // The canonical VMOV for a zero vector uses a 32-bit element size.
16445 unsigned Imm = Op.getConstantOperandVal(i: 0);
16446 unsigned EltBits;
16447 if (ARM_AM::decodeVMOVModImm(ModImm: Imm, EltBits) == 0)
16448 EltSize = 8;
16449 if (EltSize > VT.getScalarSizeInBits())
16450 return SDValue();
16451
16452 return DCI.DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT, Operand: Op);
16453}
16454
16455/// PerformVDUPCombine - Target-specific dag combine xforms for ARMISD::VDUP.
16456static SDValue PerformVDUPCombine(SDNode *N, SelectionDAG &DAG,
16457 const ARMSubtarget *Subtarget) {
16458 SDValue Op = N->getOperand(Num: 0);
16459 SDLoc dl(N);
16460
16461 if (Subtarget->hasMVEIntegerOps()) {
16462 // Convert VDUP f32 -> VDUP BITCAST i32 under MVE, as we know the value will
16463 // need to come from a GPR.
16464 if (Op.getValueType() == MVT::f32)
16465 return DAG.getNode(Opcode: ARMISD::VDUP, DL: dl, VT: N->getValueType(ResNo: 0),
16466 Operand: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: Op));
16467 else if (Op.getValueType() == MVT::f16)
16468 return DAG.getNode(Opcode: ARMISD::VDUP, DL: dl, VT: N->getValueType(ResNo: 0),
16469 Operand: DAG.getNode(Opcode: ARMISD::VMOVrh, DL: dl, VT: MVT::i32, Operand: Op));
16470 }
16471
16472 if (!Subtarget->hasNEON())
16473 return SDValue();
16474
16475 // Match VDUP(LOAD) -> VLD1DUP.
16476 // We match this pattern here rather than waiting for isel because the
16477 // transform is only legal for unindexed loads.
16478 LoadSDNode *LD = dyn_cast<LoadSDNode>(Val: Op.getNode());
16479 if (LD && Op.hasOneUse() && LD->isUnindexed() &&
16480 LD->getMemoryVT() == N->getValueType(ResNo: 0).getVectorElementType()) {
16481 SDValue Ops[] = {LD->getOperand(Num: 0), LD->getOperand(Num: 1),
16482 DAG.getConstant(Val: LD->getAlign().value(), DL: SDLoc(N), VT: MVT::i32)};
16483 SDVTList SDTys = DAG.getVTList(VT1: N->getValueType(ResNo: 0), VT2: MVT::Other);
16484 SDValue VLDDup =
16485 DAG.getMemIntrinsicNode(Opcode: ARMISD::VLD1DUP, dl: SDLoc(N), VTList: SDTys, Ops,
16486 MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand());
16487 DAG.ReplaceAllUsesOfValueWith(From: SDValue(LD, 1), To: VLDDup.getValue(R: 1));
16488 return VLDDup;
16489 }
16490
16491 return SDValue();
16492}
16493
16494static SDValue PerformLOADCombine(SDNode *N,
16495 TargetLowering::DAGCombinerInfo &DCI,
16496 const ARMSubtarget *Subtarget) {
16497 EVT VT = N->getValueType(ResNo: 0);
16498
16499 // If this is a legal vector load, try to combine it into a VLD1_UPD.
16500 if (Subtarget->hasNEON() && ISD::isNormalLoad(N) && VT.isVector() &&
16501 DCI.DAG.getTargetLoweringInfo().isTypeLegal(VT))
16502 return CombineBaseUpdate(N, DCI);
16503
16504 return SDValue();
16505}
16506
16507// Optimize trunc store (of multiple scalars) to shuffle and store. First,
16508// pack all of the elements in one place. Next, store to memory in fewer
16509// chunks.
16510static SDValue PerformTruncatingStoreCombine(StoreSDNode *St,
16511 SelectionDAG &DAG) {
16512 SDValue StVal = St->getValue();
16513 EVT VT = StVal.getValueType();
16514 if (!St->isTruncatingStore() || !VT.isVector())
16515 return SDValue();
16516 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
16517 EVT StVT = St->getMemoryVT();
16518 unsigned NumElems = VT.getVectorNumElements();
16519 assert(StVT != VT && "Cannot truncate to the same type");
16520 unsigned FromEltSz = VT.getScalarSizeInBits();
16521 unsigned ToEltSz = StVT.getScalarSizeInBits();
16522
16523 // From, To sizes and ElemCount must be pow of two
16524 if (!isPowerOf2_32(Value: NumElems * FromEltSz * ToEltSz))
16525 return SDValue();
16526
16527 // We are going to use the original vector elt for storing.
16528 // Accumulated smaller vector elements must be a multiple of the store size.
16529 if (0 != (NumElems * FromEltSz) % ToEltSz)
16530 return SDValue();
16531
16532 unsigned SizeRatio = FromEltSz / ToEltSz;
16533 assert(SizeRatio * NumElems * ToEltSz == VT.getSizeInBits());
16534
16535 // Create a type on which we perform the shuffle.
16536 EVT WideVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: StVT.getScalarType(),
16537 NumElements: NumElems * SizeRatio);
16538 assert(WideVecVT.getSizeInBits() == VT.getSizeInBits());
16539
16540 SDLoc DL(St);
16541 SDValue WideVec = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: WideVecVT, Operand: StVal);
16542 SmallVector<int, 8> ShuffleVec(NumElems * SizeRatio, -1);
16543 for (unsigned i = 0; i < NumElems; ++i)
16544 ShuffleVec[i] = DAG.getDataLayout().isBigEndian() ? (i + 1) * SizeRatio - 1
16545 : i * SizeRatio;
16546
16547 // Can't shuffle using an illegal type.
16548 if (!TLI.isTypeLegal(VT: WideVecVT))
16549 return SDValue();
16550
16551 SDValue Shuff = DAG.getVectorShuffle(
16552 VT: WideVecVT, dl: DL, N1: WideVec, N2: DAG.getUNDEF(VT: WideVec.getValueType()), Mask: ShuffleVec);
16553 // At this point all of the data is stored at the bottom of the
16554 // register. We now need to save it to mem.
16555
16556 // Find the largest store unit
16557 MVT StoreType = MVT::i8;
16558 for (MVT Tp : MVT::integer_valuetypes()) {
16559 if (TLI.isTypeLegal(VT: Tp) && Tp.getSizeInBits() <= NumElems * ToEltSz)
16560 StoreType = Tp;
16561 }
16562 // Didn't find a legal store type.
16563 if (!TLI.isTypeLegal(VT: StoreType))
16564 return SDValue();
16565
16566 // Bitcast the original vector into a vector of store-size units
16567 EVT StoreVecVT =
16568 EVT::getVectorVT(Context&: *DAG.getContext(), VT: StoreType,
16569 NumElements: VT.getSizeInBits() / EVT(StoreType).getSizeInBits());
16570 assert(StoreVecVT.getSizeInBits() == VT.getSizeInBits());
16571 SDValue ShuffWide = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: StoreVecVT, Operand: Shuff);
16572 SmallVector<SDValue, 8> Chains;
16573 SDValue Increment = DAG.getConstant(Val: StoreType.getSizeInBits() / 8, DL,
16574 VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
16575 SDValue BasePtr = St->getBasePtr();
16576
16577 // Perform one or more big stores into memory.
16578 unsigned E = (ToEltSz * NumElems) / StoreType.getSizeInBits();
16579 for (unsigned I = 0; I < E; I++) {
16580 SDValue SubVec = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: StoreType,
16581 N1: ShuffWide, N2: DAG.getIntPtrConstant(Val: I, DL));
16582 SDValue Ch =
16583 DAG.getStore(Chain: St->getChain(), dl: DL, Val: SubVec, Ptr: BasePtr, PtrInfo: St->getPointerInfo(),
16584 Alignment: St->getAlign(), MMOFlags: St->getMemOperand()->getFlags());
16585 BasePtr =
16586 DAG.getNode(Opcode: ISD::ADD, DL, VT: BasePtr.getValueType(), N1: BasePtr, N2: Increment);
16587 Chains.push_back(Elt: Ch);
16588 }
16589 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: Chains);
16590}
16591
16592// Try taking a single vector store from an fpround (which would otherwise turn
16593// into an expensive buildvector) and splitting it into a series of narrowing
16594// stores.
16595static SDValue PerformSplittingToNarrowingStores(StoreSDNode *St,
16596 SelectionDAG &DAG) {
16597 if (!St->isSimple() || St->isTruncatingStore() || !St->isUnindexed())
16598 return SDValue();
16599 SDValue Trunc = St->getValue();
16600 if (Trunc->getOpcode() != ISD::FP_ROUND)
16601 return SDValue();
16602 EVT FromVT = Trunc->getOperand(Num: 0).getValueType();
16603 EVT ToVT = Trunc.getValueType();
16604 if (!ToVT.isVector())
16605 return SDValue();
16606 assert(FromVT.getVectorNumElements() == ToVT.getVectorNumElements());
16607 EVT ToEltVT = ToVT.getVectorElementType();
16608 EVT FromEltVT = FromVT.getVectorElementType();
16609
16610 if (FromEltVT != MVT::f32 || ToEltVT != MVT::f16)
16611 return SDValue();
16612
16613 unsigned NumElements = 4;
16614 if (FromVT.getVectorNumElements() % NumElements != 0)
16615 return SDValue();
16616
16617 // Test if the Trunc will be convertible to a VMOVN with a shuffle, and if so
16618 // use the VMOVN over splitting the store. We are looking for patterns of:
16619 // !rev: 0 N 1 N+1 2 N+2 ...
16620 // rev: N 0 N+1 1 N+2 2 ...
16621 // The shuffle may either be a single source (in which case N = NumElts/2) or
16622 // two inputs extended with concat to the same size (in which case N =
16623 // NumElts).
16624 auto isVMOVNShuffle = [&](ShuffleVectorSDNode *SVN, bool Rev) {
16625 ArrayRef<int> M = SVN->getMask();
16626 unsigned NumElts = ToVT.getVectorNumElements();
16627 if (SVN->getOperand(Num: 1).isUndef())
16628 NumElts /= 2;
16629
16630 unsigned Off0 = Rev ? NumElts : 0;
16631 unsigned Off1 = Rev ? 0 : NumElts;
16632
16633 for (unsigned I = 0; I < NumElts; I += 2) {
16634 if (M[I] >= 0 && M[I] != (int)(Off0 + I / 2))
16635 return false;
16636 if (M[I + 1] >= 0 && M[I + 1] != (int)(Off1 + I / 2))
16637 return false;
16638 }
16639
16640 return true;
16641 };
16642
16643 if (auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Val: Trunc.getOperand(i: 0)))
16644 if (isVMOVNShuffle(Shuffle, false) || isVMOVNShuffle(Shuffle, true))
16645 return SDValue();
16646
16647 LLVMContext &C = *DAG.getContext();
16648 SDLoc DL(St);
16649 // Details about the old store
16650 SDValue Ch = St->getChain();
16651 SDValue BasePtr = St->getBasePtr();
16652 Align Alignment = St->getBaseAlign();
16653 MachineMemOperand::Flags MMOFlags = St->getMemOperand()->getFlags();
16654 AAMDNodes AAInfo = St->getAAInfo();
16655
16656 // We split the store into slices of NumElements. fp16 trunc stores are vcvt
16657 // and then stored as truncating integer stores.
16658 EVT NewFromVT = EVT::getVectorVT(Context&: C, VT: FromEltVT, NumElements);
16659 EVT NewToVT = EVT::getVectorVT(
16660 Context&: C, VT: EVT::getIntegerVT(Context&: C, BitWidth: ToEltVT.getSizeInBits()), NumElements);
16661
16662 SmallVector<SDValue, 4> Stores;
16663 for (unsigned i = 0; i < FromVT.getVectorNumElements() / NumElements; i++) {
16664 unsigned NewOffset = i * NumElements * ToEltVT.getSizeInBits() / 8;
16665 SDValue NewPtr =
16666 DAG.getObjectPtrOffset(SL: DL, Ptr: BasePtr, Offset: TypeSize::getFixed(ExactSize: NewOffset));
16667
16668 SDValue Extract =
16669 DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: NewFromVT, N1: Trunc.getOperand(i: 0),
16670 N2: DAG.getConstant(Val: i * NumElements, DL, VT: MVT::i32));
16671
16672 SDValue FPTrunc =
16673 DAG.getNode(Opcode: ARMISD::VCVTN, DL, VT: MVT::v8f16, N1: DAG.getUNDEF(VT: MVT::v8f16),
16674 N2: Extract, N3: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
16675 Extract = DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL, VT: MVT::v4i32, Operand: FPTrunc);
16676
16677 SDValue Store = DAG.getTruncStore(
16678 Chain: Ch, dl: DL, Val: Extract, Ptr: NewPtr, PtrInfo: St->getPointerInfo().getWithOffset(O: NewOffset),
16679 SVT: NewToVT, Alignment, MMOFlags, Metadata: AAInfo);
16680 Stores.push_back(Elt: Store);
16681 }
16682 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: Stores);
16683}
16684
16685// Try taking a single vector store from an MVETRUNC (which would otherwise turn
16686// into an expensive buildvector) and splitting it into a series of narrowing
16687// stores.
16688static SDValue PerformSplittingMVETruncToNarrowingStores(StoreSDNode *St,
16689 SelectionDAG &DAG) {
16690 if (!St->isSimple() || St->isTruncatingStore() || !St->isUnindexed())
16691 return SDValue();
16692 SDValue Trunc = St->getValue();
16693 if (Trunc->getOpcode() != ARMISD::MVETRUNC)
16694 return SDValue();
16695 EVT FromVT = Trunc->getOperand(Num: 0).getValueType();
16696 EVT ToVT = Trunc.getValueType();
16697
16698 LLVMContext &C = *DAG.getContext();
16699 SDLoc DL(St);
16700 // Details about the old store
16701 SDValue Ch = St->getChain();
16702 SDValue BasePtr = St->getBasePtr();
16703 Align Alignment = St->getBaseAlign();
16704 MachineMemOperand::Flags MMOFlags = St->getMemOperand()->getFlags();
16705 AAMDNodes AAInfo = St->getAAInfo();
16706
16707 EVT NewToVT = EVT::getVectorVT(Context&: C, VT: ToVT.getVectorElementType(),
16708 NumElements: FromVT.getVectorNumElements());
16709
16710 SmallVector<SDValue, 4> Stores;
16711 for (unsigned i = 0; i < Trunc.getNumOperands(); i++) {
16712 unsigned NewOffset =
16713 i * FromVT.getVectorNumElements() * ToVT.getScalarSizeInBits() / 8;
16714 SDValue NewPtr =
16715 DAG.getObjectPtrOffset(SL: DL, Ptr: BasePtr, Offset: TypeSize::getFixed(ExactSize: NewOffset));
16716
16717 SDValue Extract = Trunc.getOperand(i);
16718 SDValue Store = DAG.getTruncStore(
16719 Chain: Ch, dl: DL, Val: Extract, Ptr: NewPtr, PtrInfo: St->getPointerInfo().getWithOffset(O: NewOffset),
16720 SVT: NewToVT, Alignment, MMOFlags, Metadata: AAInfo);
16721 Stores.push_back(Elt: Store);
16722 }
16723 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: Stores);
16724}
16725
16726// Given a floating point store from an extracted vector, with an integer
16727// VGETLANE that already exists, store the existing VGETLANEu directly. This can
16728// help reduce fp register pressure, doesn't require the fp extract and allows
16729// use of more integer post-inc stores not available with vstr.
16730static SDValue PerformExtractFpToIntStores(StoreSDNode *St, SelectionDAG &DAG) {
16731 if (!St->isSimple() || St->isTruncatingStore() || !St->isUnindexed())
16732 return SDValue();
16733 SDValue Extract = St->getValue();
16734 EVT VT = Extract.getValueType();
16735 // For now only uses f16. This may be useful for f32 too, but that will
16736 // be bitcast(extract), not the VGETLANEu we currently check here.
16737 if (VT != MVT::f16 || Extract->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
16738 return SDValue();
16739
16740 SDNode *GetLane =
16741 DAG.getNodeIfExists(Opcode: ARMISD::VGETLANEu, VTList: DAG.getVTList(VT: MVT::i32),
16742 Ops: {Extract.getOperand(i: 0), Extract.getOperand(i: 1)});
16743 if (!GetLane)
16744 return SDValue();
16745
16746 LLVMContext &C = *DAG.getContext();
16747 SDLoc DL(St);
16748 // Create a new integer store to replace the existing floating point version.
16749 SDValue Ch = St->getChain();
16750 SDValue BasePtr = St->getBasePtr();
16751 Align Alignment = St->getBaseAlign();
16752 MachineMemOperand::Flags MMOFlags = St->getMemOperand()->getFlags();
16753 AAMDNodes AAInfo = St->getAAInfo();
16754 EVT NewToVT = EVT::getIntegerVT(Context&: C, BitWidth: VT.getSizeInBits());
16755 SDValue Store = DAG.getTruncStore(Chain: Ch, dl: DL, Val: SDValue(GetLane, 0), Ptr: BasePtr,
16756 PtrInfo: St->getPointerInfo(), SVT: NewToVT, Alignment,
16757 MMOFlags, Metadata: AAInfo);
16758
16759 return Store;
16760}
16761
16762/// PerformSTORECombine - Target-specific dag combine xforms for
16763/// ISD::STORE.
16764static SDValue PerformSTORECombine(SDNode *N,
16765 TargetLowering::DAGCombinerInfo &DCI,
16766 const ARMSubtarget *Subtarget) {
16767 StoreSDNode *St = cast<StoreSDNode>(Val: N);
16768 if (St->isVolatile())
16769 return SDValue();
16770 SDValue StVal = St->getValue();
16771 EVT VT = StVal.getValueType();
16772
16773 if (Subtarget->hasNEON())
16774 if (SDValue Store = PerformTruncatingStoreCombine(St, DAG&: DCI.DAG))
16775 return Store;
16776
16777 if (Subtarget->hasMVEFloatOps())
16778 if (SDValue NewToken = PerformSplittingToNarrowingStores(St, DAG&: DCI.DAG))
16779 return NewToken;
16780
16781 if (Subtarget->hasMVEIntegerOps()) {
16782 if (SDValue NewChain = PerformExtractFpToIntStores(St, DAG&: DCI.DAG))
16783 return NewChain;
16784 if (SDValue NewToken =
16785 PerformSplittingMVETruncToNarrowingStores(St, DAG&: DCI.DAG))
16786 return NewToken;
16787 }
16788
16789 if (!ISD::isNormalStore(N: St))
16790 return SDValue();
16791
16792 // Split a store of a VMOVDRR into two integer stores to avoid mixing NEON and
16793 // ARM stores of arguments in the same cache line.
16794 if (StVal.getOpcode() == ARMISD::VMOVDRR && StVal->hasOneUse()) {
16795 SelectionDAG &DAG = DCI.DAG;
16796 bool isBigEndian = DAG.getDataLayout().isBigEndian();
16797 SDLoc DL(St);
16798 SDValue BasePtr = St->getBasePtr();
16799 SDValue NewST1 =
16800 DAG.getStore(Chain: St->getChain(), dl: DL, Val: StVal.getOperand(i: isBigEndian ? 1 : 0),
16801 Ptr: BasePtr, PtrInfo: St->getPointerInfo(), Alignment: St->getBaseAlign(),
16802 MMOFlags: St->getMemOperand()->getFlags());
16803
16804 SDValue OffsetPtr = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i32, N1: BasePtr,
16805 N2: DAG.getConstant(Val: 4, DL, VT: MVT::i32));
16806 return DAG.getStore(Chain: NewST1.getValue(R: 0), dl: DL,
16807 Val: StVal.getOperand(i: isBigEndian ? 0 : 1), Ptr: OffsetPtr,
16808 PtrInfo: St->getPointerInfo().getWithOffset(O: 4),
16809 Alignment: St->getBaseAlign(), MMOFlags: St->getMemOperand()->getFlags());
16810 }
16811
16812 if (StVal.getValueType() == MVT::i64 &&
16813 StVal.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
16814 // Bitcast an i64 store extracted from a vector to f64.
16815 // Otherwise, the i64 value will be legalized to a pair of i32 values.
16816 SelectionDAG &DAG = DCI.DAG;
16817 SDLoc dl(StVal);
16818 SDValue IntVec = StVal.getOperand(i: 0);
16819 EVT FloatVT =
16820 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::f64,
16821 NumElements: IntVec.getValueType().getVectorNumElements());
16822 SDValue Vec = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: FloatVT, Operand: IntVec);
16823 SDValue ExtElt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: MVT::f64, N1: Vec,
16824 N2: StVal.getOperand(i: 1));
16825 dl = SDLoc(N);
16826 SDValue V = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i64, Operand: ExtElt);
16827 // Make the DAGCombiner fold the bitcasts.
16828 DCI.AddToWorklist(N: Vec.getNode());
16829 DCI.AddToWorklist(N: ExtElt.getNode());
16830 DCI.AddToWorklist(N: V.getNode());
16831 return DAG.getStore(Chain: St->getChain(), dl, Val: V, Ptr: St->getBasePtr(),
16832 PtrInfo: St->getPointerInfo(), Alignment: St->getAlign(),
16833 MMOFlags: St->getMemOperand()->getFlags(), Metadata: St->getAAInfo());
16834 }
16835
16836 // If this is a legal vector store, try to combine it into a VST1_UPD.
16837 if (Subtarget->hasNEON() && ISD::isNormalStore(N) && VT.isVector() &&
16838 DCI.DAG.getTargetLoweringInfo().isTypeLegal(VT))
16839 return CombineBaseUpdate(N, DCI);
16840
16841 return SDValue();
16842}
16843
16844/// PerformVCVTCombine - VCVT (floating-point to fixed-point, Advanced SIMD)
16845/// can replace combinations of VMUL and VCVT (floating-point to integer)
16846/// when the VMUL has a constant operand that is a power of 2.
16847///
16848/// Example (assume d17 = <float 8.000000e+00, float 8.000000e+00>):
16849/// vmul.f32 d16, d17, d16
16850/// vcvt.s32.f32 d16, d16
16851/// becomes:
16852/// vcvt.s32.f32 d16, d16, #3
16853static SDValue PerformVCVTCombine(SDNode *N, SelectionDAG &DAG,
16854 const ARMSubtarget *Subtarget) {
16855 if (!Subtarget->hasNEON())
16856 return SDValue();
16857
16858 SDValue Op = N->getOperand(Num: 0);
16859 if (!Op.getValueType().isVector() || !Op.getValueType().isSimple() ||
16860 Op.getOpcode() != ISD::FMUL)
16861 return SDValue();
16862
16863 SDValue ConstVec = Op->getOperand(Num: 1);
16864 if (!isa<BuildVectorSDNode>(Val: ConstVec))
16865 return SDValue();
16866
16867 MVT FloatTy = Op.getSimpleValueType().getVectorElementType();
16868 uint32_t FloatBits = FloatTy.getSizeInBits();
16869 MVT IntTy = N->getSimpleValueType(ResNo: 0).getVectorElementType();
16870 uint32_t IntBits = IntTy.getSizeInBits();
16871 unsigned NumLanes = Op.getValueType().getVectorNumElements();
16872 if (FloatBits != 32 || IntBits > 32 || (NumLanes != 4 && NumLanes != 2)) {
16873 // These instructions only exist converting from f32 to i32. We can handle
16874 // smaller integers by generating an extra truncate, but larger ones would
16875 // be lossy. We also can't handle anything other than 2 or 4 lanes, since
16876 // these instructions only support v2i32/v4i32 types.
16877 return SDValue();
16878 }
16879
16880 BitVector UndefElements;
16881 BuildVectorSDNode *BV = cast<BuildVectorSDNode>(Val&: ConstVec);
16882 int32_t C = BV->getConstantFPSplatPow2ToLog2Int(UndefElements: &UndefElements, BitWidth: 33);
16883 if (C == -1 || C == 0 || C > 32)
16884 return SDValue();
16885
16886 SDLoc dl(N);
16887 bool isSigned = N->getOpcode() == ISD::FP_TO_SINT;
16888 unsigned IntrinsicOpcode = isSigned ? Intrinsic::arm_neon_vcvtfp2fxs :
16889 Intrinsic::arm_neon_vcvtfp2fxu;
16890 SDValue FixConv = DAG.getNode(
16891 Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: NumLanes == 2 ? MVT::v2i32 : MVT::v4i32,
16892 N1: DAG.getConstant(Val: IntrinsicOpcode, DL: dl, VT: MVT::i32), N2: Op->getOperand(Num: 0),
16893 N3: DAG.getConstant(Val: C, DL: dl, VT: MVT::i32));
16894
16895 if (IntBits < FloatBits)
16896 FixConv = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: N->getValueType(ResNo: 0), Operand: FixConv);
16897
16898 return FixConv;
16899}
16900
16901static SDValue PerformFAddVSelectCombine(SDNode *N, SelectionDAG &DAG,
16902 const ARMSubtarget *Subtarget) {
16903 if (!Subtarget->hasMVEFloatOps())
16904 return SDValue();
16905
16906 // Turn (fadd x, (vselect c, y, -0.0)) into (vselect c, (fadd x, y), x)
16907 // The second form can be more easily turned into a predicated vadd, and
16908 // possibly combined into a fma to become a predicated vfma.
16909 SDValue Op0 = N->getOperand(Num: 0);
16910 SDValue Op1 = N->getOperand(Num: 1);
16911 EVT VT = N->getValueType(ResNo: 0);
16912 SDLoc DL(N);
16913
16914 // The identity element for a fadd is -0.0 or +0.0 when the nsz flag is set,
16915 // which these VMOV's represent.
16916 auto isIdentitySplat = [&](SDValue Op, bool NSZ) {
16917 if (Op.getOpcode() != ISD::BITCAST ||
16918 Op.getOperand(i: 0).getOpcode() != ARMISD::VMOVIMM)
16919 return false;
16920 uint64_t ImmVal = Op.getOperand(i: 0).getConstantOperandVal(i: 0);
16921 if (VT == MVT::v4f32 && (ImmVal == 1664 || (ImmVal == 0 && NSZ)))
16922 return true;
16923 if (VT == MVT::v8f16 && (ImmVal == 2688 || (ImmVal == 0 && NSZ)))
16924 return true;
16925 return false;
16926 };
16927
16928 if (Op0.getOpcode() == ISD::VSELECT && Op1.getOpcode() != ISD::VSELECT)
16929 std::swap(a&: Op0, b&: Op1);
16930
16931 if (Op1.getOpcode() != ISD::VSELECT)
16932 return SDValue();
16933
16934 SDNodeFlags FaddFlags = N->getFlags();
16935 bool NSZ = FaddFlags.hasNoSignedZeros();
16936 if (!isIdentitySplat(Op1.getOperand(i: 2), NSZ))
16937 return SDValue();
16938
16939 SDValue FAdd =
16940 DAG.getNode(Opcode: ISD::FADD, DL, VT, N1: Op0, N2: Op1.getOperand(i: 1), Flags: FaddFlags);
16941 return DAG.getNode(Opcode: ISD::VSELECT, DL, VT, N1: Op1.getOperand(i: 0), N2: FAdd, N3: Op0, Flags: FaddFlags);
16942}
16943
16944static SDValue PerformFADDVCMLACombine(SDNode *N, SelectionDAG &DAG) {
16945 SDValue LHS = N->getOperand(Num: 0);
16946 SDValue RHS = N->getOperand(Num: 1);
16947 EVT VT = N->getValueType(ResNo: 0);
16948 SDLoc DL(N);
16949
16950 if (!N->getFlags().hasAllowReassociation())
16951 return SDValue();
16952
16953 // Combine fadd(a, vcmla(b, c, d)) -> vcmla(fadd(a, b), b, c)
16954 auto ReassocComplex = [&](SDValue A, SDValue B) {
16955 if (A.getOpcode() != ISD::INTRINSIC_WO_CHAIN)
16956 return SDValue();
16957 unsigned Opc = A.getConstantOperandVal(i: 0);
16958 if (Opc != Intrinsic::arm_mve_vcmlaq)
16959 return SDValue();
16960 SDValue VCMLA = DAG.getNode(
16961 Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT, N1: A.getOperand(i: 0), N2: A.getOperand(i: 1),
16962 N3: DAG.getNode(Opcode: ISD::FADD, DL, VT, N1: A.getOperand(i: 2), N2: B, Flags: N->getFlags()),
16963 N4: A.getOperand(i: 3), N5: A.getOperand(i: 4));
16964 VCMLA->setFlags(A->getFlags());
16965 return VCMLA;
16966 };
16967 if (SDValue R = ReassocComplex(LHS, RHS))
16968 return R;
16969 if (SDValue R = ReassocComplex(RHS, LHS))
16970 return R;
16971
16972 return SDValue();
16973}
16974
16975static SDValue PerformFADDCombine(SDNode *N, SelectionDAG &DAG,
16976 const ARMSubtarget *Subtarget) {
16977 if (SDValue S = PerformFAddVSelectCombine(N, DAG, Subtarget))
16978 return S;
16979 if (SDValue S = PerformFADDVCMLACombine(N, DAG))
16980 return S;
16981 return SDValue();
16982}
16983
16984/// PerformVMulVCTPCombine - VCVT (fixed-point to floating-point, Advanced SIMD)
16985/// can replace combinations of VCVT (integer to floating-point) and VMUL
16986/// when the VMUL has a constant operand that is a power of 2.
16987///
16988/// Example (assume d17 = <float 0.125, float 0.125>):
16989/// vcvt.f32.s32 d16, d16
16990/// vmul.f32 d16, d16, d17
16991/// becomes:
16992/// vcvt.f32.s32 d16, d16, #3
16993static SDValue PerformVMulVCTPCombine(SDNode *N, SelectionDAG &DAG,
16994 const ARMSubtarget *Subtarget) {
16995 if (!Subtarget->hasNEON())
16996 return SDValue();
16997
16998 SDValue Op = N->getOperand(Num: 0);
16999 unsigned OpOpcode = Op.getNode()->getOpcode();
17000 if (!N->getValueType(ResNo: 0).isVector() || !N->getValueType(ResNo: 0).isSimple() ||
17001 (OpOpcode != ISD::SINT_TO_FP && OpOpcode != ISD::UINT_TO_FP))
17002 return SDValue();
17003
17004 SDValue ConstVec = N->getOperand(Num: 1);
17005 if (!isa<BuildVectorSDNode>(Val: ConstVec))
17006 return SDValue();
17007
17008 MVT FloatTy = N->getSimpleValueType(ResNo: 0).getVectorElementType();
17009 uint32_t FloatBits = FloatTy.getSizeInBits();
17010 MVT IntTy = Op.getOperand(i: 0).getSimpleValueType().getVectorElementType();
17011 uint32_t IntBits = IntTy.getSizeInBits();
17012 unsigned NumLanes = Op.getValueType().getVectorNumElements();
17013 if (FloatBits != 32 || IntBits > 32 || (NumLanes != 4 && NumLanes != 2)) {
17014 // These instructions only exist converting from i32 to f32. We can handle
17015 // smaller integers by generating an extra extend, but larger ones would
17016 // be lossy. We also can't handle anything other than 2 or 4 lanes, since
17017 // these instructions only support v2i32/v4i32 types.
17018 return SDValue();
17019 }
17020
17021 ConstantFPSDNode *CN = isConstOrConstSplatFP(N: ConstVec, AllowUndefs: true);
17022 APFloat Recip(0.0f);
17023 if (!CN || !CN->getValueAPF().getExactInverse(Inv: &Recip))
17024 return SDValue();
17025
17026 bool IsExact;
17027 APSInt IntVal(33);
17028 if (Recip.convertToInteger(Result&: IntVal, RM: APFloat::rmTowardZero, IsExact: &IsExact) !=
17029 APFloat::opOK ||
17030 !IsExact)
17031 return SDValue();
17032
17033 int32_t C = IntVal.exactLogBase2();
17034 if (C == -1 || C == 0 || C > 32)
17035 return SDValue();
17036
17037 SDLoc DL(N);
17038 bool isSigned = OpOpcode == ISD::SINT_TO_FP;
17039 SDValue ConvInput = Op.getOperand(i: 0);
17040 if (IntBits < FloatBits)
17041 ConvInput = DAG.getNode(Opcode: isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL,
17042 VT: NumLanes == 2 ? MVT::v2i32 : MVT::v4i32, Operand: ConvInput);
17043
17044 unsigned IntrinsicOpcode = isSigned ? Intrinsic::arm_neon_vcvtfxs2fp
17045 : Intrinsic::arm_neon_vcvtfxu2fp;
17046 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: Op.getValueType(),
17047 N1: DAG.getConstant(Val: IntrinsicOpcode, DL, VT: MVT::i32), N2: ConvInput,
17048 N3: DAG.getConstant(Val: C, DL, VT: MVT::i32));
17049}
17050
17051static SDValue PerformVECREDUCE_ADDCombine(SDNode *N, SelectionDAG &DAG,
17052 const ARMSubtarget *ST) {
17053 if (!ST->hasMVEIntegerOps())
17054 return SDValue();
17055
17056 assert(N->getOpcode() == ISD::VECREDUCE_ADD);
17057 EVT ResVT = N->getValueType(ResNo: 0);
17058 SDValue N0 = N->getOperand(Num: 0);
17059 SDLoc dl(N);
17060
17061 // Try to turn vecreduce_add(add(x, y)) into vecreduce(x) + vecreduce(y)
17062 if (ResVT == MVT::i32 && N0.getOpcode() == ISD::ADD &&
17063 (N0.getValueType() == MVT::v4i32 || N0.getValueType() == MVT::v8i16 ||
17064 N0.getValueType() == MVT::v16i8)) {
17065 SDValue Red0 = DAG.getNode(Opcode: ISD::VECREDUCE_ADD, DL: dl, VT: ResVT, Operand: N0.getOperand(i: 0));
17066 SDValue Red1 = DAG.getNode(Opcode: ISD::VECREDUCE_ADD, DL: dl, VT: ResVT, Operand: N0.getOperand(i: 1));
17067 return DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ResVT, N1: Red0, N2: Red1);
17068 }
17069
17070 // We are looking for something that will have illegal types if left alone,
17071 // but that we can convert to a single instruction under MVE. For example
17072 // vecreduce_add(sext(A, v8i32)) => VADDV.s16 A
17073 // or
17074 // vecreduce_add(mul(zext(A, v16i32), zext(B, v16i32))) => VMLADAV.u8 A, B
17075
17076 // The legal cases are:
17077 // VADDV u/s 8/16/32
17078 // VMLAV u/s 8/16/32
17079 // VADDLV u/s 32
17080 // VMLALV u/s 16/32
17081
17082 // If the input vector is smaller than legal (v4i8/v4i16 for example) we can
17083 // extend it and use v4i32 instead.
17084 auto ExtTypeMatches = [](SDValue A, ArrayRef<MVT> ExtTypes) {
17085 EVT AVT = A.getValueType();
17086 return any_of(Range&: ExtTypes, P: [&](MVT Ty) {
17087 return AVT.getVectorNumElements() == Ty.getVectorNumElements() &&
17088 AVT.bitsLE(VT: Ty);
17089 });
17090 };
17091 auto ExtendIfNeeded = [&](SDValue A, unsigned ExtendCode) {
17092 EVT AVT = A.getValueType();
17093 if (!AVT.is128BitVector())
17094 A = DAG.getNode(
17095 Opcode: ExtendCode, DL: dl,
17096 VT: AVT.changeVectorElementType(
17097 Context&: *DAG.getContext(),
17098 EltVT: MVT::getIntegerVT(BitWidth: 128 / AVT.getVectorMinNumElements())),
17099 Operand: A);
17100 return A;
17101 };
17102 auto IsVADDV = [&](MVT RetTy, unsigned ExtendCode, ArrayRef<MVT> ExtTypes) {
17103 if (ResVT != RetTy || N0->getOpcode() != ExtendCode)
17104 return SDValue();
17105 SDValue A = N0->getOperand(Num: 0);
17106 if (ExtTypeMatches(A, ExtTypes))
17107 return ExtendIfNeeded(A, ExtendCode);
17108 return SDValue();
17109 };
17110 auto IsPredVADDV = [&](MVT RetTy, unsigned ExtendCode,
17111 ArrayRef<MVT> ExtTypes, SDValue &Mask) {
17112 if (ResVT != RetTy || N0->getOpcode() != ISD::VSELECT ||
17113 !ISD::isBuildVectorAllZeros(N: N0->getOperand(Num: 2).getNode()))
17114 return SDValue();
17115 Mask = N0->getOperand(Num: 0);
17116 SDValue Ext = N0->getOperand(Num: 1);
17117 if (Ext->getOpcode() != ExtendCode)
17118 return SDValue();
17119 SDValue A = Ext->getOperand(Num: 0);
17120 if (ExtTypeMatches(A, ExtTypes))
17121 return ExtendIfNeeded(A, ExtendCode);
17122 return SDValue();
17123 };
17124 auto IsVMLAV = [&](MVT RetTy, unsigned ExtendCode, ArrayRef<MVT> ExtTypes,
17125 SDValue &A, SDValue &B) {
17126 // For a vmla we are trying to match a larger pattern:
17127 // ExtA = sext/zext A
17128 // ExtB = sext/zext B
17129 // Mul = mul ExtA, ExtB
17130 // vecreduce.add Mul
17131 // There might also be en extra extend between the mul and the addreduce, so
17132 // long as the bitwidth is high enough to make them equivalent (for example
17133 // original v8i16 might be mul at v8i32 and the reduce happens at v8i64).
17134 if (ResVT != RetTy)
17135 return false;
17136 SDValue Mul = N0;
17137 if (Mul->getOpcode() == ExtendCode &&
17138 Mul->getOperand(Num: 0).getScalarValueSizeInBits() * 2 >=
17139 ResVT.getScalarSizeInBits())
17140 Mul = Mul->getOperand(Num: 0);
17141 if (Mul->getOpcode() != ISD::MUL)
17142 return false;
17143 SDValue ExtA = Mul->getOperand(Num: 0);
17144 SDValue ExtB = Mul->getOperand(Num: 1);
17145 if (ExtA->getOpcode() != ExtendCode || ExtB->getOpcode() != ExtendCode)
17146 return false;
17147 A = ExtA->getOperand(Num: 0);
17148 B = ExtB->getOperand(Num: 0);
17149 if (ExtTypeMatches(A, ExtTypes) && ExtTypeMatches(B, ExtTypes)) {
17150 A = ExtendIfNeeded(A, ExtendCode);
17151 B = ExtendIfNeeded(B, ExtendCode);
17152 return true;
17153 }
17154 return false;
17155 };
17156 auto IsPredVMLAV = [&](MVT RetTy, unsigned ExtendCode, ArrayRef<MVT> ExtTypes,
17157 SDValue &A, SDValue &B, SDValue &Mask) {
17158 // Same as the pattern above with a select for the zero predicated lanes
17159 // ExtA = sext/zext A
17160 // ExtB = sext/zext B
17161 // Mul = mul ExtA, ExtB
17162 // N0 = select Mask, Mul, 0
17163 // vecreduce.add N0
17164 if (ResVT != RetTy || N0->getOpcode() != ISD::VSELECT ||
17165 !ISD::isBuildVectorAllZeros(N: N0->getOperand(Num: 2).getNode()))
17166 return false;
17167 Mask = N0->getOperand(Num: 0);
17168 SDValue Mul = N0->getOperand(Num: 1);
17169 if (Mul->getOpcode() == ExtendCode &&
17170 Mul->getOperand(Num: 0).getScalarValueSizeInBits() * 2 >=
17171 ResVT.getScalarSizeInBits())
17172 Mul = Mul->getOperand(Num: 0);
17173 if (Mul->getOpcode() != ISD::MUL)
17174 return false;
17175 SDValue ExtA = Mul->getOperand(Num: 0);
17176 SDValue ExtB = Mul->getOperand(Num: 1);
17177 if (ExtA->getOpcode() != ExtendCode || ExtB->getOpcode() != ExtendCode)
17178 return false;
17179 A = ExtA->getOperand(Num: 0);
17180 B = ExtB->getOperand(Num: 0);
17181 if (ExtTypeMatches(A, ExtTypes) && ExtTypeMatches(B, ExtTypes)) {
17182 A = ExtendIfNeeded(A, ExtendCode);
17183 B = ExtendIfNeeded(B, ExtendCode);
17184 return true;
17185 }
17186 return false;
17187 };
17188 auto Create64bitNode = [&](unsigned Opcode, ArrayRef<SDValue> Ops) {
17189 // Split illegal MVT::v16i8->i64 vector reductions into two legal v8i16->i64
17190 // reductions. The operands are extended with MVEEXT, but as they are
17191 // reductions the lane orders do not matter. MVEEXT may be combined with
17192 // loads to produce two extending loads, or else they will be expanded to
17193 // VREV/VMOVL.
17194 EVT VT = Ops[0].getValueType();
17195 if (VT == MVT::v16i8) {
17196 assert((Opcode == ARMISD::VMLALVs || Opcode == ARMISD::VMLALVu) &&
17197 "Unexpected illegal long reduction opcode");
17198 bool IsUnsigned = Opcode == ARMISD::VMLALVu;
17199
17200 SDValue Ext0 =
17201 DAG.getNode(Opcode: IsUnsigned ? ARMISD::MVEZEXT : ARMISD::MVESEXT, DL: dl,
17202 VTList: DAG.getVTList(VT1: MVT::v8i16, VT2: MVT::v8i16), N: Ops[0]);
17203 SDValue Ext1 =
17204 DAG.getNode(Opcode: IsUnsigned ? ARMISD::MVEZEXT : ARMISD::MVESEXT, DL: dl,
17205 VTList: DAG.getVTList(VT1: MVT::v8i16, VT2: MVT::v8i16), N: Ops[1]);
17206
17207 SDValue MLA0 = DAG.getNode(Opcode, DL: dl, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
17208 N1: Ext0, N2: Ext1);
17209 SDValue MLA1 =
17210 DAG.getNode(Opcode: IsUnsigned ? ARMISD::VMLALVAu : ARMISD::VMLALVAs, DL: dl,
17211 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N1: MLA0, N2: MLA0.getValue(R: 1),
17212 N3: Ext0.getValue(R: 1), N4: Ext1.getValue(R: 1));
17213 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: MVT::i64, N1: MLA1, N2: MLA1.getValue(R: 1));
17214 }
17215 SDValue Node = DAG.getNode(Opcode, DL: dl, ResultTys: {MVT::i32, MVT::i32}, Ops);
17216 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: MVT::i64, N1: Node,
17217 N2: SDValue(Node.getNode(), 1));
17218 };
17219
17220 SDValue A, B;
17221 SDValue Mask;
17222 if (IsVMLAV(MVT::i32, ISD::SIGN_EXTEND, {MVT::v8i16, MVT::v16i8}, A, B))
17223 return DAG.getNode(Opcode: ARMISD::VMLAVs, DL: dl, VT: ResVT, N1: A, N2: B);
17224 if (IsVMLAV(MVT::i32, ISD::ZERO_EXTEND, {MVT::v8i16, MVT::v16i8}, A, B))
17225 return DAG.getNode(Opcode: ARMISD::VMLAVu, DL: dl, VT: ResVT, N1: A, N2: B);
17226 if (IsVMLAV(MVT::i64, ISD::SIGN_EXTEND, {MVT::v16i8, MVT::v8i16, MVT::v4i32},
17227 A, B))
17228 return Create64bitNode(ARMISD::VMLALVs, {A, B});
17229 if (IsVMLAV(MVT::i64, ISD::ZERO_EXTEND, {MVT::v16i8, MVT::v8i16, MVT::v4i32},
17230 A, B))
17231 return Create64bitNode(ARMISD::VMLALVu, {A, B});
17232 if (IsVMLAV(MVT::i16, ISD::SIGN_EXTEND, {MVT::v16i8}, A, B))
17233 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: ResVT,
17234 Operand: DAG.getNode(Opcode: ARMISD::VMLAVs, DL: dl, VT: MVT::i32, N1: A, N2: B));
17235 if (IsVMLAV(MVT::i16, ISD::ZERO_EXTEND, {MVT::v16i8}, A, B))
17236 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: ResVT,
17237 Operand: DAG.getNode(Opcode: ARMISD::VMLAVu, DL: dl, VT: MVT::i32, N1: A, N2: B));
17238
17239 if (IsPredVMLAV(MVT::i32, ISD::SIGN_EXTEND, {MVT::v8i16, MVT::v16i8}, A, B,
17240 Mask))
17241 return DAG.getNode(Opcode: ARMISD::VMLAVps, DL: dl, VT: ResVT, N1: A, N2: B, N3: Mask);
17242 if (IsPredVMLAV(MVT::i32, ISD::ZERO_EXTEND, {MVT::v8i16, MVT::v16i8}, A, B,
17243 Mask))
17244 return DAG.getNode(Opcode: ARMISD::VMLAVpu, DL: dl, VT: ResVT, N1: A, N2: B, N3: Mask);
17245 if (IsPredVMLAV(MVT::i64, ISD::SIGN_EXTEND, {MVT::v8i16, MVT::v4i32}, A, B,
17246 Mask))
17247 return Create64bitNode(ARMISD::VMLALVps, {A, B, Mask});
17248 if (IsPredVMLAV(MVT::i64, ISD::ZERO_EXTEND, {MVT::v8i16, MVT::v4i32}, A, B,
17249 Mask))
17250 return Create64bitNode(ARMISD::VMLALVpu, {A, B, Mask});
17251 if (IsPredVMLAV(MVT::i16, ISD::SIGN_EXTEND, {MVT::v16i8}, A, B, Mask))
17252 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: ResVT,
17253 Operand: DAG.getNode(Opcode: ARMISD::VMLAVps, DL: dl, VT: MVT::i32, N1: A, N2: B, N3: Mask));
17254 if (IsPredVMLAV(MVT::i16, ISD::ZERO_EXTEND, {MVT::v16i8}, A, B, Mask))
17255 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: ResVT,
17256 Operand: DAG.getNode(Opcode: ARMISD::VMLAVpu, DL: dl, VT: MVT::i32, N1: A, N2: B, N3: Mask));
17257
17258 if (SDValue A = IsVADDV(MVT::i32, ISD::SIGN_EXTEND, {MVT::v8i16, MVT::v16i8}))
17259 return DAG.getNode(Opcode: ARMISD::VADDVs, DL: dl, VT: ResVT, Operand: A);
17260 if (SDValue A = IsVADDV(MVT::i32, ISD::ZERO_EXTEND, {MVT::v8i16, MVT::v16i8}))
17261 return DAG.getNode(Opcode: ARMISD::VADDVu, DL: dl, VT: ResVT, Operand: A);
17262 if (SDValue A = IsVADDV(MVT::i64, ISD::SIGN_EXTEND, {MVT::v4i32}))
17263 return Create64bitNode(ARMISD::VADDLVs, {A});
17264 if (SDValue A = IsVADDV(MVT::i64, ISD::ZERO_EXTEND, {MVT::v4i32}))
17265 return Create64bitNode(ARMISD::VADDLVu, {A});
17266 if (SDValue A = IsVADDV(MVT::i16, ISD::SIGN_EXTEND, {MVT::v16i8}))
17267 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: ResVT,
17268 Operand: DAG.getNode(Opcode: ARMISD::VADDVs, DL: dl, VT: MVT::i32, Operand: A));
17269 if (SDValue A = IsVADDV(MVT::i16, ISD::ZERO_EXTEND, {MVT::v16i8}))
17270 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: ResVT,
17271 Operand: DAG.getNode(Opcode: ARMISD::VADDVu, DL: dl, VT: MVT::i32, Operand: A));
17272
17273 if (SDValue A = IsPredVADDV(MVT::i32, ISD::SIGN_EXTEND, {MVT::v8i16, MVT::v16i8}, Mask))
17274 return DAG.getNode(Opcode: ARMISD::VADDVps, DL: dl, VT: ResVT, N1: A, N2: Mask);
17275 if (SDValue A = IsPredVADDV(MVT::i32, ISD::ZERO_EXTEND, {MVT::v8i16, MVT::v16i8}, Mask))
17276 return DAG.getNode(Opcode: ARMISD::VADDVpu, DL: dl, VT: ResVT, N1: A, N2: Mask);
17277 if (SDValue A = IsPredVADDV(MVT::i64, ISD::SIGN_EXTEND, {MVT::v4i32}, Mask))
17278 return Create64bitNode(ARMISD::VADDLVps, {A, Mask});
17279 if (SDValue A = IsPredVADDV(MVT::i64, ISD::ZERO_EXTEND, {MVT::v4i32}, Mask))
17280 return Create64bitNode(ARMISD::VADDLVpu, {A, Mask});
17281 if (SDValue A = IsPredVADDV(MVT::i16, ISD::SIGN_EXTEND, {MVT::v16i8}, Mask))
17282 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: ResVT,
17283 Operand: DAG.getNode(Opcode: ARMISD::VADDVps, DL: dl, VT: MVT::i32, N1: A, N2: Mask));
17284 if (SDValue A = IsPredVADDV(MVT::i16, ISD::ZERO_EXTEND, {MVT::v16i8}, Mask))
17285 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: ResVT,
17286 Operand: DAG.getNode(Opcode: ARMISD::VADDVpu, DL: dl, VT: MVT::i32, N1: A, N2: Mask));
17287
17288 // Some complications. We can get a case where the two inputs of the mul are
17289 // the same, then the output sext will have been helpfully converted to a
17290 // zext. Turn it back.
17291 SDValue Op = N0;
17292 if (Op->getOpcode() == ISD::VSELECT)
17293 Op = Op->getOperand(Num: 1);
17294 if (Op->getOpcode() == ISD::ZERO_EXTEND &&
17295 Op->getOperand(Num: 0)->getOpcode() == ISD::MUL) {
17296 SDValue Mul = Op->getOperand(Num: 0);
17297 if (Mul->getOperand(Num: 0) == Mul->getOperand(Num: 1) &&
17298 Mul->getOperand(Num: 0)->getOpcode() == ISD::SIGN_EXTEND) {
17299 SDValue Ext = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: N0->getValueType(ResNo: 0), Operand: Mul);
17300 if (Op != N0)
17301 Ext = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: N0->getValueType(ResNo: 0),
17302 N1: N0->getOperand(Num: 0), N2: Ext, N3: N0->getOperand(Num: 2));
17303 return DAG.getNode(Opcode: ISD::VECREDUCE_ADD, DL: dl, VT: ResVT, Operand: Ext);
17304 }
17305 }
17306
17307 return SDValue();
17308}
17309
17310// Looks for vaddv(shuffle) or vmlav(shuffle, shuffle), with a shuffle where all
17311// the lanes are used. Due to the reduction being commutative the shuffle can be
17312// removed.
17313static SDValue PerformReduceShuffleCombine(SDNode *N, SelectionDAG &DAG) {
17314 unsigned VecOp = N->getOperand(Num: 0).getValueType().isVector() ? 0 : 2;
17315 auto *Shuf = dyn_cast<ShuffleVectorSDNode>(Val: N->getOperand(Num: VecOp));
17316 if (!Shuf || !Shuf->getOperand(Num: 1).isUndef())
17317 return SDValue();
17318
17319 // Check all elements are used once in the mask.
17320 ArrayRef<int> Mask = Shuf->getMask();
17321 APInt SetElts(Mask.size(), 0);
17322 for (int E : Mask) {
17323 if (E < 0 || E >= (int)Mask.size())
17324 return SDValue();
17325 SetElts.setBit(E);
17326 }
17327 if (!SetElts.isAllOnes())
17328 return SDValue();
17329
17330 if (N->getNumOperands() != VecOp + 1) {
17331 auto *Shuf2 = dyn_cast<ShuffleVectorSDNode>(Val: N->getOperand(Num: VecOp + 1));
17332 if (!Shuf2 || !Shuf2->getOperand(Num: 1).isUndef() || Shuf2->getMask() != Mask)
17333 return SDValue();
17334 }
17335
17336 SmallVector<SDValue> Ops;
17337 for (SDValue Op : N->ops()) {
17338 if (Op.getValueType().isVector())
17339 Ops.push_back(Elt: Op.getOperand(i: 0));
17340 else
17341 Ops.push_back(Elt: Op);
17342 }
17343 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VTList: N->getVTList(), Ops);
17344}
17345
17346static SDValue PerformVMOVNCombine(SDNode *N,
17347 TargetLowering::DAGCombinerInfo &DCI) {
17348 SDValue Op0 = N->getOperand(Num: 0);
17349 SDValue Op1 = N->getOperand(Num: 1);
17350 unsigned IsTop = N->getConstantOperandVal(Num: 2);
17351
17352 // VMOVNT a undef -> a
17353 // VMOVNB a undef -> a
17354 // VMOVNB undef a -> a
17355 if (Op1->isUndef())
17356 return Op0;
17357 if (Op0->isUndef() && !IsTop)
17358 return Op1;
17359
17360 // VMOVNt(c, VQMOVNb(a, b)) => VQMOVNt(c, b)
17361 // VMOVNb(c, VQMOVNb(a, b)) => VQMOVNb(c, b)
17362 if ((Op1->getOpcode() == ARMISD::VQMOVNs ||
17363 Op1->getOpcode() == ARMISD::VQMOVNu) &&
17364 Op1->getConstantOperandVal(Num: 2) == 0)
17365 return DCI.DAG.getNode(Opcode: Op1->getOpcode(), DL: SDLoc(Op1), VT: N->getValueType(ResNo: 0),
17366 N1: Op0, N2: Op1->getOperand(Num: 1), N3: N->getOperand(Num: 2));
17367
17368 // Only the bottom lanes from Qm (Op1) and either the top or bottom lanes from
17369 // Qd (Op0) are demanded from a VMOVN, depending on whether we are inserting
17370 // into the top or bottom lanes.
17371 unsigned NumElts = N->getValueType(ResNo: 0).getVectorNumElements();
17372 APInt Op1DemandedElts = APInt::getSplat(NewLen: NumElts, V: APInt::getLowBitsSet(numBits: 2, loBitsSet: 1));
17373 APInt Op0DemandedElts =
17374 IsTop ? Op1DemandedElts
17375 : APInt::getSplat(NewLen: NumElts, V: APInt::getHighBitsSet(numBits: 2, hiBitsSet: 1));
17376
17377 const TargetLowering &TLI = DCI.DAG.getTargetLoweringInfo();
17378 if (TLI.SimplifyDemandedVectorElts(Op: Op0, DemandedElts: Op0DemandedElts, DCI))
17379 return SDValue(N, 0);
17380 if (TLI.SimplifyDemandedVectorElts(Op: Op1, DemandedElts: Op1DemandedElts, DCI))
17381 return SDValue(N, 0);
17382
17383 return SDValue();
17384}
17385
17386static SDValue PerformVQMOVNCombine(SDNode *N,
17387 TargetLowering::DAGCombinerInfo &DCI) {
17388 SDValue Op0 = N->getOperand(Num: 0);
17389 unsigned IsTop = N->getConstantOperandVal(Num: 2);
17390
17391 unsigned NumElts = N->getValueType(ResNo: 0).getVectorNumElements();
17392 APInt Op0DemandedElts =
17393 APInt::getSplat(NewLen: NumElts, V: IsTop ? APInt::getLowBitsSet(numBits: 2, loBitsSet: 1)
17394 : APInt::getHighBitsSet(numBits: 2, hiBitsSet: 1));
17395
17396 const TargetLowering &TLI = DCI.DAG.getTargetLoweringInfo();
17397 if (TLI.SimplifyDemandedVectorElts(Op: Op0, DemandedElts: Op0DemandedElts, DCI))
17398 return SDValue(N, 0);
17399 return SDValue();
17400}
17401
17402static SDValue PerformVQDMULHCombine(SDNode *N,
17403 TargetLowering::DAGCombinerInfo &DCI) {
17404 EVT VT = N->getValueType(ResNo: 0);
17405 SDValue LHS = N->getOperand(Num: 0);
17406 SDValue RHS = N->getOperand(Num: 1);
17407
17408 auto *Shuf0 = dyn_cast<ShuffleVectorSDNode>(Val&: LHS);
17409 auto *Shuf1 = dyn_cast<ShuffleVectorSDNode>(Val&: RHS);
17410 // Turn VQDMULH(shuffle, shuffle) -> shuffle(VQDMULH)
17411 if (Shuf0 && Shuf1 && Shuf0->getMask().equals(RHS: Shuf1->getMask()) &&
17412 LHS.getOperand(i: 1).isUndef() && RHS.getOperand(i: 1).isUndef() &&
17413 (LHS.hasOneUse() || RHS.hasOneUse() || LHS == RHS)) {
17414 SDLoc DL(N);
17415 SDValue NewBinOp = DCI.DAG.getNode(Opcode: N->getOpcode(), DL, VT,
17416 N1: LHS.getOperand(i: 0), N2: RHS.getOperand(i: 0));
17417 SDValue UndefV = LHS.getOperand(i: 1);
17418 return DCI.DAG.getVectorShuffle(VT, dl: DL, N1: NewBinOp, N2: UndefV, Mask: Shuf0->getMask());
17419 }
17420 return SDValue();
17421}
17422
17423static SDValue PerformLongShiftCombine(SDNode *N, SelectionDAG &DAG) {
17424 SDLoc DL(N);
17425 SDValue Op0 = N->getOperand(Num: 0);
17426 SDValue Op1 = N->getOperand(Num: 1);
17427
17428 // Turn X << -C -> X >> C and viceversa. The negative shifts can come up from
17429 // uses of the intrinsics.
17430 if (auto C = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 2))) {
17431 int ShiftAmt = C->getSExtValue();
17432 if (ShiftAmt == 0) {
17433 SDValue Merge = DAG.getMergeValues(Ops: {Op0, Op1}, dl: DL);
17434 DAG.ReplaceAllUsesWith(From: N, To: Merge.getNode());
17435 return SDValue();
17436 }
17437
17438 if (ShiftAmt >= -32 && ShiftAmt < 0) {
17439 unsigned NewOpcode =
17440 N->getOpcode() == ARMISD::LSLL ? ARMISD::LSRL : ARMISD::LSLL;
17441 SDValue NewShift = DAG.getNode(Opcode: NewOpcode, DL, VTList: N->getVTList(), N1: Op0, N2: Op1,
17442 N3: DAG.getConstant(Val: -ShiftAmt, DL, VT: MVT::i32));
17443 DAG.ReplaceAllUsesWith(From: N, To: NewShift.getNode());
17444 return NewShift;
17445 }
17446 }
17447
17448 return SDValue();
17449}
17450
17451/// PerformIntrinsicCombine - ARM-specific DAG combining for intrinsics.
17452SDValue ARMTargetLowering::PerformIntrinsicCombine(SDNode *N,
17453 DAGCombinerInfo &DCI) const {
17454 SelectionDAG &DAG = DCI.DAG;
17455 unsigned IntNo = N->getConstantOperandVal(Num: 0);
17456 switch (IntNo) {
17457 default:
17458 // Don't do anything for most intrinsics.
17459 break;
17460
17461 // Vector shifts: check for immediate versions and lower them.
17462 // Note: This is done during DAG combining instead of DAG legalizing because
17463 // the build_vectors for 64-bit vector element shift counts are generally
17464 // not legal, and it is hard to see their values after they get legalized to
17465 // loads from a constant pool.
17466 case Intrinsic::arm_neon_vshifts:
17467 case Intrinsic::arm_neon_vshiftu:
17468 case Intrinsic::arm_neon_vrshifts:
17469 case Intrinsic::arm_neon_vrshiftu:
17470 case Intrinsic::arm_neon_vrshiftn:
17471 case Intrinsic::arm_neon_vqshifts:
17472 case Intrinsic::arm_neon_vqshiftu:
17473 case Intrinsic::arm_neon_vqshiftsu:
17474 case Intrinsic::arm_neon_vqshiftns:
17475 case Intrinsic::arm_neon_vqshiftnu:
17476 case Intrinsic::arm_neon_vqshiftnsu:
17477 case Intrinsic::arm_neon_vqrshiftns:
17478 case Intrinsic::arm_neon_vqrshiftnu:
17479 case Intrinsic::arm_neon_vqrshiftnsu: {
17480 EVT VT = N->getOperand(Num: 1).getValueType();
17481 int64_t Cnt;
17482 unsigned VShiftOpc = 0;
17483
17484 switch (IntNo) {
17485 case Intrinsic::arm_neon_vshifts:
17486 case Intrinsic::arm_neon_vshiftu:
17487 if (isVShiftLImm(Op: N->getOperand(Num: 2), VT, isLong: false, Cnt)) {
17488 VShiftOpc = ARMISD::VSHLIMM;
17489 break;
17490 }
17491 if (isVShiftRImm(Op: N->getOperand(Num: 2), VT, isNarrow: false, isIntrinsic: true, Cnt)) {
17492 VShiftOpc = (IntNo == Intrinsic::arm_neon_vshifts ? ARMISD::VSHRsIMM
17493 : ARMISD::VSHRuIMM);
17494 break;
17495 }
17496 return SDValue();
17497
17498 case Intrinsic::arm_neon_vrshifts:
17499 case Intrinsic::arm_neon_vrshiftu:
17500 if (isVShiftRImm(Op: N->getOperand(Num: 2), VT, isNarrow: false, isIntrinsic: true, Cnt))
17501 break;
17502 return SDValue();
17503
17504 case Intrinsic::arm_neon_vqshifts:
17505 case Intrinsic::arm_neon_vqshiftu:
17506 if (isVShiftLImm(Op: N->getOperand(Num: 2), VT, isLong: false, Cnt))
17507 break;
17508 return SDValue();
17509
17510 case Intrinsic::arm_neon_vqshiftsu:
17511 if (isVShiftLImm(Op: N->getOperand(Num: 2), VT, isLong: false, Cnt))
17512 break;
17513 llvm_unreachable("invalid shift count for vqshlu intrinsic");
17514
17515 case Intrinsic::arm_neon_vrshiftn:
17516 case Intrinsic::arm_neon_vqshiftns:
17517 case Intrinsic::arm_neon_vqshiftnu:
17518 case Intrinsic::arm_neon_vqshiftnsu:
17519 case Intrinsic::arm_neon_vqrshiftns:
17520 case Intrinsic::arm_neon_vqrshiftnu:
17521 case Intrinsic::arm_neon_vqrshiftnsu:
17522 // Narrowing shifts require an immediate right shift.
17523 if (isVShiftRImm(Op: N->getOperand(Num: 2), VT, isNarrow: true, isIntrinsic: true, Cnt))
17524 break;
17525 llvm_unreachable("invalid shift count for narrowing vector shift "
17526 "intrinsic");
17527
17528 default:
17529 llvm_unreachable("unhandled vector shift");
17530 }
17531
17532 switch (IntNo) {
17533 case Intrinsic::arm_neon_vshifts:
17534 case Intrinsic::arm_neon_vshiftu:
17535 // Opcode already set above.
17536 break;
17537 case Intrinsic::arm_neon_vrshifts:
17538 VShiftOpc = ARMISD::VRSHRsIMM;
17539 break;
17540 case Intrinsic::arm_neon_vrshiftu:
17541 VShiftOpc = ARMISD::VRSHRuIMM;
17542 break;
17543 case Intrinsic::arm_neon_vrshiftn:
17544 VShiftOpc = ARMISD::VRSHRNIMM;
17545 break;
17546 case Intrinsic::arm_neon_vqshifts:
17547 VShiftOpc = ARMISD::VQSHLsIMM;
17548 break;
17549 case Intrinsic::arm_neon_vqshiftu:
17550 VShiftOpc = ARMISD::VQSHLuIMM;
17551 break;
17552 case Intrinsic::arm_neon_vqshiftsu:
17553 VShiftOpc = ARMISD::VQSHLsuIMM;
17554 break;
17555 case Intrinsic::arm_neon_vqshiftns:
17556 VShiftOpc = ARMISD::VQSHRNsIMM;
17557 break;
17558 case Intrinsic::arm_neon_vqshiftnu:
17559 VShiftOpc = ARMISD::VQSHRNuIMM;
17560 break;
17561 case Intrinsic::arm_neon_vqshiftnsu:
17562 VShiftOpc = ARMISD::VQSHRNsuIMM;
17563 break;
17564 case Intrinsic::arm_neon_vqrshiftns:
17565 VShiftOpc = ARMISD::VQRSHRNsIMM;
17566 break;
17567 case Intrinsic::arm_neon_vqrshiftnu:
17568 VShiftOpc = ARMISD::VQRSHRNuIMM;
17569 break;
17570 case Intrinsic::arm_neon_vqrshiftnsu:
17571 VShiftOpc = ARMISD::VQRSHRNsuIMM;
17572 break;
17573 }
17574
17575 SDLoc dl(N);
17576 return DAG.getNode(Opcode: VShiftOpc, DL: dl, VT: N->getValueType(ResNo: 0),
17577 N1: N->getOperand(Num: 1), N2: DAG.getConstant(Val: Cnt, DL: dl, VT: MVT::i32));
17578 }
17579
17580 case Intrinsic::arm_neon_vshiftins: {
17581 EVT VT = N->getOperand(Num: 1).getValueType();
17582 int64_t Cnt;
17583 unsigned VShiftOpc = 0;
17584
17585 if (isVShiftLImm(Op: N->getOperand(Num: 3), VT, isLong: false, Cnt))
17586 VShiftOpc = ARMISD::VSLIIMM;
17587 else if (isVShiftRImm(Op: N->getOperand(Num: 3), VT, isNarrow: false, isIntrinsic: true, Cnt))
17588 VShiftOpc = ARMISD::VSRIIMM;
17589 else {
17590 llvm_unreachable("invalid shift count for vsli/vsri intrinsic");
17591 }
17592
17593 SDLoc dl(N);
17594 return DAG.getNode(Opcode: VShiftOpc, DL: dl, VT: N->getValueType(ResNo: 0),
17595 N1: N->getOperand(Num: 1), N2: N->getOperand(Num: 2),
17596 N3: DAG.getConstant(Val: Cnt, DL: dl, VT: MVT::i32));
17597 }
17598
17599 case Intrinsic::arm_neon_vqrshifts:
17600 case Intrinsic::arm_neon_vqrshiftu:
17601 // No immediate versions of these to check for.
17602 break;
17603
17604 case Intrinsic::arm_neon_vbsl: {
17605 SDLoc dl(N);
17606 return DAG.getNode(Opcode: ARMISD::VBSP, DL: dl, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
17607 N2: N->getOperand(Num: 2), N3: N->getOperand(Num: 3));
17608 }
17609 case Intrinsic::arm_mve_vqdmlah:
17610 case Intrinsic::arm_mve_vqdmlash:
17611 case Intrinsic::arm_mve_vqrdmlah:
17612 case Intrinsic::arm_mve_vqrdmlash:
17613 case Intrinsic::arm_mve_vmla_n_predicated:
17614 case Intrinsic::arm_mve_vmlas_n_predicated:
17615 case Intrinsic::arm_mve_vqdmlah_predicated:
17616 case Intrinsic::arm_mve_vqdmlash_predicated:
17617 case Intrinsic::arm_mve_vqrdmlah_predicated:
17618 case Intrinsic::arm_mve_vqrdmlash_predicated: {
17619 // These intrinsics all take an i32 scalar operand which is narrowed to the
17620 // size of a single lane of the vector type they return. So we don't need
17621 // any bits of that operand above that point, which allows us to eliminate
17622 // uxth/sxth.
17623 unsigned BitWidth = N->getValueType(ResNo: 0).getScalarSizeInBits();
17624 APInt DemandedMask = APInt::getLowBitsSet(numBits: 32, loBitsSet: BitWidth);
17625 if (SimplifyDemandedBits(Op: N->getOperand(Num: 3), DemandedBits: DemandedMask, DCI))
17626 return SDValue();
17627 break;
17628 }
17629
17630 case Intrinsic::arm_mve_minv:
17631 case Intrinsic::arm_mve_maxv:
17632 case Intrinsic::arm_mve_minav:
17633 case Intrinsic::arm_mve_maxav:
17634 case Intrinsic::arm_mve_minv_predicated:
17635 case Intrinsic::arm_mve_maxv_predicated:
17636 case Intrinsic::arm_mve_minav_predicated:
17637 case Intrinsic::arm_mve_maxav_predicated: {
17638 // These intrinsics all take an i32 scalar operand which is narrowed to the
17639 // size of a single lane of the vector type they take as the other input.
17640 unsigned BitWidth = N->getOperand(Num: 2)->getValueType(ResNo: 0).getScalarSizeInBits();
17641 APInt DemandedMask = APInt::getLowBitsSet(numBits: 32, loBitsSet: BitWidth);
17642 if (SimplifyDemandedBits(Op: N->getOperand(Num: 1), DemandedBits: DemandedMask, DCI))
17643 return SDValue();
17644 break;
17645 }
17646
17647 case Intrinsic::arm_mve_addv: {
17648 // Turn this intrinsic straight into the appropriate ARMISD::VADDV node,
17649 // which allow PerformADDVecReduce to turn it into VADDLV when possible.
17650 bool Unsigned = N->getConstantOperandVal(Num: 2);
17651 unsigned Opc = Unsigned ? ARMISD::VADDVu : ARMISD::VADDVs;
17652 return DAG.getNode(Opcode: Opc, DL: SDLoc(N), VTList: N->getVTList(), N: N->getOperand(Num: 1));
17653 }
17654
17655 case Intrinsic::arm_mve_addlv:
17656 case Intrinsic::arm_mve_addlv_predicated: {
17657 // Same for these, but ARMISD::VADDLV has to be followed by a BUILD_PAIR
17658 // which recombines the two outputs into an i64
17659 bool Unsigned = N->getConstantOperandVal(Num: 2);
17660 unsigned Opc = IntNo == Intrinsic::arm_mve_addlv ?
17661 (Unsigned ? ARMISD::VADDLVu : ARMISD::VADDLVs) :
17662 (Unsigned ? ARMISD::VADDLVpu : ARMISD::VADDLVps);
17663
17664 SmallVector<SDValue, 4> Ops;
17665 for (unsigned i = 1, e = N->getNumOperands(); i < e; i++)
17666 if (i != 2) // skip the unsigned flag
17667 Ops.push_back(Elt: N->getOperand(Num: i));
17668
17669 SDLoc dl(N);
17670 SDValue val = DAG.getNode(Opcode: Opc, DL: dl, ResultTys: {MVT::i32, MVT::i32}, Ops);
17671 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: MVT::i64, N1: val.getValue(R: 0),
17672 N2: val.getValue(R: 1));
17673 }
17674 }
17675
17676 return SDValue();
17677}
17678
17679bool ARMTargetLowering::hasAndNot(SDValue Y) const {
17680 EVT VT = Y.getValueType();
17681 if (!VT.isVector())
17682 return hasAndNotCompare(V: Y);
17683 if (Subtarget->hasMVEIntegerOps())
17684 return VT.is128BitVector();
17685 if (Subtarget->hasNEON())
17686 return VT.is64BitVector() || VT.is128BitVector();
17687 return false;
17688}
17689
17690/// PerformShiftCombine - Checks for immediate versions of vector shifts and
17691/// lowers them. As with the vector shift intrinsics, this is done during DAG
17692/// combining instead of DAG legalizing because the build_vectors for 64-bit
17693/// vector element shift counts are generally not legal, and it is hard to see
17694/// their values after they get legalized to loads from a constant pool.
17695static SDValue PerformShiftCombine(SDNode *N,
17696 TargetLowering::DAGCombinerInfo &DCI,
17697 const ARMSubtarget *ST) {
17698 SelectionDAG &DAG = DCI.DAG;
17699 EVT VT = N->getValueType(ResNo: 0);
17700
17701 if (ST->isThumb1Only() && N->getOpcode() == ISD::SHL && VT == MVT::i32 &&
17702 N->getOperand(Num: 0)->getOpcode() == ISD::AND &&
17703 N->getOperand(Num: 0)->hasOneUse()) {
17704 if (DCI.isBeforeLegalize() || DCI.isCalledByLegalizer())
17705 return SDValue();
17706 // Look for the pattern (shl (and x, AndMask), ShiftAmt). This doesn't
17707 // usually show up because instcombine prefers to canonicalize it to
17708 // (and (shl x, ShiftAmt) (shl AndMask, ShiftAmt)), but the shift can come
17709 // out of GEP lowering in some cases.
17710 SDValue N0 = N->getOperand(Num: 0);
17711 ConstantSDNode *ShiftAmtNode = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
17712 if (!ShiftAmtNode)
17713 return SDValue();
17714 uint32_t ShiftAmt = static_cast<uint32_t>(ShiftAmtNode->getZExtValue());
17715 ConstantSDNode *AndMaskNode = dyn_cast<ConstantSDNode>(Val: N0->getOperand(Num: 1));
17716 if (!AndMaskNode)
17717 return SDValue();
17718 uint32_t AndMask = static_cast<uint32_t>(AndMaskNode->getZExtValue());
17719 // Don't transform uxtb/uxth.
17720 if (AndMask == 255 || AndMask == 65535)
17721 return SDValue();
17722 if (isMask_32(Value: AndMask)) {
17723 uint32_t MaskedBits = llvm::countl_zero(Val: AndMask);
17724 if (MaskedBits > ShiftAmt) {
17725 SDLoc DL(N);
17726 SDValue SHL = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i32, N1: N0->getOperand(Num: 0),
17727 N2: DAG.getConstant(Val: MaskedBits, DL, VT: MVT::i32));
17728 return DAG.getNode(
17729 Opcode: ISD::SRL, DL, VT: MVT::i32, N1: SHL,
17730 N2: DAG.getConstant(Val: MaskedBits - ShiftAmt, DL, VT: MVT::i32));
17731 }
17732 }
17733 }
17734
17735 // Nothing to be done for scalar shifts.
17736 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
17737 if (!VT.isVector() || !TLI.isTypeLegal(VT))
17738 return SDValue();
17739 if (ST->hasMVEIntegerOps())
17740 return SDValue();
17741
17742 int64_t Cnt;
17743
17744 switch (N->getOpcode()) {
17745 default: llvm_unreachable("unexpected shift opcode");
17746
17747 case ISD::SHL:
17748 if (isVShiftLImm(Op: N->getOperand(Num: 1), VT, isLong: false, Cnt)) {
17749 SDLoc dl(N);
17750 return DAG.getNode(Opcode: ARMISD::VSHLIMM, DL: dl, VT, N1: N->getOperand(Num: 0),
17751 N2: DAG.getConstant(Val: Cnt, DL: dl, VT: MVT::i32));
17752 }
17753 break;
17754
17755 case ISD::SRA:
17756 case ISD::SRL:
17757 if (isVShiftRImm(Op: N->getOperand(Num: 1), VT, isNarrow: false, isIntrinsic: false, Cnt)) {
17758 unsigned VShiftOpc =
17759 (N->getOpcode() == ISD::SRA ? ARMISD::VSHRsIMM : ARMISD::VSHRuIMM);
17760 SDLoc dl(N);
17761 return DAG.getNode(Opcode: VShiftOpc, DL: dl, VT, N1: N->getOperand(Num: 0),
17762 N2: DAG.getConstant(Val: Cnt, DL: dl, VT: MVT::i32));
17763 }
17764 }
17765 return SDValue();
17766}
17767
17768// Look for a sign/zero/fpextend extend of a larger than legal load. This can be
17769// split into multiple extending loads, which are simpler to deal with than an
17770// arbitrary extend. For fp extends we use an integer extending load and a VCVTL
17771// to convert the type to an f32.
17772static SDValue PerformSplittingToWideningLoad(SDNode *N, SelectionDAG &DAG) {
17773 SDValue N0 = N->getOperand(Num: 0);
17774 if (N0.getOpcode() != ISD::LOAD)
17775 return SDValue();
17776 LoadSDNode *LD = cast<LoadSDNode>(Val: N0.getNode());
17777 if (!LD->isSimple() || !N0.hasOneUse() || LD->isIndexed() ||
17778 LD->getExtensionType() != ISD::NON_EXTLOAD)
17779 return SDValue();
17780 EVT FromVT = LD->getValueType(ResNo: 0);
17781 EVT ToVT = N->getValueType(ResNo: 0);
17782 if (!ToVT.isVector())
17783 return SDValue();
17784 assert(FromVT.getVectorNumElements() == ToVT.getVectorNumElements());
17785 EVT ToEltVT = ToVT.getVectorElementType();
17786 EVT FromEltVT = FromVT.getVectorElementType();
17787
17788 unsigned NumElements = 0;
17789 if (ToEltVT == MVT::i32 && FromEltVT == MVT::i8)
17790 NumElements = 4;
17791 if (ToEltVT == MVT::f32 && FromEltVT == MVT::f16)
17792 NumElements = 4;
17793 if (NumElements == 0 ||
17794 (FromEltVT != MVT::f16 && FromVT.getVectorNumElements() == NumElements) ||
17795 FromVT.getVectorNumElements() % NumElements != 0 ||
17796 !isPowerOf2_32(Value: NumElements))
17797 return SDValue();
17798
17799 LLVMContext &C = *DAG.getContext();
17800 SDLoc DL(LD);
17801 // Details about the old load
17802 SDValue Ch = LD->getChain();
17803 SDValue BasePtr = LD->getBasePtr();
17804 Align Alignment = LD->getBaseAlign();
17805 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
17806 AAMDNodes AAInfo = LD->getAAInfo();
17807
17808 ISD::LoadExtType NewExtType =
17809 N->getOpcode() == ISD::SIGN_EXTEND ? ISD::SEXTLOAD : ISD::ZEXTLOAD;
17810 SDValue Offset = DAG.getPOISON(VT: BasePtr.getValueType());
17811 EVT NewFromVT = EVT::getVectorVT(
17812 Context&: C, VT: EVT::getIntegerVT(Context&: C, BitWidth: FromEltVT.getScalarSizeInBits()), NumElements);
17813 EVT NewToVT = EVT::getVectorVT(
17814 Context&: C, VT: EVT::getIntegerVT(Context&: C, BitWidth: ToEltVT.getScalarSizeInBits()), NumElements);
17815
17816 SmallVector<SDValue, 4> Loads;
17817 SmallVector<SDValue, 4> Chains;
17818 for (unsigned i = 0; i < FromVT.getVectorNumElements() / NumElements; i++) {
17819 unsigned NewOffset = (i * NewFromVT.getSizeInBits()) / 8;
17820 SDValue NewPtr =
17821 DAG.getObjectPtrOffset(SL: DL, Ptr: BasePtr, Offset: TypeSize::getFixed(ExactSize: NewOffset));
17822
17823 SDValue NewLoad =
17824 DAG.getLoad(AM: ISD::UNINDEXED, ExtType: NewExtType, VT: NewToVT, dl: DL, Chain: Ch, Ptr: NewPtr, Offset,
17825 PtrInfo: LD->getPointerInfo().getWithOffset(O: NewOffset), MemVT: NewFromVT,
17826 Alignment, MMOFlags, Metadata: AAInfo);
17827 Loads.push_back(Elt: NewLoad);
17828 Chains.push_back(Elt: SDValue(NewLoad.getNode(), 1));
17829 }
17830
17831 // Float truncs need to extended with VCVTB's into their floating point types.
17832 if (FromEltVT == MVT::f16) {
17833 SmallVector<SDValue, 4> Extends;
17834
17835 for (unsigned i = 0; i < Loads.size(); i++) {
17836 SDValue LoadBC =
17837 DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL, VT: MVT::v8f16, Operand: Loads[i]);
17838 SDValue FPExt = DAG.getNode(Opcode: ARMISD::VCVTL, DL, VT: MVT::v4f32, N1: LoadBC,
17839 N2: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
17840 Extends.push_back(Elt: FPExt);
17841 }
17842
17843 Loads = Extends;
17844 }
17845
17846 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: Chains);
17847 DAG.ReplaceAllUsesOfValueWith(From: SDValue(LD, 1), To: NewChain);
17848 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: ToVT, Ops: Loads);
17849}
17850
17851/// PerformExtendCombine - Target-specific DAG combining for ISD::SIGN_EXTEND,
17852/// ISD::ZERO_EXTEND, and ISD::ANY_EXTEND.
17853static SDValue PerformExtendCombine(SDNode *N, SelectionDAG &DAG,
17854 const ARMSubtarget *ST) {
17855 SDValue N0 = N->getOperand(Num: 0);
17856 EVT VT = N->getValueType(ResNo: 0);
17857 SDLoc DL(N);
17858
17859 // Check for sign- and zero-extensions of vector extract operations of 8- and
17860 // 16-bit vector elements. NEON and MVE support these directly. They are
17861 // handled during DAG combining because type legalization will promote them
17862 // to 32-bit types and it is messy to recognize the operations after that.
17863 if ((ST->hasNEON() || ST->hasMVEIntegerOps()) &&
17864 N0.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
17865 SDValue Vec = N0.getOperand(i: 0);
17866 SDValue Lane = N0.getOperand(i: 1);
17867 EVT EltVT = N0.getValueType();
17868 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
17869
17870 if (VT == MVT::i32 &&
17871 (EltVT == MVT::i8 || EltVT == MVT::i16) &&
17872 TLI.isTypeLegal(VT: Vec.getValueType()) &&
17873 isa<ConstantSDNode>(Val: Lane)) {
17874
17875 unsigned Opc = 0;
17876 switch (N->getOpcode()) {
17877 default: llvm_unreachable("unexpected opcode");
17878 case ISD::SIGN_EXTEND:
17879 Opc = ARMISD::VGETLANEs;
17880 break;
17881 case ISD::ZERO_EXTEND:
17882 case ISD::ANY_EXTEND:
17883 Opc = ARMISD::VGETLANEu;
17884 break;
17885 }
17886 return DAG.getNode(Opcode: Opc, DL, VT, N1: Vec, N2: Lane);
17887 }
17888 }
17889
17890 if (ST->hasMVEIntegerOps())
17891 if (SDValue NewLoad = PerformSplittingToWideningLoad(N, DAG))
17892 return NewLoad;
17893
17894 // Combine sext(buildvector(..)) to buildvector(sext(..)) to help avoid
17895 // difficult to lower i1 buildvector.
17896 if (ST->hasMVEIntegerOps() && N0.getValueType().getScalarSizeInBits() == 1 &&
17897 N0.getOpcode() == ISD::BUILD_VECTOR && VT.getScalarSizeInBits() <= 32) {
17898 SmallVector<SDValue> Ops;
17899 for (unsigned I = 0; I < N0.getNumOperands(); I++) {
17900 SDValue InReg = N0.getOperand(i: I);
17901 if (N->getOpcode() == ISD::ZERO_EXTEND)
17902 InReg = DAG.getNode(Opcode: ISD::AND, DL, VT: InReg.getValueType(), N1: InReg,
17903 N2: DAG.getConstant(Val: 1, DL, VT: InReg.getValueType()));
17904 else if (N->getOpcode() == ISD::SIGN_EXTEND)
17905 InReg = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: InReg.getValueType(),
17906 N1: InReg, N2: DAG.getValueType(MVT::i1));
17907 SDValue Ext = DAG.getNode(Opcode: N->getOpcode(), DL, VT: MVT::i32, Operand: InReg);
17908 Ops.push_back(Elt: Ext);
17909 }
17910 return DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT, Ops);
17911 }
17912
17913 return SDValue();
17914}
17915
17916static SDValue PerformFPExtendCombine(SDNode *N, SelectionDAG &DAG,
17917 const ARMSubtarget *ST) {
17918 if (ST->hasMVEFloatOps())
17919 if (SDValue NewLoad = PerformSplittingToWideningLoad(N, DAG))
17920 return NewLoad;
17921
17922 return SDValue();
17923}
17924
17925// Lower smin(smax(x, C1), C2) to ssat or usat, if they have saturating
17926// constant bounds.
17927static SDValue PerformMinMaxToSatCombine(SDValue Op, SelectionDAG &DAG,
17928 const ARMSubtarget *Subtarget) {
17929 if ((Subtarget->isThumb() || !Subtarget->hasV6Ops()) &&
17930 !Subtarget->isThumb2())
17931 return SDValue();
17932
17933 EVT VT = Op.getValueType();
17934 SDValue Op0 = Op.getOperand(i: 0);
17935
17936 if (VT != MVT::i32 ||
17937 (Op0.getOpcode() != ISD::SMIN && Op0.getOpcode() != ISD::SMAX) ||
17938 !isa<ConstantSDNode>(Val: Op.getOperand(i: 1)) ||
17939 !isa<ConstantSDNode>(Val: Op0.getOperand(i: 1)))
17940 return SDValue();
17941
17942 SDValue Min = Op;
17943 SDValue Max = Op0;
17944 SDValue Input = Op0.getOperand(i: 0);
17945 if (Min.getOpcode() == ISD::SMAX)
17946 std::swap(a&: Min, b&: Max);
17947
17948 if (Min.getOpcode() != ISD::SMIN || Max.getOpcode() != ISD::SMAX)
17949 return SDValue();
17950
17951 APInt MinC = Min.getConstantOperandAPInt(i: 1);
17952 APInt MaxC = Max.getConstantOperandAPInt(i: 1);
17953 if (MaxC.sgt(RHS: MinC))
17954 return SDValue();
17955
17956 SDLoc DL(Op);
17957
17958 // A clamp whose bounds are already a saturation range maps to a single
17959 // SSAT / USAT.
17960 if ((MinC + 1).isPowerOf2()) {
17961 if (MinC == ~MaxC)
17962 return DAG.getNode(Opcode: ARMISD::SSAT, DL, VT, N1: Input,
17963 N2: DAG.getConstant(Val: MinC.countr_one(), DL, VT));
17964 if (MaxC == 0)
17965 return DAG.getNode(Opcode: ARMISD::USAT, DL, VT, N1: Input,
17966 N2: DAG.getConstant(Val: MinC.countr_one(), DL, VT));
17967 }
17968
17969 // For power-of-two clamp widths, convert the range to be zero-centered,
17970 // apply SSAT, and convert the result back.
17971 //
17972 // Width = Hi - Lo + 1
17973 // Center = Lo + Width / 2
17974 // Result = ssat(X - Center) + Center
17975 //
17976 // The idea is to shift the input so that the clamp range is centered
17977 // around zero, apply ssat, and then shift the result back.
17978 //
17979 // For example clamp(X, -118, 137) -> Width = 256, Center = 10, so it becomes
17980 // ssat(X - 10, 8) + 10
17981
17982 APInt Width = MinC - MaxC + 1;
17983 if (!Width.isPowerOf2() || Width.isOne())
17984 return SDValue();
17985 unsigned SatBit = Width.logBase2() - 1; // ssat to SatBit + 1 signed bits
17986 APInt Center = MaxC + Width.lshr(shiftAmt: 1);
17987
17988 // The rewrite is only valid when X - Center does not overflow;
17989 SDValue NegC = DAG.getConstant(Val: -Center, DL, VT);
17990 if (DAG.computeOverflowForSignedAdd(N0: Input, N1: NegC) != SelectionDAG::OFK_Never)
17991 return SDValue();
17992
17993 SDValue Shifted = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Input, N2: NegC);
17994 SDValue Sat = DAG.getNode(Opcode: ARMISD::SSAT, DL, VT, N1: Shifted,
17995 N2: DAG.getConstant(Val: SatBit, DL, VT));
17996 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Sat, N2: DAG.getConstant(Val: Center, DL, VT));
17997}
17998
17999/// PerformMinMaxCombine - Target-specific DAG combining for creating truncating
18000/// saturates.
18001static SDValue PerformMinMaxCombine(SDNode *N, SelectionDAG &DAG,
18002 const ARMSubtarget *ST) {
18003 EVT VT = N->getValueType(ResNo: 0);
18004 SDValue N0 = N->getOperand(Num: 0);
18005
18006 if (VT == MVT::i32)
18007 return PerformMinMaxToSatCombine(Op: SDValue(N, 0), DAG, Subtarget: ST);
18008
18009 if (!ST->hasMVEIntegerOps())
18010 return SDValue();
18011
18012 if (SDValue V = PerformVQDMULHCombine(N, DAG))
18013 return V;
18014
18015 if (VT != MVT::v4i32 && VT != MVT::v8i16)
18016 return SDValue();
18017
18018 auto IsSignedSaturate = [&](SDNode *Min, SDNode *Max) {
18019 // Check one is a smin and the other is a smax
18020 if (Min->getOpcode() != ISD::SMIN)
18021 std::swap(a&: Min, b&: Max);
18022 if (Min->getOpcode() != ISD::SMIN || Max->getOpcode() != ISD::SMAX)
18023 return false;
18024
18025 APInt SaturateC;
18026 if (VT == MVT::v4i32)
18027 SaturateC = APInt(32, (1 << 15) - 1, true);
18028 else //if (VT == MVT::v8i16)
18029 SaturateC = APInt(16, (1 << 7) - 1, true);
18030
18031 APInt MinC, MaxC;
18032 if (!ISD::isConstantSplatVector(N: Min->getOperand(Num: 1).getNode(), SplatValue&: MinC) ||
18033 MinC != SaturateC)
18034 return false;
18035 if (!ISD::isConstantSplatVector(N: Max->getOperand(Num: 1).getNode(), SplatValue&: MaxC) ||
18036 MaxC != ~SaturateC)
18037 return false;
18038 return true;
18039 };
18040
18041 if (IsSignedSaturate(N, N0.getNode())) {
18042 SDLoc DL(N);
18043 MVT ExtVT, HalfVT;
18044 if (VT == MVT::v4i32) {
18045 HalfVT = MVT::v8i16;
18046 ExtVT = MVT::v4i16;
18047 } else { // if (VT == MVT::v8i16)
18048 HalfVT = MVT::v16i8;
18049 ExtVT = MVT::v8i8;
18050 }
18051
18052 // Create a VQMOVNB with undef top lanes, then signed extended into the top
18053 // half. That extend will hopefully be removed if only the bottom bits are
18054 // demanded (though a truncating store, for example).
18055 SDValue VQMOVN =
18056 DAG.getNode(Opcode: ARMISD::VQMOVNs, DL, VT: HalfVT, N1: DAG.getUNDEF(VT: HalfVT),
18057 N2: N0->getOperand(Num: 0), N3: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
18058 SDValue Bitcast = DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL, VT, Operand: VQMOVN);
18059 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT, N1: Bitcast,
18060 N2: DAG.getValueType(ExtVT));
18061 }
18062
18063 auto IsUnsignedSaturate = [&](SDNode *Min) {
18064 // For unsigned, we just need to check for <= 0xffff
18065 if (Min->getOpcode() != ISD::UMIN)
18066 return false;
18067
18068 APInt SaturateC;
18069 if (VT == MVT::v4i32)
18070 SaturateC = APInt(32, (1 << 16) - 1, true);
18071 else //if (VT == MVT::v8i16)
18072 SaturateC = APInt(16, (1 << 8) - 1, true);
18073
18074 APInt MinC;
18075 if (!ISD::isConstantSplatVector(N: Min->getOperand(Num: 1).getNode(), SplatValue&: MinC) ||
18076 MinC != SaturateC)
18077 return false;
18078 return true;
18079 };
18080
18081 if (IsUnsignedSaturate(N)) {
18082 SDLoc DL(N);
18083 MVT HalfVT;
18084 unsigned ExtConst;
18085 if (VT == MVT::v4i32) {
18086 HalfVT = MVT::v8i16;
18087 ExtConst = 0x0000FFFF;
18088 } else { //if (VT == MVT::v8i16)
18089 HalfVT = MVT::v16i8;
18090 ExtConst = 0x00FF;
18091 }
18092
18093 // Create a VQMOVNB with undef top lanes, then ZExt into the top half with
18094 // an AND. That extend will hopefully be removed if only the bottom bits are
18095 // demanded (though a truncating store, for example).
18096 SDValue VQMOVN =
18097 DAG.getNode(Opcode: ARMISD::VQMOVNu, DL, VT: HalfVT, N1: DAG.getUNDEF(VT: HalfVT), N2: N0,
18098 N3: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
18099 SDValue Bitcast = DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL, VT, Operand: VQMOVN);
18100 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Bitcast,
18101 N2: DAG.getConstant(Val: ExtConst, DL, VT));
18102 }
18103
18104 return SDValue();
18105}
18106
18107static const APInt *isPowerOf2Constant(SDValue V) {
18108 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: V);
18109 if (!C)
18110 return nullptr;
18111 const APInt *CV = &C->getAPIntValue();
18112 return CV->isPowerOf2() ? CV : nullptr;
18113}
18114
18115SDValue ARMTargetLowering::PerformCMOVToBFICombine(SDNode *CMOV, SelectionDAG &DAG) const {
18116 // If we have a CMOV, OR and AND combination such as:
18117 // if (x & CN)
18118 // y |= CM;
18119 //
18120 // And:
18121 // * CN is a single bit;
18122 // * All bits covered by CM are known zero in y
18123 //
18124 // Then we can convert this into a sequence of BFI instructions. This will
18125 // always be a win if CM is a single bit, will always be no worse than the
18126 // TST&OR sequence if CM is two bits, and for thumb will be no worse if CM is
18127 // three bits (due to the extra IT instruction).
18128
18129 SDValue Op0 = CMOV->getOperand(Num: 0);
18130 SDValue Op1 = CMOV->getOperand(Num: 1);
18131 auto CC = CMOV->getConstantOperandAPInt(Num: 2).getLimitedValue();
18132 SDValue CmpZ = CMOV->getOperand(Num: 3);
18133
18134 // The compare must be against zero.
18135 if (!isNullConstant(V: CmpZ->getOperand(Num: 1)))
18136 return SDValue();
18137
18138 assert(CmpZ->getOpcode() == ARMISD::CMPZ);
18139 SDValue And = CmpZ->getOperand(Num: 0);
18140 if (And->getOpcode() != ISD::AND)
18141 return SDValue();
18142 const APInt *AndC = isPowerOf2Constant(V: And->getOperand(Num: 1));
18143 if (!AndC)
18144 return SDValue();
18145 SDValue X = And->getOperand(Num: 0);
18146
18147 if (CC == ARMCC::EQ) {
18148 // We're performing an "equal to zero" compare. Swap the operands so we
18149 // canonicalize on a "not equal to zero" compare.
18150 std::swap(a&: Op0, b&: Op1);
18151 } else {
18152 assert(CC == ARMCC::NE && "How can a CMPZ node not be EQ or NE?");
18153 }
18154
18155 if (Op1->getOpcode() != ISD::OR)
18156 return SDValue();
18157
18158 ConstantSDNode *OrC = dyn_cast<ConstantSDNode>(Val: Op1->getOperand(Num: 1));
18159 if (!OrC)
18160 return SDValue();
18161 SDValue Y = Op1->getOperand(Num: 0);
18162
18163 if (Op0 != Y)
18164 return SDValue();
18165
18166 // Now, is it profitable to continue?
18167 APInt OrCI = OrC->getAPIntValue();
18168 unsigned Heuristic = Subtarget->isThumb() ? 3 : 2;
18169 if (OrCI.popcount() > Heuristic)
18170 return SDValue();
18171
18172 // Lastly, can we determine that the bits defined by OrCI
18173 // are zero in Y?
18174 KnownBits Known = DAG.computeKnownBits(Op: Y);
18175 if ((OrCI & Known.Zero) != OrCI)
18176 return SDValue();
18177
18178 // OK, we can do the combine.
18179 SDValue V = Y;
18180 SDLoc dl(X);
18181 EVT VT = X.getValueType();
18182 unsigned BitInX = AndC->logBase2();
18183
18184 if (BitInX != 0) {
18185 // We must shift X first.
18186 X = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: X,
18187 N2: DAG.getConstant(Val: BitInX, DL: dl, VT));
18188 }
18189
18190 for (unsigned BitInY = 0, NumActiveBits = OrCI.getActiveBits();
18191 BitInY < NumActiveBits; ++BitInY) {
18192 if (OrCI[BitInY] == 0)
18193 continue;
18194 APInt Mask(VT.getSizeInBits(), 0);
18195 Mask.setBit(BitInY);
18196 V = DAG.getNode(Opcode: ARMISD::BFI, DL: dl, VT, N1: V, N2: X,
18197 // Confusingly, the operand is an *inverted* mask.
18198 N3: DAG.getConstant(Val: ~Mask, DL: dl, VT));
18199 }
18200
18201 return V;
18202}
18203
18204// Given N, the value controlling the conditional branch, search for the loop
18205// intrinsic, returning it, along with how the value is used. We need to handle
18206// patterns such as the following:
18207// (brcond (xor (setcc (loop.decrement), 0, ne), 1), exit)
18208// (brcond (setcc (loop.decrement), 0, eq), exit)
18209// (brcond (setcc (loop.decrement), 0, ne), header)
18210static SDValue SearchLoopIntrinsic(SDValue N, ISD::CondCode &CC, int &Imm,
18211 bool &Negate) {
18212 switch (N->getOpcode()) {
18213 default:
18214 break;
18215 case ISD::XOR: {
18216 if (!isa<ConstantSDNode>(Val: N.getOperand(i: 1)))
18217 return SDValue();
18218 if (!cast<ConstantSDNode>(Val: N.getOperand(i: 1))->isOne())
18219 return SDValue();
18220 Negate = !Negate;
18221 return SearchLoopIntrinsic(N: N.getOperand(i: 0), CC, Imm, Negate);
18222 }
18223 case ISD::SETCC: {
18224 auto *Const = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1));
18225 if (!Const)
18226 return SDValue();
18227 if (Const->isZero())
18228 Imm = 0;
18229 else if (Const->isOne())
18230 Imm = 1;
18231 else
18232 return SDValue();
18233 CC = cast<CondCodeSDNode>(Val: N.getOperand(i: 2))->get();
18234 return SearchLoopIntrinsic(N: N->getOperand(Num: 0), CC, Imm, Negate);
18235 }
18236 case ISD::INTRINSIC_W_CHAIN: {
18237 unsigned IntOp = N.getConstantOperandVal(i: 1);
18238 if (IntOp != Intrinsic::test_start_loop_iterations &&
18239 IntOp != Intrinsic::loop_decrement_reg)
18240 return SDValue();
18241 return N;
18242 }
18243 }
18244 return SDValue();
18245}
18246
18247static SDValue PerformHWLoopCombine(SDNode *N,
18248 TargetLowering::DAGCombinerInfo &DCI,
18249 const ARMSubtarget *ST) {
18250
18251 // The hwloop intrinsics that we're interested are used for control-flow,
18252 // either for entering or exiting the loop:
18253 // - test.start.loop.iterations will test whether its operand is zero. If it
18254 // is zero, the proceeding branch should not enter the loop.
18255 // - loop.decrement.reg also tests whether its operand is zero. If it is
18256 // zero, the proceeding branch should not branch back to the beginning of
18257 // the loop.
18258 // So here, we need to check that how the brcond is using the result of each
18259 // of the intrinsics to ensure that we're branching to the right place at the
18260 // right time.
18261
18262 ISD::CondCode CC;
18263 SDValue Cond;
18264 int Imm = 1;
18265 bool Negate = false;
18266 SDValue Chain = N->getOperand(Num: 0);
18267 SDValue Dest;
18268
18269 if (N->getOpcode() == ISD::BRCOND) {
18270 CC = ISD::SETEQ;
18271 Cond = N->getOperand(Num: 1);
18272 Dest = N->getOperand(Num: 2);
18273 } else {
18274 assert(N->getOpcode() == ISD::BR_CC && "Expected BRCOND or BR_CC!");
18275 CC = cast<CondCodeSDNode>(Val: N->getOperand(Num: 1))->get();
18276 Cond = N->getOperand(Num: 2);
18277 Dest = N->getOperand(Num: 4);
18278 if (auto *Const = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 3))) {
18279 if (!Const->isOne() && !Const->isZero())
18280 return SDValue();
18281 Imm = Const->getZExtValue();
18282 } else
18283 return SDValue();
18284 }
18285
18286 SDValue Int = SearchLoopIntrinsic(N: Cond, CC, Imm, Negate);
18287 if (!Int)
18288 return SDValue();
18289
18290 if (Negate)
18291 CC = ISD::getSetCCInverse(Operation: CC, /* Integer inverse */ Type: MVT::i32);
18292
18293 auto IsTrueIfZero = [](ISD::CondCode CC, int Imm) {
18294 return (CC == ISD::SETEQ && Imm == 0) ||
18295 (CC == ISD::SETNE && Imm == 1) ||
18296 (CC == ISD::SETLT && Imm == 1) ||
18297 (CC == ISD::SETULT && Imm == 1);
18298 };
18299
18300 auto IsFalseIfZero = [](ISD::CondCode CC, int Imm) {
18301 return (CC == ISD::SETEQ && Imm == 1) ||
18302 (CC == ISD::SETNE && Imm == 0) ||
18303 (CC == ISD::SETGT && Imm == 0) ||
18304 (CC == ISD::SETUGT && Imm == 0) ||
18305 (CC == ISD::SETGE && Imm == 1) ||
18306 (CC == ISD::SETUGE && Imm == 1);
18307 };
18308
18309 assert((IsTrueIfZero(CC, Imm) || IsFalseIfZero(CC, Imm)) &&
18310 "unsupported condition");
18311
18312 SDLoc dl(Int);
18313 SelectionDAG &DAG = DCI.DAG;
18314 SDValue Elements = Int.getOperand(i: 2);
18315 unsigned IntOp = Int->getConstantOperandVal(Num: 1);
18316 assert((N->hasOneUse() && N->user_begin()->getOpcode() == ISD::BR) &&
18317 "expected single br user");
18318 SDNode *Br = *N->user_begin();
18319 SDValue OtherTarget = Br->getOperand(Num: 1);
18320
18321 // Update the unconditional branch to branch to the given Dest.
18322 auto UpdateUncondBr = [](SDNode *Br, SDValue Dest, SelectionDAG &DAG) {
18323 SDValue NewBrOps[] = { Br->getOperand(Num: 0), Dest };
18324 SDValue NewBr = DAG.getNode(Opcode: ISD::BR, DL: SDLoc(Br), VT: MVT::Other, Ops: NewBrOps);
18325 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Br, 0), To: NewBr);
18326 };
18327
18328 if (IntOp == Intrinsic::test_start_loop_iterations) {
18329 SDValue Res;
18330 SDValue Setup = DAG.getNode(Opcode: ARMISD::WLSSETUP, DL: dl, VT: MVT::i32, Operand: Elements);
18331 // We expect this 'instruction' to branch when the counter is zero.
18332 if (IsTrueIfZero(CC, Imm)) {
18333 SDValue Ops[] = {Chain, Setup, Dest};
18334 Res = DAG.getNode(Opcode: ARMISD::WLS, DL: dl, VT: MVT::Other, Ops);
18335 } else {
18336 // The logic is the reverse of what we need for WLS, so find the other
18337 // basic block target: the target of the proceeding br.
18338 UpdateUncondBr(Br, Dest, DAG);
18339
18340 SDValue Ops[] = {Chain, Setup, OtherTarget};
18341 Res = DAG.getNode(Opcode: ARMISD::WLS, DL: dl, VT: MVT::Other, Ops);
18342 }
18343 // Update LR count to the new value
18344 DAG.ReplaceAllUsesOfValueWith(From: Int.getValue(R: 0), To: Setup);
18345 // Update chain
18346 DAG.ReplaceAllUsesOfValueWith(From: Int.getValue(R: 2), To: Int.getOperand(i: 0));
18347 return Res;
18348 } else {
18349 SDValue Size =
18350 DAG.getTargetConstant(Val: Int.getConstantOperandVal(i: 3), DL: dl, VT: MVT::i32);
18351 SDValue Args[] = { Int.getOperand(i: 0), Elements, Size, };
18352 SDValue LoopDec = DAG.getNode(Opcode: ARMISD::LOOP_DEC, DL: dl,
18353 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other), Ops: Args);
18354 DAG.ReplaceAllUsesWith(From: Int.getNode(), To: LoopDec.getNode());
18355
18356 // We expect this instruction to branch when the count is not zero.
18357 SDValue Target = IsFalseIfZero(CC, Imm) ? Dest : OtherTarget;
18358
18359 // Update the unconditional branch to target the loop preheader if we've
18360 // found the condition has been reversed.
18361 if (Target == OtherTarget)
18362 UpdateUncondBr(Br, Dest, DAG);
18363
18364 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other,
18365 N1: SDValue(LoopDec.getNode(), 1), N2: Chain);
18366
18367 SDValue EndArgs[] = { Chain, SDValue(LoopDec.getNode(), 0), Target };
18368 return DAG.getNode(Opcode: ARMISD::LE, DL: dl, VT: MVT::Other, Ops: EndArgs);
18369 }
18370 return SDValue();
18371}
18372
18373/// PerformBRCONDCombine - Target-specific DAG combining for ARMISD::BRCOND.
18374SDValue
18375ARMTargetLowering::PerformBRCONDCombine(SDNode *N, SelectionDAG &DAG) const {
18376 SDValue Cmp = N->getOperand(Num: 3);
18377 if (Cmp.getOpcode() != ARMISD::CMPZ)
18378 // Only looking at NE cases.
18379 return SDValue();
18380
18381 SDLoc dl(N);
18382 SDValue LHS = Cmp.getOperand(i: 0);
18383 SDValue RHS = Cmp.getOperand(i: 1);
18384 SDValue Chain = N->getOperand(Num: 0);
18385 SDValue BB = N->getOperand(Num: 1);
18386 SDValue ARMcc = N->getOperand(Num: 2);
18387 ARMCC::CondCodes CC = (ARMCC::CondCodes)ARMcc->getAsZExtVal();
18388
18389 // (brcond Chain BB ne (cmpz (and (cmov 0 1 CC Flags) 1) 0))
18390 // -> (brcond Chain BB CC Flags)
18391 if (CC == ARMCC::NE && LHS.getOpcode() == ISD::AND && LHS->hasOneUse() &&
18392 LHS->getOperand(Num: 0)->getOpcode() == ARMISD::CMOV &&
18393 LHS->getOperand(Num: 0)->hasOneUse() &&
18394 isNullConstant(V: LHS->getOperand(Num: 0)->getOperand(Num: 0)) &&
18395 isOneConstant(V: LHS->getOperand(Num: 0)->getOperand(Num: 1)) &&
18396 isOneConstant(V: LHS->getOperand(Num: 1)) && isNullConstant(V: RHS)) {
18397 return DAG.getNode(Opcode: ARMISD::BRCOND, DL: dl, VT: MVT::Other, N1: Chain, N2: BB,
18398 N3: LHS->getOperand(Num: 0)->getOperand(Num: 2),
18399 N4: LHS->getOperand(Num: 0)->getOperand(Num: 3));
18400 }
18401
18402 return SDValue();
18403}
18404
18405/// PerformCMOVCombine - Target-specific DAG combining for ARMISD::CMOV.
18406SDValue
18407ARMTargetLowering::PerformCMOVCombine(SDNode *N, SelectionDAG &DAG) const {
18408 SDLoc dl(N);
18409 EVT VT = N->getValueType(ResNo: 0);
18410 SDValue FalseVal = N->getOperand(Num: 0);
18411 SDValue TrueVal = N->getOperand(Num: 1);
18412 SDValue ARMcc = N->getOperand(Num: 2);
18413 SDValue Cmp = N->getOperand(Num: 3);
18414
18415 // Try to form CSINV etc.
18416 unsigned Opcode;
18417 bool InvertCond;
18418 if (SDValue CSetOp =
18419 matchCSET(Opcode, InvertCond, TrueVal, FalseVal, Subtarget)) {
18420 if (InvertCond) {
18421 ARMCC::CondCodes CondCode =
18422 (ARMCC::CondCodes)cast<const ConstantSDNode>(Val&: ARMcc)->getZExtValue();
18423 CondCode = ARMCC::getOppositeCondition(CC: CondCode);
18424 ARMcc = DAG.getConstant(Val: CondCode, DL: SDLoc(ARMcc), VT: MVT::i32);
18425 }
18426 return DAG.getNode(Opcode, DL: dl, VT, N1: CSetOp, N2: CSetOp, N3: ARMcc, N4: Cmp);
18427 }
18428
18429 if (Cmp.getOpcode() != ARMISD::CMPZ)
18430 // Only looking at EQ and NE cases.
18431 return SDValue();
18432
18433 SDValue LHS = Cmp.getOperand(i: 0);
18434 SDValue RHS = Cmp.getOperand(i: 1);
18435 ARMCC::CondCodes CC = (ARMCC::CondCodes)ARMcc->getAsZExtVal();
18436
18437 // BFI is only available on V6T2+.
18438 if (!Subtarget->isThumb1Only() && Subtarget->hasV6T2Ops()) {
18439 SDValue R = PerformCMOVToBFICombine(CMOV: N, DAG);
18440 if (R)
18441 return R;
18442 }
18443
18444 // Simplify
18445 // mov r1, r0
18446 // cmp r1, x
18447 // mov r0, y
18448 // moveq r0, x
18449 // to
18450 // cmp r0, x
18451 // movne r0, y
18452 //
18453 // mov r1, r0
18454 // cmp r1, x
18455 // mov r0, x
18456 // movne r0, y
18457 // to
18458 // cmp r0, x
18459 // movne r0, y
18460 /// FIXME: Turn this into a target neutral optimization?
18461 SDValue Res;
18462 if (CC == ARMCC::NE && FalseVal == RHS && FalseVal != LHS) {
18463 Res = DAG.getNode(Opcode: ARMISD::CMOV, DL: dl, VT, N1: LHS, N2: TrueVal, N3: ARMcc, N4: Cmp);
18464 } else if (CC == ARMCC::EQ && TrueVal == RHS) {
18465 SDValue ARMcc;
18466 SDValue NewCmp = getARMCmp(LHS, RHS, CC: ISD::SETNE, ARMcc, DAG, dl);
18467 Res = DAG.getNode(Opcode: ARMISD::CMOV, DL: dl, VT, N1: LHS, N2: FalseVal, N3: ARMcc, N4: NewCmp);
18468 }
18469
18470 // (cmov F T ne (cmpz (cmov 0 1 CC Flags) 0))
18471 // -> (cmov F T CC Flags)
18472 if (CC == ARMCC::NE && LHS.getOpcode() == ARMISD::CMOV && LHS->hasOneUse() &&
18473 isNullConstant(V: LHS->getOperand(Num: 0)) && isOneConstant(V: LHS->getOperand(Num: 1)) &&
18474 isNullConstant(V: RHS)) {
18475 return DAG.getNode(Opcode: ARMISD::CMOV, DL: dl, VT, N1: FalseVal, N2: TrueVal,
18476 N3: LHS->getOperand(Num: 2), N4: LHS->getOperand(Num: 3));
18477 }
18478
18479 if (!VT.isInteger())
18480 return SDValue();
18481
18482 // Fold away an unnecessary CMPZ/CMOV
18483 // CMOV A, B, C1, (CMPZ (CMOV 1, 0, C2, D), 0) ->
18484 // if C1==EQ -> CMOV A, B, C2, D
18485 // if C1==NE -> CMOV A, B, NOT(C2), D
18486 if (N->getConstantOperandVal(Num: 2) == ARMCC::EQ ||
18487 N->getConstantOperandVal(Num: 2) == ARMCC::NE) {
18488 ARMCC::CondCodes Cond;
18489 if (SDValue C = IsCMPZCSINC(Cmp: N->getOperand(Num: 3).getNode(), CC&: Cond)) {
18490 if (N->getConstantOperandVal(Num: 2) == ARMCC::NE)
18491 Cond = ARMCC::getOppositeCondition(CC: Cond);
18492 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: MVT::i32, N1: N->getOperand(Num: 0),
18493 N2: N->getOperand(Num: 1),
18494 N3: DAG.getConstant(Val: Cond, DL: SDLoc(N), VT: MVT::i32), N4: C);
18495 }
18496 }
18497
18498 // Materialize a boolean comparison for integers so we can avoid branching.
18499 if (isNullConstant(V: FalseVal)) {
18500 if (CC == ARMCC::EQ && isOneConstant(V: TrueVal)) {
18501 if (!Subtarget->isThumb1Only() && Subtarget->hasV5TOps()) {
18502 // If x == y then x - y == 0 and ARM's CLZ will return 32, shifting it
18503 // right 5 bits will make that 32 be 1, otherwise it will be 0.
18504 // CMOV 0, 1, ==, (CMPZ x, y) -> SRL (CTLZ (SUB x, y)), 5
18505 SDValue Sub = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: LHS, N2: RHS);
18506 Res = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: DAG.getNode(Opcode: ISD::CTLZ, DL: dl, VT, Operand: Sub),
18507 N2: DAG.getConstant(Val: 5, DL: dl, VT: MVT::i32));
18508 } else {
18509 // CMOV 0, 1, ==, (CMPZ x, y) ->
18510 // (UADDO_CARRY (SUB x, y), t:0, t:1)
18511 // where t = (USUBO_CARRY 0, (SUB x, y), 0)
18512 //
18513 // The USUBO_CARRY computes 0 - (x - y) and this will give a borrow when
18514 // x != y. In other words, a carry C == 1 when x == y, C == 0
18515 // otherwise.
18516 // The final UADDO_CARRY computes
18517 // x - y + (0 - (x - y)) + C == C
18518 SDValue Sub = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: LHS, N2: RHS);
18519 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: MVT::i32);
18520 SDValue Neg = DAG.getNode(Opcode: ISD::USUBO, DL: dl, VTList: VTs, N1: FalseVal, N2: Sub);
18521 // ISD::USUBO_CARRY returns a borrow but we want the carry here
18522 // actually.
18523 SDValue Carry =
18524 DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32,
18525 N1: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32), N2: Neg.getValue(R: 1));
18526 Res = DAG.getNode(Opcode: ISD::UADDO_CARRY, DL: dl, VTList: VTs, N1: Sub, N2: Neg, N3: Carry);
18527 }
18528 } else if (CC == ARMCC::NE && !isNullConstant(V: RHS) &&
18529 (!Subtarget->isThumb1Only() || isPowerOf2Constant(V: TrueVal))) {
18530 // This seems pointless but will allow us to combine it further below.
18531 // CMOV 0, z, !=, (CMPZ x, y) -> CMOV (SUBC x, y), z, !=, (SUBC x, y):1
18532 SDValue Sub =
18533 DAG.getNode(Opcode: ARMISD::SUBC, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: MVT::i32), N1: LHS, N2: RHS);
18534 Res = DAG.getNode(Opcode: ARMISD::CMOV, DL: dl, VT, N1: Sub, N2: TrueVal, N3: ARMcc,
18535 N4: Sub.getValue(R: 1));
18536 FalseVal = Sub;
18537 }
18538 } else if (isNullConstant(V: TrueVal)) {
18539 if (CC == ARMCC::EQ && !isNullConstant(V: RHS) &&
18540 (!Subtarget->isThumb1Only() || isPowerOf2Constant(V: FalseVal))) {
18541 // This seems pointless but will allow us to combine it further below
18542 // Note that we change == for != as this is the dual for the case above.
18543 // CMOV z, 0, ==, (CMPZ x, y) -> CMOV (SUBC x, y), z, !=, (SUBC x, y):1
18544 SDValue Sub =
18545 DAG.getNode(Opcode: ARMISD::SUBC, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: MVT::i32), N1: LHS, N2: RHS);
18546 Res = DAG.getNode(Opcode: ARMISD::CMOV, DL: dl, VT, N1: Sub, N2: FalseVal,
18547 N3: DAG.getConstant(Val: ARMCC::NE, DL: dl, VT: MVT::i32),
18548 N4: Sub.getValue(R: 1));
18549 FalseVal = Sub;
18550 }
18551 }
18552
18553 // On Thumb1, the DAG above may be further combined if z is a power of 2
18554 // (z == 2 ^ K).
18555 // CMOV (SUBC x, y), z, !=, (SUBC x, y):1 ->
18556 // t1 = (USUBO (SUB x, y), 1)
18557 // t2 = (USUBO_CARRY (SUB x, y), t1:0, t1:1)
18558 // Result = if K != 0 then (SHL t2:0, K) else t2:0
18559 //
18560 // This also handles the special case of comparing against zero; it's
18561 // essentially, the same pattern, except there's no SUBC:
18562 // CMOV x, z, !=, (CMPZ x, 0) ->
18563 // t1 = (USUBO x, 1)
18564 // t2 = (USUBO_CARRY x, t1:0, t1:1)
18565 // Result = if K != 0 then (SHL t2:0, K) else t2:0
18566 const APInt *TrueConst;
18567 if (Subtarget->isThumb1Only() && CC == ARMCC::NE &&
18568 ((FalseVal.getOpcode() == ARMISD::SUBC && FalseVal.getOperand(i: 0) == LHS &&
18569 FalseVal.getOperand(i: 1) == RHS) ||
18570 (FalseVal == LHS && isNullConstant(V: RHS))) &&
18571 (TrueConst = isPowerOf2Constant(V: TrueVal))) {
18572 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: MVT::i32);
18573 unsigned ShiftAmount = TrueConst->logBase2();
18574 if (ShiftAmount)
18575 TrueVal = DAG.getConstant(Val: 1, DL: dl, VT);
18576 SDValue Subc = DAG.getNode(Opcode: ISD::USUBO, DL: dl, VTList: VTs, N1: FalseVal, N2: TrueVal);
18577 Res = DAG.getNode(Opcode: ISD::USUBO_CARRY, DL: dl, VTList: VTs, N1: FalseVal, N2: Subc,
18578 N3: Subc.getValue(R: 1));
18579
18580 if (ShiftAmount)
18581 Res = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Res,
18582 N2: DAG.getConstant(Val: ShiftAmount, DL: dl, VT: MVT::i32));
18583 }
18584
18585 if (Res.getNode()) {
18586 KnownBits Known = DAG.computeKnownBits(Op: SDValue(N,0));
18587 // Capture demanded bits information that would be otherwise lost.
18588 if (Known.Zero == 0xfffffffe)
18589 Res = DAG.getNode(Opcode: ISD::AssertZext, DL: dl, VT: MVT::i32, N1: Res,
18590 N2: DAG.getValueType(MVT::i1));
18591 else if (Known.Zero == 0xffffff00)
18592 Res = DAG.getNode(Opcode: ISD::AssertZext, DL: dl, VT: MVT::i32, N1: Res,
18593 N2: DAG.getValueType(MVT::i8));
18594 else if (Known.Zero == 0xffff0000)
18595 Res = DAG.getNode(Opcode: ISD::AssertZext, DL: dl, VT: MVT::i32, N1: Res,
18596 N2: DAG.getValueType(MVT::i16));
18597 }
18598
18599 return Res;
18600}
18601
18602static SDValue PerformBITCASTCombine(SDNode *N,
18603 TargetLowering::DAGCombinerInfo &DCI,
18604 const ARMSubtarget *ST) {
18605 SelectionDAG &DAG = DCI.DAG;
18606 SDValue Src = N->getOperand(Num: 0);
18607 EVT DstVT = N->getValueType(ResNo: 0);
18608
18609 // Convert v4f32 bitcast (v4i32 vdup (i32)) -> v4f32 vdup (i32) under MVE.
18610 if (ST->hasMVEIntegerOps() && Src.getOpcode() == ARMISD::VDUP) {
18611 EVT SrcVT = Src.getValueType();
18612 if (SrcVT.getScalarSizeInBits() == DstVT.getScalarSizeInBits())
18613 return DAG.getNode(Opcode: ARMISD::VDUP, DL: SDLoc(N), VT: DstVT, Operand: Src.getOperand(i: 0));
18614 }
18615
18616 // We may have a bitcast of something that has already had this bitcast
18617 // combine performed on it, so skip past any VECTOR_REG_CASTs.
18618 if (Src.getOpcode() == ARMISD::VECTOR_REG_CAST &&
18619 Src.getOperand(i: 0).getValueType().getScalarSizeInBits() <=
18620 Src.getValueType().getScalarSizeInBits())
18621 Src = Src.getOperand(i: 0);
18622
18623 // Bitcast from element-wise VMOV or VMVN doesn't need VREV if the VREV that
18624 // would be generated is at least the width of the element type.
18625 EVT SrcVT = Src.getValueType();
18626 if ((Src.getOpcode() == ARMISD::VMOVIMM ||
18627 Src.getOpcode() == ARMISD::VMVNIMM ||
18628 Src.getOpcode() == ARMISD::VMOVFPIMM) &&
18629 SrcVT.getScalarSizeInBits() <= DstVT.getScalarSizeInBits() &&
18630 DAG.getDataLayout().isBigEndian())
18631 return DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: SDLoc(N), VT: DstVT, Operand: Src);
18632
18633 // bitcast(extract(x, n)); bitcast(extract(x, n+1)) -> VMOVRRD x
18634 if (SDValue R = PerformExtractEltToVMOVRRD(N, DCI))
18635 return R;
18636
18637 return SDValue();
18638}
18639
18640// Some combines for the MVETrunc truncations legalizer helper. Also lowers the
18641// node into stack operations after legalizeOps.
18642SDValue ARMTargetLowering::PerformMVETruncCombine(
18643 SDNode *N, TargetLowering::DAGCombinerInfo &DCI) const {
18644 SelectionDAG &DAG = DCI.DAG;
18645 EVT VT = N->getValueType(ResNo: 0);
18646 SDLoc DL(N);
18647
18648 // MVETrunc(Undef, Undef) -> Undef
18649 if (all_of(Range: N->ops(), P: [](SDValue Op) { return Op.isUndef(); }))
18650 return DAG.getUNDEF(VT);
18651
18652 // MVETrunc(MVETrunc a b, MVETrunc c, d) -> MVETrunc
18653 if (N->getNumOperands() == 2 &&
18654 N->getOperand(Num: 0).getOpcode() == ARMISD::MVETRUNC &&
18655 N->getOperand(Num: 1).getOpcode() == ARMISD::MVETRUNC)
18656 return DAG.getNode(Opcode: ARMISD::MVETRUNC, DL, VT, N1: N->getOperand(Num: 0).getOperand(i: 0),
18657 N2: N->getOperand(Num: 0).getOperand(i: 1),
18658 N3: N->getOperand(Num: 1).getOperand(i: 0),
18659 N4: N->getOperand(Num: 1).getOperand(i: 1));
18660
18661 // MVETrunc(shuffle, shuffle) -> VMOVN
18662 if (N->getNumOperands() == 2 &&
18663 N->getOperand(Num: 0).getOpcode() == ISD::VECTOR_SHUFFLE &&
18664 N->getOperand(Num: 1).getOpcode() == ISD::VECTOR_SHUFFLE) {
18665 auto *S0 = cast<ShuffleVectorSDNode>(Val: N->getOperand(Num: 0).getNode());
18666 auto *S1 = cast<ShuffleVectorSDNode>(Val: N->getOperand(Num: 1).getNode());
18667
18668 if (S0->getOperand(Num: 0) == S1->getOperand(Num: 0) &&
18669 S0->getOperand(Num: 1) == S1->getOperand(Num: 1)) {
18670 // Construct complete shuffle mask
18671 SmallVector<int, 8> Mask(S0->getMask());
18672 Mask.append(in_start: S1->getMask().begin(), in_end: S1->getMask().end());
18673
18674 if (isVMOVNTruncMask(M: Mask, ToVT: VT, rev: false))
18675 return DAG.getNode(
18676 Opcode: ARMISD::VMOVN, DL, VT,
18677 N1: DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL, VT, Operand: S0->getOperand(Num: 0)),
18678 N2: DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL, VT, Operand: S0->getOperand(Num: 1)),
18679 N3: DAG.getConstant(Val: 1, DL, VT: MVT::i32));
18680 if (isVMOVNTruncMask(M: Mask, ToVT: VT, rev: true))
18681 return DAG.getNode(
18682 Opcode: ARMISD::VMOVN, DL, VT,
18683 N1: DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL, VT, Operand: S0->getOperand(Num: 1)),
18684 N2: DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL, VT, Operand: S0->getOperand(Num: 0)),
18685 N3: DAG.getConstant(Val: 1, DL, VT: MVT::i32));
18686 }
18687 }
18688
18689 // For MVETrunc of a buildvector or shuffle, it can be beneficial to lower the
18690 // truncate to a buildvector to allow the generic optimisations to kick in.
18691 if (all_of(Range: N->ops(), P: [](SDValue Op) {
18692 return Op.getOpcode() == ISD::BUILD_VECTOR ||
18693 Op.getOpcode() == ISD::VECTOR_SHUFFLE ||
18694 (Op.getOpcode() == ISD::BITCAST &&
18695 Op.getOperand(i: 0).getOpcode() == ISD::BUILD_VECTOR);
18696 })) {
18697 SmallVector<SDValue, 8> Extracts;
18698 for (unsigned Op = 0; Op < N->getNumOperands(); Op++) {
18699 SDValue O = N->getOperand(Num: Op);
18700 for (unsigned i = 0; i < O.getValueType().getVectorNumElements(); i++) {
18701 SDValue Ext = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: O,
18702 N2: DAG.getConstant(Val: i, DL, VT: MVT::i32));
18703 Extracts.push_back(Elt: Ext);
18704 }
18705 }
18706 return DAG.getBuildVector(VT, DL, Ops: Extracts);
18707 }
18708
18709 // If we are late in the legalization process and nothing has optimised
18710 // the trunc to anything better, lower it to a stack store and reload,
18711 // performing the truncation whilst keeping the lanes in the correct order:
18712 // VSTRH.32 a, stack; VSTRH.32 b, stack+8; VLDRW.32 stack;
18713 if (!DCI.isAfterLegalizeDAG())
18714 return SDValue();
18715
18716 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: TypeSize::getFixed(ExactSize: 16), Alignment: Align(4));
18717 int SPFI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
18718 int NumIns = N->getNumOperands();
18719 assert((NumIns == 2 || NumIns == 4) &&
18720 "Expected 2 or 4 inputs to an MVETrunc");
18721 EVT StoreVT = VT.getHalfNumVectorElementsVT(Context&: *DAG.getContext());
18722 if (N->getNumOperands() == 4)
18723 StoreVT = StoreVT.getHalfNumVectorElementsVT(Context&: *DAG.getContext());
18724
18725 SmallVector<SDValue> Chains;
18726 for (int I = 0; I < NumIns; I++) {
18727 SDValue Ptr = DAG.getNode(
18728 Opcode: ISD::ADD, DL, VT: StackPtr.getValueType(), N1: StackPtr,
18729 N2: DAG.getConstant(Val: I * 16 / NumIns, DL, VT: StackPtr.getValueType()));
18730 MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(
18731 MF&: DAG.getMachineFunction(), FI: SPFI, Offset: I * 16 / NumIns);
18732 SDValue Ch = DAG.getTruncStore(Chain: DAG.getEntryNode(), dl: DL, Val: N->getOperand(Num: I),
18733 Ptr, PtrInfo: MPI, SVT: StoreVT, Alignment: Align(4));
18734 Chains.push_back(Elt: Ch);
18735 }
18736
18737 SDValue Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: Chains);
18738 MachinePointerInfo MPI =
18739 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI, Offset: 0);
18740 return DAG.getLoad(VT, dl: DL, Chain, Ptr: StackPtr, PtrInfo: MPI, Alignment: Align(4));
18741}
18742
18743// Take a MVEEXT(load x) and split that into (extload x, extload x+8)
18744static SDValue PerformSplittingMVEEXTToWideningLoad(SDNode *N,
18745 SelectionDAG &DAG) {
18746 SDValue N0 = N->getOperand(Num: 0);
18747 LoadSDNode *LD = dyn_cast<LoadSDNode>(Val: N0.getNode());
18748 if (!LD || !LD->isSimple() || !N0.hasOneUse() || LD->isIndexed())
18749 return SDValue();
18750
18751 EVT FromVT = LD->getMemoryVT();
18752 EVT ToVT = N->getValueType(ResNo: 0);
18753 if (!ToVT.isVector())
18754 return SDValue();
18755 assert(FromVT.getVectorNumElements() == ToVT.getVectorNumElements() * 2);
18756 EVT ToEltVT = ToVT.getVectorElementType();
18757 EVT FromEltVT = FromVT.getVectorElementType();
18758
18759 unsigned NumElements = 0;
18760 if (ToEltVT == MVT::i32 && (FromEltVT == MVT::i16 || FromEltVT == MVT::i8))
18761 NumElements = 4;
18762 if (ToEltVT == MVT::i16 && FromEltVT == MVT::i8)
18763 NumElements = 8;
18764 assert(NumElements != 0);
18765
18766 ISD::LoadExtType NewExtType =
18767 N->getOpcode() == ARMISD::MVESEXT ? ISD::SEXTLOAD : ISD::ZEXTLOAD;
18768 if (LD->getExtensionType() != ISD::NON_EXTLOAD &&
18769 LD->getExtensionType() != ISD::EXTLOAD &&
18770 LD->getExtensionType() != NewExtType)
18771 return SDValue();
18772
18773 LLVMContext &C = *DAG.getContext();
18774 SDLoc DL(LD);
18775 // Details about the old load
18776 SDValue Ch = LD->getChain();
18777 SDValue BasePtr = LD->getBasePtr();
18778 Align Alignment = LD->getBaseAlign();
18779 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
18780 AAMDNodes AAInfo = LD->getAAInfo();
18781
18782 SDValue Offset = DAG.getPOISON(VT: BasePtr.getValueType());
18783 EVT NewFromVT = EVT::getVectorVT(
18784 Context&: C, VT: EVT::getIntegerVT(Context&: C, BitWidth: FromEltVT.getScalarSizeInBits()), NumElements);
18785 EVT NewToVT = EVT::getVectorVT(
18786 Context&: C, VT: EVT::getIntegerVT(Context&: C, BitWidth: ToEltVT.getScalarSizeInBits()), NumElements);
18787
18788 SmallVector<SDValue, 4> Loads;
18789 SmallVector<SDValue, 4> Chains;
18790 for (unsigned i = 0; i < FromVT.getVectorNumElements() / NumElements; i++) {
18791 unsigned NewOffset = (i * NewFromVT.getSizeInBits()) / 8;
18792 SDValue NewPtr =
18793 DAG.getObjectPtrOffset(SL: DL, Ptr: BasePtr, Offset: TypeSize::getFixed(ExactSize: NewOffset));
18794
18795 SDValue NewLoad =
18796 DAG.getLoad(AM: ISD::UNINDEXED, ExtType: NewExtType, VT: NewToVT, dl: DL, Chain: Ch, Ptr: NewPtr, Offset,
18797 PtrInfo: LD->getPointerInfo().getWithOffset(O: NewOffset), MemVT: NewFromVT,
18798 Alignment, MMOFlags, Metadata: AAInfo);
18799 Loads.push_back(Elt: NewLoad);
18800 Chains.push_back(Elt: SDValue(NewLoad.getNode(), 1));
18801 }
18802
18803 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: Chains);
18804 DAG.ReplaceAllUsesOfValueWith(From: SDValue(LD, 1), To: NewChain);
18805 return DAG.getMergeValues(Ops: Loads, dl: DL);
18806}
18807
18808// Perform combines for MVEEXT. If it has not be optimized to anything better
18809// before lowering, it gets converted to stack store and extloads performing the
18810// extend whilst still keeping the same lane ordering.
18811SDValue ARMTargetLowering::PerformMVEExtCombine(
18812 SDNode *N, TargetLowering::DAGCombinerInfo &DCI) const {
18813 SelectionDAG &DAG = DCI.DAG;
18814 EVT VT = N->getValueType(ResNo: 0);
18815 SDLoc DL(N);
18816 assert(N->getNumValues() == 2 && "Expected MVEEXT with 2 elements");
18817 assert((VT == MVT::v4i32 || VT == MVT::v8i16) && "Unexpected MVEEXT type");
18818
18819 EVT ExtVT = N->getOperand(Num: 0).getValueType().getHalfNumVectorElementsVT(
18820 Context&: *DAG.getContext());
18821 auto Extend = [&](SDValue V) {
18822 SDValue VVT = DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL, VT, Operand: V);
18823 return N->getOpcode() == ARMISD::MVESEXT
18824 ? DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT, N1: VVT,
18825 N2: DAG.getValueType(ExtVT))
18826 : DAG.getZeroExtendInReg(Op: VVT, DL, VT: ExtVT);
18827 };
18828
18829 // MVEEXT(VDUP) -> SIGN_EXTEND_INREG(VDUP)
18830 if (N->getOperand(Num: 0).getOpcode() == ARMISD::VDUP) {
18831 SDValue Ext = Extend(N->getOperand(Num: 0));
18832 return DAG.getMergeValues(Ops: {Ext, Ext}, dl: DL);
18833 }
18834
18835 // MVEEXT(shuffle) -> SIGN_EXTEND_INREG/ZERO_EXTEND_INREG
18836 if (auto *SVN = dyn_cast<ShuffleVectorSDNode>(Val: N->getOperand(Num: 0))) {
18837 ArrayRef<int> Mask = SVN->getMask();
18838 assert(Mask.size() == 2 * VT.getVectorNumElements());
18839 assert(Mask.size() == SVN->getValueType(0).getVectorNumElements());
18840 unsigned Rev = VT == MVT::v4i32 ? ARMISD::VREV32 : ARMISD::VREV16;
18841 SDValue Op0 = SVN->getOperand(Num: 0);
18842 SDValue Op1 = SVN->getOperand(Num: 1);
18843
18844 auto CheckInregMask = [&](int Start, int Offset) {
18845 for (int Idx = 0, E = VT.getVectorNumElements(); Idx < E; ++Idx)
18846 if (Mask[Start + Idx] >= 0 && Mask[Start + Idx] != Idx * 2 + Offset)
18847 return false;
18848 return true;
18849 };
18850 SDValue V0 = SDValue(N, 0);
18851 SDValue V1 = SDValue(N, 1);
18852 if (CheckInregMask(0, 0))
18853 V0 = Extend(Op0);
18854 else if (CheckInregMask(0, 1))
18855 V0 = Extend(DAG.getNode(Opcode: Rev, DL, VT: SVN->getValueType(ResNo: 0), Operand: Op0));
18856 else if (CheckInregMask(0, Mask.size()))
18857 V0 = Extend(Op1);
18858 else if (CheckInregMask(0, Mask.size() + 1))
18859 V0 = Extend(DAG.getNode(Opcode: Rev, DL, VT: SVN->getValueType(ResNo: 0), Operand: Op1));
18860
18861 if (CheckInregMask(VT.getVectorNumElements(), Mask.size()))
18862 V1 = Extend(Op1);
18863 else if (CheckInregMask(VT.getVectorNumElements(), Mask.size() + 1))
18864 V1 = Extend(DAG.getNode(Opcode: Rev, DL, VT: SVN->getValueType(ResNo: 0), Operand: Op1));
18865 else if (CheckInregMask(VT.getVectorNumElements(), 0))
18866 V1 = Extend(Op0);
18867 else if (CheckInregMask(VT.getVectorNumElements(), 1))
18868 V1 = Extend(DAG.getNode(Opcode: Rev, DL, VT: SVN->getValueType(ResNo: 0), Operand: Op0));
18869
18870 if (V0.getNode() != N || V1.getNode() != N)
18871 return DAG.getMergeValues(Ops: {V0, V1}, dl: DL);
18872 }
18873
18874 // MVEEXT(load) -> extload, extload
18875 if (N->getOperand(Num: 0)->getOpcode() == ISD::LOAD)
18876 if (SDValue L = PerformSplittingMVEEXTToWideningLoad(N, DAG))
18877 return L;
18878
18879 if (!DCI.isAfterLegalizeDAG())
18880 return SDValue();
18881
18882 // Lower to a stack store and reload:
18883 // VSTRW.32 a, stack; VLDRH.32 stack; VLDRH.32 stack+8;
18884 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: TypeSize::getFixed(ExactSize: 16), Alignment: Align(4));
18885 int SPFI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
18886 int NumOuts = N->getNumValues();
18887 assert((NumOuts == 2 || NumOuts == 4) &&
18888 "Expected 2 or 4 outputs to an MVEEXT");
18889 EVT LoadVT = N->getOperand(Num: 0).getValueType().getHalfNumVectorElementsVT(
18890 Context&: *DAG.getContext());
18891 if (N->getNumOperands() == 4)
18892 LoadVT = LoadVT.getHalfNumVectorElementsVT(Context&: *DAG.getContext());
18893
18894 MachinePointerInfo MPI =
18895 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI, Offset: 0);
18896 SDValue Chain = DAG.getStore(Chain: DAG.getEntryNode(), dl: DL, Val: N->getOperand(Num: 0),
18897 Ptr: StackPtr, PtrInfo: MPI, Alignment: Align(4));
18898
18899 SmallVector<SDValue> Loads;
18900 for (int I = 0; I < NumOuts; I++) {
18901 SDValue Ptr = DAG.getNode(
18902 Opcode: ISD::ADD, DL, VT: StackPtr.getValueType(), N1: StackPtr,
18903 N2: DAG.getConstant(Val: I * 16 / NumOuts, DL, VT: StackPtr.getValueType()));
18904 MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(
18905 MF&: DAG.getMachineFunction(), FI: SPFI, Offset: I * 16 / NumOuts);
18906 SDValue Load = DAG.getExtLoad(
18907 ExtType: N->getOpcode() == ARMISD::MVESEXT ? ISD::SEXTLOAD : ISD::ZEXTLOAD, dl: DL,
18908 VT, Chain, Ptr, PtrInfo: MPI, MemVT: LoadVT, Alignment: Align(4));
18909 Loads.push_back(Elt: Load);
18910 }
18911
18912 return DAG.getMergeValues(Ops: Loads, dl: DL);
18913}
18914
18915SDValue ARMTargetLowering::PerformDAGCombine(SDNode *N,
18916 DAGCombinerInfo &DCI) const {
18917 switch (N->getOpcode()) {
18918 default: break;
18919 case ISD::SELECT_CC:
18920 case ISD::SELECT: return PerformSELECTCombine(N, DCI, Subtarget);
18921 case ISD::VSELECT: return PerformVSELECTCombine(N, DCI, Subtarget);
18922 case ISD::SETCC: return PerformVSetCCToVCTPCombine(N, DCI, Subtarget);
18923 case ARMISD::ADDE: return PerformADDECombine(N, DCI, Subtarget);
18924 case ARMISD::UMLAL: return PerformUMLALCombine(N, DAG&: DCI.DAG, Subtarget);
18925 case ISD::ADD: return PerformADDCombine(N, DCI, Subtarget);
18926 case ISD::SUB: return PerformSUBCombine(N, DCI, Subtarget);
18927 case ISD::MUL: return PerformMULCombine(N, DCI, Subtarget);
18928 case ISD::OR: return PerformORCombine(N, DCI, Subtarget);
18929 case ISD::XOR: return PerformXORCombine(N, DCI, Subtarget);
18930 case ISD::AND: return PerformANDCombine(N, DCI, Subtarget);
18931 case ISD::BRCOND:
18932 case ISD::BR_CC: return PerformHWLoopCombine(N, DCI, ST: Subtarget);
18933 case ARMISD::ADDC:
18934 case ARMISD::SUBC: return PerformAddcSubcCombine(N, DCI, Subtarget);
18935 case ARMISD::SUBE: return PerformAddeSubeCombine(N, DCI, Subtarget);
18936 case ARMISD::BFI: return PerformBFICombine(N, DAG&: DCI.DAG);
18937 case ARMISD::VMOVRRD: return PerformVMOVRRDCombine(N, DCI, Subtarget);
18938 case ARMISD::VMOVDRR: return PerformVMOVDRRCombine(N, DAG&: DCI.DAG);
18939 case ARMISD::VMOVhr: return PerformVMOVhrCombine(N, DCI);
18940 case ARMISD::VMOVrh: return PerformVMOVrhCombine(N, DAG&: DCI.DAG);
18941 case ISD::STORE: return PerformSTORECombine(N, DCI, Subtarget);
18942 case ISD::BUILD_VECTOR: return PerformBUILD_VECTORCombine(N, DCI, Subtarget);
18943 case ISD::INSERT_VECTOR_ELT: return PerformInsertEltCombine(N, DCI);
18944 case ISD::EXTRACT_VECTOR_ELT:
18945 return PerformExtractEltCombine(N, DCI, ST: Subtarget);
18946 case ISD::SIGN_EXTEND_INREG: return PerformSignExtendInregCombine(N, DAG&: DCI.DAG);
18947 case ISD::INSERT_SUBVECTOR: return PerformInsertSubvectorCombine(N, DCI);
18948 case ISD::VECTOR_SHUFFLE: return PerformVECTOR_SHUFFLECombine(N, DAG&: DCI.DAG);
18949 case ARMISD::VDUPLANE: return PerformVDUPLANECombine(N, DCI, Subtarget);
18950 case ARMISD::VDUP: return PerformVDUPCombine(N, DAG&: DCI.DAG, Subtarget);
18951 case ISD::FP_TO_SINT:
18952 case ISD::FP_TO_UINT:
18953 return PerformVCVTCombine(N, DAG&: DCI.DAG, Subtarget);
18954 case ISD::FADD:
18955 return PerformFADDCombine(N, DAG&: DCI.DAG, Subtarget);
18956 case ISD::FMUL:
18957 return PerformVMulVCTPCombine(N, DAG&: DCI.DAG, Subtarget);
18958 case ISD::INTRINSIC_WO_CHAIN:
18959 return PerformIntrinsicCombine(N, DCI);
18960 case ISD::SHL:
18961 case ISD::SRA:
18962 case ISD::SRL:
18963 return PerformShiftCombine(N, DCI, ST: Subtarget);
18964 case ISD::SIGN_EXTEND:
18965 case ISD::ZERO_EXTEND:
18966 case ISD::ANY_EXTEND:
18967 return PerformExtendCombine(N, DAG&: DCI.DAG, ST: Subtarget);
18968 case ISD::FP_EXTEND:
18969 return PerformFPExtendCombine(N, DAG&: DCI.DAG, ST: Subtarget);
18970 case ISD::SMIN:
18971 case ISD::UMIN:
18972 case ISD::SMAX:
18973 case ISD::UMAX:
18974 return PerformMinMaxCombine(N, DAG&: DCI.DAG, ST: Subtarget);
18975 case ARMISD::CMOV:
18976 return PerformCMOVCombine(N, DAG&: DCI.DAG);
18977 case ARMISD::BRCOND:
18978 return PerformBRCONDCombine(N, DAG&: DCI.DAG);
18979 case ARMISD::CMPZ:
18980 return PerformCMPZCombine(N, DAG&: DCI.DAG);
18981 case ARMISD::CSINC:
18982 case ARMISD::CSINV:
18983 case ARMISD::CSNEG:
18984 return PerformCSETCombine(N, DAG&: DCI.DAG);
18985 case ISD::LOAD:
18986 return PerformLOADCombine(N, DCI, Subtarget);
18987 case ARMISD::VLD1DUP:
18988 case ARMISD::VLD2DUP:
18989 case ARMISD::VLD3DUP:
18990 case ARMISD::VLD4DUP:
18991 return PerformVLDCombine(N, DCI);
18992 case ARMISD::BUILD_VECTOR:
18993 return PerformARMBUILD_VECTORCombine(N, DCI);
18994 case ISD::BITCAST:
18995 return PerformBITCASTCombine(N, DCI, ST: Subtarget);
18996 case ARMISD::PREDICATE_CAST:
18997 return PerformPREDICATE_CASTCombine(N, DCI);
18998 case ARMISD::VECTOR_REG_CAST:
18999 return PerformVECTOR_REG_CASTCombine(N, DAG&: DCI.DAG, ST: Subtarget);
19000 case ARMISD::MVETRUNC:
19001 return PerformMVETruncCombine(N, DCI);
19002 case ARMISD::MVESEXT:
19003 case ARMISD::MVEZEXT:
19004 return PerformMVEExtCombine(N, DCI);
19005 case ARMISD::VCMP:
19006 return PerformVCMPCombine(N, DAG&: DCI.DAG, Subtarget);
19007 case ISD::VECREDUCE_ADD:
19008 return PerformVECREDUCE_ADDCombine(N, DAG&: DCI.DAG, ST: Subtarget);
19009 case ARMISD::VADDVs:
19010 case ARMISD::VADDVu:
19011 case ARMISD::VADDLVs:
19012 case ARMISD::VADDLVu:
19013 case ARMISD::VADDLVAs:
19014 case ARMISD::VADDLVAu:
19015 case ARMISD::VMLAVs:
19016 case ARMISD::VMLAVu:
19017 case ARMISD::VMLALVs:
19018 case ARMISD::VMLALVu:
19019 case ARMISD::VMLALVAs:
19020 case ARMISD::VMLALVAu:
19021 return PerformReduceShuffleCombine(N, DAG&: DCI.DAG);
19022 case ARMISD::VMOVN:
19023 return PerformVMOVNCombine(N, DCI);
19024 case ARMISD::VQMOVNs:
19025 case ARMISD::VQMOVNu:
19026 return PerformVQMOVNCombine(N, DCI);
19027 case ARMISD::VQDMULH:
19028 return PerformVQDMULHCombine(N, DCI);
19029 case ARMISD::ASRL:
19030 case ARMISD::LSRL:
19031 case ARMISD::LSLL:
19032 return PerformLongShiftCombine(N, DAG&: DCI.DAG);
19033 case ARMISD::SMULWB: {
19034 unsigned BitWidth = N->getValueType(ResNo: 0).getSizeInBits();
19035 APInt DemandedMask = APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: 16);
19036 if (SimplifyDemandedBits(Op: N->getOperand(Num: 1), DemandedBits: DemandedMask, DCI))
19037 return SDValue();
19038 break;
19039 }
19040 case ARMISD::SMULWT: {
19041 unsigned BitWidth = N->getValueType(ResNo: 0).getSizeInBits();
19042 APInt DemandedMask = APInt::getHighBitsSet(numBits: BitWidth, hiBitsSet: 16);
19043 if (SimplifyDemandedBits(Op: N->getOperand(Num: 1), DemandedBits: DemandedMask, DCI))
19044 return SDValue();
19045 break;
19046 }
19047 case ARMISD::SMLALBB:
19048 case ARMISD::QADD16b:
19049 case ARMISD::QSUB16b:
19050 case ARMISD::UQADD16b:
19051 case ARMISD::UQSUB16b: {
19052 unsigned BitWidth = N->getValueType(ResNo: 0).getSizeInBits();
19053 APInt DemandedMask = APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: 16);
19054 if ((SimplifyDemandedBits(Op: N->getOperand(Num: 0), DemandedBits: DemandedMask, DCI)) ||
19055 (SimplifyDemandedBits(Op: N->getOperand(Num: 1), DemandedBits: DemandedMask, DCI)))
19056 return SDValue();
19057 break;
19058 }
19059 case ARMISD::SMLALBT: {
19060 unsigned LowWidth = N->getOperand(Num: 0).getValueType().getSizeInBits();
19061 APInt LowMask = APInt::getLowBitsSet(numBits: LowWidth, loBitsSet: 16);
19062 unsigned HighWidth = N->getOperand(Num: 1).getValueType().getSizeInBits();
19063 APInt HighMask = APInt::getHighBitsSet(numBits: HighWidth, hiBitsSet: 16);
19064 if ((SimplifyDemandedBits(Op: N->getOperand(Num: 0), DemandedBits: LowMask, DCI)) ||
19065 (SimplifyDemandedBits(Op: N->getOperand(Num: 1), DemandedBits: HighMask, DCI)))
19066 return SDValue();
19067 break;
19068 }
19069 case ARMISD::SMLALTB: {
19070 unsigned HighWidth = N->getOperand(Num: 0).getValueType().getSizeInBits();
19071 APInt HighMask = APInt::getHighBitsSet(numBits: HighWidth, hiBitsSet: 16);
19072 unsigned LowWidth = N->getOperand(Num: 1).getValueType().getSizeInBits();
19073 APInt LowMask = APInt::getLowBitsSet(numBits: LowWidth, loBitsSet: 16);
19074 if ((SimplifyDemandedBits(Op: N->getOperand(Num: 0), DemandedBits: HighMask, DCI)) ||
19075 (SimplifyDemandedBits(Op: N->getOperand(Num: 1), DemandedBits: LowMask, DCI)))
19076 return SDValue();
19077 break;
19078 }
19079 case ARMISD::SMLALTT: {
19080 unsigned BitWidth = N->getValueType(ResNo: 0).getSizeInBits();
19081 APInt DemandedMask = APInt::getHighBitsSet(numBits: BitWidth, hiBitsSet: 16);
19082 if ((SimplifyDemandedBits(Op: N->getOperand(Num: 0), DemandedBits: DemandedMask, DCI)) ||
19083 (SimplifyDemandedBits(Op: N->getOperand(Num: 1), DemandedBits: DemandedMask, DCI)))
19084 return SDValue();
19085 break;
19086 }
19087 case ARMISD::QADD8b:
19088 case ARMISD::QSUB8b:
19089 case ARMISD::UQADD8b:
19090 case ARMISD::UQSUB8b: {
19091 unsigned BitWidth = N->getValueType(ResNo: 0).getSizeInBits();
19092 APInt DemandedMask = APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: 8);
19093 if ((SimplifyDemandedBits(Op: N->getOperand(Num: 0), DemandedBits: DemandedMask, DCI)) ||
19094 (SimplifyDemandedBits(Op: N->getOperand(Num: 1), DemandedBits: DemandedMask, DCI)))
19095 return SDValue();
19096 break;
19097 }
19098 case ARMISD::VBSP:
19099 if (N->getOperand(Num: 1) == N->getOperand(Num: 2))
19100 return N->getOperand(Num: 1);
19101 return SDValue();
19102 case ISD::INTRINSIC_VOID:
19103 case ISD::INTRINSIC_W_CHAIN:
19104 switch (N->getConstantOperandVal(Num: 1)) {
19105 case Intrinsic::arm_neon_vld1:
19106 case Intrinsic::arm_neon_vld1x2:
19107 case Intrinsic::arm_neon_vld1x3:
19108 case Intrinsic::arm_neon_vld1x4:
19109 case Intrinsic::arm_neon_vld2:
19110 case Intrinsic::arm_neon_vld3:
19111 case Intrinsic::arm_neon_vld4:
19112 case Intrinsic::arm_neon_vld2lane:
19113 case Intrinsic::arm_neon_vld3lane:
19114 case Intrinsic::arm_neon_vld4lane:
19115 case Intrinsic::arm_neon_vld2dup:
19116 case Intrinsic::arm_neon_vld3dup:
19117 case Intrinsic::arm_neon_vld4dup:
19118 case Intrinsic::arm_neon_vst1:
19119 case Intrinsic::arm_neon_vst1x2:
19120 case Intrinsic::arm_neon_vst1x3:
19121 case Intrinsic::arm_neon_vst1x4:
19122 case Intrinsic::arm_neon_vst2:
19123 case Intrinsic::arm_neon_vst3:
19124 case Intrinsic::arm_neon_vst4:
19125 case Intrinsic::arm_neon_vst2lane:
19126 case Intrinsic::arm_neon_vst3lane:
19127 case Intrinsic::arm_neon_vst4lane:
19128 return PerformVLDCombine(N, DCI);
19129 case Intrinsic::arm_mve_vld2q:
19130 case Intrinsic::arm_mve_vld4q:
19131 case Intrinsic::arm_mve_vst2q:
19132 case Intrinsic::arm_mve_vst4q:
19133 return PerformMVEVLDCombine(N, DCI);
19134 default: break;
19135 }
19136 break;
19137 }
19138 return SDValue();
19139}
19140
19141bool ARMTargetLowering::isDesirableToTransformToIntegerOp(unsigned Opc,
19142 EVT VT) const {
19143 return (VT == MVT::f32) && (Opc == ISD::LOAD || Opc == ISD::STORE);
19144}
19145
19146bool ARMTargetLowering::allowsMisalignedMemoryAccesses(EVT VT, unsigned,
19147 Align Alignment,
19148 MachineMemOperand::Flags,
19149 unsigned *Fast) const {
19150 // Depends what it gets converted into if the type is weird.
19151 if (!VT.isSimple())
19152 return false;
19153
19154 // The AllowsUnaligned flag models the SCTLR.A setting in ARM cpus
19155 bool AllowsUnaligned = Subtarget->allowsUnalignedMem();
19156 auto Ty = VT.getSimpleVT().SimpleTy;
19157
19158 if (Ty == MVT::i8 || Ty == MVT::i16 || Ty == MVT::i32) {
19159 // Unaligned access can use (for example) LRDB, LRDH, LDR
19160 if (AllowsUnaligned) {
19161 if (Fast)
19162 *Fast = Subtarget->hasV7Ops();
19163 return true;
19164 }
19165 }
19166
19167 if (Ty == MVT::f64 || Ty == MVT::v2f64) {
19168 // For any little-endian targets with neon, we can support unaligned ld/st
19169 // of D and Q (e.g. {D0,D1}) registers by using vld1.i8/vst1.i8.
19170 // A big-endian target may also explicitly support unaligned accesses
19171 if (Subtarget->hasNEON() && (AllowsUnaligned || Subtarget->isLittle())) {
19172 if (Fast)
19173 *Fast = 1;
19174 return true;
19175 }
19176 }
19177
19178 if (!Subtarget->hasMVEIntegerOps())
19179 return false;
19180
19181 // These are for predicates
19182 if ((Ty == MVT::v16i1 || Ty == MVT::v8i1 || Ty == MVT::v4i1 ||
19183 Ty == MVT::v2i1)) {
19184 if (Fast)
19185 *Fast = 1;
19186 return true;
19187 }
19188
19189 // These are for truncated stores/narrowing loads. They are fine so long as
19190 // the alignment is at least the size of the item being loaded
19191 if ((Ty == MVT::v4i8 || Ty == MVT::v8i8 || Ty == MVT::v4i16) &&
19192 Alignment >= VT.getScalarSizeInBits() / 8) {
19193 if (Fast)
19194 *Fast = true;
19195 return true;
19196 }
19197
19198 // In little-endian MVE, the store instructions VSTRB.U8, VSTRH.U16 and
19199 // VSTRW.U32 all store the vector register in exactly the same format, and
19200 // differ only in the range of their immediate offset field and the required
19201 // alignment. So there is always a store that can be used, regardless of
19202 // actual type.
19203 //
19204 // For big endian, that is not the case. But can still emit a (VSTRB.U8;
19205 // VREV64.8) pair and get the same effect. This will likely be better than
19206 // aligning the vector through the stack.
19207 if (Ty == MVT::v16i8 || Ty == MVT::v8i16 || Ty == MVT::v8f16 ||
19208 Ty == MVT::v4i32 || Ty == MVT::v4f32 || Ty == MVT::v2i64 ||
19209 Ty == MVT::v2f64) {
19210 if (Fast)
19211 *Fast = 1;
19212 return true;
19213 }
19214
19215 return false;
19216}
19217
19218EVT ARMTargetLowering::getOptimalMemOpType(
19219 LLVMContext &Context, const MemOp &Op,
19220 const AttributeList &FuncAttributes) const {
19221 // See if we can use NEON instructions for this...
19222 if ((Op.isMemcpyOrMemmove() || Op.isZeroMemset()) && Subtarget->hasNEON() &&
19223 !FuncAttributes.hasFnAttr(Kind: Attribute::NoImplicitFloat)) {
19224 unsigned Fast;
19225 if (Op.size() >= 16 &&
19226 (Op.isAligned(AlignCheck: Align(16)) ||
19227 (allowsMisalignedMemoryAccesses(VT: MVT::v2f64, 0, Alignment: Align(1),
19228 MachineMemOperand::MONone, Fast: &Fast) &&
19229 Fast))) {
19230 return MVT::v2f64;
19231 } else if (Op.size() >= 8 &&
19232 (Op.isAligned(AlignCheck: Align(8)) ||
19233 (allowsMisalignedMemoryAccesses(
19234 VT: MVT::f64, 0, Alignment: Align(1), MachineMemOperand::MONone, Fast: &Fast) &&
19235 Fast))) {
19236 return MVT::f64;
19237 }
19238 }
19239
19240 // Let the target-independent logic figure it out.
19241 return MVT::Other;
19242}
19243
19244// 64-bit integers are split into their high and low parts and held in two
19245// different registers, so the trunc is free since the low register can just
19246// be used.
19247bool ARMTargetLowering::isTruncateFree(Type *SrcTy, Type *DstTy) const {
19248 if (!SrcTy->isIntegerTy() || !DstTy->isIntegerTy())
19249 return false;
19250 unsigned SrcBits = SrcTy->getPrimitiveSizeInBits();
19251 unsigned DestBits = DstTy->getPrimitiveSizeInBits();
19252 return (SrcBits == 64 && DestBits == 32);
19253}
19254
19255bool ARMTargetLowering::isTruncateFree(EVT SrcVT, EVT DstVT) const {
19256 if (SrcVT.isVector() || DstVT.isVector() || !SrcVT.isInteger() ||
19257 !DstVT.isInteger())
19258 return false;
19259 unsigned SrcBits = SrcVT.getSizeInBits();
19260 unsigned DestBits = DstVT.getSizeInBits();
19261 return (SrcBits == 64 && DestBits == 32);
19262}
19263
19264bool ARMTargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
19265 if (Val.getOpcode() != ISD::LOAD)
19266 return false;
19267
19268 EVT VT1 = Val.getValueType();
19269 if (!VT1.isSimple() || !VT1.isInteger() ||
19270 !VT2.isSimple() || !VT2.isInteger())
19271 return false;
19272
19273 switch (VT1.getSimpleVT().SimpleTy) {
19274 default: break;
19275 case MVT::i1:
19276 case MVT::i8:
19277 case MVT::i16:
19278 // 8-bit and 16-bit loads implicitly zero-extend to 32-bits.
19279 return true;
19280 }
19281
19282 return false;
19283}
19284
19285bool ARMTargetLowering::isFNegFree(EVT VT) const {
19286 if (!VT.isSimple())
19287 return false;
19288
19289 // There are quite a few FP16 instructions (e.g. VNMLA, VNMLS, etc.) that
19290 // negate values directly (fneg is free). So, we don't want to let the DAG
19291 // combiner rewrite fneg into xors and some other instructions. For f16 and
19292 // FullFP16 argument passing, some bitcast nodes may be introduced,
19293 // triggering this DAG combine rewrite, so we are avoiding that with this.
19294 switch (VT.getSimpleVT().SimpleTy) {
19295 default: break;
19296 case MVT::f16:
19297 return Subtarget->hasFullFP16();
19298 }
19299
19300 return false;
19301}
19302
19303Type *ARMTargetLowering::shouldConvertSplatType(ShuffleVectorInst *SVI) const {
19304 if (!Subtarget->hasMVEIntegerOps())
19305 return nullptr;
19306 Type *SVIType = SVI->getType();
19307 Type *ScalarType = SVIType->getScalarType();
19308
19309 if (ScalarType->isFloatTy())
19310 return Type::getInt32Ty(C&: SVIType->getContext());
19311 if (ScalarType->isHalfTy())
19312 return Type::getInt16Ty(C&: SVIType->getContext());
19313 return nullptr;
19314}
19315
19316bool ARMTargetLowering::isVectorLoadExtDesirable(SDValue ExtVal) const {
19317 EVT VT = ExtVal.getValueType();
19318
19319 if (!isTypeLegal(VT))
19320 return false;
19321
19322 if (auto *Ld = dyn_cast<MaskedLoadSDNode>(Val: ExtVal.getOperand(i: 0))) {
19323 if (Ld->isExpandingLoad())
19324 return false;
19325 }
19326
19327 if (Subtarget->hasMVEIntegerOps())
19328 return true;
19329
19330 // Don't create a loadext if we can fold the extension into a wide/long
19331 // instruction.
19332 // If there's more than one user instruction, the loadext is desirable no
19333 // matter what. There can be two uses by the same instruction.
19334 if (ExtVal->use_empty() ||
19335 !ExtVal->user_begin()->isOnlyUserOf(N: ExtVal.getNode()))
19336 return true;
19337
19338 SDNode *U = *ExtVal->user_begin();
19339 if ((U->getOpcode() == ISD::ADD || U->getOpcode() == ISD::SUB ||
19340 U->getOpcode() == ISD::SHL || U->getOpcode() == ARMISD::VSHLIMM))
19341 return false;
19342
19343 return true;
19344}
19345
19346bool ARMTargetLowering::allowTruncateForTailCall(Type *Ty1, Type *Ty2) const {
19347 if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy())
19348 return false;
19349
19350 if (!isTypeLegal(VT: EVT::getEVT(Ty: Ty1)))
19351 return false;
19352
19353 assert(Ty1->getPrimitiveSizeInBits() <= 64 && "i128 is probably not a noop");
19354
19355 // Assuming the caller doesn't have a zeroext or signext return parameter,
19356 // truncation all the way down to i1 is valid.
19357 return true;
19358}
19359
19360/// isFMAFasterThanFMulAndFAdd - Return true if an FMA operation is faster
19361/// than a pair of fmul and fadd instructions. fmuladd intrinsics will be
19362/// expanded to FMAs when this method returns true, otherwise fmuladd is
19363/// expanded to fmul + fadd.
19364///
19365/// ARM supports both fused and unfused multiply-add operations; we already
19366/// lower a pair of fmul and fadd to the latter so it's not clear that there
19367/// would be a gain or that the gain would be worthwhile enough to risk
19368/// correctness bugs.
19369///
19370/// For MVE, we set this to true as it helps simplify the need for some
19371/// patterns (and we don't have the non-fused floating point instruction).
19372bool ARMTargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
19373 EVT VT) const {
19374 if (Subtarget->useSoftFloat())
19375 return false;
19376
19377 if (!VT.isSimple())
19378 return false;
19379
19380 switch (VT.getSimpleVT().SimpleTy) {
19381 case MVT::v4f32:
19382 case MVT::v8f16:
19383 return Subtarget->hasMVEFloatOps();
19384 case MVT::f16:
19385 return Subtarget->useFPVFMx16();
19386 case MVT::f32:
19387 return Subtarget->useFPVFMx();
19388 case MVT::f64:
19389 return Subtarget->useFPVFMx64();
19390 default:
19391 break;
19392 }
19393
19394 return false;
19395}
19396
19397static bool isLegalT1AddressImmediate(int64_t V, EVT VT) {
19398 if (V < 0)
19399 return false;
19400
19401 unsigned Scale = 1;
19402 switch (VT.getSimpleVT().SimpleTy) {
19403 case MVT::i1:
19404 case MVT::i8:
19405 // Scale == 1;
19406 break;
19407 case MVT::i16:
19408 // Scale == 2;
19409 Scale = 2;
19410 break;
19411 default:
19412 // On thumb1 we load most things (i32, i64, floats, etc) with a LDR
19413 // Scale == 4;
19414 Scale = 4;
19415 break;
19416 }
19417
19418 if ((V & (Scale - 1)) != 0)
19419 return false;
19420 return isUInt<5>(x: V / Scale);
19421}
19422
19423static bool isLegalT2AddressImmediate(int64_t V, EVT VT,
19424 const ARMSubtarget *Subtarget) {
19425 if (!VT.isInteger() && !VT.isFloatingPoint())
19426 return false;
19427 if (VT.isVector() && Subtarget->hasNEON())
19428 return false;
19429 if (VT.isVector() && VT.isFloatingPoint() && Subtarget->hasMVEIntegerOps() &&
19430 !Subtarget->hasMVEFloatOps())
19431 return false;
19432
19433 bool IsNeg = false;
19434 if (V < 0) {
19435 IsNeg = true;
19436 V = -V;
19437 }
19438
19439 unsigned NumBytes = std::max(a: (unsigned)VT.getSizeInBits() / 8, b: 1U);
19440
19441 // MVE: size * imm7
19442 if (VT.isVector() && Subtarget->hasMVEIntegerOps()) {
19443 switch (VT.getSimpleVT().getVectorElementType().SimpleTy) {
19444 case MVT::i32:
19445 case MVT::f32:
19446 return isShiftedUInt<7,2>(x: V);
19447 case MVT::i16:
19448 case MVT::f16:
19449 return isShiftedUInt<7,1>(x: V);
19450 case MVT::i8:
19451 return isUInt<7>(x: V);
19452 default:
19453 return false;
19454 }
19455 }
19456
19457 // half VLDR: 2 * imm8
19458 if (VT.isFloatingPoint() && NumBytes == 2 && Subtarget->hasFPRegs16())
19459 return isShiftedUInt<8, 1>(x: V);
19460 // VLDR and LDRD: 4 * imm8
19461 if ((VT.isFloatingPoint() && Subtarget->hasVFP2Base()) || NumBytes == 8)
19462 return isShiftedUInt<8, 2>(x: V);
19463
19464 if (NumBytes == 1 || NumBytes == 2 || NumBytes == 4) {
19465 // + imm12 or - imm8
19466 if (IsNeg)
19467 return isUInt<8>(x: V);
19468 return isUInt<12>(x: V);
19469 }
19470
19471 return false;
19472}
19473
19474/// isLegalAddressImmediate - Return true if the integer value can be used
19475/// as the offset of the target addressing mode for load / store of the
19476/// given type.
19477static bool isLegalAddressImmediate(int64_t V, EVT VT,
19478 const ARMSubtarget *Subtarget) {
19479 if (V == 0)
19480 return true;
19481
19482 if (!VT.isSimple())
19483 return false;
19484
19485 if (Subtarget->isThumb1Only())
19486 return isLegalT1AddressImmediate(V, VT);
19487 else if (Subtarget->isThumb2())
19488 return isLegalT2AddressImmediate(V, VT, Subtarget);
19489
19490 // ARM mode.
19491 if (V < 0)
19492 V = - V;
19493 switch (VT.getSimpleVT().SimpleTy) {
19494 default: return false;
19495 case MVT::i1:
19496 case MVT::i8:
19497 case MVT::i32:
19498 // +- imm12
19499 return isUInt<12>(x: V);
19500 case MVT::i16:
19501 // +- imm8
19502 return isUInt<8>(x: V);
19503 case MVT::f32:
19504 case MVT::f64:
19505 if (!Subtarget->hasVFP2Base()) // FIXME: NEON?
19506 return false;
19507 return isShiftedUInt<8, 2>(x: V);
19508 }
19509}
19510
19511bool ARMTargetLowering::isLegalT2ScaledAddressingMode(const AddrMode &AM,
19512 EVT VT) const {
19513 int Scale = AM.Scale;
19514 if (Scale < 0)
19515 return false;
19516
19517 switch (VT.getSimpleVT().SimpleTy) {
19518 default: return false;
19519 case MVT::i1:
19520 case MVT::i8:
19521 case MVT::i16:
19522 case MVT::i32:
19523 if (Scale == 1)
19524 return true;
19525 // r + r << imm
19526 Scale = Scale & ~1;
19527 return Scale == 2 || Scale == 4 || Scale == 8;
19528 case MVT::i64:
19529 // FIXME: What are we trying to model here? ldrd doesn't have an r + r
19530 // version in Thumb mode.
19531 // r + r
19532 if (Scale == 1)
19533 return true;
19534 // r * 2 (this can be lowered to r + r).
19535 if (!AM.HasBaseReg && Scale == 2)
19536 return true;
19537 return false;
19538 case MVT::isVoid:
19539 // Note, we allow "void" uses (basically, uses that aren't loads or
19540 // stores), because arm allows folding a scale into many arithmetic
19541 // operations. This should be made more precise and revisited later.
19542
19543 // Allow r << imm, but the imm has to be a multiple of two.
19544 if (Scale & 1) return false;
19545 return isPowerOf2_32(Value: Scale);
19546 }
19547}
19548
19549bool ARMTargetLowering::isLegalT1ScaledAddressingMode(const AddrMode &AM,
19550 EVT VT) const {
19551 const int Scale = AM.Scale;
19552
19553 // Negative scales are not supported in Thumb1.
19554 if (Scale < 0)
19555 return false;
19556
19557 // Thumb1 addressing modes do not support register scaling excepting the
19558 // following cases:
19559 // 1. Scale == 1 means no scaling.
19560 // 2. Scale == 2 this can be lowered to r + r if there is no base register.
19561 return (Scale == 1) || (!AM.HasBaseReg && Scale == 2);
19562}
19563
19564/// isLegalAddressingMode - Return true if the addressing mode represented
19565/// by AM is legal for this target, for a load/store of the specified type.
19566bool ARMTargetLowering::isLegalAddressingMode(const DataLayout &DL,
19567 const AddrMode &AM, Type *Ty,
19568 unsigned AS, Instruction *I) const {
19569 EVT VT = getValueType(DL, Ty, AllowUnknown: true);
19570 if (!isLegalAddressImmediate(V: AM.BaseOffs, VT, Subtarget))
19571 return false;
19572
19573 // Can never fold addr of global into load/store.
19574 if (AM.BaseGV)
19575 return false;
19576
19577 switch (AM.Scale) {
19578 case 0: // no scale reg, must be "r+i" or "r", or "i".
19579 break;
19580 default:
19581 // ARM doesn't support any R+R*scale+imm addr modes.
19582 if (AM.BaseOffs)
19583 return false;
19584
19585 if (!VT.isSimple())
19586 return false;
19587
19588 if (Subtarget->isThumb1Only())
19589 return isLegalT1ScaledAddressingMode(AM, VT);
19590
19591 if (Subtarget->isThumb2())
19592 return isLegalT2ScaledAddressingMode(AM, VT);
19593
19594 int Scale = AM.Scale;
19595 switch (VT.getSimpleVT().SimpleTy) {
19596 default: return false;
19597 case MVT::i1:
19598 case MVT::i8:
19599 case MVT::i32:
19600 if (Scale < 0) Scale = -Scale;
19601 if (Scale == 1)
19602 return true;
19603 // r + r << imm
19604 return isPowerOf2_32(Value: Scale & ~1);
19605 case MVT::i16:
19606 case MVT::i64:
19607 // r +/- r
19608 if (Scale == 1 || (AM.HasBaseReg && Scale == -1))
19609 return true;
19610 // r * 2 (this can be lowered to r + r).
19611 if (!AM.HasBaseReg && Scale == 2)
19612 return true;
19613 return false;
19614
19615 case MVT::isVoid:
19616 // Note, we allow "void" uses (basically, uses that aren't loads or
19617 // stores), because arm allows folding a scale into many arithmetic
19618 // operations. This should be made more precise and revisited later.
19619
19620 // Allow r << imm, but the imm has to be a multiple of two.
19621 if (Scale & 1) return false;
19622 return isPowerOf2_32(Value: Scale);
19623 }
19624 }
19625 return true;
19626}
19627
19628/// isLegalICmpImmediate - Return true if the specified immediate is legal
19629/// icmp immediate, that is the target has icmp instructions which can compare
19630/// a register against the immediate without having to materialize the
19631/// immediate into a register.
19632bool ARMTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
19633 // Thumb2 and ARM modes can use cmn for negative immediates.
19634 if (!Subtarget->isThumb())
19635 return ARM_AM::getSOImmVal(Arg: (uint32_t)Imm) != -1 ||
19636 ARM_AM::getSOImmVal(Arg: -(uint32_t)Imm) != -1;
19637 if (Subtarget->isThumb2())
19638 return ARM_AM::getT2SOImmVal(Arg: (uint32_t)Imm) != -1 ||
19639 ARM_AM::getT2SOImmVal(Arg: -(uint32_t)Imm) != -1;
19640 // Thumb1 doesn't have cmn, and only 8-bit immediates.
19641 return Imm >= 0 && Imm <= 255;
19642}
19643
19644/// isLegalAddImmediate - Return true if the specified immediate is a legal add
19645/// *or sub* immediate, that is the target has add or sub instructions which can
19646/// add a register with the immediate without having to materialize the
19647/// immediate into a register.
19648bool ARMTargetLowering::isLegalAddImmediate(int64_t Imm) const {
19649 // Same encoding for add/sub, just flip the sign.
19650 uint64_t AbsImm = AbsoluteValue(X: Imm);
19651 if (!Subtarget->isThumb())
19652 return ARM_AM::getSOImmVal(Arg: AbsImm) != -1;
19653 if (Subtarget->isThumb2())
19654 return ARM_AM::getT2SOImmVal(Arg: AbsImm) != -1;
19655 // Thumb1 only has 8-bit unsigned immediate.
19656 return AbsImm <= 255;
19657}
19658
19659// Return false to prevent folding
19660// (mul (add r, c0), c1) -> (add (mul r, c1), c0*c1) in DAGCombine,
19661// if the folding leads to worse code.
19662bool ARMTargetLowering::isMulAddWithConstProfitable(SDValue AddNode,
19663 SDValue ConstNode) const {
19664 // Let the DAGCombiner decide for vector types and large types.
19665 const EVT VT = AddNode.getValueType();
19666 if (VT.isVector() || VT.getScalarSizeInBits() > 32)
19667 return true;
19668
19669 // It is worse if c0 is legal add immediate, while c1*c0 is not
19670 // and has to be composed by at least two instructions.
19671 const ConstantSDNode *C0Node = cast<ConstantSDNode>(Val: AddNode.getOperand(i: 1));
19672 const ConstantSDNode *C1Node = cast<ConstantSDNode>(Val&: ConstNode);
19673 const int64_t C0 = C0Node->getSExtValue();
19674 APInt CA = C0Node->getAPIntValue() * C1Node->getAPIntValue();
19675 if (!isLegalAddImmediate(Imm: C0) || isLegalAddImmediate(Imm: CA.getSExtValue()))
19676 return true;
19677 if (ConstantMaterializationCost(Val: (unsigned)CA.getZExtValue(), Subtarget) > 1)
19678 return false;
19679
19680 // Default to true and let the DAGCombiner decide.
19681 return true;
19682}
19683
19684static bool getARMIndexedAddressParts(SDNode *Ptr, EVT VT,
19685 bool isSEXTLoad, SDValue &Base,
19686 SDValue &Offset, bool &isInc,
19687 SelectionDAG &DAG) {
19688 if (Ptr->getOpcode() != ISD::ADD && Ptr->getOpcode() != ISD::SUB)
19689 return false;
19690
19691 if (VT == MVT::i16 || ((VT == MVT::i8 || VT == MVT::i1) && isSEXTLoad)) {
19692 // AddressingMode 3
19693 Base = Ptr->getOperand(Num: 0);
19694 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Val: Ptr->getOperand(Num: 1))) {
19695 int RHSC = (int)RHS->getZExtValue();
19696 if (RHSC < 0 && RHSC > -256) {
19697 assert(Ptr->getOpcode() == ISD::ADD);
19698 isInc = false;
19699 Offset = DAG.getConstant(Val: -RHSC, DL: SDLoc(Ptr), VT: RHS->getValueType(ResNo: 0));
19700 return true;
19701 }
19702 }
19703 isInc = (Ptr->getOpcode() == ISD::ADD);
19704 Offset = Ptr->getOperand(Num: 1);
19705 return true;
19706 } else if (VT == MVT::i32 || VT == MVT::i8 || VT == MVT::i1) {
19707 // AddressingMode 2
19708 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Val: Ptr->getOperand(Num: 1))) {
19709 int RHSC = (int)RHS->getZExtValue();
19710 if (RHSC < 0 && RHSC > -0x1000) {
19711 assert(Ptr->getOpcode() == ISD::ADD);
19712 isInc = false;
19713 Offset = DAG.getConstant(Val: -RHSC, DL: SDLoc(Ptr), VT: RHS->getValueType(ResNo: 0));
19714 Base = Ptr->getOperand(Num: 0);
19715 return true;
19716 }
19717 }
19718
19719 if (Ptr->getOpcode() == ISD::ADD) {
19720 isInc = true;
19721 ARM_AM::ShiftOpc ShOpcVal=
19722 ARM_AM::getShiftOpcForNode(Opcode: Ptr->getOperand(Num: 0).getOpcode());
19723 if (ShOpcVal != ARM_AM::no_shift) {
19724 Base = Ptr->getOperand(Num: 1);
19725 Offset = Ptr->getOperand(Num: 0);
19726 } else {
19727 Base = Ptr->getOperand(Num: 0);
19728 Offset = Ptr->getOperand(Num: 1);
19729 }
19730 return true;
19731 }
19732
19733 isInc = (Ptr->getOpcode() == ISD::ADD);
19734 Base = Ptr->getOperand(Num: 0);
19735 Offset = Ptr->getOperand(Num: 1);
19736 return true;
19737 }
19738
19739 // FIXME: Use VLDM / VSTM to emulate indexed FP load / store.
19740 return false;
19741}
19742
19743static bool getT2IndexedAddressParts(SDNode *Ptr, EVT VT,
19744 bool isSEXTLoad, SDValue &Base,
19745 SDValue &Offset, bool &isInc,
19746 SelectionDAG &DAG) {
19747 if (Ptr->getOpcode() != ISD::ADD && Ptr->getOpcode() != ISD::SUB)
19748 return false;
19749
19750 Base = Ptr->getOperand(Num: 0);
19751 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Val: Ptr->getOperand(Num: 1))) {
19752 int RHSC = (int)RHS->getZExtValue();
19753 if (RHSC < 0 && RHSC > -0x100) { // 8 bits.
19754 assert(Ptr->getOpcode() == ISD::ADD);
19755 isInc = false;
19756 Offset = DAG.getConstant(Val: -RHSC, DL: SDLoc(Ptr), VT: RHS->getValueType(ResNo: 0));
19757 return true;
19758 } else if (RHSC > 0 && RHSC < 0x100) { // 8 bit, no zero.
19759 isInc = Ptr->getOpcode() == ISD::ADD;
19760 Offset = DAG.getConstant(Val: RHSC, DL: SDLoc(Ptr), VT: RHS->getValueType(ResNo: 0));
19761 return true;
19762 }
19763 }
19764
19765 return false;
19766}
19767
19768static bool getMVEIndexedAddressParts(SDNode *Ptr, EVT VT, Align Alignment,
19769 bool isSEXTLoad, bool IsMasked, bool isLE,
19770 SDValue &Base, SDValue &Offset,
19771 bool &isInc, SelectionDAG &DAG) {
19772 if (Ptr->getOpcode() != ISD::ADD && Ptr->getOpcode() != ISD::SUB)
19773 return false;
19774 if (!isa<ConstantSDNode>(Val: Ptr->getOperand(Num: 1)))
19775 return false;
19776
19777 // We allow LE non-masked loads to change the type (for example use a vldrb.8
19778 // as opposed to a vldrw.32). This can allow extra addressing modes or
19779 // alignments for what is otherwise an equivalent instruction.
19780 bool CanChangeType = isLE && !IsMasked;
19781
19782 ConstantSDNode *RHS = cast<ConstantSDNode>(Val: Ptr->getOperand(Num: 1));
19783 int RHSC = (int)RHS->getZExtValue();
19784
19785 auto IsInRange = [&](int RHSC, int Limit, int Scale) {
19786 if (RHSC < 0 && RHSC > -Limit * Scale && RHSC % Scale == 0) {
19787 assert(Ptr->getOpcode() == ISD::ADD);
19788 isInc = false;
19789 Offset = DAG.getConstant(Val: -RHSC, DL: SDLoc(Ptr), VT: RHS->getValueType(ResNo: 0));
19790 return true;
19791 } else if (RHSC > 0 && RHSC < Limit * Scale && RHSC % Scale == 0) {
19792 isInc = Ptr->getOpcode() == ISD::ADD;
19793 Offset = DAG.getConstant(Val: RHSC, DL: SDLoc(Ptr), VT: RHS->getValueType(ResNo: 0));
19794 return true;
19795 }
19796 return false;
19797 };
19798
19799 // Try to find a matching instruction based on s/zext, Alignment, Offset and
19800 // (in BE/masked) type.
19801 Base = Ptr->getOperand(Num: 0);
19802 if (VT == MVT::v4i16) {
19803 if (Alignment >= 2 && IsInRange(RHSC, 0x80, 2))
19804 return true;
19805 } else if (VT == MVT::v4i8 || VT == MVT::v8i8) {
19806 if (IsInRange(RHSC, 0x80, 1))
19807 return true;
19808 } else if (Alignment >= 4 &&
19809 (CanChangeType || VT == MVT::v4i32 || VT == MVT::v4f32) &&
19810 IsInRange(RHSC, 0x80, 4))
19811 return true;
19812 else if (Alignment >= 2 &&
19813 (CanChangeType || VT == MVT::v8i16 || VT == MVT::v8f16) &&
19814 IsInRange(RHSC, 0x80, 2))
19815 return true;
19816 else if ((CanChangeType || VT == MVT::v16i8) && IsInRange(RHSC, 0x80, 1))
19817 return true;
19818 return false;
19819}
19820
19821/// getPreIndexedAddressParts - returns true by value, base pointer and
19822/// offset pointer and addressing mode by reference if the node's address
19823/// can be legally represented as pre-indexed load / store address.
19824bool
19825ARMTargetLowering::getPreIndexedAddressParts(SDNode *N, SDValue &Base,
19826 SDValue &Offset,
19827 ISD::MemIndexedMode &AM,
19828 SelectionDAG &DAG) const {
19829 if (Subtarget->isThumb1Only())
19830 return false;
19831
19832 EVT VT;
19833 SDValue Ptr;
19834 Align Alignment;
19835 unsigned AS = 0;
19836 bool isSEXTLoad = false;
19837 bool IsMasked = false;
19838 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Val: N)) {
19839 Ptr = LD->getBasePtr();
19840 VT = LD->getMemoryVT();
19841 Alignment = LD->getAlign();
19842 AS = LD->getAddressSpace();
19843 isSEXTLoad = LD->getExtensionType() == ISD::SEXTLOAD;
19844 } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(Val: N)) {
19845 Ptr = ST->getBasePtr();
19846 VT = ST->getMemoryVT();
19847 Alignment = ST->getAlign();
19848 AS = ST->getAddressSpace();
19849 } else if (MaskedLoadSDNode *LD = dyn_cast<MaskedLoadSDNode>(Val: N)) {
19850 Ptr = LD->getBasePtr();
19851 VT = LD->getMemoryVT();
19852 Alignment = LD->getAlign();
19853 AS = LD->getAddressSpace();
19854 isSEXTLoad = LD->getExtensionType() == ISD::SEXTLOAD;
19855 IsMasked = true;
19856 } else if (MaskedStoreSDNode *ST = dyn_cast<MaskedStoreSDNode>(Val: N)) {
19857 Ptr = ST->getBasePtr();
19858 VT = ST->getMemoryVT();
19859 Alignment = ST->getAlign();
19860 AS = ST->getAddressSpace();
19861 IsMasked = true;
19862 } else
19863 return false;
19864
19865 unsigned Fast = 0;
19866 if (!allowsMisalignedMemoryAccesses(VT, AS, Alignment,
19867 MachineMemOperand::MONone, Fast: &Fast)) {
19868 // Only generate post-increment or pre-increment forms when a real
19869 // hardware instruction exists for them. Do not emit postinc/preinc
19870 // if the operation will end up as a libcall.
19871 return false;
19872 }
19873
19874 bool isInc;
19875 bool isLegal = false;
19876 if (VT.isVector())
19877 isLegal = Subtarget->hasMVEIntegerOps() &&
19878 getMVEIndexedAddressParts(
19879 Ptr: Ptr.getNode(), VT, Alignment, isSEXTLoad, IsMasked,
19880 isLE: Subtarget->isLittle(), Base, Offset, isInc, DAG);
19881 else {
19882 if (Subtarget->isThumb2())
19883 isLegal = getT2IndexedAddressParts(Ptr: Ptr.getNode(), VT, isSEXTLoad, Base,
19884 Offset, isInc, DAG);
19885 else
19886 isLegal = getARMIndexedAddressParts(Ptr: Ptr.getNode(), VT, isSEXTLoad, Base,
19887 Offset, isInc, DAG);
19888 }
19889 if (!isLegal)
19890 return false;
19891
19892 AM = isInc ? ISD::PRE_INC : ISD::PRE_DEC;
19893 return true;
19894}
19895
19896/// getPostIndexedAddressParts - returns true by value, base pointer and
19897/// offset pointer and addressing mode by reference if this node can be
19898/// combined with a load / store to form a post-indexed load / store.
19899bool ARMTargetLowering::getPostIndexedAddressParts(SDNode *N, SDNode *Op,
19900 SDValue &Base,
19901 SDValue &Offset,
19902 ISD::MemIndexedMode &AM,
19903 SelectionDAG &DAG) const {
19904 EVT VT;
19905 SDValue Ptr;
19906 Align Alignment;
19907 bool isSEXTLoad = false, isNonExt;
19908 bool IsMasked = false;
19909 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Val: N)) {
19910 VT = LD->getMemoryVT();
19911 Ptr = LD->getBasePtr();
19912 Alignment = LD->getAlign();
19913 isSEXTLoad = LD->getExtensionType() == ISD::SEXTLOAD;
19914 isNonExt = LD->getExtensionType() == ISD::NON_EXTLOAD;
19915 } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(Val: N)) {
19916 VT = ST->getMemoryVT();
19917 Ptr = ST->getBasePtr();
19918 Alignment = ST->getAlign();
19919 isNonExt = !ST->isTruncatingStore();
19920 } else if (MaskedLoadSDNode *LD = dyn_cast<MaskedLoadSDNode>(Val: N)) {
19921 VT = LD->getMemoryVT();
19922 Ptr = LD->getBasePtr();
19923 Alignment = LD->getAlign();
19924 isSEXTLoad = LD->getExtensionType() == ISD::SEXTLOAD;
19925 isNonExt = LD->getExtensionType() == ISD::NON_EXTLOAD;
19926 IsMasked = true;
19927 } else if (MaskedStoreSDNode *ST = dyn_cast<MaskedStoreSDNode>(Val: N)) {
19928 VT = ST->getMemoryVT();
19929 Ptr = ST->getBasePtr();
19930 Alignment = ST->getAlign();
19931 isNonExt = !ST->isTruncatingStore();
19932 IsMasked = true;
19933 } else
19934 return false;
19935
19936 if (Subtarget->isThumb1Only()) {
19937 // Thumb-1 can do a limited post-inc load or store as an updating LDM. It
19938 // must be non-extending/truncating, i32, with an offset of 4.
19939 assert(Op->getValueType(0) == MVT::i32 && "Non-i32 post-inc op?!");
19940 if (Op->getOpcode() != ISD::ADD || !isNonExt)
19941 return false;
19942 auto *RHS = dyn_cast<ConstantSDNode>(Val: Op->getOperand(Num: 1));
19943 if (!RHS || RHS->getZExtValue() != 4)
19944 return false;
19945 if (Alignment < Align(4))
19946 return false;
19947
19948 Offset = Op->getOperand(Num: 1);
19949 Base = Op->getOperand(Num: 0);
19950 AM = ISD::POST_INC;
19951 return true;
19952 }
19953
19954 bool isInc;
19955 bool isLegal = false;
19956 if (VT.isVector())
19957 isLegal = Subtarget->hasMVEIntegerOps() &&
19958 getMVEIndexedAddressParts(Ptr: Op, VT, Alignment, isSEXTLoad, IsMasked,
19959 isLE: Subtarget->isLittle(), Base, Offset,
19960 isInc, DAG);
19961 else {
19962 if (Subtarget->isThumb2())
19963 isLegal = getT2IndexedAddressParts(Ptr: Op, VT, isSEXTLoad, Base, Offset,
19964 isInc, DAG);
19965 else
19966 isLegal = getARMIndexedAddressParts(Ptr: Op, VT, isSEXTLoad, Base, Offset,
19967 isInc, DAG);
19968 }
19969 if (!isLegal)
19970 return false;
19971
19972 if (Ptr != Base) {
19973 // Swap base ptr and offset to catch more post-index load / store when
19974 // it's legal. In Thumb2 mode, offset must be an immediate.
19975 if (Ptr == Offset && Op->getOpcode() == ISD::ADD &&
19976 !Subtarget->isThumb2())
19977 std::swap(a&: Base, b&: Offset);
19978
19979 // Post-indexed load / store update the base pointer.
19980 if (Ptr != Base)
19981 return false;
19982 }
19983
19984 AM = isInc ? ISD::POST_INC : ISD::POST_DEC;
19985 return true;
19986}
19987
19988void ARMTargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
19989 KnownBits &Known,
19990 const APInt &DemandedElts,
19991 const SelectionDAG &DAG,
19992 unsigned Depth) const {
19993 unsigned BitWidth = Known.getBitWidth();
19994 Known.resetAll();
19995 switch (Op.getOpcode()) {
19996 default: break;
19997 case ARMISD::ADDC:
19998 case ARMISD::ADDE:
19999 case ARMISD::SUBC:
20000 case ARMISD::SUBE:
20001 // Special cases when we convert a carry to a boolean.
20002 if (Op.getResNo() == 0) {
20003 SDValue LHS = Op.getOperand(i: 0);
20004 SDValue RHS = Op.getOperand(i: 1);
20005 // (ADDE 0, 0, C) will give us a single bit.
20006 if (Op->getOpcode() == ARMISD::ADDE && isNullConstant(V: LHS) &&
20007 isNullConstant(V: RHS)) {
20008 Known.Zero |= APInt::getHighBitsSet(numBits: BitWidth, hiBitsSet: BitWidth - 1);
20009 return;
20010 }
20011 }
20012 break;
20013 case ARMISD::CMOV: {
20014 // Bits are known zero/one if known on the LHS and RHS.
20015 Known = DAG.computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth+1);
20016 if (Known.isUnknown())
20017 return;
20018
20019 KnownBits KnownRHS = DAG.computeKnownBits(Op: Op.getOperand(i: 1), Depth: Depth+1);
20020 Known = Known.intersectWith(RHS: KnownRHS);
20021 return;
20022 }
20023 case ISD::INTRINSIC_W_CHAIN: {
20024 Intrinsic::ID IntID =
20025 static_cast<Intrinsic::ID>(Op->getConstantOperandVal(Num: 1));
20026 switch (IntID) {
20027 default: return;
20028 case Intrinsic::arm_ldaex:
20029 case Intrinsic::arm_ldrex: {
20030 EVT VT = cast<MemIntrinsicSDNode>(Val: Op)->getMemoryVT();
20031 unsigned MemBits = VT.getScalarSizeInBits();
20032 Known.Zero |= APInt::getHighBitsSet(numBits: BitWidth, hiBitsSet: BitWidth - MemBits);
20033 return;
20034 }
20035 }
20036 }
20037 case ARMISD::BFI: {
20038 // Conservatively, we can recurse down the first operand
20039 // and just mask out all affected bits.
20040 Known = DAG.computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
20041
20042 // The operand to BFI is already a mask suitable for removing the bits it
20043 // sets.
20044 const APInt &Mask = Op.getConstantOperandAPInt(i: 2);
20045 Known.Zero &= Mask;
20046 Known.One &= Mask;
20047 return;
20048 }
20049 case ARMISD::VGETLANEs:
20050 case ARMISD::VGETLANEu: {
20051 const SDValue &SrcSV = Op.getOperand(i: 0);
20052 EVT VecVT = SrcSV.getValueType();
20053 assert(VecVT.isVector() && "VGETLANE expected a vector type");
20054 const unsigned NumSrcElts = VecVT.getVectorNumElements();
20055 ConstantSDNode *Pos = cast<ConstantSDNode>(Val: Op.getOperand(i: 1).getNode());
20056 assert(Pos->getAPIntValue().ult(NumSrcElts) &&
20057 "VGETLANE index out of bounds");
20058 unsigned Idx = Pos->getZExtValue();
20059 APInt DemandedElt = APInt::getOneBitSet(numBits: NumSrcElts, BitNo: Idx);
20060 Known = DAG.computeKnownBits(Op: SrcSV, DemandedElts: DemandedElt, Depth: Depth + 1);
20061
20062 EVT VT = Op.getValueType();
20063 const unsigned DstSz = VT.getScalarSizeInBits();
20064 const unsigned SrcSz = VecVT.getVectorElementType().getSizeInBits();
20065 (void)SrcSz;
20066 assert(SrcSz == Known.getBitWidth());
20067 assert(DstSz > SrcSz);
20068 if (Op.getOpcode() == ARMISD::VGETLANEs)
20069 Known = Known.sext(BitWidth: DstSz);
20070 else {
20071 Known = Known.zext(BitWidth: DstSz);
20072 }
20073 assert(DstSz == Known.getBitWidth());
20074 break;
20075 }
20076 case ARMISD::VMOVrh: {
20077 KnownBits KnownOp = DAG.computeKnownBits(Op: Op->getOperand(Num: 0), Depth: Depth + 1);
20078 assert(KnownOp.getBitWidth() == 16);
20079 Known = KnownOp.zext(BitWidth: 32);
20080 break;
20081 }
20082 case ARMISD::CSINC:
20083 case ARMISD::CSINV:
20084 case ARMISD::CSNEG: {
20085 KnownBits KnownOp0 = DAG.computeKnownBits(Op: Op->getOperand(Num: 0), Depth: Depth + 1);
20086 KnownBits KnownOp1 = DAG.computeKnownBits(Op: Op->getOperand(Num: 1), Depth: Depth + 1);
20087
20088 // The result is either:
20089 // CSINC: KnownOp0 or KnownOp1 + 1
20090 // CSINV: KnownOp0 or ~KnownOp1
20091 // CSNEG: KnownOp0 or KnownOp1 * -1
20092 if (Op.getOpcode() == ARMISD::CSINC)
20093 KnownOp1 =
20094 KnownBits::add(LHS: KnownOp1, RHS: KnownBits::makeConstant(C: APInt(32, 1)));
20095 else if (Op.getOpcode() == ARMISD::CSINV)
20096 std::swap(a&: KnownOp1.Zero, b&: KnownOp1.One);
20097 else if (Op.getOpcode() == ARMISD::CSNEG)
20098 KnownOp1 = KnownBits::mul(LHS: KnownOp1,
20099 RHS: KnownBits::makeConstant(C: APInt::getAllOnes(numBits: 32)));
20100
20101 Known = KnownOp0.intersectWith(RHS: KnownOp1);
20102 break;
20103 }
20104 case ARMISD::VORRIMM:
20105 case ARMISD::VBICIMM: {
20106 unsigned Encoded = Op.getConstantOperandVal(i: 1);
20107 unsigned DecEltBits = 0;
20108 uint64_t DecodedVal = ARM_AM::decodeVMOVModImm(ModImm: Encoded, EltBits&: DecEltBits);
20109
20110 unsigned EltBits = Op.getScalarValueSizeInBits();
20111 if (EltBits != DecEltBits) {
20112 // Be conservative: only update Known when EltBits == DecEltBits.
20113 // This is believed to always be true for VORRIMM/VBICIMM today, but if
20114 // that changes in the future, doing nothing here is safer than risking
20115 // subtle bugs.
20116 break;
20117 }
20118
20119 KnownBits KnownLHS = DAG.computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
20120 bool IsVORR = Op.getOpcode() == ARMISD::VORRIMM;
20121 APInt Imm(DecEltBits, DecodedVal);
20122
20123 Known.One = IsVORR ? (KnownLHS.One | Imm) : (KnownLHS.One & ~Imm);
20124 Known.Zero = IsVORR ? (KnownLHS.Zero & ~Imm) : (KnownLHS.Zero | Imm);
20125 break;
20126 }
20127 }
20128}
20129
20130static bool isLegalLogicalImmediate(unsigned Imm,
20131 const ARMSubtarget *Subtarget) {
20132 if (!Subtarget->isThumb())
20133 return ARM_AM::getSOImmVal(Arg: Imm) != -1;
20134 if (Subtarget->isThumb2())
20135 return ARM_AM::getT2SOImmVal(Arg: Imm) != -1;
20136 // Thumb1 only has 8-bit unsigned immediate.
20137 return Imm <= 255;
20138}
20139
20140/// Refine i32 AND/OR/XOR with a constant RHS using demanded bits: replace the
20141/// immediate with an equivalent constant that ARM/Thumb can encode as a
20142/// logical immediate (or that selects better lowering), without changing the
20143/// computed result on those demanded bits.
20144static bool optimizeLogicalImm(SDValue Op, unsigned Imm,
20145 const APInt &DemandedBits,
20146 const ARMSubtarget *Subtarget,
20147 TargetLowering::TargetLoweringOpt &TLO) {
20148
20149 if (Imm == 0 || Imm == ~0U)
20150 return false;
20151
20152 unsigned Opc = Op.getOpcode();
20153 unsigned Demanded = DemandedBits.getZExtValue();
20154 EVT VT = Op.getValueType();
20155
20156 unsigned ShrunkImm = Imm & Demanded;
20157 unsigned ExpandedImm = Imm | ~Demanded;
20158
20159 auto IsLegalImm = [ShrunkImm, ExpandedImm](unsigned CandidateImm) -> bool {
20160 return (ShrunkImm & CandidateImm) == ShrunkImm &&
20161 (~ExpandedImm & CandidateImm) == 0;
20162 };
20163 auto UseImm = [Imm, Opc, Op, VT, &TLO](unsigned NewImm) -> bool {
20164 if (NewImm == Imm)
20165 return true;
20166 SDLoc DL(Op);
20167 SDValue NewC = TLO.DAG.getConstant(Val: NewImm, DL, VT);
20168 SDValue NewOp =
20169 TLO.DAG.getNode(Opcode: Opc, DL, VT, N1: Op.getOperand(i: 0), N2: NewC, Flags: Op->getFlags());
20170 return TLO.CombineTo(O: Op, N: NewOp);
20171 };
20172
20173 // Shrunk immediate is 0: AND becomes zero; OR/XOR with 0 leaves the other
20174 // operand (still valid on demanded bits).
20175 if (ShrunkImm == 0) {
20176 ++NumOptimizedImms;
20177 return UseImm(ShrunkImm);
20178 }
20179
20180 // If the immediate is all ones: for AND this removes the operation; for
20181 // OR/XOR it remains a transform valid on demanded bits. (Target-independent
20182 // shrink may not fold this, so keep it to avoid obscure combine loops.)
20183 if (ExpandedImm == ~0U) {
20184 ++NumOptimizedImms;
20185 return UseImm(ExpandedImm);
20186 }
20187
20188 // Thumb1: prefer 0xFF / 0xFFFF when they fit the demanded-bit envelope so
20189 // lowering can match uxtb / uxth (AND immediates only; OR/XOR do not use
20190 // that). Run this before strict ShrunkImm: a tight 8-bit ShrunkImm can be
20191 // legal while 0xFF still matches the envelope and yields better isel (uxtb).
20192 if (Opc == ISD::AND && Subtarget->hasV6Ops()) {
20193 if (IsLegalImm(0xFF)) {
20194 ++NumOptimizedImms;
20195 return UseImm(0xFF);
20196 }
20197
20198 if (IsLegalImm(0xFFFF)) {
20199 ++NumOptimizedImms;
20200 return UseImm(0xFFFF);
20201 }
20202 }
20203
20204 // Don't optimize if it is legal.
20205 if (isLegalLogicalImmediate(Imm, Subtarget))
20206 return false;
20207
20208 // FIXME: Check for BIC being legal causes infinite loop due to target
20209 // independent DAG combine undoing this.
20210
20211 // Prefer strict shrink when ShrunkImm encodes for this target, before
20212 // complement expansion.
20213 if (isLegalLogicalImmediate(Imm: ShrunkImm, Subtarget)) {
20214 ++NumOptimizedImms;
20215 return UseImm(ShrunkImm);
20216 }
20217
20218 // Complement expansion: if all undemanded bits are already one, ExpandedImm
20219 // is Imm with every non-demanded bit set. When (~ExpandedImm) < 256, the
20220 // complement fits in an 8-bit unsigned value, i.e. bits 8–31 of ExpandedImm
20221 // are all ones; only the low byte may differ from ~0. Use that expanded
20222 // constant so isel sees a mask shape that fits logical-immediate patterns.
20223 if ((~ExpandedImm) < 256) {
20224 ++NumOptimizedImms;
20225 return UseImm(ExpandedImm);
20226 }
20227
20228 // FIXME: The check for v6 is because this interferes with some ubfx
20229 // optimizations.
20230 if (Opc == ISD::AND && isLegalLogicalImmediate(Imm: ~ExpandedImm, Subtarget) &&
20231 !Subtarget->hasV6Ops()) {
20232 ++NumOptimizedImms;
20233 return UseImm(ExpandedImm);
20234 }
20235
20236 // Potential improvements:
20237 //
20238 // We could try to recognize lsls+lsrs or lsrs+lsls pairs here.
20239 // We could try to prefer Thumb1 immediates which can be lowered to a
20240 // two-instruction sequence.
20241
20242 return false;
20243}
20244
20245bool ARMTargetLowering::targetShrinkDemandedConstant(
20246 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
20247 TargetLoweringOpt &TLO) const {
20248 // Delay this optimization to as late as possible.
20249 if (!TLO.LegalOps)
20250 return false;
20251
20252 EVT VT = Op.getValueType();
20253
20254 // Ignore vectors.
20255 if (VT.isVector())
20256 return false;
20257
20258 unsigned Size = VT.getSizeInBits();
20259
20260 if (Size != 32)
20261 return false;
20262
20263 // Exit early if we demand all bits.
20264 if (DemandedBits.isAllOnes())
20265 return false;
20266
20267 switch (Op.getOpcode()) {
20268 default:
20269 return false;
20270 case ISD::AND:
20271 case ISD::OR:
20272 case ISD::XOR:
20273 break;
20274 }
20275 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1));
20276 if (!C)
20277 return false;
20278 unsigned Imm = C->getZExtValue();
20279 return optimizeLogicalImm(Op, Imm, DemandedBits, Subtarget, TLO);
20280}
20281
20282bool ARMTargetLowering::SimplifyDemandedBitsForTargetNode(
20283 SDValue Op, const APInt &OriginalDemandedBits,
20284 const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO,
20285 unsigned Depth) const {
20286 unsigned Opc = Op.getOpcode();
20287
20288 switch (Opc) {
20289 case ARMISD::ASRL:
20290 case ARMISD::LSRL: {
20291 // If this is result 0 and the other result is unused, see if the demand
20292 // bits allow us to shrink this long shift into a standard small shift in
20293 // the opposite direction.
20294 if (Op.getResNo() == 0 && !Op->hasAnyUseOfValue(Value: 1) &&
20295 isa<ConstantSDNode>(Val: Op->getOperand(Num: 2))) {
20296 unsigned ShAmt = Op->getConstantOperandVal(Num: 2);
20297 if (ShAmt < 32 && OriginalDemandedBits.isSubsetOf(RHS: APInt::getAllOnes(numBits: 32)
20298 << (32 - ShAmt)))
20299 return TLO.CombineTo(
20300 O: Op, N: TLO.DAG.getNode(
20301 Opcode: ISD::SHL, DL: SDLoc(Op), VT: MVT::i32, N1: Op.getOperand(i: 1),
20302 N2: TLO.DAG.getConstant(Val: 32 - ShAmt, DL: SDLoc(Op), VT: MVT::i32)));
20303 }
20304 break;
20305 }
20306 case ARMISD::VBICIMM: {
20307 SDValue Op0 = Op.getOperand(i: 0);
20308 unsigned ModImm = Op.getConstantOperandVal(i: 1);
20309 unsigned EltBits = 0;
20310 uint64_t Mask = ARM_AM::decodeVMOVModImm(ModImm, EltBits);
20311 if ((OriginalDemandedBits & Mask) == 0)
20312 return TLO.CombineTo(O: Op, N: Op0);
20313 }
20314 }
20315
20316 return TargetLowering::SimplifyDemandedBitsForTargetNode(
20317 Op, DemandedBits: OriginalDemandedBits, DemandedElts: OriginalDemandedElts, Known, TLO, Depth);
20318}
20319
20320//===----------------------------------------------------------------------===//
20321// ARM Inline Assembly Support
20322//===----------------------------------------------------------------------===//
20323
20324const char *ARMTargetLowering::LowerXConstraint(EVT ConstraintVT) const {
20325 // At this point, we have to lower this constraint to something else, so we
20326 // lower it to an "r" or "w". However, by doing this we will force the result
20327 // to be in register, while the X constraint is much more permissive.
20328 //
20329 // Although we are correct (we are free to emit anything, without
20330 // constraints), we might break use cases that would expect us to be more
20331 // efficient and emit something else.
20332 if (!Subtarget->hasVFP2Base())
20333 return "r";
20334 if (ConstraintVT.isFloatingPoint())
20335 return "w";
20336 if (ConstraintVT.isVector() && Subtarget->hasNEON() &&
20337 (ConstraintVT.getSizeInBits() == 64 ||
20338 ConstraintVT.getSizeInBits() == 128))
20339 return "w";
20340
20341 return "r";
20342}
20343
20344/// getConstraintType - Given a constraint letter, return the type of
20345/// constraint it is for this target.
20346ARMTargetLowering::ConstraintType
20347ARMTargetLowering::getConstraintType(StringRef Constraint) const {
20348 unsigned S = Constraint.size();
20349 if (S == 1) {
20350 switch (Constraint[0]) {
20351 default: break;
20352 case 'l': return C_RegisterClass;
20353 case 'w': return C_RegisterClass;
20354 case 'h': return C_RegisterClass;
20355 case 'x': return C_RegisterClass;
20356 case 't': return C_RegisterClass;
20357 case 'j': return C_Immediate; // Constant for movw.
20358 // An address with a single base register. Due to the way we
20359 // currently handle addresses it is the same as an 'r' memory constraint.
20360 case 'Q': return C_Memory;
20361 }
20362 } else if (S == 2) {
20363 switch (Constraint[0]) {
20364 default: break;
20365 case 'T': return C_RegisterClass;
20366 // All 'U+' constraints are addresses.
20367 case 'U': return C_Memory;
20368 }
20369 }
20370 return TargetLowering::getConstraintType(Constraint);
20371}
20372
20373/// Examine constraint type and operand type and determine a weight value.
20374/// This object must already have been set up with the operand type
20375/// and the current alternative constraint selected.
20376TargetLowering::ConstraintWeight
20377ARMTargetLowering::getSingleConstraintMatchWeight(
20378 AsmOperandInfo &info, const char *constraint) const {
20379 ConstraintWeight weight = CW_Invalid;
20380 Value *CallOperandVal = info.CallOperandVal;
20381 // If we don't have a value, we can't do a match,
20382 // but allow it at the lowest weight.
20383 if (!CallOperandVal)
20384 return CW_Default;
20385 Type *type = CallOperandVal->getType();
20386 // Look at the constraint type.
20387 switch (*constraint) {
20388 default:
20389 weight = TargetLowering::getSingleConstraintMatchWeight(info, constraint);
20390 break;
20391 case 'l':
20392 if (type->isIntegerTy()) {
20393 if (Subtarget->isThumb())
20394 weight = CW_SpecificReg;
20395 else
20396 weight = CW_Register;
20397 }
20398 break;
20399 case 'w':
20400 if (type->isFloatingPointTy())
20401 weight = CW_Register;
20402 break;
20403 }
20404 return weight;
20405}
20406
20407static bool isIncompatibleReg(const MCPhysReg &PR, MVT VT) {
20408 if (PR == 0 || VT == MVT::Other)
20409 return false;
20410 if (ARM::SPRRegClass.contains(Reg: PR))
20411 return VT != MVT::f32 && VT != MVT::f16 && VT != MVT::i32;
20412 if (ARM::DPRRegClass.contains(Reg: PR))
20413 return VT != MVT::f64 && !VT.is64BitVector();
20414 return false;
20415}
20416
20417using RCPair = std::pair<unsigned, const TargetRegisterClass *>;
20418
20419RCPair ARMTargetLowering::getRegForInlineAsmConstraint(
20420 const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const {
20421 switch (Constraint.size()) {
20422 case 1:
20423 // GCC ARM Constraint Letters
20424 switch (Constraint[0]) {
20425 case 'l': // Low regs or general regs.
20426 if (Subtarget->isThumb())
20427 return RCPair(0U, &ARM::tGPRRegClass);
20428 return RCPair(0U, &ARM::GPRRegClass);
20429 case 'h': // High regs or no regs.
20430 if (Subtarget->isThumb())
20431 return RCPair(0U, &ARM::hGPRRegClass);
20432 break;
20433 case 'r':
20434 if (Subtarget->isThumb1Only())
20435 return RCPair(0U, &ARM::tGPRRegClass);
20436 return RCPair(0U, &ARM::GPRRegClass);
20437 case 'w':
20438 if (VT == MVT::Other)
20439 break;
20440 if (VT == MVT::f32 || VT == MVT::f16 || VT == MVT::bf16)
20441 return RCPair(0U, &ARM::SPRRegClass);
20442 if (VT.getSizeInBits() == 64)
20443 return RCPair(0U, &ARM::DPRRegClass);
20444 if (VT.getSizeInBits() == 128)
20445 return RCPair(0U, &ARM::QPRRegClass);
20446 break;
20447 case 'x':
20448 if (VT == MVT::Other)
20449 break;
20450 if (VT == MVT::f32 || VT == MVT::f16 || VT == MVT::bf16)
20451 return RCPair(0U, &ARM::SPR_8RegClass);
20452 if (VT.getSizeInBits() == 64)
20453 return RCPair(0U, &ARM::DPR_8RegClass);
20454 if (VT.getSizeInBits() == 128)
20455 return RCPair(0U, &ARM::QPR_8RegClass);
20456 break;
20457 case 't':
20458 if (VT == MVT::Other)
20459 break;
20460 if (VT == MVT::f32 || VT == MVT::i32 || VT == MVT::f16 || VT == MVT::bf16)
20461 return RCPair(0U, &ARM::SPRRegClass);
20462 if (VT.getSizeInBits() == 64)
20463 return RCPair(0U, &ARM::DPR_VFP2RegClass);
20464 if (VT.getSizeInBits() == 128)
20465 return RCPair(0U, &ARM::QPR_VFP2RegClass);
20466 break;
20467 }
20468 break;
20469
20470 case 2:
20471 if (Constraint[0] == 'T') {
20472 switch (Constraint[1]) {
20473 default:
20474 break;
20475 case 'e':
20476 return RCPair(0U, &ARM::tGPREvenRegClass);
20477 case 'o':
20478 return RCPair(0U, &ARM::tGPROddRegClass);
20479 }
20480 }
20481 break;
20482
20483 default:
20484 break;
20485 }
20486
20487 if (StringRef("{cc}").equals_insensitive(RHS: Constraint))
20488 return std::make_pair(x: unsigned(ARM::CPSR), y: &ARM::CCRRegClass);
20489
20490 // r14 is an alias of lr.
20491 if (StringRef("{r14}").equals_insensitive(RHS: Constraint))
20492 Constraint = "{lr}";
20493
20494 auto RCP = TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
20495 if (isIncompatibleReg(PR: RCP.first, VT))
20496 return {0, nullptr};
20497 return RCP;
20498}
20499
20500/// LowerAsmOperandForConstraint - Lower the specified operand into the Ops
20501/// vector. If it is invalid, don't add anything to Ops.
20502void ARMTargetLowering::LowerAsmOperandForConstraint(SDValue Op,
20503 StringRef Constraint,
20504 std::vector<SDValue> &Ops,
20505 SelectionDAG &DAG) const {
20506 SDValue Result;
20507
20508 // Currently only support length 1 constraints.
20509 if (Constraint.size() != 1)
20510 return;
20511
20512 char ConstraintLetter = Constraint[0];
20513 switch (ConstraintLetter) {
20514 default: break;
20515 case 'j':
20516 case 'I': case 'J': case 'K': case 'L':
20517 case 'M': case 'N': case 'O':
20518 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: Op);
20519 if (!C)
20520 return;
20521
20522 int64_t CVal64 = C->getSExtValue();
20523 int CVal = (int) CVal64;
20524 // None of these constraints allow values larger than 32 bits. Check
20525 // that the value fits in an int.
20526 if (CVal != CVal64)
20527 return;
20528
20529 switch (ConstraintLetter) {
20530 case 'j':
20531 // Constant suitable for movw, must be between 0 and
20532 // 65535.
20533 if (Subtarget->hasV6T2Ops() || (Subtarget->hasV8MBaselineOps()))
20534 if (CVal >= 0 && CVal <= 65535)
20535 break;
20536 return;
20537 case 'I':
20538 if (Subtarget->isThumb1Only()) {
20539 // This must be a constant between 0 and 255, for ADD
20540 // immediates.
20541 if (CVal >= 0 && CVal <= 255)
20542 break;
20543 } else if (Subtarget->isThumb2()) {
20544 // A constant that can be used as an immediate value in a
20545 // data-processing instruction.
20546 if (ARM_AM::getT2SOImmVal(Arg: CVal) != -1)
20547 break;
20548 } else {
20549 // A constant that can be used as an immediate value in a
20550 // data-processing instruction.
20551 if (ARM_AM::getSOImmVal(Arg: CVal) != -1)
20552 break;
20553 }
20554 return;
20555
20556 case 'J':
20557 if (Subtarget->isThumb1Only()) {
20558 // This must be a constant between -255 and -1, for negated ADD
20559 // immediates. This can be used in GCC with an "n" modifier that
20560 // prints the negated value, for use with SUB instructions. It is
20561 // not useful otherwise but is implemented for compatibility.
20562 if (CVal >= -255 && CVal <= -1)
20563 break;
20564 } else {
20565 // This must be a constant between -4095 and 4095. This is suitable
20566 // for use as the immediate offset field in LDR and STR instructions
20567 // such as LDR r0,[r1,#offset].
20568 if (CVal >= -4095 && CVal <= 4095)
20569 break;
20570 }
20571 return;
20572
20573 case 'K':
20574 if (Subtarget->isThumb1Only()) {
20575 // A 32-bit value where only one byte has a nonzero value. Exclude
20576 // zero to match GCC. This constraint is used by GCC internally for
20577 // constants that can be loaded with a move/shift combination.
20578 // It is not useful otherwise but is implemented for compatibility.
20579 if (CVal != 0 && ARM_AM::isThumbImmShiftedVal(V: CVal))
20580 break;
20581 } else if (Subtarget->isThumb2()) {
20582 // A constant whose bitwise inverse can be used as an immediate
20583 // value in a data-processing instruction. This can be used in GCC
20584 // with a "B" modifier that prints the inverted value, for use with
20585 // BIC and MVN instructions. It is not useful otherwise but is
20586 // implemented for compatibility.
20587 if (ARM_AM::getT2SOImmVal(Arg: ~CVal) != -1)
20588 break;
20589 } else {
20590 // A constant whose bitwise inverse can be used as an immediate
20591 // value in a data-processing instruction. This can be used in GCC
20592 // with a "B" modifier that prints the inverted value, for use with
20593 // BIC and MVN instructions. It is not useful otherwise but is
20594 // implemented for compatibility.
20595 if (ARM_AM::getSOImmVal(Arg: ~CVal) != -1)
20596 break;
20597 }
20598 return;
20599
20600 case 'L':
20601 if (Subtarget->isThumb1Only()) {
20602 // This must be a constant between -7 and 7,
20603 // for 3-operand ADD/SUB immediate instructions.
20604 if (CVal >= -7 && CVal < 7)
20605 break;
20606 } else if (Subtarget->isThumb2()) {
20607 // A constant whose negation can be used as an immediate value in a
20608 // data-processing instruction. This can be used in GCC with an "n"
20609 // modifier that prints the negated value, for use with SUB
20610 // instructions. It is not useful otherwise but is implemented for
20611 // compatibility.
20612 if (ARM_AM::getT2SOImmVal(Arg: -CVal) != -1)
20613 break;
20614 } else {
20615 // A constant whose negation can be used as an immediate value in a
20616 // data-processing instruction. This can be used in GCC with an "n"
20617 // modifier that prints the negated value, for use with SUB
20618 // instructions. It is not useful otherwise but is implemented for
20619 // compatibility.
20620 if (ARM_AM::getSOImmVal(Arg: -CVal) != -1)
20621 break;
20622 }
20623 return;
20624
20625 case 'M':
20626 if (Subtarget->isThumb1Only()) {
20627 // This must be a multiple of 4 between 0 and 1020, for
20628 // ADD sp + immediate.
20629 if ((CVal >= 0 && CVal <= 1020) && ((CVal & 3) == 0))
20630 break;
20631 } else {
20632 // A power of two or a constant between 0 and 32. This is used in
20633 // GCC for the shift amount on shifted register operands, but it is
20634 // useful in general for any shift amounts.
20635 if ((CVal >= 0 && CVal <= 32) || ((CVal & (CVal - 1)) == 0))
20636 break;
20637 }
20638 return;
20639
20640 case 'N':
20641 if (Subtarget->isThumb1Only()) {
20642 // This must be a constant between 0 and 31, for shift amounts.
20643 if (CVal >= 0 && CVal <= 31)
20644 break;
20645 }
20646 return;
20647
20648 case 'O':
20649 if (Subtarget->isThumb1Only()) {
20650 // This must be a multiple of 4 between -508 and 508, for
20651 // ADD/SUB sp = sp + immediate.
20652 if ((CVal >= -508 && CVal <= 508) && ((CVal & 3) == 0))
20653 break;
20654 }
20655 return;
20656 }
20657 Result = DAG.getSignedTargetConstant(Val: CVal, DL: SDLoc(Op), VT: Op.getValueType());
20658 break;
20659 }
20660
20661 if (Result.getNode()) {
20662 Ops.push_back(x: Result);
20663 return;
20664 }
20665 return TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
20666}
20667
20668static RTLIB::Libcall getDivRemLibcall(
20669 const SDNode *N, MVT::SimpleValueType SVT) {
20670 assert((N->getOpcode() == ISD::SDIVREM || N->getOpcode() == ISD::UDIVREM ||
20671 N->getOpcode() == ISD::SREM || N->getOpcode() == ISD::UREM) &&
20672 "Unhandled Opcode in getDivRemLibcall");
20673 bool isSigned = N->getOpcode() == ISD::SDIVREM ||
20674 N->getOpcode() == ISD::SREM;
20675 RTLIB::Libcall LC;
20676 switch (SVT) {
20677 default: llvm_unreachable("Unexpected request for libcall!");
20678 case MVT::i8: LC = isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break;
20679 case MVT::i16: LC = isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
20680 case MVT::i32: LC = isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
20681 case MVT::i64: LC = isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
20682 }
20683 return LC;
20684}
20685
20686static TargetLowering::ArgListTy
20687getDivRemArgList(const SDNode *N, FunctionType *FuncTy,
20688 const AttributeList &FuncAttrs, RTLIB::LibcallImpl LCImpl) {
20689 assert((N->getOpcode() == ISD::SDIVREM || N->getOpcode() == ISD::UDIVREM ||
20690 N->getOpcode() == ISD::SREM || N->getOpcode() == ISD::UREM) &&
20691 "Unhandled Opcode in getDivRemArgList");
20692 SDValue Ops[2] = {N->getOperand(Num: 0), N->getOperand(Num: 1)};
20693
20694 // The Windows __rt_*div* helpers take the divisor before the dividend.
20695 switch (LCImpl) {
20696 case RTLIB::impl___rt_sdiv:
20697 case RTLIB::impl___rt_udiv:
20698 case RTLIB::impl___rt_sdiv64:
20699 case RTLIB::impl___rt_udiv64:
20700 std::swap(a&: Ops[0], b&: Ops[1]);
20701 break;
20702 default:
20703 break;
20704 }
20705
20706 return TargetLowering::getArgListForFunctionType(FuncTy, FuncAttrs, Ops);
20707}
20708
20709SDValue ARMTargetLowering::LowerDivRem(SDValue Op, SelectionDAG &DAG) const {
20710 assert((Subtarget->isTargetAEABI() || Subtarget->isTargetAndroid() ||
20711 Subtarget->isTargetGNUAEABI() || Subtarget->isTargetMuslAEABI() ||
20712 Subtarget->isTargetFuchsia() || Subtarget->isTargetWindows()) &&
20713 "Register-based DivRem lowering only");
20714 unsigned Opcode = Op->getOpcode();
20715 assert((Opcode == ISD::SDIVREM || Opcode == ISD::UDIVREM) &&
20716 "Invalid opcode for Div/Rem lowering");
20717 bool isSigned = (Opcode == ISD::SDIVREM);
20718 EVT VT = Op->getValueType(ResNo: 0);
20719 SDLoc dl(Op);
20720
20721 if (VT == MVT::i64 && isa<ConstantSDNode>(Val: Op.getOperand(i: 1))) {
20722 SmallVector<SDValue> Result;
20723 if (expandDIVREMByConstant(N: Op.getNode(), Result, HiLoVT: MVT::i32, DAG)) {
20724 SDValue Res0 =
20725 DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT, N1: Result[0], N2: Result[1]);
20726 SDValue Res1 =
20727 DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT, N1: Result[2], N2: Result[3]);
20728 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: dl, VTList: Op->getVTList(),
20729 Ops: {Res0, Res1});
20730 }
20731 }
20732
20733 // If the target has hardware divide, use divide + multiply + subtract:
20734 // div = a / b
20735 // rem = a - b * div
20736 // return {div, rem}
20737 // This should be lowered into UDIV/SDIV + MLS later on.
20738 bool hasDivide = Subtarget->isThumb() ? Subtarget->hasDivideInThumbMode()
20739 : Subtarget->hasDivideInARMMode();
20740 if (hasDivide && Op->getValueType(ResNo: 0).isSimple() &&
20741 Op->getSimpleValueType(ResNo: 0) == MVT::i32) {
20742 unsigned DivOpcode = isSigned ? ISD::SDIV : ISD::UDIV;
20743 const SDValue Dividend = Op->getOperand(Num: 0);
20744 const SDValue Divisor = Op->getOperand(Num: 1);
20745 SDValue Div = DAG.getNode(Opcode: DivOpcode, DL: dl, VT, N1: Dividend, N2: Divisor);
20746 SDValue Mul = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: Div, N2: Divisor);
20747 SDValue Rem = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Dividend, N2: Mul);
20748
20749 SDValue Values[2] = {Div, Rem};
20750 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: VT), Ops: Values);
20751 }
20752
20753 RTLIB::Libcall LC = getDivRemLibcall(N: Op.getNode(),
20754 SVT: VT.getSimpleVT().SimpleTy);
20755 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
20756 if (LCImpl == RTLIB::Unsupported)
20757 return SDValue();
20758
20759 auto [FuncTy, FuncAttrs] =
20760 DAG.getLibcalls().getRuntimeLibcallsInfo().getFunctionTy(
20761 Ctx&: *DAG.getContext(), TT: getTM().getTargetTriple(), DL: DAG.getDataLayout(),
20762 LibcallImpl: LCImpl);
20763 Type *RetTy = FuncTy->getReturnType();
20764
20765 SDValue InChain = DAG.getEntryNode();
20766
20767 TargetLowering::ArgListTy Args =
20768 getDivRemArgList(N: Op.getNode(), FuncTy, FuncAttrs, LCImpl);
20769
20770 SDValue Callee =
20771 DAG.getExternalSymbol(LCImpl, VT: getPointerTy(DL: DAG.getDataLayout()));
20772
20773 if (getTM().getTargetTriple().isOSWindows())
20774 InChain = WinDBZCheckDenominator(DAG, N: Op.getNode(), InChain);
20775
20776 TargetLowering::CallLoweringInfo CLI(DAG);
20777 CLI.setDebugLoc(dl)
20778 .setChain(InChain)
20779 .setCallee(CC: DAG.getLibcalls().getLibcallImplCallingConv(Call: LCImpl), ResultType: RetTy,
20780 Target: Callee, ArgsList: std::move(Args))
20781 .setInRegister()
20782 .setSExtResult(isSigned)
20783 .setZExtResult(!isSigned);
20784
20785 std::pair<SDValue, SDValue> CallInfo = LowerCallTo(CLI);
20786 return CallInfo.first;
20787}
20788
20789// Lowers REM using divmod helpers
20790// see RTABI section 4.2/4.3
20791SDValue ARMTargetLowering::LowerREM(SDNode *N, SelectionDAG &DAG) const {
20792 EVT VT = N->getValueType(ResNo: 0);
20793
20794 if (VT == MVT::i64 && isa<ConstantSDNode>(Val: N->getOperand(Num: 1))) {
20795 SmallVector<SDValue> Result;
20796 if (expandDIVREMByConstant(N, Result, HiLoVT: MVT::i32, DAG))
20797 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
20798 N1: Result[0], N2: Result[1]);
20799 }
20800
20801 RTLIB::Libcall LC = getDivRemLibcall(N, SVT: N->getValueType(ResNo: 0).getSimpleVT().
20802 SimpleTy);
20803 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
20804 if (LCImpl == RTLIB::Unsupported)
20805 return SDValue();
20806
20807 auto [FuncTy, FuncAttrs] =
20808 DAG.getLibcalls().getRuntimeLibcallsInfo().getFunctionTy(
20809 Ctx&: *DAG.getContext(), TT: getTM().getTargetTriple(), DL: DAG.getDataLayout(),
20810 LibcallImpl: LCImpl);
20811 Type *RetTy = FuncTy->getReturnType();
20812
20813 SDValue InChain = DAG.getEntryNode();
20814 TargetLowering::ArgListTy Args =
20815 getDivRemArgList(N, FuncTy, FuncAttrs, LCImpl);
20816 bool isSigned = N->getOpcode() == ISD::SREM;
20817
20818 SDValue Callee =
20819 DAG.getExternalSymbol(LCImpl, VT: getPointerTy(DL: DAG.getDataLayout()));
20820
20821 if (getTM().getTargetTriple().isOSWindows())
20822 InChain = WinDBZCheckDenominator(DAG, N, InChain);
20823
20824 // Lower call
20825 CallLoweringInfo CLI(DAG);
20826 CLI.setChain(InChain)
20827 .setCallee(CC: DAG.getLibcalls().getLibcallImplCallingConv(Call: LCImpl), ResultType: RetTy,
20828 Target: Callee, ArgsList: std::move(Args))
20829 .setSExtResult(isSigned)
20830 .setZExtResult(!isSigned)
20831 .setDebugLoc(SDLoc(N));
20832 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
20833
20834 // Return second (rem) result operand (first contains div)
20835 SDNode *ResNode = CallResult.first.getNode();
20836 assert(ResNode->getNumOperands() == 2 && "divmod should return two operands");
20837 return ResNode->getOperand(Num: 1);
20838}
20839
20840SDValue
20841ARMTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const {
20842 assert(getTM().getTargetTriple().isOSWindows() &&
20843 "unsupported target platform");
20844 SDLoc DL(Op);
20845
20846 // Get the inputs.
20847 SDValue Chain = Op.getOperand(i: 0);
20848 SDValue Size = Op.getOperand(i: 1);
20849
20850 if (DAG.getMachineFunction().getFunction().hasFnAttribute(
20851 Kind: "no-stack-arg-probe")) {
20852 MaybeAlign Align =
20853 cast<ConstantSDNode>(Val: Op.getOperand(i: 2))->getMaybeAlignValue();
20854 SDValue SP = DAG.getCopyFromReg(Chain, dl: DL, Reg: ARM::SP, VT: MVT::i32);
20855 Chain = SP.getValue(R: 1);
20856 SP = DAG.getNode(Opcode: ISD::SUB, DL, VT: MVT::i32, N1: SP, N2: Size);
20857 if (Align)
20858 SP = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: SP.getValue(R: 0),
20859 N2: DAG.getSignedConstant(Val: -Align->value(), DL, VT: MVT::i32));
20860 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: ARM::SP, N: SP);
20861 SDValue Ops[2] = { SP, Chain };
20862 return DAG.getMergeValues(Ops, dl: DL);
20863 }
20864
20865 SDValue Words = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: Size,
20866 N2: DAG.getConstant(Val: 2, DL, VT: MVT::i32));
20867
20868 SDValue Glue;
20869 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: ARM::R4, N: Words, Glue);
20870 Glue = Chain.getValue(R: 1);
20871
20872 SDVTList NodeTys = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
20873 Chain = DAG.getNode(Opcode: ARMISD::WIN__CHKSTK, DL, VTList: NodeTys, N1: Chain, N2: Glue);
20874
20875 SDValue NewSP = DAG.getCopyFromReg(Chain, dl: DL, Reg: ARM::SP, VT: MVT::i32);
20876 Chain = NewSP.getValue(R: 1);
20877
20878 SDValue Ops[2] = { NewSP, Chain };
20879 return DAG.getMergeValues(Ops, dl: DL);
20880}
20881
20882SDValue ARMTargetLowering::LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const {
20883 bool IsStrict = Op->isStrictFPOpcode();
20884 SDValue SrcVal = Op.getOperand(i: IsStrict ? 1 : 0);
20885 const unsigned DstSz = Op.getValueType().getSizeInBits();
20886 const unsigned SrcSz = SrcVal.getValueType().getSizeInBits();
20887 assert(DstSz > SrcSz && DstSz <= 64 && SrcSz >= 16 &&
20888 "Unexpected type for custom-lowering FP_EXTEND");
20889
20890 assert((!Subtarget->hasFP64() || !Subtarget->hasFPARMv8Base()) &&
20891 "With both FP DP and 16, any FP conversion is legal!");
20892
20893 assert(!(DstSz == 32 && Subtarget->hasFP16()) &&
20894 "With FP16, 16 to 32 conversion is legal!");
20895
20896 // Converting from 32 -> 64 is valid if we have FP64.
20897 if (SrcSz == 32 && DstSz == 64 && Subtarget->hasFP64()) {
20898 // FIXME: Remove this when we have strict fp instruction selection patterns
20899 if (IsStrict) {
20900 SDLoc Loc(Op);
20901 SDValue Result = DAG.getNode(Opcode: ISD::FP_EXTEND,
20902 DL: Loc, VT: Op.getValueType(), Operand: SrcVal);
20903 return DAG.getMergeValues(Ops: {Result, Op.getOperand(i: 0)}, dl: Loc);
20904 }
20905 return Op;
20906 }
20907
20908 // Either we are converting from 16 -> 64, without FP16 and/or
20909 // FP.double-precision or without Armv8-fp. So we must do it in two
20910 // steps.
20911 // Or we are converting from 32 -> 64 without fp.double-precision or 16 -> 32
20912 // without FP16. So we must do a function call.
20913 SDLoc Loc(Op);
20914 RTLIB::Libcall LC;
20915 MakeLibCallOptions CallOptions;
20916 SDValue Chain = IsStrict ? Op.getOperand(i: 0) : SDValue();
20917 for (unsigned Sz = SrcSz; Sz <= 32 && Sz < DstSz; Sz *= 2) {
20918 bool Supported = (Sz == 16 ? Subtarget->hasFP16() : Subtarget->hasFP64());
20919 MVT SrcVT = (Sz == 16 ? MVT::f16 : MVT::f32);
20920 MVT DstVT = (Sz == 16 ? MVT::f32 : MVT::f64);
20921 if (Supported) {
20922 if (IsStrict) {
20923 SrcVal = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: Loc,
20924 ResultTys: {DstVT, MVT::Other}, Ops: {Chain, SrcVal});
20925 Chain = SrcVal.getValue(R: 1);
20926 } else {
20927 SrcVal = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: Loc, VT: DstVT, Operand: SrcVal);
20928 }
20929 } else {
20930 LC = RTLIB::getFPEXT(OpVT: SrcVT, RetVT: DstVT);
20931 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
20932 "Unexpected type for custom-lowering FP_EXTEND");
20933 std::tie(args&: SrcVal, args&: Chain) = makeLibCall(DAG, LC, RetVT: DstVT, Ops: SrcVal, CallOptions,
20934 dl: Loc, Chain);
20935 }
20936 }
20937
20938 return IsStrict ? DAG.getMergeValues(Ops: {SrcVal, Chain}, dl: Loc) : SrcVal;
20939}
20940
20941SDValue ARMTargetLowering::LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const {
20942 bool IsStrict = Op->isStrictFPOpcode();
20943
20944 SDValue SrcVal = Op.getOperand(i: IsStrict ? 1 : 0);
20945 EVT SrcVT = SrcVal.getValueType();
20946 EVT DstVT = Op.getValueType();
20947
20948 if (DstVT == MVT::bf16) {
20949 if (Subtarget->hasBF16() && SrcVT == MVT::f32)
20950 return Op;
20951 return SDValue();
20952 }
20953
20954 const unsigned DstSz = Op.getValueType().getSizeInBits();
20955 const unsigned SrcSz = SrcVT.getSizeInBits();
20956 (void)DstSz;
20957 assert(DstSz < SrcSz && SrcSz <= 64 && DstSz >= 16 &&
20958 "Unexpected type for custom-lowering FP_ROUND");
20959
20960 assert((!Subtarget->hasFP64() || !Subtarget->hasFPARMv8Base()) &&
20961 "With both FP DP and 16, any FP conversion is legal!");
20962
20963 SDLoc Loc(Op);
20964
20965 // Instruction from 32 -> 16 if hasFP16 is valid
20966 if (SrcSz == 32 && Subtarget->hasFP16())
20967 return Op;
20968
20969 // Lib call from 32 -> 16 / 64 -> [32, 16]
20970 RTLIB::Libcall LC = RTLIB::getFPROUND(OpVT: SrcVT, RetVT: DstVT);
20971 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
20972 "Unexpected type for custom-lowering FP_ROUND");
20973 MakeLibCallOptions CallOptions;
20974 SDValue Chain = IsStrict ? Op.getOperand(i: 0) : SDValue();
20975 SDValue Result;
20976 std::tie(args&: Result, args&: Chain) = makeLibCall(DAG, LC, RetVT: DstVT, Ops: SrcVal, CallOptions,
20977 dl: Loc, Chain);
20978 return IsStrict ? DAG.getMergeValues(Ops: {Result, Chain}, dl: Loc) : Result;
20979}
20980
20981bool
20982ARMTargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const {
20983 // The ARM target isn't yet aware of offsets.
20984 return false;
20985}
20986
20987bool ARM::isBitFieldInvertedMask(unsigned v) {
20988 if (v == 0xffffffff)
20989 return false;
20990
20991 // there can be 1's on either or both "outsides", all the "inside"
20992 // bits must be 0's
20993 return isShiftedMask_32(Value: ~v);
20994}
20995
20996/// isFPImmLegal - Returns true if the target can instruction select the
20997/// specified FP immediate natively. If false, the legalizer will
20998/// materialize the FP immediate as a load from a constant pool.
20999bool ARMTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
21000 bool ForCodeSize) const {
21001 if (!Subtarget->hasVFP3Base())
21002 return false;
21003 if (VT == MVT::f16 && Subtarget->hasFullFP16())
21004 return ARM_AM::getFP16Imm(FPImm: Imm) != -1;
21005 if (VT == MVT::f32 && Subtarget->hasFullFP16() &&
21006 ARM_AM::getFP32FP16Imm(FPImm: Imm) != -1)
21007 return true;
21008 if (VT == MVT::f32)
21009 return ARM_AM::getFP32Imm(FPImm: Imm) != -1;
21010 if (VT == MVT::f64 && Subtarget->hasFP64())
21011 return ARM_AM::getFP64Imm(FPImm: Imm) != -1;
21012 return false;
21013}
21014
21015/// getTgtMemIntrinsic - Represent NEON load and store intrinsics as
21016/// MemIntrinsicNodes. The associated MachineMemOperands record the alignment
21017/// specified in the intrinsic calls.
21018void ARMTargetLowering::getTgtMemIntrinsic(
21019 SmallVectorImpl<IntrinsicInfo> &Infos, const CallBase &I,
21020 MachineFunction &MF, unsigned Intrinsic) const {
21021 IntrinsicInfo Info;
21022 switch (Intrinsic) {
21023 case Intrinsic::arm_neon_vld1:
21024 case Intrinsic::arm_neon_vld2:
21025 case Intrinsic::arm_neon_vld3:
21026 case Intrinsic::arm_neon_vld4:
21027 case Intrinsic::arm_neon_vld2lane:
21028 case Intrinsic::arm_neon_vld3lane:
21029 case Intrinsic::arm_neon_vld4lane:
21030 case Intrinsic::arm_neon_vld2dup:
21031 case Intrinsic::arm_neon_vld3dup:
21032 case Intrinsic::arm_neon_vld4dup: {
21033 Info.opc = ISD::INTRINSIC_W_CHAIN;
21034 // Conservatively set memVT to the entire set of vectors loaded.
21035 auto &DL = I.getDataLayout();
21036 uint64_t NumElts = DL.getTypeSizeInBits(Ty: I.getType()) / 64;
21037 Info.memVT = EVT::getVectorVT(Context&: I.getType()->getContext(), VT: MVT::i64, NumElements: NumElts);
21038 Info.ptrVal = I.getArgOperand(i: 0);
21039 Info.offset = 0;
21040 Value *AlignArg = I.getArgOperand(i: I.arg_size() - 1);
21041 Info.align = cast<ConstantInt>(Val: AlignArg)->getMaybeAlignValue();
21042 // volatile loads with NEON intrinsics not supported
21043 Info.flags = MachineMemOperand::MOLoad;
21044 Infos.push_back(Elt: Info);
21045 return;
21046 }
21047 case Intrinsic::arm_neon_vld1x2:
21048 case Intrinsic::arm_neon_vld1x3:
21049 case Intrinsic::arm_neon_vld1x4: {
21050 Info.opc = ISD::INTRINSIC_W_CHAIN;
21051 // Conservatively set memVT to the entire set of vectors loaded.
21052 auto &DL = I.getDataLayout();
21053 uint64_t NumElts = DL.getTypeSizeInBits(Ty: I.getType()) / 64;
21054 Info.memVT = EVT::getVectorVT(Context&: I.getType()->getContext(), VT: MVT::i64, NumElements: NumElts);
21055 Info.ptrVal = I.getArgOperand(i: I.arg_size() - 1);
21056 Info.offset = 0;
21057 Info.align = I.getParamAlign(ArgNo: I.arg_size() - 1).valueOrOne();
21058 // volatile loads with NEON intrinsics not supported
21059 Info.flags = MachineMemOperand::MOLoad;
21060 Infos.push_back(Elt: Info);
21061 return;
21062 }
21063 case Intrinsic::arm_neon_vst1:
21064 case Intrinsic::arm_neon_vst2:
21065 case Intrinsic::arm_neon_vst3:
21066 case Intrinsic::arm_neon_vst4:
21067 case Intrinsic::arm_neon_vst2lane:
21068 case Intrinsic::arm_neon_vst3lane:
21069 case Intrinsic::arm_neon_vst4lane: {
21070 Info.opc = ISD::INTRINSIC_VOID;
21071 // Conservatively set memVT to the entire set of vectors stored.
21072 auto &DL = I.getDataLayout();
21073 unsigned NumElts = 0;
21074 for (unsigned ArgI = 1, ArgE = I.arg_size(); ArgI < ArgE; ++ArgI) {
21075 Type *ArgTy = I.getArgOperand(i: ArgI)->getType();
21076 if (!ArgTy->isVectorTy())
21077 break;
21078 NumElts += DL.getTypeSizeInBits(Ty: ArgTy) / 64;
21079 }
21080 Info.memVT = EVT::getVectorVT(Context&: I.getType()->getContext(), VT: MVT::i64, NumElements: NumElts);
21081 Info.ptrVal = I.getArgOperand(i: 0);
21082 Info.offset = 0;
21083 Value *AlignArg = I.getArgOperand(i: I.arg_size() - 1);
21084 Info.align = cast<ConstantInt>(Val: AlignArg)->getMaybeAlignValue();
21085 // volatile stores with NEON intrinsics not supported
21086 Info.flags = MachineMemOperand::MOStore;
21087 Infos.push_back(Elt: Info);
21088 return;
21089 }
21090 case Intrinsic::arm_neon_vst1x2:
21091 case Intrinsic::arm_neon_vst1x3:
21092 case Intrinsic::arm_neon_vst1x4: {
21093 Info.opc = ISD::INTRINSIC_VOID;
21094 // Conservatively set memVT to the entire set of vectors stored.
21095 auto &DL = I.getDataLayout();
21096 unsigned NumElts = 0;
21097 for (unsigned ArgI = 1, ArgE = I.arg_size(); ArgI < ArgE; ++ArgI) {
21098 Type *ArgTy = I.getArgOperand(i: ArgI)->getType();
21099 if (!ArgTy->isVectorTy())
21100 break;
21101 NumElts += DL.getTypeSizeInBits(Ty: ArgTy) / 64;
21102 }
21103 Info.memVT = EVT::getVectorVT(Context&: I.getType()->getContext(), VT: MVT::i64, NumElements: NumElts);
21104 Info.ptrVal = I.getArgOperand(i: 0);
21105 Info.offset = 0;
21106 Info.align = I.getParamAlign(ArgNo: 0).valueOrOne();
21107 // volatile stores with NEON intrinsics not supported
21108 Info.flags = MachineMemOperand::MOStore;
21109 Infos.push_back(Elt: Info);
21110 return;
21111 }
21112 case Intrinsic::arm_mve_vld2q:
21113 case Intrinsic::arm_mve_vld4q: {
21114 Info.opc = ISD::INTRINSIC_W_CHAIN;
21115 // Conservatively set memVT to the entire set of vectors loaded.
21116 Type *VecTy = cast<StructType>(Val: I.getType())->getElementType(N: 1);
21117 unsigned Factor = Intrinsic == Intrinsic::arm_mve_vld2q ? 2 : 4;
21118 Info.memVT = EVT::getVectorVT(Context&: VecTy->getContext(), VT: MVT::i64, NumElements: Factor * 2);
21119 Info.ptrVal = I.getArgOperand(i: 0);
21120 Info.offset = 0;
21121 Info.align = Align(VecTy->getScalarSizeInBits() / 8);
21122 // volatile loads with MVE intrinsics not supported
21123 Info.flags = MachineMemOperand::MOLoad;
21124 Infos.push_back(Elt: Info);
21125 return;
21126 }
21127 case Intrinsic::arm_mve_vst2q:
21128 case Intrinsic::arm_mve_vst4q: {
21129 Info.opc = ISD::INTRINSIC_VOID;
21130 // Conservatively set memVT to the entire set of vectors stored.
21131 Type *VecTy = I.getArgOperand(i: 1)->getType();
21132 unsigned Factor = Intrinsic == Intrinsic::arm_mve_vst2q ? 2 : 4;
21133 Info.memVT = EVT::getVectorVT(Context&: VecTy->getContext(), VT: MVT::i64, NumElements: Factor * 2);
21134 Info.ptrVal = I.getArgOperand(i: 0);
21135 Info.offset = 0;
21136 Info.align = Align(VecTy->getScalarSizeInBits() / 8);
21137 // volatile stores with MVE intrinsics not supported
21138 Info.flags = MachineMemOperand::MOStore;
21139 Infos.push_back(Elt: Info);
21140 return;
21141 }
21142 case Intrinsic::arm_mve_vldr_gather_base:
21143 case Intrinsic::arm_mve_vldr_gather_base_predicated: {
21144 Info.opc = ISD::INTRINSIC_W_CHAIN;
21145 Info.ptrVal = nullptr;
21146 Info.memVT = MVT::getVT(Ty: I.getType());
21147 Info.align = Align(1);
21148 Info.flags |= MachineMemOperand::MOLoad;
21149 Infos.push_back(Elt: Info);
21150 return;
21151 }
21152 case Intrinsic::arm_mve_vldr_gather_base_wb:
21153 case Intrinsic::arm_mve_vldr_gather_base_wb_predicated: {
21154 Info.opc = ISD::INTRINSIC_W_CHAIN;
21155 Info.ptrVal = nullptr;
21156 Info.memVT = MVT::getVT(Ty: I.getType()->getContainedType(i: 0));
21157 Info.align = Align(1);
21158 Info.flags |= MachineMemOperand::MOLoad;
21159 Infos.push_back(Elt: Info);
21160 return;
21161 }
21162 case Intrinsic::arm_mve_vldr_gather_offset:
21163 case Intrinsic::arm_mve_vldr_gather_offset_predicated: {
21164 Info.opc = ISD::INTRINSIC_W_CHAIN;
21165 Info.ptrVal = nullptr;
21166 MVT DataVT = MVT::getVT(Ty: I.getType());
21167 unsigned MemSize = cast<ConstantInt>(Val: I.getArgOperand(i: 2))->getZExtValue();
21168 Info.memVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: MemSize),
21169 NumElements: DataVT.getVectorNumElements());
21170 Info.align = Align(1);
21171 Info.flags |= MachineMemOperand::MOLoad;
21172 Infos.push_back(Elt: Info);
21173 return;
21174 }
21175 case Intrinsic::arm_mve_vstr_scatter_base:
21176 case Intrinsic::arm_mve_vstr_scatter_base_predicated: {
21177 Info.opc = ISD::INTRINSIC_VOID;
21178 Info.ptrVal = nullptr;
21179 Info.memVT = MVT::getVT(Ty: I.getArgOperand(i: 2)->getType());
21180 Info.align = Align(1);
21181 Info.flags |= MachineMemOperand::MOStore;
21182 Infos.push_back(Elt: Info);
21183 return;
21184 }
21185 case Intrinsic::arm_mve_vstr_scatter_base_wb:
21186 case Intrinsic::arm_mve_vstr_scatter_base_wb_predicated: {
21187 Info.opc = ISD::INTRINSIC_W_CHAIN;
21188 Info.ptrVal = nullptr;
21189 Info.memVT = MVT::getVT(Ty: I.getArgOperand(i: 2)->getType());
21190 Info.align = Align(1);
21191 Info.flags |= MachineMemOperand::MOStore;
21192 Infos.push_back(Elt: Info);
21193 return;
21194 }
21195 case Intrinsic::arm_mve_vstr_scatter_offset:
21196 case Intrinsic::arm_mve_vstr_scatter_offset_predicated: {
21197 Info.opc = ISD::INTRINSIC_VOID;
21198 Info.ptrVal = nullptr;
21199 MVT DataVT = MVT::getVT(Ty: I.getArgOperand(i: 2)->getType());
21200 unsigned MemSize = cast<ConstantInt>(Val: I.getArgOperand(i: 3))->getZExtValue();
21201 Info.memVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: MemSize),
21202 NumElements: DataVT.getVectorNumElements());
21203 Info.align = Align(1);
21204 Info.flags |= MachineMemOperand::MOStore;
21205 Infos.push_back(Elt: Info);
21206 return;
21207 }
21208 case Intrinsic::arm_ldaex:
21209 case Intrinsic::arm_ldrex: {
21210 auto &DL = I.getDataLayout();
21211 Type *ValTy = I.getParamElementType(ArgNo: 0);
21212 Info.opc = ISD::INTRINSIC_W_CHAIN;
21213 Info.memVT = MVT::getVT(Ty: ValTy);
21214 Info.ptrVal = I.getArgOperand(i: 0);
21215 Info.offset = 0;
21216 Info.align = DL.getABITypeAlign(Ty: ValTy);
21217 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile;
21218 Infos.push_back(Elt: Info);
21219 return;
21220 }
21221 case Intrinsic::arm_stlex:
21222 case Intrinsic::arm_strex: {
21223 auto &DL = I.getDataLayout();
21224 Type *ValTy = I.getParamElementType(ArgNo: 1);
21225 Info.opc = ISD::INTRINSIC_W_CHAIN;
21226 Info.memVT = MVT::getVT(Ty: ValTy);
21227 Info.ptrVal = I.getArgOperand(i: 1);
21228 Info.offset = 0;
21229 Info.align = DL.getABITypeAlign(Ty: ValTy);
21230 Info.flags = MachineMemOperand::MOStore | MachineMemOperand::MOVolatile;
21231 Infos.push_back(Elt: Info);
21232 return;
21233 }
21234 case Intrinsic::arm_stlexd:
21235 case Intrinsic::arm_strexd:
21236 Info.opc = ISD::INTRINSIC_W_CHAIN;
21237 Info.memVT = MVT::i64;
21238 Info.ptrVal = I.getArgOperand(i: 2);
21239 Info.offset = 0;
21240 Info.align = Align(8);
21241 Info.flags = MachineMemOperand::MOStore | MachineMemOperand::MOVolatile;
21242 Infos.push_back(Elt: Info);
21243 return;
21244
21245 case Intrinsic::arm_ldaexd:
21246 case Intrinsic::arm_ldrexd:
21247 Info.opc = ISD::INTRINSIC_W_CHAIN;
21248 Info.memVT = MVT::i64;
21249 Info.ptrVal = I.getArgOperand(i: 0);
21250 Info.offset = 0;
21251 Info.align = Align(8);
21252 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile;
21253 Infos.push_back(Elt: Info);
21254 return;
21255
21256 default:
21257 break;
21258 }
21259}
21260
21261/// Returns true if it is beneficial to convert a load of a constant
21262/// to just the constant itself.
21263bool ARMTargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
21264 Type *Ty) const {
21265 assert(Ty->isIntegerTy());
21266
21267 unsigned Bits = Ty->getPrimitiveSizeInBits();
21268 if (Bits == 0 || Bits > 32)
21269 return false;
21270 return true;
21271}
21272
21273TargetLowering::ExtractSubvectorCost
21274ARMTargetLowering::getExtractSubvectorCost(EVT ResVT, EVT SrcVT,
21275 unsigned Index) const {
21276 if (!isOperationLegalOrCustom(Op: ISD::EXTRACT_SUBVECTOR, VT: ResVT))
21277 return ExtractSubvectorCost::Expensive;
21278
21279 if (Index == 0 || Index == ResVT.getVectorNumElements())
21280 return ExtractSubvectorCost::Free;
21281 return ExtractSubvectorCost::Expensive;
21282}
21283
21284Instruction *ARMTargetLowering::makeDMB(IRBuilderBase &Builder,
21285 ARM_MB::MemBOpt Domain) const {
21286 // First, if the target has no DMB, see what fallback we can use.
21287 if (!Subtarget->hasDataBarrier()) {
21288 // Some ARMv6 cpus can support data barriers with an mcr instruction.
21289 // Thumb1 and pre-v6 ARM mode use a libcall instead and should never get
21290 // here.
21291 if (Subtarget->hasV6Ops() && !Subtarget->isThumb()) {
21292 Value* args[6] = {Builder.getInt32(C: 15), Builder.getInt32(C: 0),
21293 Builder.getInt32(C: 0), Builder.getInt32(C: 7),
21294 Builder.getInt32(C: 10), Builder.getInt32(C: 5)};
21295 return Builder.CreateIntrinsicWithoutFolding(ID: Intrinsic::arm_mcr, Args: args);
21296 }
21297 // Instead of using barriers, atomic accesses on these subtargets use
21298 // libcalls.
21299 llvm_unreachable("makeDMB on a target so old that it has no barriers");
21300 } else {
21301 // Only a full system barrier exists in the M-class architectures.
21302 Domain = Subtarget->isMClass() ? ARM_MB::SY : Domain;
21303 Constant *CDomain = Builder.getInt32(C: Domain);
21304 return Builder.CreateIntrinsicWithoutFolding(ID: Intrinsic::arm_dmb, Args: CDomain);
21305 }
21306}
21307
21308// Based on http://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html
21309Instruction *ARMTargetLowering::emitLeadingFence(IRBuilderBase &Builder,
21310 Instruction *Inst,
21311 AtomicOrdering Ord) const {
21312 switch (Ord) {
21313 case AtomicOrdering::NotAtomic:
21314 case AtomicOrdering::Unordered:
21315 llvm_unreachable("Invalid fence: unordered/non-atomic");
21316 case AtomicOrdering::Monotonic:
21317 case AtomicOrdering::Acquire:
21318 return nullptr; // Nothing to do
21319 case AtomicOrdering::SequentiallyConsistent:
21320 if (!Inst->hasAtomicStore())
21321 return nullptr; // Nothing to do
21322 [[fallthrough]];
21323 case AtomicOrdering::Release:
21324 case AtomicOrdering::AcquireRelease:
21325 if (Subtarget->preferISHSTBarriers())
21326 return makeDMB(Builder, Domain: ARM_MB::ISHST);
21327 // FIXME: add a comment with a link to documentation justifying this.
21328 else
21329 return makeDMB(Builder, Domain: ARM_MB::ISH);
21330 }
21331 llvm_unreachable("Unknown fence ordering in emitLeadingFence");
21332}
21333
21334Instruction *ARMTargetLowering::emitTrailingFence(IRBuilderBase &Builder,
21335 Instruction *Inst,
21336 AtomicOrdering Ord) const {
21337 switch (Ord) {
21338 case AtomicOrdering::NotAtomic:
21339 case AtomicOrdering::Unordered:
21340 llvm_unreachable("Invalid fence: unordered/not-atomic");
21341 case AtomicOrdering::Monotonic:
21342 case AtomicOrdering::Release:
21343 return nullptr; // Nothing to do
21344 case AtomicOrdering::Acquire:
21345 case AtomicOrdering::AcquireRelease:
21346 case AtomicOrdering::SequentiallyConsistent:
21347 return makeDMB(Builder, Domain: ARM_MB::ISH);
21348 }
21349 llvm_unreachable("Unknown fence ordering in emitTrailingFence");
21350}
21351
21352// Loads and stores less than 64-bits are already atomic; ones above that
21353// are doomed anyway, so defer to the default libcall and blame the OS when
21354// things go wrong. Cortex M doesn't have ldrexd/strexd though, so don't emit
21355// anything for those.
21356TargetLoweringBase::AtomicExpansionKind
21357ARMTargetLowering::shouldExpandAtomicStoreInIR(StoreInst *SI) const {
21358 bool has64BitAtomicStore;
21359 if (Subtarget->isMClass())
21360 has64BitAtomicStore = false;
21361 else if (Subtarget->isThumb())
21362 has64BitAtomicStore = Subtarget->hasV7Ops();
21363 else
21364 has64BitAtomicStore = Subtarget->hasV6Ops();
21365
21366 unsigned Size = SI->getValueOperand()->getType()->getPrimitiveSizeInBits();
21367 return Size == 64 && has64BitAtomicStore ? AtomicExpansionKind::Expand
21368 : AtomicExpansionKind::None;
21369}
21370
21371// Loads and stores less than 64-bits are already atomic; ones above that
21372// are doomed anyway, so defer to the default libcall and blame the OS when
21373// things go wrong. Cortex M doesn't have ldrexd/strexd though, so don't emit
21374// anything for those.
21375// FIXME: ldrd and strd are atomic if the CPU has LPAE (e.g. A15 has that
21376// guarantee, see DDI0406C ARM architecture reference manual,
21377// sections A8.8.72-74 LDRD)
21378TargetLowering::AtomicExpansionKind
21379ARMTargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const {
21380 bool has64BitAtomicLoad;
21381 if (Subtarget->isMClass())
21382 has64BitAtomicLoad = false;
21383 else if (Subtarget->isThumb())
21384 has64BitAtomicLoad = Subtarget->hasV7Ops();
21385 else
21386 has64BitAtomicLoad = Subtarget->hasV6Ops();
21387
21388 unsigned Size = LI->getType()->getPrimitiveSizeInBits();
21389 return (Size == 64 && has64BitAtomicLoad) ? AtomicExpansionKind::LLOnly
21390 : AtomicExpansionKind::None;
21391}
21392
21393// For the real atomic operations, we have ldrex/strex up to 32 bits,
21394// and up to 64 bits on the non-M profiles
21395TargetLowering::AtomicExpansionKind
21396ARMTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
21397 if (AI->isFloatingPointOperation())
21398 return AtomicExpansionKind::CmpXChg;
21399
21400 unsigned Size = AI->getType()->getPrimitiveSizeInBits();
21401 bool hasAtomicRMW;
21402 if (Subtarget->isMClass())
21403 hasAtomicRMW = Subtarget->hasV8MBaselineOps();
21404 else if (Subtarget->isThumb())
21405 hasAtomicRMW = Subtarget->hasV7Ops();
21406 else
21407 hasAtomicRMW = Subtarget->hasV6Ops();
21408 if (Size <= (Subtarget->isMClass() ? 32U : 64U) && hasAtomicRMW) {
21409 // At -O0, fast-regalloc cannot cope with the live vregs necessary to
21410 // implement atomicrmw without spilling. If the target address is also on
21411 // the stack and close enough to the spill slot, this can lead to a
21412 // situation where the monitor always gets cleared and the atomic operation
21413 // can never succeed. So at -O0 lower this operation to a CAS loop.
21414 if (getTargetMachine().getOptLevel() == CodeGenOptLevel::None)
21415 return AtomicExpansionKind::CmpXChg;
21416 return AtomicExpansionKind::LLSC;
21417 }
21418 return AtomicExpansionKind::None;
21419}
21420
21421// Similar to shouldExpandAtomicRMWInIR, ldrex/strex can be used up to 32
21422// bits, and up to 64 bits on the non-M profiles.
21423TargetLowering::AtomicExpansionKind
21424ARMTargetLowering::shouldExpandAtomicCmpXchgInIR(
21425 const AtomicCmpXchgInst *AI) const {
21426 // At -O0, fast-regalloc cannot cope with the live vregs necessary to
21427 // implement cmpxchg without spilling. If the address being exchanged is also
21428 // on the stack and close enough to the spill slot, this can lead to a
21429 // situation where the monitor always gets cleared and the atomic operation
21430 // can never succeed. So at -O0 we need a late-expanded pseudo-inst instead.
21431 unsigned Size = AI->getOperand(i_nocapture: 1)->getType()->getPrimitiveSizeInBits();
21432 bool HasAtomicCmpXchg;
21433 if (Subtarget->isMClass())
21434 HasAtomicCmpXchg = Subtarget->hasV8MBaselineOps();
21435 else if (Subtarget->isThumb())
21436 HasAtomicCmpXchg = Subtarget->hasV7Ops();
21437 else
21438 HasAtomicCmpXchg = Subtarget->hasV6Ops();
21439 if (getTargetMachine().getOptLevel() != CodeGenOptLevel::None &&
21440 HasAtomicCmpXchg && Size <= (Subtarget->isMClass() ? 32U : 64U))
21441 return AtomicExpansionKind::LLSC;
21442 return AtomicExpansionKind::None;
21443}
21444
21445bool ARMTargetLowering::shouldInsertFencesForAtomic(
21446 const Instruction *I) const {
21447 return InsertFencesForAtomic;
21448}
21449
21450bool ARMTargetLowering::useLoadStackGuardNode(const Module &M) const {
21451 // ROPI/RWPI are not supported currently.
21452 return !Subtarget->isROPI() && !Subtarget->isRWPI();
21453}
21454
21455void ARMTargetLowering::insertSSPDeclarations(
21456 Module &M, const LibcallLoweringInfo &Libcalls) const {
21457 // MSVC CRT provides functionalities for stack protection.
21458 RTLIB::LibcallImpl SecurityCheckCookieLibcall =
21459 Libcalls.getLibcallImpl(Call: RTLIB::SECURITY_CHECK_COOKIE);
21460
21461 RTLIB::LibcallImpl SecurityCookieVar =
21462 Libcalls.getLibcallImpl(Call: RTLIB::STACK_CHECK_GUARD);
21463 if (SecurityCheckCookieLibcall != RTLIB::Unsupported &&
21464 SecurityCookieVar != RTLIB::Unsupported) {
21465 // MSVC CRT has a global variable holding security cookie.
21466 M.getOrInsertGlobal(Name: getLibcallImplName(Call: SecurityCookieVar),
21467 Ty: PointerType::getUnqual(C&: M.getContext()));
21468
21469 // MSVC CRT has a function to validate security cookie.
21470 FunctionCallee SecurityCheckCookie =
21471 M.getOrInsertFunction(Name: getLibcallImplName(Call: SecurityCheckCookieLibcall),
21472 RetTy: Type::getVoidTy(C&: M.getContext()),
21473 Args: PointerType::getUnqual(C&: M.getContext()));
21474 if (Function *F = dyn_cast<Function>(Val: SecurityCheckCookie.getCallee()))
21475 F->addParamAttr(ArgNo: 0, Kind: Attribute::AttrKind::InReg);
21476 }
21477
21478 TargetLowering::insertSSPDeclarations(M, Libcalls);
21479}
21480
21481bool ARMTargetLowering::canCombineStoreAndExtract(Type *VectorTy, Value *Idx,
21482 unsigned &Cost) const {
21483 // If we do not have NEON, vector types are not natively supported.
21484 if (!Subtarget->hasNEON())
21485 return false;
21486
21487 // Floating point values and vector values map to the same register file.
21488 // Therefore, although we could do a store extract of a vector type, this is
21489 // better to leave at float as we have more freedom in the addressing mode for
21490 // those.
21491 if (VectorTy->isFPOrFPVectorTy())
21492 return false;
21493
21494 // If the index is unknown at compile time, this is very expensive to lower
21495 // and it is not possible to combine the store with the extract.
21496 if (!isa<ConstantInt>(Val: Idx))
21497 return false;
21498
21499 assert(VectorTy->isVectorTy() && "VectorTy is not a vector type");
21500 unsigned BitWidth = VectorTy->getPrimitiveSizeInBits().getFixedValue();
21501 // We can do a store + vector extract on any vector that fits perfectly in a D
21502 // or Q register.
21503 if (BitWidth == 64 || BitWidth == 128) {
21504 Cost = 0;
21505 return true;
21506 }
21507 return false;
21508}
21509
21510bool ARMTargetLowering::canCreateUndefOrPoisonForTargetNode(
21511 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
21512 UndefPoisonKind Kind, bool ConsiderFlags, unsigned Depth) const {
21513 unsigned Opcode = Op.getOpcode();
21514 switch (Opcode) {
21515 case ARMISD::VORRIMM:
21516 case ARMISD::VBICIMM:
21517 return false;
21518 }
21519 return TargetLowering::canCreateUndefOrPoisonForTargetNode(
21520 Op, DemandedElts, DAG, Kind, ConsiderFlags, Depth);
21521}
21522
21523bool ARMTargetLowering::isCheapToSpeculateCttz(Type *Ty) const {
21524 return Subtarget->hasV5TOps() && !Subtarget->isThumb1Only();
21525}
21526
21527bool ARMTargetLowering::isCheapToSpeculateCtlz(Type *Ty) const {
21528 return Subtarget->hasV5TOps() && !Subtarget->isThumb1Only();
21529}
21530
21531bool ARMTargetLowering::isMaskAndCmp0FoldingBeneficial(
21532 const Instruction &AndI) const {
21533 if (!Subtarget->hasV7Ops())
21534 return false;
21535
21536 // Sink the `and` instruction only if the mask would fit into a modified
21537 // immediate operand.
21538 ConstantInt *Mask = dyn_cast<ConstantInt>(Val: AndI.getOperand(i: 1));
21539 if (!Mask || Mask->getValue().getBitWidth() > 32u)
21540 return false;
21541 auto MaskVal = unsigned(Mask->getValue().getZExtValue());
21542 return (Subtarget->isThumb2() ? ARM_AM::getT2SOImmVal(Arg: MaskVal)
21543 : ARM_AM::getSOImmVal(Arg: MaskVal)) != -1;
21544}
21545
21546TargetLowering::ShiftLegalizationStrategy
21547ARMTargetLowering::preferredShiftLegalizationStrategy(
21548 SelectionDAG &DAG, SDNode *N, unsigned ExpansionFactor) const {
21549 if (Subtarget->hasMinSize() && !getTM().getTargetTriple().isOSWindows())
21550 return ShiftLegalizationStrategy::LowerToLibcall;
21551 return TargetLowering::preferredShiftLegalizationStrategy(DAG, N,
21552 ExpansionFactor);
21553}
21554
21555Value *ARMTargetLowering::emitLoadLinked(IRBuilderBase &Builder, Type *ValueTy,
21556 Value *Addr,
21557 AtomicOrdering Ord) const {
21558 Module *M = Builder.getModule();
21559 bool IsAcquire = isAcquireOrStronger(AO: Ord);
21560
21561 // Since i64 isn't legal and intrinsics don't get type-lowered, the ldrexd
21562 // intrinsic must return {i32, i32} and we have to recombine them into a
21563 // single i64 here.
21564 if (ValueTy->getPrimitiveSizeInBits() == 64) {
21565 Intrinsic::ID Int =
21566 IsAcquire ? Intrinsic::arm_ldaexd : Intrinsic::arm_ldrexd;
21567
21568 Value *LoHi =
21569 Builder.CreateIntrinsic(ID: Int, Args: Addr, /*FMFSource=*/nullptr, Name: "lohi");
21570
21571 Value *Lo = Builder.CreateExtractValue(Agg: LoHi, Idxs: 0, Name: "lo");
21572 Value *Hi = Builder.CreateExtractValue(Agg: LoHi, Idxs: 1, Name: "hi");
21573 if (!Subtarget->isLittle())
21574 std::swap (a&: Lo, b&: Hi);
21575 Lo = Builder.CreateZExt(V: Lo, DestTy: ValueTy, Name: "lo64");
21576 Hi = Builder.CreateZExt(V: Hi, DestTy: ValueTy, Name: "hi64");
21577 return Builder.CreateOr(
21578 LHS: Lo, RHS: Builder.CreateShl(LHS: Hi, RHS: ConstantInt::get(Ty: ValueTy, V: 32)), Name: "val64");
21579 }
21580
21581 Type *Tys[] = { Addr->getType() };
21582 Intrinsic::ID Int = IsAcquire ? Intrinsic::arm_ldaex : Intrinsic::arm_ldrex;
21583 CallInst *CI = Builder.CreateIntrinsicWithoutFolding(ID: Int, OverloadTypes: Tys, Args: Addr);
21584
21585 CI->addParamAttr(
21586 ArgNo: 0, Attr: Attribute::get(Context&: M->getContext(), Kind: Attribute::ElementType, Ty: ValueTy));
21587 return Builder.CreateTruncOrBitCast(V: CI, DestTy: ValueTy);
21588}
21589
21590void ARMTargetLowering::emitAtomicCmpXchgNoStoreLLBalance(
21591 IRBuilderBase &Builder) const {
21592 if (!Subtarget->hasV7Ops())
21593 return;
21594 Builder.CreateIntrinsic(ID: Intrinsic::arm_clrex, Args: {});
21595}
21596
21597Value *ARMTargetLowering::emitStoreConditional(IRBuilderBase &Builder,
21598 Value *Val, Value *Addr,
21599 AtomicOrdering Ord) const {
21600 Module *M = Builder.getModule();
21601 bool IsRelease = isReleaseOrStronger(AO: Ord);
21602
21603 // Since the intrinsics must have legal type, the i64 intrinsics take two
21604 // parameters: "i32, i32". We must marshal Val into the appropriate form
21605 // before the call.
21606 if (Val->getType()->getPrimitiveSizeInBits() == 64) {
21607 Intrinsic::ID Int =
21608 IsRelease ? Intrinsic::arm_stlexd : Intrinsic::arm_strexd;
21609 Type *Int32Ty = Type::getInt32Ty(C&: M->getContext());
21610
21611 Value *Lo = Builder.CreateTrunc(V: Val, DestTy: Int32Ty, Name: "lo");
21612 Value *Hi = Builder.CreateTrunc(V: Builder.CreateLShr(LHS: Val, RHS: 32), DestTy: Int32Ty, Name: "hi");
21613 if (!Subtarget->isLittle())
21614 std::swap(a&: Lo, b&: Hi);
21615 return Builder.CreateIntrinsic(ID: Int, Args: {Lo, Hi, Addr});
21616 }
21617
21618 Intrinsic::ID Int = IsRelease ? Intrinsic::arm_stlex : Intrinsic::arm_strex;
21619 Type *Tys[] = { Addr->getType() };
21620 Function *Strex = Intrinsic::getOrInsertDeclaration(M, id: Int, OverloadTys: Tys);
21621
21622 CallInst *CI = Builder.CreateCall(
21623 Callee: Strex, Args: {Builder.CreateZExtOrBitCast(
21624 V: Val, DestTy: Strex->getFunctionType()->getParamType(i: 0)),
21625 Addr});
21626 CI->addParamAttr(ArgNo: 1, Attr: Attribute::get(Context&: M->getContext(), Kind: Attribute::ElementType,
21627 Ty: Val->getType()));
21628 return CI;
21629}
21630
21631
21632bool ARMTargetLowering::alignLoopsWithOptSize() const {
21633 return Subtarget->isMClass();
21634}
21635
21636/// A helper function for determining the number of interleaved accesses we
21637/// will generate when lowering accesses of the given type.
21638unsigned
21639ARMTargetLowering::getNumInterleavedAccesses(VectorType *VecTy,
21640 const DataLayout &DL) const {
21641 return (DL.getTypeSizeInBits(Ty: VecTy) + 127) / 128;
21642}
21643
21644bool ARMTargetLowering::isLegalInterleavedAccessType(
21645 unsigned Factor, FixedVectorType *VecTy, Align Alignment,
21646 const DataLayout &DL) const {
21647
21648 unsigned VecSize = DL.getTypeSizeInBits(Ty: VecTy);
21649 unsigned ElSize = DL.getTypeSizeInBits(Ty: VecTy->getElementType());
21650
21651 if (!Subtarget->hasNEON() && !Subtarget->hasMVEIntegerOps())
21652 return false;
21653
21654 // Ensure the vector doesn't have f16 elements. Even though we could do an
21655 // i16 vldN, we can't hold the f16 vectors and will end up converting via
21656 // f32.
21657 if (Subtarget->hasNEON() && VecTy->getElementType()->isHalfTy())
21658 return false;
21659 if (Subtarget->hasMVEIntegerOps() && Factor == 3)
21660 return false;
21661
21662 // Ensure the number of vector elements is greater than 1.
21663 if (VecTy->getNumElements() < 2)
21664 return false;
21665
21666 // Ensure the element type is legal.
21667 if (ElSize != 8 && ElSize != 16 && ElSize != 32)
21668 return false;
21669 // And the alignment if high enough under MVE.
21670 if (Subtarget->hasMVEIntegerOps() && Alignment < ElSize / 8)
21671 return false;
21672
21673 // Ensure the total vector size is 64 or a multiple of 128. Types larger than
21674 // 128 will be split into multiple interleaved accesses.
21675 if (Subtarget->hasNEON() && VecSize == 64)
21676 return true;
21677 return VecSize % 128 == 0;
21678}
21679
21680unsigned ARMTargetLowering::getMaxSupportedInterleaveFactor() const {
21681 if (Subtarget->hasNEON())
21682 return 4;
21683 if (Subtarget->hasMVEIntegerOps())
21684 return MVEMaxSupportedInterleaveFactor;
21685 return TargetLoweringBase::getMaxSupportedInterleaveFactor();
21686}
21687
21688/// Lower an interleaved load into a vldN intrinsic.
21689///
21690/// E.g. Lower an interleaved load (Factor = 2):
21691/// %wide.vec = load <8 x i32>, <8 x i32>* %ptr, align 4
21692/// %v0 = shuffle %wide.vec, undef, <0, 2, 4, 6> ; Extract even elements
21693/// %v1 = shuffle %wide.vec, undef, <1, 3, 5, 7> ; Extract odd elements
21694///
21695/// Into:
21696/// %vld2 = { <4 x i32>, <4 x i32> } call llvm.arm.neon.vld2(%ptr, 4)
21697/// %vec0 = extractelement { <4 x i32>, <4 x i32> } %vld2, i32 0
21698/// %vec1 = extractelement { <4 x i32>, <4 x i32> } %vld2, i32 1
21699bool ARMTargetLowering::lowerInterleavedLoad(
21700 Instruction *Load, Value *Mask, ArrayRef<ShuffleVectorInst *> Shuffles,
21701 ArrayRef<unsigned> Indices, unsigned Factor, const APInt &GapMask) const {
21702 assert(Factor >= 2 && Factor <= getMaxSupportedInterleaveFactor() &&
21703 "Invalid interleave factor");
21704 assert(!Shuffles.empty() && "Empty shufflevector input");
21705 assert(Shuffles.size() == Indices.size() &&
21706 "Unmatched number of shufflevectors and indices");
21707
21708 auto *LI = dyn_cast<LoadInst>(Val: Load);
21709 if (!LI)
21710 return false;
21711 assert(!Mask && GapMask.popcount() == Factor && "Unexpected mask on a load");
21712
21713 auto *VecTy = cast<FixedVectorType>(Val: Shuffles[0]->getType());
21714 Type *EltTy = VecTy->getElementType();
21715
21716 const DataLayout &DL = LI->getDataLayout();
21717 Align Alignment = LI->getAlign();
21718
21719 // Skip if we do not have NEON and skip illegal vector types. We can
21720 // "legalize" wide vector types into multiple interleaved accesses as long as
21721 // the vector types are divisible by 128.
21722 if (!isLegalInterleavedAccessType(Factor, VecTy, Alignment, DL))
21723 return false;
21724
21725 unsigned NumLoads = getNumInterleavedAccesses(VecTy, DL);
21726
21727 // A pointer vector can not be the return type of the ldN intrinsics. Need to
21728 // load integer vectors first and then convert to pointer vectors.
21729 if (EltTy->isPointerTy())
21730 VecTy = FixedVectorType::get(ElementType: DL.getIntPtrType(EltTy), FVTy: VecTy);
21731
21732 IRBuilder<> Builder(LI);
21733
21734 // The base address of the load.
21735 Value *BaseAddr = LI->getPointerOperand();
21736
21737 if (NumLoads > 1) {
21738 // If we're going to generate more than one load, reset the sub-vector type
21739 // to something legal.
21740 VecTy = FixedVectorType::get(ElementType: VecTy->getElementType(),
21741 NumElts: VecTy->getNumElements() / NumLoads);
21742 }
21743
21744 assert(isTypeLegal(EVT::getEVT(VecTy)) && "Illegal vldN vector type!");
21745
21746 auto createLoadIntrinsic = [&](Value *BaseAddr) {
21747 if (Subtarget->hasNEON()) {
21748 Type *PtrTy = Builder.getPtrTy(AddrSpace: LI->getPointerAddressSpace());
21749 Type *Tys[] = {VecTy, PtrTy};
21750 static const Intrinsic::ID LoadInts[3] = {Intrinsic::arm_neon_vld2,
21751 Intrinsic::arm_neon_vld3,
21752 Intrinsic::arm_neon_vld4};
21753
21754 SmallVector<Value *, 2> Ops;
21755 Ops.push_back(Elt: BaseAddr);
21756 Ops.push_back(Elt: Builder.getInt32(C: LI->getAlign().value()));
21757
21758 return Builder.CreateIntrinsic(ID: LoadInts[Factor - 2], OverloadTypes: Tys, Args: Ops,
21759 /*FMFSource=*/nullptr, Name: "vldN");
21760 } else {
21761 assert((Factor == 2 || Factor == 4) &&
21762 "expected interleave factor of 2 or 4 for MVE");
21763 Intrinsic::ID LoadInts =
21764 Factor == 2 ? Intrinsic::arm_mve_vld2q : Intrinsic::arm_mve_vld4q;
21765 Type *PtrTy = Builder.getPtrTy(AddrSpace: LI->getPointerAddressSpace());
21766 Type *Tys[] = {VecTy, PtrTy};
21767
21768 SmallVector<Value *, 2> Ops;
21769 Ops.push_back(Elt: BaseAddr);
21770 return Builder.CreateIntrinsic(ID: LoadInts, OverloadTypes: Tys, Args: Ops, /*FMFSource=*/nullptr,
21771 Name: "vldN");
21772 }
21773 };
21774
21775 // Holds sub-vectors extracted from the load intrinsic return values. The
21776 // sub-vectors are associated with the shufflevector instructions they will
21777 // replace.
21778 DenseMap<ShuffleVectorInst *, SmallVector<Value *, 4>> SubVecs;
21779
21780 for (unsigned LoadCount = 0; LoadCount < NumLoads; ++LoadCount) {
21781 // If we're generating more than one load, compute the base address of
21782 // subsequent loads as an offset from the previous.
21783 if (LoadCount > 0)
21784 BaseAddr = Builder.CreateConstGEP1_32(Ty: VecTy->getElementType(), Ptr: BaseAddr,
21785 Idx0: VecTy->getNumElements() * Factor);
21786
21787 Value *VldN = createLoadIntrinsic(BaseAddr);
21788
21789 // Replace uses of each shufflevector with the corresponding vector loaded
21790 // by ldN.
21791 for (unsigned i = 0; i < Shuffles.size(); i++) {
21792 ShuffleVectorInst *SV = Shuffles[i];
21793 unsigned Index = Indices[i];
21794
21795 Value *SubVec = Builder.CreateExtractValue(Agg: VldN, Idxs: Index);
21796
21797 // Convert the integer vector to pointer vector if the element is pointer.
21798 if (EltTy->isPointerTy())
21799 SubVec = Builder.CreateIntToPtr(
21800 V: SubVec,
21801 DestTy: FixedVectorType::get(ElementType: SV->getType()->getElementType(), FVTy: VecTy));
21802
21803 SubVecs[SV].push_back(Elt: SubVec);
21804 }
21805 }
21806
21807 // Replace uses of the shufflevector instructions with the sub-vectors
21808 // returned by the load intrinsic. If a shufflevector instruction is
21809 // associated with more than one sub-vector, those sub-vectors will be
21810 // concatenated into a single wide vector.
21811 for (ShuffleVectorInst *SVI : Shuffles) {
21812 auto &SubVec = SubVecs[SVI];
21813 auto *WideVec =
21814 SubVec.size() > 1 ? concatenateVectors(Builder, Vecs: SubVec) : SubVec[0];
21815 SVI->replaceAllUsesWith(V: WideVec);
21816 }
21817
21818 return true;
21819}
21820
21821/// Lower an interleaved store into a vstN intrinsic.
21822///
21823/// E.g. Lower an interleaved store (Factor = 3):
21824/// %i.vec = shuffle <8 x i32> %v0, <8 x i32> %v1,
21825/// <0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11>
21826/// store <12 x i32> %i.vec, <12 x i32>* %ptr, align 4
21827///
21828/// Into:
21829/// %sub.v0 = shuffle <8 x i32> %v0, <8 x i32> v1, <0, 1, 2, 3>
21830/// %sub.v1 = shuffle <8 x i32> %v0, <8 x i32> v1, <4, 5, 6, 7>
21831/// %sub.v2 = shuffle <8 x i32> %v0, <8 x i32> v1, <8, 9, 10, 11>
21832/// call void llvm.arm.neon.vst3(%ptr, %sub.v0, %sub.v1, %sub.v2, 4)
21833///
21834/// Note that the new shufflevectors will be removed and we'll only generate one
21835/// vst3 instruction in CodeGen.
21836///
21837/// Example for a more general valid mask (Factor 3). Lower:
21838/// %i.vec = shuffle <32 x i32> %v0, <32 x i32> %v1,
21839/// <4, 32, 16, 5, 33, 17, 6, 34, 18, 7, 35, 19>
21840/// store <12 x i32> %i.vec, <12 x i32>* %ptr
21841///
21842/// Into:
21843/// %sub.v0 = shuffle <32 x i32> %v0, <32 x i32> v1, <4, 5, 6, 7>
21844/// %sub.v1 = shuffle <32 x i32> %v0, <32 x i32> v1, <32, 33, 34, 35>
21845/// %sub.v2 = shuffle <32 x i32> %v0, <32 x i32> v1, <16, 17, 18, 19>
21846/// call void llvm.arm.neon.vst3(%ptr, %sub.v0, %sub.v1, %sub.v2, 4)
21847bool ARMTargetLowering::lowerInterleavedStore(Instruction *Store,
21848 Value *LaneMask,
21849 ShuffleVectorInst *SVI,
21850 unsigned Factor,
21851 const APInt &GapMask) const {
21852 assert(Factor >= 2 && Factor <= getMaxSupportedInterleaveFactor() &&
21853 "Invalid interleave factor");
21854 auto *SI = dyn_cast<StoreInst>(Val: Store);
21855 if (!SI)
21856 return false;
21857 assert(!LaneMask && GapMask.popcount() == Factor &&
21858 "Unexpected mask on store");
21859
21860 auto *VecTy = cast<FixedVectorType>(Val: SVI->getType());
21861 assert(VecTy->getNumElements() % Factor == 0 && "Invalid interleaved store");
21862
21863 unsigned LaneLen = VecTy->getNumElements() / Factor;
21864 Type *EltTy = VecTy->getElementType();
21865 auto *SubVecTy = FixedVectorType::get(ElementType: EltTy, NumElts: LaneLen);
21866
21867 const DataLayout &DL = SI->getDataLayout();
21868 Align Alignment = SI->getAlign();
21869
21870 // Skip if we do not have NEON and skip illegal vector types. We can
21871 // "legalize" wide vector types into multiple interleaved accesses as long as
21872 // the vector types are divisible by 128.
21873 if (!isLegalInterleavedAccessType(Factor, VecTy: SubVecTy, Alignment, DL))
21874 return false;
21875
21876 unsigned NumStores = getNumInterleavedAccesses(VecTy: SubVecTy, DL);
21877
21878 Value *Op0 = SVI->getOperand(i_nocapture: 0);
21879 Value *Op1 = SVI->getOperand(i_nocapture: 1);
21880 IRBuilder<> Builder(SI);
21881
21882 // StN intrinsics don't support pointer vectors as arguments. Convert pointer
21883 // vectors to integer vectors.
21884 if (EltTy->isPointerTy()) {
21885 Type *IntTy = DL.getIntPtrType(EltTy);
21886
21887 // Convert to the corresponding integer vector.
21888 auto *IntVecTy =
21889 FixedVectorType::get(ElementType: IntTy, FVTy: cast<FixedVectorType>(Val: Op0->getType()));
21890 Op0 = Builder.CreatePtrToInt(V: Op0, DestTy: IntVecTy);
21891 Op1 = Builder.CreatePtrToInt(V: Op1, DestTy: IntVecTy);
21892
21893 SubVecTy = FixedVectorType::get(ElementType: IntTy, NumElts: LaneLen);
21894 }
21895
21896 // The base address of the store.
21897 Value *BaseAddr = SI->getPointerOperand();
21898
21899 if (NumStores > 1) {
21900 // If we're going to generate more than one store, reset the lane length
21901 // and sub-vector type to something legal.
21902 LaneLen /= NumStores;
21903 SubVecTy = FixedVectorType::get(ElementType: SubVecTy->getElementType(), NumElts: LaneLen);
21904 }
21905
21906 assert(isTypeLegal(EVT::getEVT(SubVecTy)) && "Illegal vstN vector type!");
21907
21908 auto Mask = SVI->getShuffleMask();
21909
21910 auto createStoreIntrinsic = [&](Value *BaseAddr,
21911 SmallVectorImpl<Value *> &Shuffles) {
21912 if (Subtarget->hasNEON()) {
21913 static const Intrinsic::ID StoreInts[3] = {Intrinsic::arm_neon_vst2,
21914 Intrinsic::arm_neon_vst3,
21915 Intrinsic::arm_neon_vst4};
21916 Type *PtrTy = Builder.getPtrTy(AddrSpace: SI->getPointerAddressSpace());
21917 Type *Tys[] = {PtrTy, SubVecTy};
21918
21919 SmallVector<Value *, 6> Ops;
21920 Ops.push_back(Elt: BaseAddr);
21921 append_range(C&: Ops, R&: Shuffles);
21922 Ops.push_back(Elt: Builder.getInt32(C: SI->getAlign().value()));
21923 Builder.CreateIntrinsic(ID: StoreInts[Factor - 2], OverloadTypes: Tys, Args: Ops);
21924 } else {
21925 assert((Factor == 2 || Factor == 4) &&
21926 "expected interleave factor of 2 or 4 for MVE");
21927 Intrinsic::ID StoreInts =
21928 Factor == 2 ? Intrinsic::arm_mve_vst2q : Intrinsic::arm_mve_vst4q;
21929 Type *PtrTy = Builder.getPtrTy(AddrSpace: SI->getPointerAddressSpace());
21930 Type *Tys[] = {PtrTy, SubVecTy};
21931
21932 SmallVector<Value *, 6> Ops;
21933 Ops.push_back(Elt: BaseAddr);
21934 append_range(C&: Ops, R&: Shuffles);
21935 for (unsigned F = 0; F < Factor; F++) {
21936 Ops.push_back(Elt: Builder.getInt32(C: F));
21937 Builder.CreateIntrinsic(ID: StoreInts, OverloadTypes: Tys, Args: Ops);
21938 Ops.pop_back();
21939 }
21940 }
21941 };
21942
21943 for (unsigned StoreCount = 0; StoreCount < NumStores; ++StoreCount) {
21944 // If we generating more than one store, we compute the base address of
21945 // subsequent stores as an offset from the previous.
21946 if (StoreCount > 0)
21947 BaseAddr = Builder.CreateConstGEP1_32(Ty: SubVecTy->getElementType(),
21948 Ptr: BaseAddr, Idx0: LaneLen * Factor);
21949
21950 SmallVector<Value *, 4> Shuffles;
21951
21952 // Split the shufflevector operands into sub vectors for the new vstN call.
21953 for (unsigned i = 0; i < Factor; i++) {
21954 unsigned IdxI = StoreCount * LaneLen * Factor + i;
21955 if (Mask[IdxI] >= 0) {
21956 Shuffles.push_back(Elt: Builder.CreateShuffleVector(
21957 V1: Op0, V2: Op1, Mask: createSequentialMask(Start: Mask[IdxI], NumInts: LaneLen, NumUndefs: 0)));
21958 } else {
21959 unsigned StartMask = 0;
21960 for (unsigned j = 1; j < LaneLen; j++) {
21961 unsigned IdxJ = StoreCount * LaneLen * Factor + j;
21962 if (Mask[IdxJ * Factor + IdxI] >= 0) {
21963 StartMask = Mask[IdxJ * Factor + IdxI] - IdxJ;
21964 break;
21965 }
21966 }
21967 // Note: If all elements in a chunk are undefs, StartMask=0!
21968 // Note: Filling undef gaps with random elements is ok, since
21969 // those elements were being written anyway (with undefs).
21970 // In the case of all undefs we're defaulting to using elems from 0
21971 // Note: StartMask cannot be negative, it's checked in
21972 // isReInterleaveMask
21973 Shuffles.push_back(Elt: Builder.CreateShuffleVector(
21974 V1: Op0, V2: Op1, Mask: createSequentialMask(Start: StartMask, NumInts: LaneLen, NumUndefs: 0)));
21975 }
21976 }
21977
21978 createStoreIntrinsic(BaseAddr, Shuffles);
21979 }
21980 return true;
21981}
21982
21983enum HABaseType {
21984 HA_UNKNOWN = 0,
21985 HA_FLOAT,
21986 HA_DOUBLE,
21987 HA_VECT64,
21988 HA_VECT128
21989};
21990
21991static bool isHomogeneousAggregate(Type *Ty, HABaseType &Base,
21992 uint64_t &Members) {
21993 if (auto *ST = dyn_cast<StructType>(Val: Ty)) {
21994 for (unsigned i = 0; i < ST->getNumElements(); ++i) {
21995 uint64_t SubMembers = 0;
21996 if (!isHomogeneousAggregate(Ty: ST->getElementType(N: i), Base, Members&: SubMembers))
21997 return false;
21998 Members += SubMembers;
21999 }
22000 } else if (auto *AT = dyn_cast<ArrayType>(Val: Ty)) {
22001 uint64_t SubMembers = 0;
22002 if (!isHomogeneousAggregate(Ty: AT->getElementType(), Base, Members&: SubMembers))
22003 return false;
22004 Members += SubMembers * AT->getNumElements();
22005 } else if (Ty->isFloatTy()) {
22006 if (Base != HA_UNKNOWN && Base != HA_FLOAT)
22007 return false;
22008 Members = 1;
22009 Base = HA_FLOAT;
22010 } else if (Ty->isDoubleTy()) {
22011 if (Base != HA_UNKNOWN && Base != HA_DOUBLE)
22012 return false;
22013 Members = 1;
22014 Base = HA_DOUBLE;
22015 } else if (auto *VT = dyn_cast<VectorType>(Val: Ty)) {
22016 Members = 1;
22017 switch (Base) {
22018 case HA_FLOAT:
22019 case HA_DOUBLE:
22020 return false;
22021 case HA_VECT64:
22022 return VT->getPrimitiveSizeInBits().getFixedValue() == 64;
22023 case HA_VECT128:
22024 return VT->getPrimitiveSizeInBits().getFixedValue() == 128;
22025 case HA_UNKNOWN:
22026 switch (VT->getPrimitiveSizeInBits().getFixedValue()) {
22027 case 64:
22028 Base = HA_VECT64;
22029 return true;
22030 case 128:
22031 Base = HA_VECT128;
22032 return true;
22033 default:
22034 return false;
22035 }
22036 }
22037 }
22038
22039 return (Members > 0 && Members <= 4);
22040}
22041
22042/// Return the correct alignment for the current calling convention.
22043Align ARMTargetLowering::getABIAlignmentForCallingConv(
22044 Type *ArgTy, const DataLayout &DL) const {
22045 const Align ABITypeAlign = DL.getABITypeAlign(Ty: ArgTy);
22046 if (!ArgTy->isVectorTy())
22047 return ABITypeAlign;
22048
22049 // Avoid over-aligning vector parameters. It would require realigning the
22050 // stack and waste space for no real benefit.
22051 MaybeAlign StackAlign = DL.getStackAlignment();
22052 assert(StackAlign && "data layout string is missing stack alignment");
22053 return std::min(a: ABITypeAlign, b: *StackAlign);
22054}
22055
22056/// Return true if a type is an AAPCS-VFP homogeneous aggregate or one of
22057/// [N x i32] or [N x i64]. This allows front-ends to skip emitting padding when
22058/// passing according to AAPCS rules.
22059bool ARMTargetLowering::functionArgumentNeedsConsecutiveRegisters(
22060 Type *Ty, CallingConv::ID CallConv, bool isVarArg,
22061 const DataLayout &DL) const {
22062 if (getEffectiveCallingConv(CC: CallConv, isVarArg) !=
22063 CallingConv::ARM_AAPCS_VFP)
22064 return false;
22065
22066 HABaseType Base = HA_UNKNOWN;
22067 uint64_t Members = 0;
22068 bool IsHA = isHomogeneousAggregate(Ty, Base, Members);
22069 LLVM_DEBUG(dbgs() << "isHA: " << IsHA << " "; Ty->dump());
22070
22071 bool IsIntArray = Ty->isArrayTy() && Ty->getArrayElementType()->isIntegerTy();
22072 return IsHA || IsIntArray;
22073}
22074
22075Register ARMTargetLowering::getExceptionPointerRegister(
22076 ExceptionHandling EH, const Constant *PersonalityFn) const {
22077 // Platforms which do not use SjLj EH may return values in these registers
22078 // via the personality function.
22079 return EH == ExceptionHandling::SjLj ? Register() : ARM::R0;
22080}
22081
22082Register ARMTargetLowering::getExceptionSelectorRegister(
22083 ExceptionHandling EH, const Constant *PersonalityFn) const {
22084 // Platforms which do not use SjLj EH may return values in these registers
22085 // via the personality function.
22086 return EH == ExceptionHandling::SjLj ? Register() : ARM::R1;
22087}
22088
22089void ARMTargetLowering::initializeSplitCSR(MachineBasicBlock *Entry) const {
22090 // Update IsSplitCSR in ARMFunctionInfo.
22091 ARMFunctionInfo *AFI = Entry->getParent()->getInfo<ARMFunctionInfo>();
22092 AFI->setIsSplitCSR(true);
22093}
22094
22095void ARMTargetLowering::insertCopiesSplitCSR(
22096 MachineBasicBlock *Entry,
22097 const SmallVectorImpl<MachineBasicBlock *> &Exits) const {
22098 const ARMBaseRegisterInfo *TRI = Subtarget->getRegisterInfo();
22099 const MCPhysReg *IStart = TRI->getCalleeSavedRegsViaCopy(MF: Entry->getParent());
22100 if (!IStart)
22101 return;
22102
22103 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
22104 MachineRegisterInfo *MRI = &Entry->getParent()->getRegInfo();
22105 MachineBasicBlock::iterator MBBI = Entry->begin();
22106 for (const MCPhysReg *I = IStart; *I; ++I) {
22107 const TargetRegisterClass *RC = nullptr;
22108 if (ARM::GPRRegClass.contains(Reg: *I))
22109 RC = &ARM::GPRRegClass;
22110 else if (ARM::DPRRegClass.contains(Reg: *I))
22111 RC = &ARM::DPRRegClass;
22112 else
22113 llvm_unreachable("Unexpected register class in CSRsViaCopy!");
22114
22115 Register NewVR = MRI->createVirtualRegister(RegClass: RC);
22116 // Create copy from CSR to a virtual register.
22117 // FIXME: this currently does not emit CFI pseudo-instructions, it works
22118 // fine for CXX_FAST_TLS since the C++-style TLS access functions should be
22119 // nounwind. If we want to generalize this later, we may need to emit
22120 // CFI pseudo-instructions.
22121 assert(Entry->getParent()->getFunction().hasFnAttribute(
22122 Attribute::NoUnwind) &&
22123 "Function should be nounwind in insertCopiesSplitCSR!");
22124 Entry->addLiveIn(PhysReg: *I);
22125 BuildMI(BB&: *Entry, I: MBBI, MIMD: DebugLoc(), MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: NewVR)
22126 .addReg(RegNo: *I);
22127
22128 // Insert the copy-back instructions right before the terminator.
22129 for (auto *Exit : Exits)
22130 BuildMI(BB&: *Exit, I: Exit->getFirstTerminator(), MIMD: DebugLoc(),
22131 MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: *I)
22132 .addReg(RegNo: NewVR);
22133 }
22134}
22135
22136void ARMTargetLowering::finalizeLowering(MachineFunction &MF) const {
22137 MF.getFrameInfo().computeMaxCallFrameSize(MF);
22138 TargetLoweringBase::finalizeLowering(MF);
22139}
22140
22141bool ARMTargetLowering::isComplexDeinterleavingSupported() const {
22142 return Subtarget->hasMVEIntegerOps();
22143}
22144
22145bool ARMTargetLowering::isComplexDeinterleavingOperationSupported(
22146 ComplexDeinterleavingOperation Operation, Type *Ty) const {
22147 auto *VTy = dyn_cast<FixedVectorType>(Val: Ty);
22148 if (!VTy)
22149 return false;
22150
22151 auto *ScalarTy = VTy->getScalarType();
22152 unsigned NumElements = VTy->getNumElements();
22153
22154 unsigned VTyWidth = VTy->getScalarSizeInBits() * NumElements;
22155 if (VTyWidth < 128 || !llvm::isPowerOf2_32(Value: VTyWidth))
22156 return false;
22157
22158 // Both VCADD and VCMUL/VCMLA support the same types, F16 and F32
22159 if (ScalarTy->isHalfTy() || ScalarTy->isFloatTy())
22160 return Subtarget->hasMVEFloatOps();
22161
22162 if (Operation != ComplexDeinterleavingOperation::CAdd)
22163 return false;
22164
22165 return Subtarget->hasMVEIntegerOps() &&
22166 (ScalarTy->isIntegerTy(BitWidth: 8) || ScalarTy->isIntegerTy(BitWidth: 16) ||
22167 ScalarTy->isIntegerTy(BitWidth: 32));
22168}
22169
22170ArrayRef<MCPhysReg> ARMTargetLowering::getRoundingControlRegisters() const {
22171 static const MCPhysReg RCRegs[] = {ARM::FPSCR_RM};
22172 return RCRegs;
22173}
22174
22175Value *ARMTargetLowering::createComplexDeinterleavingIR(
22176 IRBuilderBase &B, ComplexDeinterleavingOperation OperationType,
22177 ComplexDeinterleavingRotation Rotation, Value *InputA, Value *InputB,
22178 Value *Accumulator) const {
22179
22180 FixedVectorType *Ty = cast<FixedVectorType>(Val: InputA->getType());
22181
22182 unsigned TyWidth = Ty->getScalarSizeInBits() * Ty->getNumElements();
22183
22184 assert(TyWidth >= 128 && "Width of vector type must be at least 128 bits");
22185
22186 if (TyWidth > 128) {
22187 int Stride = Ty->getNumElements() / 2;
22188 auto SplitSeq = llvm::seq<int>(Begin: 0, End: Ty->getNumElements());
22189 auto SplitSeqVec = llvm::to_vector(Range&: SplitSeq);
22190 ArrayRef<int> LowerSplitMask(&SplitSeqVec[0], Stride);
22191 ArrayRef<int> UpperSplitMask(&SplitSeqVec[Stride], Stride);
22192
22193 auto *LowerSplitA = B.CreateShuffleVector(V: InputA, Mask: LowerSplitMask);
22194 auto *LowerSplitB = B.CreateShuffleVector(V: InputB, Mask: LowerSplitMask);
22195 auto *UpperSplitA = B.CreateShuffleVector(V: InputA, Mask: UpperSplitMask);
22196 auto *UpperSplitB = B.CreateShuffleVector(V: InputB, Mask: UpperSplitMask);
22197 Value *LowerSplitAcc = nullptr;
22198 Value *UpperSplitAcc = nullptr;
22199
22200 if (Accumulator) {
22201 LowerSplitAcc = B.CreateShuffleVector(V: Accumulator, Mask: LowerSplitMask);
22202 UpperSplitAcc = B.CreateShuffleVector(V: Accumulator, Mask: UpperSplitMask);
22203 }
22204
22205 auto *LowerSplitInt = createComplexDeinterleavingIR(
22206 B, OperationType, Rotation, InputA: LowerSplitA, InputB: LowerSplitB, Accumulator: LowerSplitAcc);
22207 auto *UpperSplitInt = createComplexDeinterleavingIR(
22208 B, OperationType, Rotation, InputA: UpperSplitA, InputB: UpperSplitB, Accumulator: UpperSplitAcc);
22209
22210 ArrayRef<int> JoinMask(&SplitSeqVec[0], Ty->getNumElements());
22211 return B.CreateShuffleVector(V1: LowerSplitInt, V2: UpperSplitInt, Mask: JoinMask);
22212 }
22213
22214 auto *IntTy = Type::getInt32Ty(C&: B.getContext());
22215
22216 ConstantInt *ConstRotation = nullptr;
22217 if (OperationType == ComplexDeinterleavingOperation::CMulPartial) {
22218 ConstRotation = ConstantInt::get(Ty: IntTy, V: (int)Rotation);
22219
22220 if (Accumulator)
22221 return B.CreateIntrinsic(ID: Intrinsic::arm_mve_vcmlaq, OverloadTypes: Ty,
22222 Args: {ConstRotation, Accumulator, InputB, InputA});
22223 return B.CreateIntrinsic(ID: Intrinsic::arm_mve_vcmulq, OverloadTypes: Ty,
22224 Args: {ConstRotation, InputB, InputA});
22225 }
22226
22227 if (OperationType == ComplexDeinterleavingOperation::CAdd) {
22228 // 1 means the value is not halved.
22229 auto *ConstHalving = ConstantInt::get(Ty: IntTy, V: 1);
22230
22231 if (Rotation == ComplexDeinterleavingRotation::Rotation_90)
22232 ConstRotation = ConstantInt::get(Ty: IntTy, V: 0);
22233 else if (Rotation == ComplexDeinterleavingRotation::Rotation_270)
22234 ConstRotation = ConstantInt::get(Ty: IntTy, V: 1);
22235
22236 if (!ConstRotation)
22237 return nullptr; // Invalid rotation for arm_mve_vcaddq
22238
22239 return B.CreateIntrinsic(ID: Intrinsic::arm_mve_vcaddq, OverloadTypes: Ty,
22240 Args: {ConstHalving, ConstRotation, InputA, InputB});
22241 }
22242
22243 return nullptr;
22244}
22245