1//===- ARMISelLowering.cpp - ARM DAG Lowering Implementation --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the interfaces that ARM uses to lower LLVM code into a
10// selection DAG.
11//
12//===----------------------------------------------------------------------===//
13
14#include "ARMISelLowering.h"
15#include "ARMBaseInstrInfo.h"
16#include "ARMBaseRegisterInfo.h"
17#include "ARMCallingConv.h"
18#include "ARMConstantPoolValue.h"
19#include "ARMMachineFunctionInfo.h"
20#include "ARMPerfectShuffle.h"
21#include "ARMRegisterInfo.h"
22#include "ARMSelectionDAGInfo.h"
23#include "ARMSubtarget.h"
24#include "ARMTargetTransformInfo.h"
25#include "MCTargetDesc/ARMAddressingModes.h"
26#include "MCTargetDesc/ARMBaseInfo.h"
27#include "Utils/ARMBaseInfo.h"
28#include "llvm/ADT/APFloat.h"
29#include "llvm/ADT/APInt.h"
30#include "llvm/ADT/ArrayRef.h"
31#include "llvm/ADT/BitVector.h"
32#include "llvm/ADT/DenseMap.h"
33#include "llvm/ADT/STLExtras.h"
34#include "llvm/ADT/SmallPtrSet.h"
35#include "llvm/ADT/SmallVector.h"
36#include "llvm/ADT/Statistic.h"
37#include "llvm/ADT/StringExtras.h"
38#include "llvm/ADT/StringRef.h"
39#include "llvm/ADT/StringSwitch.h"
40#include "llvm/ADT/Twine.h"
41#include "llvm/Analysis/VectorUtils.h"
42#include "llvm/CodeGen/CallingConvLower.h"
43#include "llvm/CodeGen/ComplexDeinterleavingPass.h"
44#include "llvm/CodeGen/ISDOpcodes.h"
45#include "llvm/CodeGen/MachineBasicBlock.h"
46#include "llvm/CodeGen/MachineConstantPool.h"
47#include "llvm/CodeGen/MachineFrameInfo.h"
48#include "llvm/CodeGen/MachineFunction.h"
49#include "llvm/CodeGen/MachineInstr.h"
50#include "llvm/CodeGen/MachineInstrBuilder.h"
51#include "llvm/CodeGen/MachineJumpTableInfo.h"
52#include "llvm/CodeGen/MachineMemOperand.h"
53#include "llvm/CodeGen/MachineOperand.h"
54#include "llvm/CodeGen/MachineRegisterInfo.h"
55#include "llvm/CodeGen/RuntimeLibcallUtil.h"
56#include "llvm/CodeGen/SDPatternMatch.h"
57#include "llvm/CodeGen/SelectionDAG.h"
58#include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
59#include "llvm/CodeGen/SelectionDAGNodes.h"
60#include "llvm/CodeGen/TargetInstrInfo.h"
61#include "llvm/CodeGen/TargetLowering.h"
62#include "llvm/CodeGen/TargetOpcodes.h"
63#include "llvm/CodeGen/TargetRegisterInfo.h"
64#include "llvm/CodeGen/TargetSubtargetInfo.h"
65#include "llvm/CodeGen/ValueTypes.h"
66#include "llvm/CodeGenTypes/MachineValueType.h"
67#include "llvm/IR/Attributes.h"
68#include "llvm/IR/CallingConv.h"
69#include "llvm/IR/Constant.h"
70#include "llvm/IR/Constants.h"
71#include "llvm/IR/DataLayout.h"
72#include "llvm/IR/DebugLoc.h"
73#include "llvm/IR/DerivedTypes.h"
74#include "llvm/IR/Function.h"
75#include "llvm/IR/GlobalAlias.h"
76#include "llvm/IR/GlobalValue.h"
77#include "llvm/IR/GlobalVariable.h"
78#include "llvm/IR/IRBuilder.h"
79#include "llvm/IR/InlineAsm.h"
80#include "llvm/IR/Instruction.h"
81#include "llvm/IR/Instructions.h"
82#include "llvm/IR/IntrinsicInst.h"
83#include "llvm/IR/Intrinsics.h"
84#include "llvm/IR/IntrinsicsARM.h"
85#include "llvm/IR/Module.h"
86#include "llvm/IR/Type.h"
87#include "llvm/IR/User.h"
88#include "llvm/IR/Value.h"
89#include "llvm/MC/MCInstrDesc.h"
90#include "llvm/MC/MCInstrItineraries.h"
91#include "llvm/MC/MCSchedule.h"
92#include "llvm/Support/AtomicOrdering.h"
93#include "llvm/Support/BranchProbability.h"
94#include "llvm/Support/Casting.h"
95#include "llvm/Support/CodeGen.h"
96#include "llvm/Support/CommandLine.h"
97#include "llvm/Support/Compiler.h"
98#include "llvm/Support/Debug.h"
99#include "llvm/Support/ErrorHandling.h"
100#include "llvm/Support/KnownBits.h"
101#include "llvm/Support/MathExtras.h"
102#include "llvm/Support/raw_ostream.h"
103#include "llvm/Target/TargetMachine.h"
104#include "llvm/Target/TargetOptions.h"
105#include "llvm/TargetParser/Triple.h"
106#include <algorithm>
107#include <cassert>
108#include <cstdint>
109#include <iterator>
110#include <limits>
111#include <optional>
112#include <tuple>
113#include <utility>
114#include <vector>
115
116using namespace llvm;
117using namespace llvm::SDPatternMatch;
118
119#define DEBUG_TYPE "arm-isel"
120
121STATISTIC(NumTailCalls, "Number of tail calls");
122STATISTIC(NumOptimizedImms, "Number of times immediates were optimized");
123STATISTIC(NumMovwMovt, "Number of GAs materialized with movw + movt");
124STATISTIC(NumLoopByVals, "Number of loops generated for byval arguments");
125STATISTIC(NumConstpoolPromoted,
126 "Number of constants with their storage promoted into constant pools");
127
128static cl::opt<bool>
129ARMInterworking("arm-interworking", cl::Hidden,
130 cl::desc("Enable / disable ARM interworking (for debugging only)"),
131 cl::init(Val: true));
132
133static cl::opt<bool> EnableConstpoolPromotion(
134 "arm-promote-constant", cl::Hidden,
135 cl::desc("Enable / disable promotion of unnamed_addr constants into "
136 "constant pools"),
137 cl::init(Val: false)); // FIXME: set to true by default once PR32780 is fixed
138static cl::opt<unsigned> ConstpoolPromotionMaxSize(
139 "arm-promote-constant-max-size", cl::Hidden,
140 cl::desc("Maximum size of constant to promote into a constant pool"),
141 cl::init(Val: 64));
142static cl::opt<unsigned> ConstpoolPromotionMaxTotal(
143 "arm-promote-constant-max-total", cl::Hidden,
144 cl::desc("Maximum size of ALL constants to promote into a constant pool"),
145 cl::init(Val: 128));
146
147cl::opt<unsigned>
148MVEMaxSupportedInterleaveFactor("mve-max-interleave-factor", cl::Hidden,
149 cl::desc("Maximum interleave factor for MVE VLDn to generate."),
150 cl::init(Val: 2));
151
152cl::opt<unsigned> ArmMaxBaseUpdatesToCheck(
153 "arm-max-base-updates-to-check", cl::Hidden,
154 cl::desc("Maximum number of base-updates to check generating postindex."),
155 cl::init(Val: 64));
156
157/// Value type used for "flags" operands / results (either CPSR or FPSCR_NZCV).
158constexpr MVT FlagsVT = MVT::i32;
159
160// The APCS parameter registers.
161static const MCPhysReg GPRArgRegs[] = {
162 ARM::R0, ARM::R1, ARM::R2, ARM::R3
163};
164
165static SDValue handleCMSEValue(const SDValue &Value, const ISD::InputArg &Arg,
166 SelectionDAG &DAG, const SDLoc &DL) {
167 assert(Arg.ArgVT.isScalarInteger());
168 assert(Arg.ArgVT.bitsLT(MVT::i32));
169 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: Arg.ArgVT, Operand: Value);
170 SDValue Ext =
171 DAG.getNode(Opcode: Arg.Flags.isSExt() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL,
172 VT: MVT::i32, Operand: Trunc);
173 return Ext;
174}
175
176void ARMTargetLowering::addTypeForNEON(MVT VT, MVT PromotedLdStVT) {
177 if (VT != PromotedLdStVT) {
178 setOperationAction(Op: ISD::LOAD, VT, Action: Promote);
179 AddPromotedToType (Opc: ISD::LOAD, OrigVT: VT, DestVT: PromotedLdStVT);
180
181 setOperationAction(Op: ISD::STORE, VT, Action: Promote);
182 AddPromotedToType (Opc: ISD::STORE, OrigVT: VT, DestVT: PromotedLdStVT);
183 }
184
185 MVT ElemTy = VT.getVectorElementType();
186 if (ElemTy != MVT::f64)
187 setOperationAction(Op: ISD::SETCC, VT, Action: Custom);
188 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT, Action: Custom);
189 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT, Action: Custom);
190 if (ElemTy == MVT::i32) {
191 setOperationAction(Op: ISD::SINT_TO_FP, VT, Action: Custom);
192 setOperationAction(Op: ISD::UINT_TO_FP, VT, Action: Custom);
193 setOperationAction(Op: ISD::FP_TO_SINT, VT, Action: Custom);
194 setOperationAction(Op: ISD::FP_TO_UINT, VT, Action: Custom);
195 } else {
196 setOperationAction(Op: ISD::SINT_TO_FP, VT, Action: Expand);
197 setOperationAction(Op: ISD::UINT_TO_FP, VT, Action: Expand);
198 setOperationAction(Op: ISD::FP_TO_SINT, VT, Action: Expand);
199 setOperationAction(Op: ISD::FP_TO_UINT, VT, Action: Expand);
200 }
201 setOperationAction(Op: ISD::BUILD_VECTOR, VT, Action: Custom);
202 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT, Action: Custom);
203 setOperationAction(Op: ISD::CONCAT_VECTORS, VT, Action: Legal);
204 setOperationAction(Op: ISD::EXTRACT_SUBVECTOR, VT, Action: Legal);
205 setOperationAction(Op: ISD::SELECT, VT, Action: Expand);
206 setOperationAction(Op: ISD::SELECT_CC, VT, Action: Expand);
207 setOperationAction(Op: ISD::VSELECT, VT, Action: Expand);
208 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT, Action: Expand);
209 if (VT.isInteger()) {
210 setOperationAction(Op: ISD::SHL, VT, Action: Custom);
211 setOperationAction(Op: ISD::SRA, VT, Action: Custom);
212 setOperationAction(Op: ISD::SRL, VT, Action: Custom);
213 }
214
215 // Neon does not support vector divide/remainder operations.
216 setOperationAction(Op: ISD::SDIV, VT, Action: Expand);
217 setOperationAction(Op: ISD::UDIV, VT, Action: Expand);
218 setOperationAction(Op: ISD::FDIV, VT, Action: Expand);
219 setOperationAction(Op: ISD::SREM, VT, Action: Expand);
220 setOperationAction(Op: ISD::UREM, VT, Action: Expand);
221 setOperationAction(Op: ISD::FREM, VT, Action: Expand);
222 setOperationAction(Op: ISD::SDIVREM, VT, Action: Expand);
223 setOperationAction(Op: ISD::UDIVREM, VT, Action: Expand);
224
225 if (!VT.isFloatingPoint() && VT != MVT::v2i64 && VT != MVT::v1i64)
226 for (auto Opcode : {ISD::ABS, ISD::ABDS, ISD::ABDU, ISD::SMIN, ISD::SMAX,
227 ISD::UMIN, ISD::UMAX, ISD::CTLS})
228 setOperationAction(Op: Opcode, VT, Action: Legal);
229 if (!VT.isFloatingPoint())
230 for (auto Opcode : {ISD::SADDSAT, ISD::UADDSAT, ISD::SSUBSAT, ISD::USUBSAT})
231 setOperationAction(Op: Opcode, VT, Action: Legal);
232}
233
234void ARMTargetLowering::addDRTypeForNEON(MVT VT) {
235 addRegisterClass(VT, RC: &ARM::DPRRegClass);
236 addTypeForNEON(VT, PromotedLdStVT: MVT::f64);
237}
238
239void ARMTargetLowering::addQRTypeForNEON(MVT VT) {
240 addRegisterClass(VT, RC: &ARM::DPairRegClass);
241 addTypeForNEON(VT, PromotedLdStVT: MVT::v2f64);
242}
243
244void ARMTargetLowering::setAllExpand(MVT VT) {
245 for (unsigned Opc = 0; Opc < ISD::BUILTIN_OP_END; ++Opc)
246 setOperationAction(Op: Opc, VT, Action: Expand);
247
248 // We support these really simple operations even on types where all
249 // the actual arithmetic has to be broken down into simpler
250 // operations or turned into library calls.
251 setOperationAction(Op: ISD::BITCAST, VT, Action: Legal);
252 setOperationAction(Op: ISD::LOAD, VT, Action: Legal);
253 setOperationAction(Op: ISD::STORE, VT, Action: Legal);
254 setOperationAction(Ops: {ISD::UNDEF, ISD::POISON}, VT, Action: Legal);
255}
256
257void ARMTargetLowering::addAllExtLoads(const MVT From, const MVT To,
258 LegalizeAction Action) {
259 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: From, MemVT: To, Action);
260 setLoadExtAction(ExtType: ISD::ZEXTLOAD, ValVT: From, MemVT: To, Action);
261 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: From, MemVT: To, Action);
262}
263
264void ARMTargetLowering::addMVEVectorTypes(bool HasMVEFP) {
265 const MVT IntTypes[] = { MVT::v16i8, MVT::v8i16, MVT::v4i32 };
266
267 for (auto VT : IntTypes) {
268 addRegisterClass(VT, RC: &ARM::MQPRRegClass);
269 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT, Action: Custom);
270 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT, Action: Custom);
271 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT, Action: Custom);
272 setOperationAction(Op: ISD::BUILD_VECTOR, VT, Action: Custom);
273 setOperationAction(Op: ISD::SHL, VT, Action: Custom);
274 setOperationAction(Op: ISD::SRA, VT, Action: Custom);
275 setOperationAction(Op: ISD::SRL, VT, Action: Custom);
276 setOperationAction(Op: ISD::SMIN, VT, Action: Legal);
277 setOperationAction(Op: ISD::SMAX, VT, Action: Legal);
278 setOperationAction(Op: ISD::UMIN, VT, Action: Legal);
279 setOperationAction(Op: ISD::UMAX, VT, Action: Legal);
280 setOperationAction(Op: ISD::ABS, VT, Action: Legal);
281 setOperationAction(Op: ISD::CTLS, VT, Action: Legal);
282 setOperationAction(Op: ISD::SETCC, VT, Action: Custom);
283 setOperationAction(Op: ISD::MLOAD, VT, Action: Custom);
284 setOperationAction(Op: ISD::MSTORE, VT, Action: Legal);
285 setOperationAction(Op: ISD::CTLZ, VT, Action: Legal);
286 setOperationAction(Op: ISD::CTTZ, VT, Action: Custom);
287 setOperationAction(Op: ISD::BITREVERSE, VT, Action: Legal);
288 setOperationAction(Op: ISD::BSWAP, VT, Action: Legal);
289 setOperationAction(Op: ISD::SADDSAT, VT, Action: Legal);
290 setOperationAction(Op: ISD::UADDSAT, VT, Action: Legal);
291 setOperationAction(Op: ISD::SSUBSAT, VT, Action: Legal);
292 setOperationAction(Op: ISD::USUBSAT, VT, Action: Legal);
293 setOperationAction(Op: ISD::ABDS, VT, Action: Legal);
294 setOperationAction(Op: ISD::ABDU, VT, Action: Legal);
295 setOperationAction(Op: ISD::AVGFLOORS, VT, Action: Legal);
296 setOperationAction(Op: ISD::AVGFLOORU, VT, Action: Legal);
297 setOperationAction(Op: ISD::AVGCEILS, VT, Action: Legal);
298 setOperationAction(Op: ISD::AVGCEILU, VT, Action: Legal);
299
300 // No native support for these.
301 setOperationAction(Op: ISD::UDIV, VT, Action: Expand);
302 setOperationAction(Op: ISD::SDIV, VT, Action: Expand);
303 setOperationAction(Op: ISD::UREM, VT, Action: Expand);
304 setOperationAction(Op: ISD::SREM, VT, Action: Expand);
305 setOperationAction(Op: ISD::UDIVREM, VT, Action: Expand);
306 setOperationAction(Op: ISD::SDIVREM, VT, Action: Expand);
307 setOperationAction(Op: ISD::CTPOP, VT, Action: Expand);
308 setOperationAction(Op: ISD::SELECT, VT, Action: Expand);
309 setOperationAction(Op: ISD::SELECT_CC, VT, Action: Expand);
310
311 // Vector reductions
312 setOperationAction(Op: ISD::VECREDUCE_ADD, VT, Action: Legal);
313 setOperationAction(Op: ISD::VECREDUCE_SMAX, VT, Action: Legal);
314 setOperationAction(Op: ISD::VECREDUCE_UMAX, VT, Action: Legal);
315 setOperationAction(Op: ISD::VECREDUCE_SMIN, VT, Action: Legal);
316 setOperationAction(Op: ISD::VECREDUCE_UMIN, VT, Action: Legal);
317 setOperationAction(Op: ISD::VECREDUCE_MUL, VT, Action: Custom);
318 setOperationAction(Op: ISD::VECREDUCE_AND, VT, Action: Custom);
319 setOperationAction(Op: ISD::VECREDUCE_OR, VT, Action: Custom);
320 setOperationAction(Op: ISD::VECREDUCE_XOR, VT, Action: Custom);
321
322 if (!HasMVEFP) {
323 setOperationAction(Op: ISD::SINT_TO_FP, VT, Action: Expand);
324 setOperationAction(Op: ISD::UINT_TO_FP, VT, Action: Expand);
325 setOperationAction(Op: ISD::FP_TO_SINT, VT, Action: Expand);
326 setOperationAction(Op: ISD::FP_TO_UINT, VT, Action: Expand);
327 } else {
328 setOperationAction(Op: ISD::FP_TO_SINT_SAT, VT, Action: Custom);
329 setOperationAction(Op: ISD::FP_TO_UINT_SAT, VT, Action: Custom);
330 }
331
332 // Pre and Post inc are supported on loads and stores
333 for (unsigned im = (unsigned)ISD::PRE_INC;
334 im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) {
335 setIndexedLoadAction(IdxModes: im, VT, Action: Legal);
336 setIndexedStoreAction(IdxModes: im, VT, Action: Legal);
337 setIndexedMaskedLoadAction(IdxMode: im, VT, Action: Legal);
338 setIndexedMaskedStoreAction(IdxMode: im, VT, Action: Legal);
339 }
340 }
341
342 const MVT FloatTypes[] = { MVT::v8f16, MVT::v4f32 };
343 for (auto VT : FloatTypes) {
344 addRegisterClass(VT, RC: &ARM::MQPRRegClass);
345 if (!HasMVEFP)
346 setAllExpand(VT);
347
348 // These are legal or custom whether we have MVE.fp or not
349 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT, Action: Custom);
350 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT, Action: Custom);
351 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: VT.getVectorElementType(), Action: Custom);
352 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT, Action: Custom);
353 setOperationAction(Op: ISD::BUILD_VECTOR, VT, Action: Custom);
354 setOperationAction(Op: ISD::BUILD_VECTOR, VT: VT.getVectorElementType(), Action: Custom);
355 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT, Action: Legal);
356 setOperationAction(Op: ISD::SETCC, VT, Action: Custom);
357 setOperationAction(Op: ISD::MLOAD, VT, Action: Custom);
358 setOperationAction(Op: ISD::MSTORE, VT, Action: Legal);
359 setOperationAction(Op: ISD::SELECT, VT, Action: Expand);
360 setOperationAction(Op: ISD::SELECT_CC, VT, Action: Expand);
361
362 // Pre and Post inc are supported on loads and stores
363 for (unsigned im = (unsigned)ISD::PRE_INC;
364 im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) {
365 setIndexedLoadAction(IdxModes: im, VT, Action: Legal);
366 setIndexedStoreAction(IdxModes: im, VT, Action: Legal);
367 setIndexedMaskedLoadAction(IdxMode: im, VT, Action: Legal);
368 setIndexedMaskedStoreAction(IdxMode: im, VT, Action: Legal);
369 }
370
371 if (HasMVEFP) {
372 setOperationAction(Op: ISD::FMINNUM, VT, Action: Legal);
373 setOperationAction(Op: ISD::FMAXNUM, VT, Action: Legal);
374 for (auto Op :
375 {ISD::FROUND, ISD::STRICT_FROUND, ISD::FROUNDEVEN,
376 ISD::STRICT_FROUNDEVEN, ISD::FTRUNC, ISD::STRICT_FTRUNC, ISD::FRINT,
377 ISD::FFLOOR, ISD::STRICT_FFLOOR, ISD::FCEIL, ISD::STRICT_FCEIL}) {
378 setOperationAction(Op, VT, Action: Legal);
379 }
380 setOperationAction(Op: ISD::VECREDUCE_FADD, VT, Action: Custom);
381 setOperationAction(Op: ISD::VECREDUCE_FMUL, VT, Action: Custom);
382 setOperationAction(Op: ISD::VECREDUCE_FMIN, VT, Action: Custom);
383 setOperationAction(Op: ISD::VECREDUCE_FMAX, VT, Action: Custom);
384
385 // No native support for these.
386 setOperationAction(Op: ISD::FDIV, VT, Action: Expand);
387 setOperationAction(Op: ISD::FREM, VT, Action: Expand);
388 setOperationAction(Op: ISD::FSQRT, VT, Action: Expand);
389 setOperationAction(Op: ISD::FSIN, VT, Action: Expand);
390 setOperationAction(Op: ISD::FCOS, VT, Action: Expand);
391 setOperationAction(Op: ISD::FTAN, VT, Action: Expand);
392 setOperationAction(Op: ISD::FPOW, VT, Action: Expand);
393 setOperationAction(Op: ISD::FLOG, VT, Action: Expand);
394 setOperationAction(Op: ISD::FLOG2, VT, Action: Expand);
395 setOperationAction(Op: ISD::FLOG10, VT, Action: Expand);
396 setOperationAction(Op: ISD::FEXP, VT, Action: Expand);
397 setOperationAction(Op: ISD::FEXP2, VT, Action: Expand);
398 setOperationAction(Op: ISD::FEXP10, VT, Action: Expand);
399 setOperationAction(Op: ISD::FNEARBYINT, VT, Action: Expand);
400 }
401 }
402
403 // Custom Expand smaller than legal vector reductions to prevent false zero
404 // items being added.
405 setOperationAction(Op: ISD::VECREDUCE_FADD, VT: MVT::v4f16, Action: Custom);
406 setOperationAction(Op: ISD::VECREDUCE_FMUL, VT: MVT::v4f16, Action: Custom);
407 setOperationAction(Op: ISD::VECREDUCE_FMIN, VT: MVT::v4f16, Action: Custom);
408 setOperationAction(Op: ISD::VECREDUCE_FMAX, VT: MVT::v4f16, Action: Custom);
409 setOperationAction(Op: ISD::VECREDUCE_FADD, VT: MVT::v2f16, Action: Custom);
410 setOperationAction(Op: ISD::VECREDUCE_FMUL, VT: MVT::v2f16, Action: Custom);
411 setOperationAction(Op: ISD::VECREDUCE_FMIN, VT: MVT::v2f16, Action: Custom);
412 setOperationAction(Op: ISD::VECREDUCE_FMAX, VT: MVT::v2f16, Action: Custom);
413
414 // We 'support' these types up to bitcast/load/store level, regardless of
415 // MVE integer-only / float support. Only doing FP data processing on the FP
416 // vector types is inhibited at integer-only level.
417 const MVT LongTypes[] = { MVT::v2i64, MVT::v2f64 };
418 for (auto VT : LongTypes) {
419 addRegisterClass(VT, RC: &ARM::MQPRRegClass);
420 setAllExpand(VT);
421 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT, Action: Custom);
422 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT, Action: Custom);
423 setOperationAction(Op: ISD::BUILD_VECTOR, VT, Action: Custom);
424 setOperationAction(Op: ISD::VSELECT, VT, Action: Legal);
425 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT, Action: Custom);
426 }
427 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v2f64, Action: Legal);
428
429 // We can do bitwise operations on v2i64 vectors
430 setOperationAction(Op: ISD::AND, VT: MVT::v2i64, Action: Legal);
431 setOperationAction(Op: ISD::OR, VT: MVT::v2i64, Action: Legal);
432 setOperationAction(Op: ISD::XOR, VT: MVT::v2i64, Action: Legal);
433
434 // It is legal to extload from v4i8 to v4i16 or v4i32.
435 addAllExtLoads(From: MVT::v8i16, To: MVT::v8i8, Action: Legal);
436 addAllExtLoads(From: MVT::v4i32, To: MVT::v4i16, Action: Legal);
437 addAllExtLoads(From: MVT::v4i32, To: MVT::v4i8, Action: Legal);
438
439 // It is legal to sign extend from v4i8/v4i16 to v4i32 or v8i8 to v8i16.
440 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::v4i8, Action: Legal);
441 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::v4i16, Action: Legal);
442 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::v4i32, Action: Legal);
443 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::v8i8, Action: Legal);
444 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::v8i16, Action: Legal);
445
446 // Some truncating stores are legal too.
447 setTruncStoreAction(ValVT: MVT::v4i32, MemVT: MVT::v4i16, Action: Legal);
448 setTruncStoreAction(ValVT: MVT::v4i32, MemVT: MVT::v4i8, Action: Legal);
449 setTruncStoreAction(ValVT: MVT::v8i16, MemVT: MVT::v8i8, Action: Legal);
450
451 // Pre and Post inc on these are legal, given the correct extends
452 for (unsigned im = (unsigned)ISD::PRE_INC;
453 im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) {
454 for (auto VT : {MVT::v8i8, MVT::v4i8, MVT::v4i16}) {
455 setIndexedLoadAction(IdxModes: im, VT, Action: Legal);
456 setIndexedStoreAction(IdxModes: im, VT, Action: Legal);
457 setIndexedMaskedLoadAction(IdxMode: im, VT, Action: Legal);
458 setIndexedMaskedStoreAction(IdxMode: im, VT, Action: Legal);
459 }
460 }
461
462 // Predicate types
463 const MVT pTypes[] = {MVT::v16i1, MVT::v8i1, MVT::v4i1, MVT::v2i1};
464 for (auto VT : pTypes) {
465 addRegisterClass(VT, RC: &ARM::VCCRRegClass);
466 setOperationAction(Op: ISD::BUILD_VECTOR, VT, Action: Custom);
467 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT, Action: Custom);
468 setOperationAction(Op: ISD::EXTRACT_SUBVECTOR, VT, Action: Custom);
469 setOperationAction(Op: ISD::CONCAT_VECTORS, VT, Action: Custom);
470 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT, Action: Custom);
471 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT, Action: Custom);
472 setOperationAction(Op: ISD::SETCC, VT, Action: Custom);
473 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT, Action: Expand);
474 setOperationAction(Op: ISD::LOAD, VT, Action: Custom);
475 setOperationAction(Op: ISD::STORE, VT, Action: Custom);
476 setOperationAction(Op: ISD::TRUNCATE, VT, Action: Custom);
477 setOperationAction(Op: ISD::VSELECT, VT, Action: Expand);
478 setOperationAction(Op: ISD::SELECT, VT, Action: Expand);
479 setOperationAction(Op: ISD::SELECT_CC, VT, Action: Expand);
480
481 if (!HasMVEFP) {
482 setOperationAction(Op: ISD::SINT_TO_FP, VT, Action: Expand);
483 setOperationAction(Op: ISD::UINT_TO_FP, VT, Action: Expand);
484 setOperationAction(Op: ISD::FP_TO_SINT, VT, Action: Expand);
485 setOperationAction(Op: ISD::FP_TO_UINT, VT, Action: Expand);
486 }
487 }
488 setOperationAction(Op: ISD::SETCC, VT: MVT::v2i1, Action: Expand);
489 setOperationAction(Op: ISD::TRUNCATE, VT: MVT::v2i1, Action: Expand);
490 setOperationAction(Op: ISD::AND, VT: MVT::v2i1, Action: Expand);
491 setOperationAction(Op: ISD::OR, VT: MVT::v2i1, Action: Expand);
492 setOperationAction(Op: ISD::XOR, VT: MVT::v2i1, Action: Expand);
493 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v2i1, Action: Expand);
494 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v2i1, Action: Expand);
495 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::v2i1, Action: Expand);
496 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::v2i1, Action: Expand);
497
498 setOperationAction(Op: ISD::SIGN_EXTEND, VT: MVT::v8i32, Action: Custom);
499 setOperationAction(Op: ISD::SIGN_EXTEND, VT: MVT::v16i16, Action: Custom);
500 setOperationAction(Op: ISD::SIGN_EXTEND, VT: MVT::v16i32, Action: Custom);
501 setOperationAction(Op: ISD::ZERO_EXTEND, VT: MVT::v8i32, Action: Custom);
502 setOperationAction(Op: ISD::ZERO_EXTEND, VT: MVT::v16i16, Action: Custom);
503 setOperationAction(Op: ISD::ZERO_EXTEND, VT: MVT::v16i32, Action: Custom);
504 setOperationAction(Op: ISD::TRUNCATE, VT: MVT::v8i32, Action: Custom);
505 setOperationAction(Op: ISD::TRUNCATE, VT: MVT::v16i16, Action: Custom);
506}
507
508static bool hasNativeAtomics(const ARMSubtarget &ST) {
509 return ST.hasAnyDataBarrier() && (!ST.isThumb() || ST.hasV8MBaselineOps());
510}
511
512const ARMBaseTargetMachine &ARMTargetLowering::getTM() const {
513 return static_cast<const ARMBaseTargetMachine &>(getTargetMachine());
514}
515
516ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM_,
517 const ARMSubtarget &STI)
518 : TargetLowering(TM_, STI), Subtarget(&STI),
519 RegInfo(Subtarget->getRegisterInfo()),
520 Itins(Subtarget->getInstrItineraryData()) {
521 const auto &TM = static_cast<const ARMBaseTargetMachine &>(TM_);
522
523 setBooleanContents(ZeroOrOneBooleanContent);
524 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
525
526 const Triple &TT = TM.getTargetTriple();
527
528 if (Subtarget->isThumb1Only())
529 addRegisterClass(VT: MVT::i32, RC: &ARM::tGPRRegClass);
530 else
531 addRegisterClass(VT: MVT::i32, RC: &ARM::GPRRegClass);
532
533 if (!Subtarget->useSoftFloat() && !Subtarget->isThumb1Only() &&
534 Subtarget->hasFPRegs()) {
535 addRegisterClass(VT: MVT::f32, RC: &ARM::SPRRegClass);
536 addRegisterClass(VT: MVT::f64, RC: &ARM::DPRRegClass);
537
538 if (!Subtarget->hasVFP2Base()) {
539 setAllExpand(MVT::f32);
540 } else {
541 setOperationAction(Op: ISD::FP_TO_SINT_SAT, VT: MVT::i32, Action: Custom);
542 setOperationAction(Op: ISD::FP_TO_UINT_SAT, VT: MVT::i32, Action: Custom);
543
544 for (auto Op : {ISD::STRICT_FADD, ISD::STRICT_FSUB, ISD::STRICT_FMUL,
545 ISD::STRICT_FDIV, ISD::STRICT_FMA, ISD::STRICT_FSQRT})
546 setOperationAction(Op, VT: MVT::f32, Action: Legal);
547 }
548 if (!Subtarget->hasFP64()) {
549 setAllExpand(MVT::f64);
550 } else {
551 for (auto Op : {ISD::STRICT_FADD, ISD::STRICT_FSUB, ISD::STRICT_FMUL,
552 ISD::STRICT_FDIV, ISD::STRICT_FMA, ISD::STRICT_FSQRT})
553 setOperationAction(Op, VT: MVT::f64, Action: Legal);
554
555 setOperationAction(Op: ISD::STRICT_FP_ROUND, VT: MVT::f32, Action: Legal);
556 }
557 }
558
559 if (Subtarget->hasFullFP16()) {
560 for (auto Op : {ISD::STRICT_FADD, ISD::STRICT_FSUB, ISD::STRICT_FMUL,
561 ISD::STRICT_FDIV, ISD::STRICT_FMA, ISD::STRICT_FSQRT})
562 setOperationAction(Op, VT: MVT::f16, Action: Legal);
563
564 addRegisterClass(VT: MVT::f16, RC: &ARM::HPRRegClass);
565 setOperationAction(Op: ISD::BITCAST, VT: MVT::i16, Action: Custom);
566 setOperationAction(Op: ISD::BITCAST, VT: MVT::f16, Action: Custom);
567
568 setOperationAction(Op: ISD::FMINNUM, VT: MVT::f16, Action: Legal);
569 setOperationAction(Op: ISD::FMAXNUM, VT: MVT::f16, Action: Legal);
570 setOperationAction(Op: ISD::STRICT_FMINNUM, VT: MVT::f16, Action: Legal);
571 setOperationAction(Op: ISD::STRICT_FMAXNUM, VT: MVT::f16, Action: Legal);
572 }
573
574 if (Subtarget->hasBF16()) {
575 addRegisterClass(VT: MVT::bf16, RC: &ARM::HPRRegClass);
576 setAllExpand(MVT::bf16);
577 if (!Subtarget->hasFullFP16()) {
578 setOperationAction(Op: ISD::BITCAST, VT: MVT::bf16, Action: Custom);
579 setOperationAction(Op: ISD::BITCAST, VT: MVT::f16, Action: Custom);
580 setOperationAction(Op: ISD::BITCAST, VT: MVT::i16, Action: Custom);
581 }
582 setOperationAction(Op: ISD::FP_ROUND, VT: MVT::bf16, Action: Custom);
583 setOperationAction(Op: ISD::STRICT_FP_ROUND, VT: MVT::bf16, Action: Custom);
584 setOperationAction(Op: ISD::BF16_TO_FP, VT: MVT::f32, Action: Expand);
585 setOperationAction(Op: ISD::BF16_TO_FP, VT: MVT::f64, Action: Expand);
586 } else {
587 setOperationAction(Op: ISD::BF16_TO_FP, VT: MVT::f32, Action: Expand);
588 setOperationAction(Op: ISD::BF16_TO_FP, VT: MVT::f64, Action: Expand);
589 setOperationAction(Op: ISD::FP_TO_BF16, VT: MVT::f32, Action: Custom);
590 setOperationAction(Op: ISD::FP_TO_BF16, VT: MVT::f64, Action: Custom);
591 }
592
593 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
594 for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) {
595 setTruncStoreAction(ValVT: VT, MemVT: InnerVT, Action: Expand);
596 addAllExtLoads(From: VT, To: InnerVT, Action: Expand);
597 }
598
599 setOperationAction(Op: ISD::SMUL_LOHI, VT, Action: Expand);
600 setOperationAction(Op: ISD::UMUL_LOHI, VT, Action: Expand);
601
602 setOperationAction(Op: ISD::BSWAP, VT, Action: Expand);
603 }
604
605 if (!Subtarget->isThumb1Only() && !Subtarget->hasV8_1MMainlineOps())
606 setOperationAction(Op: ISD::SCMP, VT: MVT::i32, Action: Custom);
607
608 if (!Subtarget->hasV8_1MMainlineOps())
609 setOperationAction(Op: ISD::UCMP, VT: MVT::i32, Action: Custom);
610
611 if (!Subtarget->isThumb1Only())
612 setOperationAction(Op: ISD::ABS, VT: MVT::i32, Action: Custom);
613
614 setOperationAction(Op: ISD::ConstantFP, VT: MVT::f32, Action: Custom);
615 setOperationAction(Op: ISD::ConstantFP, VT: MVT::f64, Action: Custom);
616
617 setOperationAction(Op: ISD::READ_REGISTER, VT: MVT::i64, Action: Custom);
618 setOperationAction(Op: ISD::WRITE_REGISTER, VT: MVT::i64, Action: Custom);
619
620 if (Subtarget->hasMVEIntegerOps())
621 addMVEVectorTypes(HasMVEFP: Subtarget->hasMVEFloatOps());
622
623 // Combine low-overhead loop intrinsics so that we can lower i1 types.
624 if (Subtarget->hasLOB()) {
625 setTargetDAGCombine({ISD::BRCOND, ISD::BR_CC});
626 }
627
628 if (Subtarget->hasNEON()) {
629 addDRTypeForNEON(VT: MVT::v2f32);
630 addDRTypeForNEON(VT: MVT::v8i8);
631 addDRTypeForNEON(VT: MVT::v4i16);
632 addDRTypeForNEON(VT: MVT::v2i32);
633 addDRTypeForNEON(VT: MVT::v1i64);
634
635 addQRTypeForNEON(VT: MVT::v4f32);
636 addQRTypeForNEON(VT: MVT::v2f64);
637 addQRTypeForNEON(VT: MVT::v16i8);
638 addQRTypeForNEON(VT: MVT::v8i16);
639 addQRTypeForNEON(VT: MVT::v4i32);
640 addQRTypeForNEON(VT: MVT::v2i64);
641
642 if (Subtarget->hasFullFP16()) {
643 addQRTypeForNEON(VT: MVT::v8f16);
644 addDRTypeForNEON(VT: MVT::v4f16);
645 }
646
647 if (Subtarget->hasBF16()) {
648 addQRTypeForNEON(VT: MVT::v8bf16);
649 addDRTypeForNEON(VT: MVT::v4bf16);
650 }
651 }
652
653 if (Subtarget->hasMVEIntegerOps() || Subtarget->hasNEON()) {
654 // v2f64 is legal so that QR subregs can be extracted as f64 elements, but
655 // none of Neon, MVE or VFP supports any arithmetic operations on it.
656 setOperationAction(Op: ISD::FADD, VT: MVT::v2f64, Action: Expand);
657 setOperationAction(Op: ISD::FSUB, VT: MVT::v2f64, Action: Expand);
658 setOperationAction(Op: ISD::FMUL, VT: MVT::v2f64, Action: Expand);
659 // FIXME: Code duplication: FDIV and FREM are expanded always, see
660 // ARMTargetLowering::addTypeForNEON method for details.
661 setOperationAction(Op: ISD::FDIV, VT: MVT::v2f64, Action: Expand);
662 setOperationAction(Op: ISD::FREM, VT: MVT::v2f64, Action: Expand);
663 // FIXME: Create unittest.
664 // In another words, find a way when "copysign" appears in DAG with vector
665 // operands.
666 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::v2f64, Action: Expand);
667 // FIXME: Code duplication: SETCC has custom operation action, see
668 // ARMTargetLowering::addTypeForNEON method for details.
669 setOperationAction(Op: ISD::SETCC, VT: MVT::v2f64, Action: Expand);
670 // FIXME: Create unittest for FNEG and for FABS.
671 setOperationAction(Op: ISD::FNEG, VT: MVT::v2f64, Action: Expand);
672 setOperationAction(Op: ISD::FABS, VT: MVT::v2f64, Action: Expand);
673 setOperationAction(Op: ISD::FSQRT, VT: MVT::v2f64, Action: Expand);
674 setOperationAction(Op: ISD::FSIN, VT: MVT::v2f64, Action: Expand);
675 setOperationAction(Op: ISD::FCOS, VT: MVT::v2f64, Action: Expand);
676 setOperationAction(Op: ISD::FTAN, VT: MVT::v2f64, Action: Expand);
677 setOperationAction(Op: ISD::FPOW, VT: MVT::v2f64, Action: Expand);
678 setOperationAction(Op: ISD::FLOG, VT: MVT::v2f64, Action: Expand);
679 setOperationAction(Op: ISD::FLOG2, VT: MVT::v2f64, Action: Expand);
680 setOperationAction(Op: ISD::FLOG10, VT: MVT::v2f64, Action: Expand);
681 setOperationAction(Op: ISD::FEXP, VT: MVT::v2f64, Action: Expand);
682 setOperationAction(Op: ISD::FEXP2, VT: MVT::v2f64, Action: Expand);
683 setOperationAction(Op: ISD::FEXP10, VT: MVT::v2f64, Action: Expand);
684 setOperationAction(Op: ISD::FCEIL, VT: MVT::v2f64, Action: Expand);
685 setOperationAction(Op: ISD::FTRUNC, VT: MVT::v2f64, Action: Expand);
686 setOperationAction(Op: ISD::FRINT, VT: MVT::v2f64, Action: Expand);
687 setOperationAction(Op: ISD::FROUNDEVEN, VT: MVT::v2f64, Action: Expand);
688 setOperationAction(Op: ISD::FNEARBYINT, VT: MVT::v2f64, Action: Expand);
689 setOperationAction(Op: ISD::FFLOOR, VT: MVT::v2f64, Action: Expand);
690 setOperationAction(Op: ISD::FMA, VT: MVT::v2f64, Action: Expand);
691 }
692
693 if (Subtarget->hasNEON()) {
694 // The same with v4f32. But keep in mind that vadd, vsub, vmul are natively
695 // supported for v4f32.
696 setOperationAction(Op: ISD::FSQRT, VT: MVT::v4f32, Action: Expand);
697 setOperationAction(Op: ISD::FSIN, VT: MVT::v4f32, Action: Expand);
698 setOperationAction(Op: ISD::FCOS, VT: MVT::v4f32, Action: Expand);
699 setOperationAction(Op: ISD::FTAN, VT: MVT::v4f32, Action: Expand);
700 setOperationAction(Op: ISD::FPOW, VT: MVT::v4f32, Action: Expand);
701 setOperationAction(Op: ISD::FLOG, VT: MVT::v4f32, Action: Expand);
702 setOperationAction(Op: ISD::FLOG2, VT: MVT::v4f32, Action: Expand);
703 setOperationAction(Op: ISD::FLOG10, VT: MVT::v4f32, Action: Expand);
704 setOperationAction(Op: ISD::FEXP, VT: MVT::v4f32, Action: Expand);
705 setOperationAction(Op: ISD::FEXP2, VT: MVT::v4f32, Action: Expand);
706 setOperationAction(Op: ISD::FEXP10, VT: MVT::v4f32, Action: Expand);
707 setOperationAction(Op: ISD::FCEIL, VT: MVT::v4f32, Action: Expand);
708 setOperationAction(Op: ISD::FTRUNC, VT: MVT::v4f32, Action: Expand);
709 setOperationAction(Op: ISD::FRINT, VT: MVT::v4f32, Action: Expand);
710 setOperationAction(Op: ISD::FROUNDEVEN, VT: MVT::v4f32, Action: Expand);
711 setOperationAction(Op: ISD::FNEARBYINT, VT: MVT::v4f32, Action: Expand);
712 setOperationAction(Op: ISD::FFLOOR, VT: MVT::v4f32, Action: Expand);
713
714 // Mark v2f32 intrinsics.
715 setOperationAction(Op: ISD::FSQRT, VT: MVT::v2f32, Action: Expand);
716 setOperationAction(Op: ISD::FSIN, VT: MVT::v2f32, Action: Expand);
717 setOperationAction(Op: ISD::FCOS, VT: MVT::v2f32, Action: Expand);
718 setOperationAction(Op: ISD::FTAN, VT: MVT::v2f32, Action: Expand);
719 setOperationAction(Op: ISD::FPOW, VT: MVT::v2f32, Action: Expand);
720 setOperationAction(Op: ISD::FLOG, VT: MVT::v2f32, Action: Expand);
721 setOperationAction(Op: ISD::FLOG2, VT: MVT::v2f32, Action: Expand);
722 setOperationAction(Op: ISD::FLOG10, VT: MVT::v2f32, Action: Expand);
723 setOperationAction(Op: ISD::FEXP, VT: MVT::v2f32, Action: Expand);
724 setOperationAction(Op: ISD::FEXP2, VT: MVT::v2f32, Action: Expand);
725 setOperationAction(Op: ISD::FEXP10, VT: MVT::v2f32, Action: Expand);
726 setOperationAction(Op: ISD::FCEIL, VT: MVT::v2f32, Action: Expand);
727 setOperationAction(Op: ISD::FTRUNC, VT: MVT::v2f32, Action: Expand);
728 setOperationAction(Op: ISD::FRINT, VT: MVT::v2f32, Action: Expand);
729 setOperationAction(Op: ISD::FROUNDEVEN, VT: MVT::v2f32, Action: Expand);
730 setOperationAction(Op: ISD::FNEARBYINT, VT: MVT::v2f32, Action: Expand);
731 setOperationAction(Op: ISD::FFLOOR, VT: MVT::v2f32, Action: Expand);
732
733 for (ISD::NodeType Op : {ISD::FFLOOR, ISD::FNEARBYINT, ISD::FCEIL,
734 ISD::FRINT, ISD::FTRUNC, ISD::FROUNDEVEN}) {
735 setOperationAction(Op, VT: MVT::v4f16, Action: Expand);
736 setOperationAction(Op, VT: MVT::v8f16, Action: Expand);
737 }
738
739 // Neon does not support some operations on v1i64 and v2i64 types.
740 setOperationAction(Op: ISD::MUL, VT: MVT::v1i64, Action: Expand);
741 // Custom handling for some quad-vector types to detect VMULL.
742 setOperationAction(Op: ISD::MUL, VT: MVT::v8i16, Action: Custom);
743 setOperationAction(Op: ISD::MUL, VT: MVT::v4i32, Action: Custom);
744 setOperationAction(Op: ISD::MUL, VT: MVT::v2i64, Action: Custom);
745 // Custom handling for some vector types to avoid expensive expansions
746 setOperationAction(Op: ISD::SDIV, VT: MVT::v4i16, Action: Custom);
747 setOperationAction(Op: ISD::SDIV, VT: MVT::v8i8, Action: Custom);
748 setOperationAction(Op: ISD::UDIV, VT: MVT::v4i16, Action: Custom);
749 setOperationAction(Op: ISD::UDIV, VT: MVT::v8i8, Action: Custom);
750 // Neon does not have single instruction SINT_TO_FP and UINT_TO_FP with
751 // a destination type that is wider than the source, and nor does
752 // it have a FP_TO_[SU]INT instruction with a narrower destination than
753 // source.
754 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v4i16, Action: Custom);
755 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v8i16, Action: Custom);
756 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v4i16, Action: Custom);
757 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v8i16, Action: Custom);
758 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::v4i16, Action: Custom);
759 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::v8i16, Action: Custom);
760 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::v4i16, Action: Custom);
761 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::v8i16, Action: Custom);
762
763 setOperationAction(Op: ISD::FP_ROUND, VT: MVT::v2f32, Action: Expand);
764 setOperationAction(Op: ISD::FP_EXTEND, VT: MVT::v2f64, Action: Expand);
765
766 // NEON does not have single instruction CTPOP for vectors with element
767 // types wider than 8-bits. However, custom lowering can leverage the
768 // v8i8/v16i8 vcnt instruction.
769 setOperationAction(Op: ISD::CTPOP, VT: MVT::v2i32, Action: Custom);
770 setOperationAction(Op: ISD::CTPOP, VT: MVT::v4i32, Action: Custom);
771 setOperationAction(Op: ISD::CTPOP, VT: MVT::v4i16, Action: Custom);
772 setOperationAction(Op: ISD::CTPOP, VT: MVT::v8i16, Action: Custom);
773 setOperationAction(Op: ISD::CTPOP, VT: MVT::v1i64, Action: Custom);
774 setOperationAction(Op: ISD::CTPOP, VT: MVT::v2i64, Action: Custom);
775
776 setOperationAction(Op: ISD::CTLZ, VT: MVT::v1i64, Action: Expand);
777 setOperationAction(Op: ISD::CTLZ, VT: MVT::v2i64, Action: Expand);
778
779 // NEON does not have single instruction CTTZ for vectors.
780 setOperationAction(Op: ISD::CTTZ, VT: MVT::v8i8, Action: Custom);
781 setOperationAction(Op: ISD::CTTZ, VT: MVT::v4i16, Action: Custom);
782 setOperationAction(Op: ISD::CTTZ, VT: MVT::v2i32, Action: Custom);
783 setOperationAction(Op: ISD::CTTZ, VT: MVT::v1i64, Action: Custom);
784
785 setOperationAction(Op: ISD::CTTZ, VT: MVT::v16i8, Action: Custom);
786 setOperationAction(Op: ISD::CTTZ, VT: MVT::v8i16, Action: Custom);
787 setOperationAction(Op: ISD::CTTZ, VT: MVT::v4i32, Action: Custom);
788 setOperationAction(Op: ISD::CTTZ, VT: MVT::v2i64, Action: Custom);
789
790 setOperationAction(Op: ISD::CTTZ_ZERO_POISON, VT: MVT::v8i8, Action: Custom);
791 setOperationAction(Op: ISD::CTTZ_ZERO_POISON, VT: MVT::v4i16, Action: Custom);
792 setOperationAction(Op: ISD::CTTZ_ZERO_POISON, VT: MVT::v2i32, Action: Custom);
793 setOperationAction(Op: ISD::CTTZ_ZERO_POISON, VT: MVT::v1i64, Action: Custom);
794
795 setOperationAction(Op: ISD::CTTZ_ZERO_POISON, VT: MVT::v16i8, Action: Custom);
796 setOperationAction(Op: ISD::CTTZ_ZERO_POISON, VT: MVT::v8i16, Action: Custom);
797 setOperationAction(Op: ISD::CTTZ_ZERO_POISON, VT: MVT::v4i32, Action: Custom);
798 setOperationAction(Op: ISD::CTTZ_ZERO_POISON, VT: MVT::v2i64, Action: Custom);
799
800 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
801 setOperationAction(Op: ISD::MULHS, VT, Action: Expand);
802 setOperationAction(Op: ISD::MULHU, VT, Action: Expand);
803 }
804
805 // NEON only has FMA instructions as of VFP4.
806 if (!Subtarget->hasVFP4Base()) {
807 setOperationAction(Op: ISD::FMA, VT: MVT::v2f32, Action: Expand);
808 setOperationAction(Op: ISD::FMA, VT: MVT::v4f32, Action: Expand);
809 }
810
811 setTargetDAGCombine({ISD::SHL, ISD::SRL, ISD::SRA, ISD::FP_TO_SINT,
812 ISD::FP_TO_UINT, ISD::FMUL, ISD::LOAD});
813
814 // It is legal to extload from v4i8 to v4i16 or v4i32.
815 for (MVT Ty : {MVT::v8i8, MVT::v4i8, MVT::v2i8, MVT::v4i16, MVT::v2i16,
816 MVT::v2i32}) {
817 for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) {
818 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: Ty, Action: Legal);
819 setLoadExtAction(ExtType: ISD::ZEXTLOAD, ValVT: VT, MemVT: Ty, Action: Legal);
820 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: VT, MemVT: Ty, Action: Legal);
821 }
822 }
823
824 for (auto VT : {MVT::v8i8, MVT::v4i16, MVT::v2i32, MVT::v16i8, MVT::v8i16,
825 MVT::v4i32}) {
826 setOperationAction(Op: ISD::VECREDUCE_SMAX, VT, Action: Custom);
827 setOperationAction(Op: ISD::VECREDUCE_UMAX, VT, Action: Custom);
828 setOperationAction(Op: ISD::VECREDUCE_SMIN, VT, Action: Custom);
829 setOperationAction(Op: ISD::VECREDUCE_UMIN, VT, Action: Custom);
830 }
831 }
832
833 if (Subtarget->hasNEON() || Subtarget->hasMVEIntegerOps()) {
834 setTargetDAGCombine(
835 {ISD::BUILD_VECTOR, ISD::VECTOR_SHUFFLE, ISD::VECTOR_DEINTERLEAVE,
836 ISD::INSERT_SUBVECTOR, ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT,
837 ISD::SIGN_EXTEND_INREG, ISD::STORE, ISD::SIGN_EXTEND, ISD::ZERO_EXTEND,
838 ISD::ANY_EXTEND, ISD::INTRINSIC_WO_CHAIN, ISD::INTRINSIC_W_CHAIN,
839 ISD::INTRINSIC_VOID, ISD::VECREDUCE_ADD, ISD::ADD, ISD::BITCAST});
840 }
841 if (Subtarget->hasMVEIntegerOps()) {
842 setTargetDAGCombine({ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX,
843 ISD::FP_EXTEND, ISD::SELECT, ISD::SELECT_CC,
844 ISD::SETCC});
845 }
846 if (Subtarget->hasMVEFloatOps()) {
847 setTargetDAGCombine(ISD::FADD);
848 }
849
850 if (!Subtarget->hasFP64()) {
851 // When targeting a floating-point unit with only single-precision
852 // operations, f64 is legal for the few double-precision instructions which
853 // are present However, no double-precision operations other than moves,
854 // loads and stores are provided by the hardware.
855 setOperationAction(Op: ISD::FADD, VT: MVT::f64, Action: Expand);
856 setOperationAction(Op: ISD::FSUB, VT: MVT::f64, Action: Expand);
857 setOperationAction(Op: ISD::FMUL, VT: MVT::f64, Action: Expand);
858 setOperationAction(Op: ISD::FMA, VT: MVT::f64, Action: Expand);
859 setOperationAction(Op: ISD::FDIV, VT: MVT::f64, Action: Expand);
860 setOperationAction(Op: ISD::FREM, VT: MVT::f64, Action: LibCall);
861 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::f64, Action: Expand);
862 setOperationAction(Op: ISD::FGETSIGN, VT: MVT::f64, Action: Expand);
863 setOperationAction(Op: ISD::FNEG, VT: MVT::f64, Action: Expand);
864 setOperationAction(Op: ISD::FABS, VT: MVT::f64, Action: Expand);
865 setOperationAction(Op: ISD::FSQRT, VT: MVT::f64, Action: Expand);
866 setOperationAction(Op: ISD::FSIN, VT: MVT::f64, Action: Expand);
867 setOperationAction(Op: ISD::FCOS, VT: MVT::f64, Action: Expand);
868 setOperationAction(Op: ISD::FPOW, VT: MVT::f64, Action: Expand);
869 setOperationAction(Op: ISD::FLOG, VT: MVT::f64, Action: Expand);
870 setOperationAction(Op: ISD::FLOG2, VT: MVT::f64, Action: Expand);
871 setOperationAction(Op: ISD::FLOG10, VT: MVT::f64, Action: Expand);
872 setOperationAction(Op: ISD::FEXP, VT: MVT::f64, Action: Expand);
873 setOperationAction(Op: ISD::FEXP2, VT: MVT::f64, Action: Expand);
874 setOperationAction(Op: ISD::FEXP10, VT: MVT::f64, Action: Expand);
875 setOperationAction(Op: ISD::FCEIL, VT: MVT::f64, Action: Expand);
876 setOperationAction(Op: ISD::FTRUNC, VT: MVT::f64, Action: Expand);
877 setOperationAction(Op: ISD::FRINT, VT: MVT::f64, Action: Expand);
878 setOperationAction(Op: ISD::FROUNDEVEN, VT: MVT::f64, Action: Expand);
879 setOperationAction(Op: ISD::FNEARBYINT, VT: MVT::f64, Action: Expand);
880 setOperationAction(Op: ISD::FFLOOR, VT: MVT::f64, Action: Expand);
881 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i32, Action: Custom);
882 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::i32, Action: Custom);
883 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::i32, Action: Custom);
884 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i32, Action: Custom);
885 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::f64, Action: Custom);
886 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::f64, Action: Custom);
887 setOperationAction(Op: ISD::FP_ROUND, VT: MVT::f32, Action: Custom);
888 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::f64, Action: Custom);
889 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::f64, Action: Custom);
890 setOperationAction(Op: ISD::STRICT_FP_ROUND, VT: MVT::f32, Action: Custom);
891 }
892
893 // STRICT_(U/S)INT_TO_FP specifically use the input MVT to register with
894 // setOperationAction() as opposed to other opcodes that use the output MVT
895 // All inputs should be i32 due to type legalization
896 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::i32, Action: Custom);
897 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::i32, Action: Custom);
898
899 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::i32, Action: Custom);
900 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::i32, Action: Custom);
901
902 if (!Subtarget->hasFP64() || !Subtarget->hasFPARMv8Base()) {
903 if (Subtarget->hasFullFP16()) {
904 setOperationAction(Op: ISD::FP_ROUND, VT: MVT::f16, Action: Custom);
905 setOperationAction(Op: ISD::STRICT_FP_ROUND, VT: MVT::f16, Action: Custom);
906 }
907 }
908
909 if (Subtarget->hasFP16()) {
910 setOperationAction(Op: ISD::STRICT_FP_ROUND, VT: MVT::f16, Action: Legal);
911 }
912
913 setOperationAction(Op: ISD::FP_EXTEND, VT: MVT::f32, Action: Custom);
914 setOperationAction(Op: ISD::FP_EXTEND, VT: MVT::f64, Action: Custom);
915 setOperationAction(Op: ISD::STRICT_FP_EXTEND, VT: MVT::f64, Action: Custom);
916 setOperationAction(Op: ISD::STRICT_FP_EXTEND, VT: MVT::f32, Action: Custom);
917
918 computeRegisterProperties(TRI: Subtarget->getRegisterInfo());
919
920 // ARM does not have floating-point extending loads.
921 for (MVT VT : MVT::fp_valuetypes()) {
922 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: MVT::f32, Action: Expand);
923 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: MVT::f16, Action: Expand);
924 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: MVT::bf16, Action: Expand);
925 }
926
927 // ... or truncating stores
928 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f32, Action: Expand);
929 setTruncStoreAction(ValVT: MVT::f32, MemVT: MVT::f16, Action: Expand);
930 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f16, Action: Expand);
931 setTruncStoreAction(ValVT: MVT::f32, MemVT: MVT::bf16, Action: Expand);
932 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::bf16, Action: Expand);
933
934 // ARM does not have i1 sign extending load.
935 for (MVT VT : MVT::integer_valuetypes())
936 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: VT, MemVT: MVT::i1, Action: Promote);
937
938 // ARM supports all 4 flavors of integer indexed load / store.
939 if (!Subtarget->isThumb1Only()) {
940 for (unsigned im = (unsigned)ISD::PRE_INC;
941 im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) {
942 setIndexedLoadAction(IdxModes: im, VT: MVT::i1, Action: Legal);
943 setIndexedLoadAction(IdxModes: im, VT: MVT::i8, Action: Legal);
944 setIndexedLoadAction(IdxModes: im, VT: MVT::i16, Action: Legal);
945 setIndexedLoadAction(IdxModes: im, VT: MVT::i32, Action: Legal);
946 setIndexedStoreAction(IdxModes: im, VT: MVT::i1, Action: Legal);
947 setIndexedStoreAction(IdxModes: im, VT: MVT::i8, Action: Legal);
948 setIndexedStoreAction(IdxModes: im, VT: MVT::i16, Action: Legal);
949 setIndexedStoreAction(IdxModes: im, VT: MVT::i32, Action: Legal);
950 }
951 } else {
952 // Thumb-1 has limited post-inc load/store support - LDM r0!, {r1}.
953 setIndexedLoadAction(IdxModes: ISD::POST_INC, VT: MVT::i32, Action: Legal);
954 setIndexedStoreAction(IdxModes: ISD::POST_INC, VT: MVT::i32, Action: Legal);
955 }
956
957 // Custom loads/stores to possible use __aeabi_uread/write*
958 if (TT.isTargetAEABI() && !Subtarget->allowsUnalignedMem()) {
959 setOperationAction(Op: ISD::STORE, VT: MVT::i32, Action: Custom);
960 setOperationAction(Op: ISD::STORE, VT: MVT::i64, Action: Custom);
961 setOperationAction(Op: ISD::LOAD, VT: MVT::i32, Action: Custom);
962 setOperationAction(Op: ISD::LOAD, VT: MVT::i64, Action: Custom);
963 }
964
965 setOperationAction(Op: ISD::SADDO, VT: MVT::i32, Action: Custom);
966 setOperationAction(Op: ISD::UADDO, VT: MVT::i32, Action: Custom);
967 setOperationAction(Op: ISD::SSUBO, VT: MVT::i32, Action: Custom);
968 setOperationAction(Op: ISD::USUBO, VT: MVT::i32, Action: Custom);
969
970 if (!Subtarget->isThumb1Only()) {
971 setOperationAction(Op: ISD::UMULO, VT: MVT::i32, Action: Custom);
972 setOperationAction(Op: ISD::SMULO, VT: MVT::i32, Action: Custom);
973 }
974
975 setOperationAction(Op: ISD::UADDO_CARRY, VT: MVT::i32, Action: Custom);
976 setOperationAction(Op: ISD::USUBO_CARRY, VT: MVT::i32, Action: Custom);
977 setOperationAction(Op: ISD::SADDO_CARRY, VT: MVT::i32, Action: Custom);
978 setOperationAction(Op: ISD::SSUBO_CARRY, VT: MVT::i32, Action: Custom);
979 if (Subtarget->hasDSP()) {
980 setOperationAction(Op: ISD::SADDSAT, VT: MVT::i8, Action: Custom);
981 setOperationAction(Op: ISD::SSUBSAT, VT: MVT::i8, Action: Custom);
982 setOperationAction(Op: ISD::SADDSAT, VT: MVT::i16, Action: Custom);
983 setOperationAction(Op: ISD::SSUBSAT, VT: MVT::i16, Action: Custom);
984 setOperationAction(Op: ISD::UADDSAT, VT: MVT::i8, Action: Custom);
985 setOperationAction(Op: ISD::USUBSAT, VT: MVT::i8, Action: Custom);
986 setOperationAction(Op: ISD::UADDSAT, VT: MVT::i16, Action: Custom);
987 setOperationAction(Op: ISD::USUBSAT, VT: MVT::i16, Action: Custom);
988 }
989 if (Subtarget->hasBaseDSP()) {
990 setOperationAction(Op: ISD::SADDSAT, VT: MVT::i32, Action: Legal);
991 setOperationAction(Op: ISD::SSUBSAT, VT: MVT::i32, Action: Legal);
992 }
993
994 // i64 operation support.
995 setOperationAction(Op: ISD::MUL, VT: MVT::i64, Action: Expand);
996 setOperationAction(Op: ISD::MULHU, VT: MVT::i32, Action: Expand);
997 if (Subtarget->isThumb1Only()) {
998 setOperationAction(Op: ISD::UMUL_LOHI, VT: MVT::i32, Action: Expand);
999 setOperationAction(Op: ISD::SMUL_LOHI, VT: MVT::i32, Action: Expand);
1000 }
1001 if (Subtarget->isThumb1Only() || !Subtarget->hasV6Ops()
1002 || (Subtarget->isThumb2() && !Subtarget->hasDSP()))
1003 setOperationAction(Op: ISD::MULHS, VT: MVT::i32, Action: Expand);
1004
1005 setOperationAction(Op: ISD::SHL_PARTS, VT: MVT::i32, Action: Custom);
1006 setOperationAction(Op: ISD::SRA_PARTS, VT: MVT::i32, Action: Custom);
1007 setOperationAction(Op: ISD::SRL_PARTS, VT: MVT::i32, Action: Custom);
1008 setOperationAction(Op: ISD::SRL, VT: MVT::i64, Action: Custom);
1009 setOperationAction(Op: ISD::SRA, VT: MVT::i64, Action: Custom);
1010 setOperationAction(Op: ISD::INTRINSIC_VOID, VT: MVT::Other, Action: Custom);
1011 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::i64, Action: Custom);
1012 setOperationAction(Op: ISD::LOAD, VT: MVT::i64, Action: Custom);
1013 setOperationAction(Op: ISD::STORE, VT: MVT::i64, Action: Custom);
1014
1015 // MVE lowers 64 bit shifts to lsll and lsrl
1016 // assuming that ISD::SRL and SRA of i64 are already marked custom
1017 if (Subtarget->hasMVEIntegerOps())
1018 setOperationAction(Op: ISD::SHL, VT: MVT::i64, Action: Custom);
1019
1020 // Expand to __aeabi_l{lsl,lsr,asr} calls for Thumb1.
1021 if (Subtarget->isThumb1Only()) {
1022 setOperationAction(Op: ISD::SHL_PARTS, VT: MVT::i32, Action: Expand);
1023 setOperationAction(Op: ISD::SRA_PARTS, VT: MVT::i32, Action: Expand);
1024 setOperationAction(Op: ISD::SRL_PARTS, VT: MVT::i32, Action: Expand);
1025 }
1026
1027 if (!Subtarget->isThumb1Only() && Subtarget->hasV6T2Ops())
1028 setOperationAction(Op: ISD::BITREVERSE, VT: MVT::i32, Action: Legal);
1029
1030 // ARM does not have ROTL.
1031 setOperationAction(Op: ISD::ROTL, VT: MVT::i32, Action: Expand);
1032 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
1033 setOperationAction(Op: ISD::ROTL, VT, Action: Expand);
1034 setOperationAction(Op: ISD::ROTR, VT, Action: Expand);
1035 }
1036 setOperationAction(Op: ISD::CTTZ, VT: MVT::i32, Action: Custom);
1037 // TODO: These two should be set to LibCall, but this currently breaks
1038 // the Linux kernel build. See #101786.
1039 setOperationAction(Op: ISD::CTPOP, VT: MVT::i32, Action: Expand);
1040 setOperationAction(Op: ISD::CTPOP, VT: MVT::i64, Action: Expand);
1041 if (!Subtarget->hasV5TOps() || Subtarget->isThumb1Only()) {
1042 setOperationAction(Op: ISD::CTLZ, VT: MVT::i32, Action: Expand);
1043 setOperationAction(Op: ISD::CTLZ_ZERO_POISON, VT: MVT::i32, Action: LibCall);
1044 }
1045
1046 // @llvm.readcyclecounter requires the Performance Monitors extension.
1047 // Default to the 0 expansion on unsupported platforms.
1048 // FIXME: Technically there are older ARM CPUs that have
1049 // implementation-specific ways of obtaining this information.
1050 if (Subtarget->hasPerfMon())
1051 setOperationAction(Op: ISD::READCYCLECOUNTER, VT: MVT::i64, Action: Custom);
1052
1053 // Only ARMv6 has BSWAP.
1054 if (!Subtarget->hasV6Ops())
1055 setOperationAction(Op: ISD::BSWAP, VT: MVT::i32, Action: Expand);
1056
1057 bool hasDivide = Subtarget->isThumb() ? Subtarget->hasDivideInThumbMode()
1058 : Subtarget->hasDivideInARMMode();
1059 if (!hasDivide) {
1060 // These are expanded into libcalls if the cpu doesn't have HW divider.
1061 setOperationAction(Op: ISD::SDIV, VT: MVT::i32, Action: LibCall);
1062 setOperationAction(Op: ISD::UDIV, VT: MVT::i32, Action: LibCall);
1063 }
1064
1065 if (TT.isOSWindows() && !Subtarget->hasDivideInThumbMode()) {
1066 setOperationAction(Op: ISD::SDIV, VT: MVT::i32, Action: Custom);
1067 setOperationAction(Op: ISD::UDIV, VT: MVT::i32, Action: Custom);
1068
1069 setOperationAction(Op: ISD::SDIV, VT: MVT::i64, Action: Custom);
1070 setOperationAction(Op: ISD::UDIV, VT: MVT::i64, Action: Custom);
1071 }
1072
1073 setOperationAction(Op: ISD::SREM, VT: MVT::i32, Action: Expand);
1074 setOperationAction(Op: ISD::UREM, VT: MVT::i32, Action: Expand);
1075
1076 // Register based DivRem for AEABI (RTABI 4.2)
1077 if (TT.isTargetAEABI() || TT.isAndroid() || TT.isTargetGNUAEABI() ||
1078 TT.isTargetMuslAEABI() || TT.isOSFuchsia() || TT.isOSWindows()) {
1079 setOperationAction(Op: ISD::SREM, VT: MVT::i64, Action: Custom);
1080 setOperationAction(Op: ISD::UREM, VT: MVT::i64, Action: Custom);
1081 HasStandaloneRem = false;
1082
1083 setOperationAction(Op: ISD::SDIVREM, VT: MVT::i32, Action: Custom);
1084 setOperationAction(Op: ISD::UDIVREM, VT: MVT::i32, Action: Custom);
1085 setOperationAction(Op: ISD::SDIVREM, VT: MVT::i64, Action: Custom);
1086 setOperationAction(Op: ISD::UDIVREM, VT: MVT::i64, Action: Custom);
1087 } else {
1088 setOperationAction(Op: ISD::SDIVREM, VT: MVT::i32, Action: Expand);
1089 setOperationAction(Op: ISD::UDIVREM, VT: MVT::i32, Action: Expand);
1090 }
1091
1092 setOperationAction(Op: ISD::GlobalAddress, VT: MVT::i32, Action: Custom);
1093 setOperationAction(Op: ISD::ConstantPool, VT: MVT::i32, Action: Custom);
1094 setOperationAction(Op: ISD::GlobalTLSAddress, VT: MVT::i32, Action: Custom);
1095 setOperationAction(Op: ISD::BlockAddress, VT: MVT::i32, Action: Custom);
1096
1097 setOperationAction(Op: ISD::TRAP, VT: MVT::Other, Action: Legal);
1098 setOperationAction(Op: ISD::DEBUGTRAP, VT: MVT::Other, Action: Legal);
1099
1100 // Use the default implementation.
1101 setOperationAction(Op: ISD::VASTART, VT: MVT::Other, Action: Custom);
1102 setOperationAction(Op: ISD::VAARG, VT: MVT::Other, Action: Expand);
1103 setOperationAction(Op: ISD::VACOPY, VT: MVT::Other, Action: Expand);
1104 setOperationAction(Op: ISD::VAEND, VT: MVT::Other, Action: Expand);
1105 setOperationAction(Op: ISD::STACKSAVE, VT: MVT::Other, Action: Expand);
1106 setOperationAction(Op: ISD::STACKRESTORE, VT: MVT::Other, Action: Expand);
1107
1108 if (TT.isOSWindows())
1109 setOperationAction(Op: ISD::DYNAMIC_STACKALLOC, VT: MVT::i32, Action: Custom);
1110 else
1111 setOperationAction(Op: ISD::DYNAMIC_STACKALLOC, VT: MVT::i32, Action: Expand);
1112
1113 // ARMv6 Thumb1 (except for CPUs that support dmb / dsb) and earlier use
1114 // the default expansion.
1115 InsertFencesForAtomic = false;
1116 if (hasNativeAtomics(ST: *Subtarget)) {
1117 // ATOMIC_FENCE needs custom lowering; the others should have been expanded
1118 // to ldrex/strex loops already.
1119 setOperationAction(Op: ISD::ATOMIC_FENCE, VT: MVT::Other, Action: Custom);
1120 if (!Subtarget->isThumb() || !Subtarget->isMClass())
1121 setOperationAction(Op: ISD::ATOMIC_CMP_SWAP, VT: MVT::i64, Action: Custom);
1122
1123 // On v8, we have particularly efficient implementations of atomic fences
1124 // if they can be combined with nearby atomic loads and stores.
1125 if (!Subtarget->hasAcquireRelease() ||
1126 getTargetMachine().getOptLevel() == CodeGenOptLevel::None) {
1127 // Automatically insert fences (dmb ish) around ATOMIC_SWAP etc.
1128 InsertFencesForAtomic = true;
1129 }
1130 } else {
1131 // If there's anything we can use as a barrier, go through custom lowering
1132 // for ATOMIC_FENCE.
1133 // If target has DMB in thumb, Fences can be inserted.
1134 if (Subtarget->hasDataBarrier())
1135 InsertFencesForAtomic = true;
1136
1137 setOperationAction(Op: ISD::ATOMIC_FENCE, VT: MVT::Other,
1138 Action: Subtarget->hasAnyDataBarrier() ? Custom : Expand);
1139
1140 // Set them all for libcall, which will force libcalls.
1141 setOperationAction(
1142 Ops: {ISD::ATOMIC_CMP_SWAP, ISD::ATOMIC_SWAP, ISD::ATOMIC_LOAD_ADD,
1143 ISD::ATOMIC_LOAD_SUB, ISD::ATOMIC_LOAD_AND, ISD::ATOMIC_LOAD_OR,
1144 ISD::ATOMIC_LOAD_XOR, ISD::ATOMIC_LOAD_NAND, ISD::ATOMIC_LOAD_MIN,
1145 ISD::ATOMIC_LOAD_MAX, ISD::ATOMIC_LOAD_UMIN, ISD::ATOMIC_LOAD_UMAX},
1146 VTs: {MVT::i8, MVT::i16, MVT::i32, MVT::i64}, Action: LibCall);
1147 setOperationAction(Ops: {ISD::ATOMIC_LOAD, ISD::ATOMIC_STORE}, VT: MVT::i64,
1148 Action: LibCall);
1149 }
1150
1151 // Compute supported atomic widths.
1152 if (TT.isOSLinux() || (!Subtarget->isMClass() && Subtarget->hasV6Ops())) {
1153 // For targets where __sync_* routines are reliably available, we use them
1154 // if necessary.
1155 //
1156 // ARM Linux always supports 64-bit atomics through kernel-assisted atomic
1157 // routines (kernel 3.1 or later). FIXME: Not with compiler-rt?
1158 //
1159 // ARMv6 targets have native instructions in ARM mode. For Thumb mode,
1160 // such targets should provide __sync_* routines, which use the ARM mode
1161 // instructions. (ARMv6 doesn't have dmb, but it has an equivalent
1162 // encoding; see ARMISD::MEMBARRIER_MCR.)
1163 setMaxAtomicSizeInBitsSupported(64);
1164 } else if ((Subtarget->isMClass() && Subtarget->hasV8MBaselineOps()) ||
1165 Subtarget->hasForced32BitAtomics()) {
1166 // Cortex-M (besides Cortex-M0) have 32-bit atomics.
1167 setMaxAtomicSizeInBitsSupported(32);
1168 } else {
1169 // We can't assume anything about other targets; just use libatomic
1170 // routines.
1171 setMaxAtomicSizeInBitsSupported(0);
1172 }
1173
1174 setMaxDivRemBitWidthSupported(64);
1175
1176 setOperationAction(Op: ISD::PREFETCH, VT: MVT::Other, Action: Custom);
1177
1178 // Requires SXTB/SXTH, available on v6 and up in both ARM and Thumb modes.
1179 if (!Subtarget->hasV6Ops()) {
1180 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i16, Action: Expand);
1181 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i8, Action: Expand);
1182 }
1183 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i1, Action: Expand);
1184
1185 if (!Subtarget->useSoftFloat() && Subtarget->hasFPRegs() &&
1186 !Subtarget->isThumb1Only()) {
1187 // Turn f64->i64 into VMOVRRD, i64 -> f64 to VMOVDRR
1188 // iff target supports vfp2.
1189 setOperationAction(Op: ISD::BITCAST, VT: MVT::i64, Action: Custom);
1190 setOperationAction(Op: ISD::GET_ROUNDING, VT: MVT::i32, Action: Custom);
1191 setOperationAction(Op: ISD::SET_ROUNDING, VT: MVT::Other, Action: Custom);
1192 setOperationAction(Op: ISD::GET_FPENV, VT: MVT::i32, Action: Legal);
1193 setOperationAction(Op: ISD::SET_FPENV, VT: MVT::i32, Action: Legal);
1194 setOperationAction(Op: ISD::RESET_FPENV, VT: MVT::Other, Action: Legal);
1195 setOperationAction(Op: ISD::GET_FPMODE, VT: MVT::i32, Action: Legal);
1196 setOperationAction(Op: ISD::SET_FPMODE, VT: MVT::i32, Action: Custom);
1197 setOperationAction(Op: ISD::RESET_FPMODE, VT: MVT::Other, Action: Custom);
1198 }
1199
1200 // We want to custom lower some of our intrinsics.
1201 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::Other, Action: Custom);
1202 setOperationAction(Op: ISD::EH_SJLJ_SETJMP, VT: MVT::i32, Action: Custom);
1203 setOperationAction(Op: ISD::EH_SJLJ_LONGJMP, VT: MVT::Other, Action: Custom);
1204 setOperationAction(Op: ISD::EH_SJLJ_SETUP_DISPATCH, VT: MVT::Other, Action: Custom);
1205
1206 setOperationAction(Op: ISD::SETCC, VT: MVT::i32, Action: Expand);
1207 setOperationAction(Op: ISD::SETCC, VT: MVT::f32, Action: Expand);
1208 setOperationAction(Op: ISD::SETCC, VT: MVT::f64, Action: Expand);
1209 setOperationAction(Op: ISD::SELECT, VT: MVT::i32, Action: Custom);
1210 setOperationAction(Op: ISD::SELECT, VT: MVT::f32, Action: Custom);
1211 setOperationAction(Op: ISD::SELECT, VT: MVT::f64, Action: Custom);
1212 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::i32, Action: Custom);
1213 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::f32, Action: Custom);
1214 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::f64, Action: Custom);
1215 if (Subtarget->hasFullFP16()) {
1216 setOperationAction(Op: ISD::SETCC, VT: MVT::f16, Action: Expand);
1217 setOperationAction(Op: ISD::SELECT, VT: MVT::f16, Action: Custom);
1218 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::f16, Action: Custom);
1219 }
1220
1221 setOperationAction(Op: ISD::SETCCCARRY, VT: MVT::i32, Action: Custom);
1222
1223 setOperationAction(Op: ISD::BRCOND, VT: MVT::Other, Action: Custom);
1224 setOperationAction(Op: ISD::BR_CC, VT: MVT::i32, Action: Custom);
1225 if (Subtarget->hasFullFP16())
1226 setOperationAction(Op: ISD::BR_CC, VT: MVT::f16, Action: Custom);
1227 setOperationAction(Op: ISD::BR_CC, VT: MVT::f32, Action: Custom);
1228 setOperationAction(Op: ISD::BR_CC, VT: MVT::f64, Action: Custom);
1229 setOperationAction(Op: ISD::BR_JT, VT: MVT::Other, Action: Custom);
1230
1231 // We don't support sin/cos/fmod/copysign/pow
1232 setOperationAction(Op: ISD::FSIN, VT: MVT::f64, Action: Expand);
1233 setOperationAction(Op: ISD::FSIN, VT: MVT::f32, Action: Expand);
1234 setOperationAction(Op: ISD::FCOS, VT: MVT::f32, Action: Expand);
1235 setOperationAction(Op: ISD::FCOS, VT: MVT::f64, Action: Expand);
1236 setOperationAction(Op: ISD::FSINCOS, VT: MVT::f64, Action: Expand);
1237 setOperationAction(Op: ISD::FSINCOS, VT: MVT::f32, Action: Expand);
1238 setOperationAction(Op: ISD::FREM, VT: MVT::f64, Action: LibCall);
1239 setOperationAction(Op: ISD::FREM, VT: MVT::f32, Action: LibCall);
1240 if (!Subtarget->useSoftFloat() && Subtarget->hasVFP2Base() &&
1241 !Subtarget->isThumb1Only()) {
1242 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::f64, Action: Custom);
1243 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::f32, Action: Custom);
1244 }
1245 setOperationAction(Op: ISD::FPOW, VT: MVT::f64, Action: Expand);
1246 setOperationAction(Op: ISD::FPOW, VT: MVT::f32, Action: Expand);
1247
1248 if (!Subtarget->hasVFP4Base()) {
1249 setOperationAction(Op: ISD::FMA, VT: MVT::f64, Action: Expand);
1250 setOperationAction(Op: ISD::FMA, VT: MVT::f32, Action: Expand);
1251 }
1252
1253 // Various VFP goodness
1254 if (!Subtarget->useSoftFloat() && !Subtarget->isThumb1Only()) {
1255 // FP-ARMv8 adds f64 <-> f16 conversion. Before that it should be expanded.
1256 if (!Subtarget->hasFPARMv8Base() || !Subtarget->hasFP64()) {
1257 setOperationAction(Op: ISD::FP16_TO_FP, VT: MVT::f64, Action: Expand);
1258 setOperationAction(Op: ISD::FP_TO_FP16, VT: MVT::f64, Action: Expand);
1259 setOperationAction(Op: ISD::STRICT_FP16_TO_FP, VT: MVT::f64, Action: Expand);
1260 setOperationAction(Op: ISD::STRICT_FP_TO_FP16, VT: MVT::f64, Action: Expand);
1261 }
1262
1263 // fp16 is a special v7 extension that adds f16 <-> f32 conversions.
1264 if (!Subtarget->hasFP16()) {
1265 setOperationAction(Op: ISD::FP16_TO_FP, VT: MVT::f32, Action: Expand);
1266 setOperationAction(Op: ISD::FP_TO_FP16, VT: MVT::f32, Action: Expand);
1267 setOperationAction(Op: ISD::STRICT_FP16_TO_FP, VT: MVT::f32, Action: Expand);
1268 setOperationAction(Op: ISD::STRICT_FP_TO_FP16, VT: MVT::f32, Action: Expand);
1269 }
1270
1271 // Strict floating-point comparisons need custom lowering.
1272 setOperationAction(Op: ISD::STRICT_FSETCC, VT: MVT::f16, Action: Custom);
1273 setOperationAction(Op: ISD::STRICT_FSETCCS, VT: MVT::f16, Action: Custom);
1274 setOperationAction(Op: ISD::STRICT_FSETCC, VT: MVT::f32, Action: Custom);
1275 setOperationAction(Op: ISD::STRICT_FSETCCS, VT: MVT::f32, Action: Custom);
1276 setOperationAction(Op: ISD::STRICT_FSETCC, VT: MVT::f64, Action: Custom);
1277 setOperationAction(Op: ISD::STRICT_FSETCCS, VT: MVT::f64, Action: Custom);
1278 }
1279
1280 // FP-ARMv8 implements a lot of rounding-like FP operations.
1281 if (Subtarget->hasFPARMv8Base()) {
1282 for (auto Op :
1283 {ISD::FFLOOR, ISD::FCEIL, ISD::FROUND,
1284 ISD::FTRUNC, ISD::FNEARBYINT, ISD::FRINT,
1285 ISD::FROUNDEVEN, ISD::FMINNUM, ISD::FMAXNUM,
1286 ISD::STRICT_FFLOOR, ISD::STRICT_FCEIL, ISD::STRICT_FROUND,
1287 ISD::STRICT_FTRUNC, ISD::STRICT_FNEARBYINT, ISD::STRICT_FRINT,
1288 ISD::STRICT_FROUNDEVEN, ISD::STRICT_FMINNUM, ISD::STRICT_FMAXNUM}) {
1289 setOperationAction(Op, VT: MVT::f32, Action: Legal);
1290
1291 if (Subtarget->hasFP64())
1292 setOperationAction(Op, VT: MVT::f64, Action: Legal);
1293 }
1294
1295 if (Subtarget->hasNEON()) {
1296 setOperationAction(Op: ISD::FMINNUM, VT: MVT::v2f32, Action: Legal);
1297 setOperationAction(Op: ISD::FMAXNUM, VT: MVT::v2f32, Action: Legal);
1298 setOperationAction(Op: ISD::FMINNUM, VT: MVT::v4f32, Action: Legal);
1299 setOperationAction(Op: ISD::FMAXNUM, VT: MVT::v4f32, Action: Legal);
1300 }
1301 }
1302
1303 // FP16 often need to be promoted to call lib functions
1304 // clang-format off
1305 if (Subtarget->hasFullFP16()) {
1306 setOperationAction(Op: ISD::LRINT, VT: MVT::f16, Action: Expand);
1307 setOperationAction(Op: ISD::LROUND, VT: MVT::f16, Action: Expand);
1308 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::f16, Action: Expand);
1309
1310 for (auto Op : {ISD::FREM, ISD::FPOW, ISD::FPOWI,
1311 ISD::FCOS, ISD::FSIN, ISD::FSINCOS,
1312 ISD::FSINCOSPI, ISD::FMODF, ISD::FACOS,
1313 ISD::FASIN, ISD::FATAN, ISD::FATAN2,
1314 ISD::FCOSH, ISD::FSINH, ISD::FTANH,
1315 ISD::FTAN, ISD::FEXP, ISD::FEXP2,
1316 ISD::FEXP10, ISD::FLOG, ISD::FLOG2,
1317 ISD::FLOG10, ISD::STRICT_FREM, ISD::STRICT_FPOW,
1318 ISD::STRICT_FPOWI, ISD::STRICT_FCOS, ISD::STRICT_FSIN,
1319 ISD::STRICT_FACOS, ISD::STRICT_FASIN, ISD::STRICT_FATAN,
1320 ISD::STRICT_FATAN2, ISD::STRICT_FCOSH, ISD::STRICT_FSINH,
1321 ISD::STRICT_FTANH, ISD::STRICT_FEXP, ISD::STRICT_FEXP2,
1322 ISD::STRICT_FLOG, ISD::STRICT_FLOG2, ISD::STRICT_FLOG10,
1323 ISD::STRICT_FTAN}) {
1324 setOperationAction(Op, VT: MVT::f16, Action: Promote);
1325 }
1326
1327 // Round-to-integer need custom lowering for fp16, as Promote doesn't work
1328 // because the result type is integer.
1329 for (auto Op : {ISD::STRICT_LROUND, ISD::STRICT_LLROUND, ISD::STRICT_LRINT, ISD::STRICT_LLRINT})
1330 setOperationAction(Op, VT: MVT::f16, Action: Custom);
1331
1332 for (auto Op : {ISD::FROUND, ISD::FROUNDEVEN, ISD::FTRUNC,
1333 ISD::FNEARBYINT, ISD::FRINT, ISD::FFLOOR,
1334 ISD::FCEIL, ISD::STRICT_FROUND, ISD::STRICT_FROUNDEVEN,
1335 ISD::STRICT_FTRUNC, ISD::STRICT_FNEARBYINT, ISD::STRICT_FRINT,
1336 ISD::STRICT_FFLOOR, ISD::STRICT_FCEIL}) {
1337 setOperationAction(Op, VT: MVT::f16, Action: Legal);
1338 }
1339 // clang-format on
1340 }
1341
1342 if (Subtarget->hasNEON()) {
1343 // vmin and vmax aren't available in a scalar form, so we can use
1344 // a NEON instruction with an undef lane instead.
1345 setOperationAction(Op: ISD::FMINIMUM, VT: MVT::f32, Action: Legal);
1346 setOperationAction(Op: ISD::FMAXIMUM, VT: MVT::f32, Action: Legal);
1347 setOperationAction(Op: ISD::FMINIMUM, VT: MVT::f16, Action: Legal);
1348 setOperationAction(Op: ISD::FMAXIMUM, VT: MVT::f16, Action: Legal);
1349 setOperationAction(Op: ISD::FMINIMUM, VT: MVT::v2f32, Action: Legal);
1350 setOperationAction(Op: ISD::FMAXIMUM, VT: MVT::v2f32, Action: Legal);
1351 setOperationAction(Op: ISD::FMINIMUM, VT: MVT::v4f32, Action: Legal);
1352 setOperationAction(Op: ISD::FMAXIMUM, VT: MVT::v4f32, Action: Legal);
1353
1354 if (Subtarget->hasV8Ops()) {
1355 for (auto Op :
1356 {ISD::FROUND, ISD::STRICT_FROUND, ISD::FROUNDEVEN,
1357 ISD::STRICT_FROUNDEVEN, ISD::FTRUNC, ISD::STRICT_FTRUNC, ISD::FRINT,
1358 ISD::FFLOOR, ISD::STRICT_FFLOOR, ISD::FCEIL, ISD::STRICT_FCEIL}) {
1359 setOperationAction(Op, VT: MVT::v2f32, Action: Legal);
1360 setOperationAction(Op, VT: MVT::v4f32, Action: Legal);
1361 }
1362 }
1363
1364 if (Subtarget->hasFullFP16()) {
1365 setOperationAction(Op: ISD::FMINNUM, VT: MVT::v4f16, Action: Legal);
1366 setOperationAction(Op: ISD::FMAXNUM, VT: MVT::v4f16, Action: Legal);
1367 setOperationAction(Op: ISD::FMINNUM, VT: MVT::v8f16, Action: Legal);
1368 setOperationAction(Op: ISD::FMAXNUM, VT: MVT::v8f16, Action: Legal);
1369
1370 setOperationAction(Op: ISD::FMINIMUM, VT: MVT::v4f16, Action: Legal);
1371 setOperationAction(Op: ISD::FMAXIMUM, VT: MVT::v4f16, Action: Legal);
1372 setOperationAction(Op: ISD::FMINIMUM, VT: MVT::v8f16, Action: Legal);
1373 setOperationAction(Op: ISD::FMAXIMUM, VT: MVT::v8f16, Action: Legal);
1374
1375 for (auto Op :
1376 {ISD::FROUND, ISD::STRICT_FROUND, ISD::FROUNDEVEN,
1377 ISD::STRICT_FROUNDEVEN, ISD::FTRUNC, ISD::STRICT_FTRUNC, ISD::FRINT,
1378 ISD::FFLOOR, ISD::STRICT_FFLOOR, ISD::FCEIL, ISD::STRICT_FCEIL}) {
1379 setOperationAction(Op, VT: MVT::v4f16, Action: Legal);
1380 setOperationAction(Op, VT: MVT::v8f16, Action: Legal);
1381 }
1382 }
1383 }
1384
1385 // On MSVC, both 32-bit and 64-bit, ldexpf(f32) is not defined. MinGW has
1386 // it, but it's just a wrapper around ldexp.
1387 if (TT.isOSWindows()) {
1388 for (ISD::NodeType Op : {ISD::FLDEXP, ISD::STRICT_FLDEXP, ISD::FFREXP})
1389 if (isOperationExpand(Op, VT: MVT::f32))
1390 setOperationAction(Op, VT: MVT::f32, Action: Promote);
1391 }
1392
1393 // LegalizeDAG currently can't expand fp16 LDEXP/FREXP on targets where i16
1394 // isn't legal.
1395 for (ISD::NodeType Op : {ISD::FLDEXP, ISD::STRICT_FLDEXP, ISD::FFREXP})
1396 if (isOperationExpand(Op, VT: MVT::f16))
1397 setOperationAction(Op, VT: MVT::f16, Action: Promote);
1398
1399 // We have target-specific dag combine patterns for the following nodes:
1400 // ARMISD::VMOVRRD - No need to call setTargetDAGCombine
1401 setTargetDAGCombine(
1402 {ISD::ADD, ISD::SUB, ISD::MUL, ISD::AND, ISD::OR, ISD::XOR});
1403
1404 if (Subtarget->hasMVEIntegerOps())
1405 setTargetDAGCombine(ISD::VSELECT);
1406
1407 if (Subtarget->hasV6Ops())
1408 setTargetDAGCombine(ISD::SRL);
1409 if (Subtarget->isThumb1Only())
1410 setTargetDAGCombine(ISD::SHL);
1411 // Attempt to lower smin/smax to ssat/usat
1412 if ((!Subtarget->isThumb() && Subtarget->hasV6Ops()) ||
1413 Subtarget->isThumb2()) {
1414 setTargetDAGCombine({ISD::SMIN, ISD::SMAX});
1415 }
1416
1417 setStackPointerRegisterToSaveRestore(ARM::SP);
1418
1419 if (Subtarget->useSoftFloat() || Subtarget->isThumb1Only() ||
1420 !Subtarget->hasVFP2Base() || Subtarget->hasMinSize())
1421 setSchedulingPreference(Sched::RegPressure);
1422 else
1423 setSchedulingPreference(Sched::Hybrid);
1424
1425 //// temporary - rewrite interface to use type
1426 MaxStoresPerMemset = 8;
1427 MaxStoresPerMemsetOptSize = 4;
1428 MaxStoresPerMemcpy = 4; // For @llvm.memcpy -> sequence of stores
1429 MaxStoresPerMemcpyOptSize = 2;
1430 MaxStoresPerMemmove = 4; // For @llvm.memmove -> sequence of stores
1431 MaxStoresPerMemmoveOptSize = 2;
1432
1433 // On ARM arguments smaller than 4 bytes are extended, so all arguments
1434 // are at least 4 bytes aligned.
1435 setMinStackArgumentAlignment(Align(4));
1436
1437 // Prefer likely predicted branches to selects on out-of-order cores.
1438 PredictableSelectIsExpensive = Subtarget->getSchedModel().isOutOfOrder();
1439
1440 setPrefLoopAlignment(
1441 Align::fromLog2(Log2Value: Subtarget->getPreferBranchLogAlignment()));
1442 setPrefFunctionAlignment(
1443 Align::fromLog2(Log2Value: Subtarget->getPreferBranchLogAlignment()));
1444
1445 setMinFunctionAlignment(Subtarget->isThumb() ? Align(2) : Align(4));
1446
1447 IsStrictFPEnabled = true;
1448}
1449
1450bool ARMTargetLowering::useSoftFloat() const {
1451 return Subtarget->useSoftFloat();
1452}
1453
1454bool ARMTargetLowering::preferSelectsOverBooleanArithmetic(EVT VT) const {
1455 return !Subtarget->isThumb1Only() && VT.getSizeInBits() <= 32;
1456}
1457
1458// FIXME: It might make sense to define the representative register class as the
1459// nearest super-register that has a non-null superset. For example, DPR_VFP2 is
1460// a super-register of SPR, and DPR is a superset if DPR_VFP2. Consequently,
1461// SPR's representative would be DPR_VFP2. This should work well if register
1462// pressure tracking were modified such that a register use would increment the
1463// pressure of the register class's representative and all of it's super
1464// classes' representatives transitively. We have not implemented this because
1465// of the difficulty prior to coalescing of modeling operand register classes
1466// due to the common occurrence of cross class copies and subregister insertions
1467// and extractions.
1468std::pair<const TargetRegisterClass *, uint8_t>
1469ARMTargetLowering::findRepresentativeClass(const TargetRegisterInfo *TRI,
1470 MVT VT) const {
1471 const TargetRegisterClass *RRC = nullptr;
1472 uint8_t Cost = 1;
1473 switch (VT.SimpleTy) {
1474 default:
1475 return TargetLowering::findRepresentativeClass(TRI, VT);
1476 // Use DPR as representative register class for all floating point
1477 // and vector types. Since there are 32 SPR registers and 32 DPR registers so
1478 // the cost is 1 for both f32 and f64.
1479 case MVT::f32: case MVT::f64: case MVT::v8i8: case MVT::v4i16:
1480 case MVT::v2i32: case MVT::v1i64: case MVT::v2f32:
1481 RRC = &ARM::DPRRegClass;
1482 // When NEON is used for SP, only half of the register file is available
1483 // because operations that define both SP and DP results will be constrained
1484 // to the VFP2 class (D0-D15). We currently model this constraint prior to
1485 // coalescing by double-counting the SP regs. See the FIXME above.
1486 if (Subtarget->useNEONForSinglePrecisionFP())
1487 Cost = 2;
1488 break;
1489 case MVT::v16i8: case MVT::v8i16: case MVT::v4i32: case MVT::v2i64:
1490 case MVT::v4f32: case MVT::v2f64:
1491 RRC = &ARM::DPRRegClass;
1492 Cost = 2;
1493 break;
1494 case MVT::v4i64:
1495 RRC = &ARM::DPRRegClass;
1496 Cost = 4;
1497 break;
1498 case MVT::v8i64:
1499 RRC = &ARM::DPRRegClass;
1500 Cost = 8;
1501 break;
1502 }
1503 return std::make_pair(x&: RRC, y&: Cost);
1504}
1505
1506EVT ARMTargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &C,
1507 EVT VT) const {
1508 if (!VT.isVector())
1509 return getPointerTy(DL);
1510
1511 // MVE has a predicate register.
1512 if (Subtarget->hasMVEIntegerOps())
1513 return EVT::getVectorVT(Context&: C, VT: MVT::i1, EC: VT.getVectorElementCount());
1514
1515 return VT.changeVectorElementTypeToInteger();
1516}
1517
1518/// getRegClassFor - Return the register class that should be used for the
1519/// specified value type.
1520const TargetRegisterClass *
1521ARMTargetLowering::getRegClassFor(MVT VT, bool isDivergent) const {
1522 (void)isDivergent;
1523 // Map v4i64 to QQ registers but do not make the type legal. Similarly map
1524 // v8i64 to QQQQ registers. v4i64 and v8i64 are only used for REG_SEQUENCE to
1525 // load / store 4 to 8 consecutive NEON D registers, or 2 to 4 consecutive
1526 // MVE Q registers.
1527 if (Subtarget->hasNEON()) {
1528 if (VT == MVT::v4i64)
1529 return &ARM::QQPRRegClass;
1530 if (VT == MVT::v8i64)
1531 return &ARM::QQQQPRRegClass;
1532 }
1533 if (Subtarget->hasMVEIntegerOps()) {
1534 if (VT == MVT::v4i64)
1535 return &ARM::MQQPRRegClass;
1536 if (VT == MVT::v8i64)
1537 return &ARM::MQQQQPRRegClass;
1538 }
1539 return TargetLowering::getRegClassFor(VT);
1540}
1541
1542// memcpy, and other memory intrinsics, typically tries to use LDM/STM if the
1543// source/dest is aligned and the copy size is large enough. We therefore want
1544// to align such objects passed to memory intrinsics.
1545bool ARMTargetLowering::shouldAlignPointerArgs(CallInst *CI, unsigned &MinSize,
1546 Align &PrefAlign) const {
1547 if (!isa<MemIntrinsic>(Val: CI))
1548 return false;
1549 MinSize = 8;
1550 // On ARM11 onwards (excluding M class) 8-byte aligned LDM is typically 1
1551 // cycle faster than 4-byte aligned LDM.
1552 PrefAlign =
1553 (Subtarget->hasV6Ops() && !Subtarget->isMClass() ? Align(8) : Align(4));
1554 return true;
1555}
1556
1557// Create a fast isel object.
1558FastISel *ARMTargetLowering::createFastISel(
1559 FunctionLoweringInfo &funcInfo, const TargetLibraryInfo *libInfo,
1560 const LibcallLoweringInfo *libcallLowering) const {
1561 return ARM::createFastISel(funcInfo, libInfo, libcallLowering);
1562}
1563
1564Sched::Preference ARMTargetLowering::getSchedulingPreference(SDNode *N) const {
1565 unsigned NumVals = N->getNumValues();
1566 if (!NumVals)
1567 return Sched::RegPressure;
1568
1569 for (unsigned i = 0; i != NumVals; ++i) {
1570 EVT VT = N->getValueType(ResNo: i);
1571 if (VT == MVT::Glue || VT == MVT::Other)
1572 continue;
1573 if (VT.isFloatingPoint() || VT.isVector())
1574 return Sched::ILP;
1575 }
1576
1577 if (!N->isMachineOpcode())
1578 return Sched::RegPressure;
1579
1580 // Load are scheduled for latency even if there instruction itinerary
1581 // is not available.
1582 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
1583 const MCInstrDesc &MCID = TII->get(Opcode: N->getMachineOpcode());
1584
1585 if (MCID.getNumDefs() == 0)
1586 return Sched::RegPressure;
1587 if (!Itins->isEmpty() &&
1588 Itins->getOperandCycle(ItinClassIndx: MCID.getSchedClass(), OperandIdx: 0) > 2U)
1589 return Sched::ILP;
1590
1591 return Sched::RegPressure;
1592}
1593
1594//===----------------------------------------------------------------------===//
1595// Lowering Code
1596//===----------------------------------------------------------------------===//
1597
1598static bool isSRL16(const SDValue &Op) {
1599 if (Op.getOpcode() != ISD::SRL)
1600 return false;
1601 if (auto Const = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1)))
1602 return Const->getZExtValue() == 16;
1603 return false;
1604}
1605
1606static bool isSRA16(const SDValue &Op) {
1607 if (Op.getOpcode() != ISD::SRA)
1608 return false;
1609 if (auto Const = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1)))
1610 return Const->getZExtValue() == 16;
1611 return false;
1612}
1613
1614static bool isSHL16(const SDValue &Op) {
1615 if (Op.getOpcode() != ISD::SHL)
1616 return false;
1617 if (auto Const = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1)))
1618 return Const->getZExtValue() == 16;
1619 return false;
1620}
1621
1622// Check for a signed 16-bit value. We special case SRA because it makes it
1623// more simple when also looking for SRAs that aren't sign extending a
1624// smaller value. Without the check, we'd need to take extra care with
1625// checking order for some operations.
1626static bool isS16(const SDValue &Op, SelectionDAG &DAG) {
1627 if (isSRA16(Op))
1628 return isSHL16(Op: Op.getOperand(i: 0));
1629 return DAG.ComputeNumSignBits(Op) == 17;
1630}
1631
1632/// IntCCToARMCC - Convert a DAG integer condition code to an ARM CC
1633static ARMCC::CondCodes IntCCToARMCC(ISD::CondCode CC) {
1634 switch (CC) {
1635 default: llvm_unreachable("Unknown condition code!");
1636 case ISD::SETNE: return ARMCC::NE;
1637 case ISD::SETEQ: return ARMCC::EQ;
1638 case ISD::SETGT: return ARMCC::GT;
1639 case ISD::SETGE: return ARMCC::GE;
1640 case ISD::SETLT: return ARMCC::LT;
1641 case ISD::SETLE: return ARMCC::LE;
1642 case ISD::SETUGT: return ARMCC::HI;
1643 case ISD::SETUGE: return ARMCC::HS;
1644 case ISD::SETULT: return ARMCC::LO;
1645 case ISD::SETULE: return ARMCC::LS;
1646 }
1647}
1648
1649/// FPCCToARMCC - Convert a DAG fp condition code to an ARM CC.
1650static void FPCCToARMCC(ISD::CondCode CC, ARMCC::CondCodes &CondCode,
1651 ARMCC::CondCodes &CondCode2) {
1652 CondCode2 = ARMCC::AL;
1653 switch (CC) {
1654 default: llvm_unreachable("Unknown FP condition!");
1655 case ISD::SETEQ:
1656 case ISD::SETOEQ: CondCode = ARMCC::EQ; break;
1657 case ISD::SETGT:
1658 case ISD::SETOGT: CondCode = ARMCC::GT; break;
1659 case ISD::SETGE:
1660 case ISD::SETOGE: CondCode = ARMCC::GE; break;
1661 case ISD::SETOLT: CondCode = ARMCC::MI; break;
1662 case ISD::SETOLE: CondCode = ARMCC::LS; break;
1663 case ISD::SETONE: CondCode = ARMCC::MI; CondCode2 = ARMCC::GT; break;
1664 case ISD::SETO: CondCode = ARMCC::VC; break;
1665 case ISD::SETUO: CondCode = ARMCC::VS; break;
1666 case ISD::SETUEQ: CondCode = ARMCC::EQ; CondCode2 = ARMCC::VS; break;
1667 case ISD::SETUGT: CondCode = ARMCC::HI; break;
1668 case ISD::SETUGE: CondCode = ARMCC::PL; break;
1669 case ISD::SETLT:
1670 case ISD::SETULT: CondCode = ARMCC::LT; break;
1671 case ISD::SETLE:
1672 case ISD::SETULE: CondCode = ARMCC::LE; break;
1673 case ISD::SETNE:
1674 case ISD::SETUNE: CondCode = ARMCC::NE; break;
1675 }
1676}
1677
1678//===----------------------------------------------------------------------===//
1679// Calling Convention Implementation
1680//===----------------------------------------------------------------------===//
1681
1682/// getEffectiveCallingConv - Get the effective calling convention, taking into
1683/// account presence of floating point hardware and calling convention
1684/// limitations, such as support for variadic functions.
1685CallingConv::ID
1686ARMTargetLowering::getEffectiveCallingConv(CallingConv::ID CC,
1687 bool isVarArg) const {
1688 switch (CC) {
1689 default:
1690 // Unknown CCs are rejected when calling convention lowering is required.
1691 case CallingConv::ARM_AAPCS:
1692 case CallingConv::ARM_APCS:
1693 case CallingConv::GHC:
1694 case CallingConv::CFGuard_Check:
1695 return CC;
1696 case CallingConv::PreserveMost:
1697 return CallingConv::PreserveMost;
1698 case CallingConv::PreserveAll:
1699 return CallingConv::PreserveAll;
1700 case CallingConv::ARM_AAPCS_VFP:
1701 case CallingConv::Swift:
1702 case CallingConv::SwiftTail:
1703 return isVarArg ? CallingConv::ARM_AAPCS : CallingConv::ARM_AAPCS_VFP;
1704 case CallingConv::C:
1705 case CallingConv::Tail:
1706 if (!Subtarget->isAAPCS_ABI())
1707 return CallingConv::ARM_APCS;
1708 else if (Subtarget->isTargetHardFloat() && !isVarArg)
1709 return CallingConv::ARM_AAPCS_VFP;
1710 else
1711 return CallingConv::ARM_AAPCS;
1712 case CallingConv::Fast:
1713 case CallingConv::CXX_FAST_TLS:
1714 if (!Subtarget->isAAPCS_ABI()) {
1715 if (Subtarget->hasFPRegs() && !Subtarget->isThumb1Only() && !isVarArg)
1716 return CallingConv::Fast;
1717 return CallingConv::ARM_APCS;
1718 } else if (Subtarget->hasFPRegs() && !Subtarget->isThumb1Only() &&
1719 !Subtarget->useSoftFloat() && !isVarArg)
1720 return CallingConv::ARM_AAPCS_VFP;
1721 else
1722 return CallingConv::ARM_AAPCS;
1723 }
1724}
1725
1726CCAssignFn *ARMTargetLowering::CCAssignFnForCall(CallingConv::ID CC,
1727 bool isVarArg) const {
1728 return CCAssignFnForNode(CC, Return: false, isVarArg);
1729}
1730
1731CCAssignFn *ARMTargetLowering::CCAssignFnForReturn(CallingConv::ID CC,
1732 bool isVarArg) const {
1733 return CCAssignFnForNode(CC, Return: true, isVarArg);
1734}
1735
1736/// CCAssignFnForNode - Selects the correct CCAssignFn for the given
1737/// CallingConvention.
1738CCAssignFn *ARMTargetLowering::CCAssignFnForNode(CallingConv::ID CC,
1739 bool Return,
1740 bool isVarArg) const {
1741 switch (getEffectiveCallingConv(CC, isVarArg)) {
1742 default:
1743 report_fatal_error(reason: "Unsupported calling convention");
1744 case CallingConv::ARM_APCS:
1745 return (Return ? RetCC_ARM_APCS : CC_ARM_APCS);
1746 case CallingConv::ARM_AAPCS:
1747 return (Return ? RetCC_ARM_AAPCS : CC_ARM_AAPCS);
1748 case CallingConv::ARM_AAPCS_VFP:
1749 return (Return ? RetCC_ARM_AAPCS_VFP : CC_ARM_AAPCS_VFP);
1750 case CallingConv::Fast:
1751 return (Return ? RetFastCC_ARM_APCS : FastCC_ARM_APCS);
1752 case CallingConv::GHC:
1753 return (Return ? RetCC_ARM_APCS : CC_ARM_APCS_GHC);
1754 case CallingConv::PreserveMost:
1755 return (Return ? RetCC_ARM_AAPCS : CC_ARM_AAPCS);
1756 case CallingConv::PreserveAll:
1757 return (Return ? RetCC_ARM_AAPCS : CC_ARM_AAPCS);
1758 case CallingConv::CFGuard_Check:
1759 return (Return ? RetCC_ARM_AAPCS : CC_ARM_Win32_CFGuard_Check);
1760 }
1761}
1762
1763SDValue ARMTargetLowering::MoveToHPR(const SDLoc &dl, SelectionDAG &DAG,
1764 MVT LocVT, MVT ValVT, SDValue Val) const {
1765 Val = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::getIntegerVT(BitWidth: LocVT.getSizeInBits()),
1766 Operand: Val);
1767 if (Subtarget->hasFullFP16()) {
1768 Val = DAG.getNode(Opcode: ARMISD::VMOVhr, DL: dl, VT: ValVT, Operand: Val);
1769 } else {
1770 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl,
1771 VT: MVT::getIntegerVT(BitWidth: ValVT.getSizeInBits()), Operand: Val);
1772 Val = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: ValVT, Operand: Val);
1773 }
1774 return Val;
1775}
1776
1777SDValue ARMTargetLowering::MoveFromHPR(const SDLoc &dl, SelectionDAG &DAG,
1778 MVT LocVT, MVT ValVT,
1779 SDValue Val) const {
1780 if (Subtarget->hasFullFP16()) {
1781 Val = DAG.getNode(Opcode: ARMISD::VMOVrh, DL: dl,
1782 VT: MVT::getIntegerVT(BitWidth: LocVT.getSizeInBits()), Operand: Val);
1783 } else {
1784 Val = DAG.getNode(Opcode: ISD::BITCAST, DL: dl,
1785 VT: MVT::getIntegerVT(BitWidth: ValVT.getSizeInBits()), Operand: Val);
1786 Val = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl,
1787 VT: MVT::getIntegerVT(BitWidth: LocVT.getSizeInBits()), Operand: Val);
1788 }
1789 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LocVT, Operand: Val);
1790}
1791
1792/// LowerCallResult - Lower the result values of a call into the
1793/// appropriate copies out of appropriate physical registers.
1794SDValue ARMTargetLowering::LowerCallResult(
1795 SDValue Chain, SDValue InGlue, CallingConv::ID CallConv, bool isVarArg,
1796 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
1797 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals, bool isThisReturn,
1798 SDValue ThisVal, bool isCmseNSCall) const {
1799 // Assign locations to each value returned by this call.
1800 SmallVector<CCValAssign, 16> RVLocs;
1801 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
1802 *DAG.getContext());
1803 CCInfo.AnalyzeCallResult(Ins, Fn: CCAssignFnForReturn(CC: CallConv, isVarArg));
1804
1805 // Copy all of the result registers out of their specified physreg.
1806 for (unsigned i = 0; i != RVLocs.size(); ++i) {
1807 CCValAssign VA = RVLocs[i];
1808
1809 // Pass 'this' value directly from the argument to return value, to avoid
1810 // reg unit interference
1811 if (i == 0 && isThisReturn) {
1812 assert(!VA.needsCustom() && VA.getLocVT() == MVT::i32 &&
1813 "unexpected return calling convention register assignment");
1814 InVals.push_back(Elt: ThisVal);
1815 continue;
1816 }
1817
1818 SDValue Val;
1819 if (VA.needsCustom() &&
1820 (VA.getLocVT() == MVT::f64 || VA.getLocVT() == MVT::v2f64)) {
1821 // Handle f64 or half of a v2f64.
1822 SDValue Lo = DAG.getCopyFromReg(Chain, dl, Reg: VA.getLocReg(), VT: MVT::i32,
1823 Glue: InGlue);
1824 Chain = Lo.getValue(R: 1);
1825 InGlue = Lo.getValue(R: 2);
1826 VA = RVLocs[++i]; // skip ahead to next loc
1827 SDValue Hi = DAG.getCopyFromReg(Chain, dl, Reg: VA.getLocReg(), VT: MVT::i32,
1828 Glue: InGlue);
1829 Chain = Hi.getValue(R: 1);
1830 InGlue = Hi.getValue(R: 2);
1831 if (!Subtarget->isLittle())
1832 std::swap (a&: Lo, b&: Hi);
1833 Val = DAG.getNode(Opcode: ARMISD::VMOVDRR, DL: dl, VT: MVT::f64, N1: Lo, N2: Hi);
1834
1835 if (VA.getLocVT() == MVT::v2f64) {
1836 SDValue Vec = DAG.getNode(Opcode: ISD::UNDEF, DL: dl, VT: MVT::v2f64);
1837 Vec = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: MVT::v2f64, N1: Vec, N2: Val,
1838 N3: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
1839
1840 VA = RVLocs[++i]; // skip ahead to next loc
1841 Lo = DAG.getCopyFromReg(Chain, dl, Reg: VA.getLocReg(), VT: MVT::i32, Glue: InGlue);
1842 Chain = Lo.getValue(R: 1);
1843 InGlue = Lo.getValue(R: 2);
1844 VA = RVLocs[++i]; // skip ahead to next loc
1845 Hi = DAG.getCopyFromReg(Chain, dl, Reg: VA.getLocReg(), VT: MVT::i32, Glue: InGlue);
1846 Chain = Hi.getValue(R: 1);
1847 InGlue = Hi.getValue(R: 2);
1848 if (!Subtarget->isLittle())
1849 std::swap (a&: Lo, b&: Hi);
1850 Val = DAG.getNode(Opcode: ARMISD::VMOVDRR, DL: dl, VT: MVT::f64, N1: Lo, N2: Hi);
1851 Val = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: MVT::v2f64, N1: Vec, N2: Val,
1852 N3: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
1853 }
1854 } else {
1855 Val = DAG.getCopyFromReg(Chain, dl, Reg: VA.getLocReg(), VT: VA.getLocVT(),
1856 Glue: InGlue);
1857 Chain = Val.getValue(R: 1);
1858 InGlue = Val.getValue(R: 2);
1859 }
1860
1861 switch (VA.getLocInfo()) {
1862 default: llvm_unreachable("Unknown loc info!");
1863 case CCValAssign::Full: break;
1864 case CCValAssign::BCvt:
1865 Val = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: VA.getValVT(), Operand: Val);
1866 break;
1867 }
1868
1869 // f16 arguments have their size extended to 4 bytes and passed as if they
1870 // had been copied to the LSBs of a 32-bit register.
1871 // For that, it's passed extended to i32 (soft ABI) or to f32 (hard ABI)
1872 if (VA.needsCustom() &&
1873 (VA.getValVT() == MVT::f16 || VA.getValVT() == MVT::bf16))
1874 Val = MoveToHPR(dl, DAG, LocVT: VA.getLocVT(), ValVT: VA.getValVT(), Val);
1875
1876 // On CMSE Non-secure Calls, call results (returned values) whose bitwidth
1877 // is less than 32 bits must be sign- or zero-extended after the call for
1878 // security reasons. Although the ABI mandates an extension done by the
1879 // callee, the latter cannot be trusted to follow the rules of the ABI.
1880 const ISD::InputArg &Arg = Ins[VA.getValNo()];
1881 if (isCmseNSCall && Arg.ArgVT.isScalarInteger() &&
1882 VA.getLocVT().isScalarInteger() && Arg.ArgVT.bitsLT(VT: MVT::i32))
1883 Val = handleCMSEValue(Value: Val, Arg, DAG, DL: dl);
1884
1885 InVals.push_back(Elt: Val);
1886 }
1887
1888 return Chain;
1889}
1890
1891std::pair<SDValue, MachinePointerInfo> ARMTargetLowering::computeAddrForCallArg(
1892 const SDLoc &dl, SelectionDAG &DAG, const CCValAssign &VA, SDValue StackPtr,
1893 bool IsTailCall, int SPDiff) const {
1894 SDValue DstAddr;
1895 MachinePointerInfo DstInfo;
1896 int32_t Offset = VA.getLocMemOffset();
1897 MachineFunction &MF = DAG.getMachineFunction();
1898
1899 if (IsTailCall) {
1900 Offset += SPDiff;
1901 auto PtrVT = getPointerTy(DL: DAG.getDataLayout());
1902 int Size = VA.getLocVT().getFixedSizeInBits() / 8;
1903 int FI = MF.getFrameInfo().CreateFixedObject(Size, SPOffset: Offset, IsImmutable: true);
1904 DstAddr = DAG.getFrameIndex(FI, VT: PtrVT);
1905 DstInfo =
1906 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI);
1907 } else {
1908 SDValue PtrOff = DAG.getIntPtrConstant(Val: Offset, DL: dl);
1909 DstAddr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: getPointerTy(DL: DAG.getDataLayout()),
1910 N1: StackPtr, N2: PtrOff);
1911 DstInfo =
1912 MachinePointerInfo::getStack(MF&: DAG.getMachineFunction(), Offset);
1913 }
1914
1915 return std::make_pair(x&: DstAddr, y&: DstInfo);
1916}
1917
1918// Returns the type of copying which is required to set up a byval argument to
1919// a tail-called function. This isn't needed for non-tail calls, because they
1920// always need the equivalent of CopyOnce, but tail-calls sometimes need two to
1921// avoid clobbering another argument (CopyViaTemp), and sometimes can be
1922// optimised to zero copies when forwarding an argument from the caller's
1923// caller (NoCopy).
1924ARMTargetLowering::ByValCopyKind ARMTargetLowering::ByValNeedsCopyForTailCall(
1925 SelectionDAG &DAG, SDValue Src, SDValue Dst, ISD::ArgFlagsTy Flags) const {
1926 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
1927 ARMFunctionInfo *AFI = DAG.getMachineFunction().getInfo<ARMFunctionInfo>();
1928
1929 // Globals are always safe to copy from.
1930 if (isa<GlobalAddressSDNode>(Val: Src) || isa<ExternalSymbolSDNode>(Val: Src))
1931 return CopyOnce;
1932
1933 // Can only analyse frame index nodes, conservatively assume we need a
1934 // temporary.
1935 auto *SrcFrameIdxNode = dyn_cast<FrameIndexSDNode>(Val&: Src);
1936 auto *DstFrameIdxNode = dyn_cast<FrameIndexSDNode>(Val&: Dst);
1937 if (!SrcFrameIdxNode || !DstFrameIdxNode)
1938 return CopyViaTemp;
1939
1940 int SrcFI = SrcFrameIdxNode->getIndex();
1941 int DstFI = DstFrameIdxNode->getIndex();
1942 assert(MFI.isFixedObjectIndex(DstFI) &&
1943 "byval passed in non-fixed stack slot");
1944
1945 int64_t SrcOffset = MFI.getObjectOffset(ObjectIdx: SrcFI);
1946 int64_t DstOffset = MFI.getObjectOffset(ObjectIdx: DstFI);
1947
1948 // If the source is in the local frame, then the copy to the argument memory
1949 // is always valid.
1950 bool FixedSrc = MFI.isFixedObjectIndex(ObjectIdx: SrcFI);
1951 if (!FixedSrc ||
1952 (FixedSrc && SrcOffset < -(int64_t)AFI->getArgRegsSaveSize()))
1953 return CopyOnce;
1954
1955 // In the case of byval arguments split between registers and the stack,
1956 // computeAddrForCallArg returns a FrameIndex which corresponds only to the
1957 // stack portion, but the Src SDValue will refer to the full value, including
1958 // the local stack memory that the register portion gets stored into. We only
1959 // need to compare them for equality, so normalise on the full value version.
1960 uint64_t RegSize = Flags.getByValSize() - MFI.getObjectSize(ObjectIdx: DstFI);
1961 DstOffset -= RegSize;
1962
1963 // If the value is already in the correct location, then no copying is
1964 // needed. If not, then we need to copy via a temporary.
1965 if (SrcOffset == DstOffset)
1966 return NoCopy;
1967 else
1968 return CopyViaTemp;
1969}
1970
1971void ARMTargetLowering::PassF64ArgInRegs(const SDLoc &dl, SelectionDAG &DAG,
1972 SDValue Chain, SDValue &Arg,
1973 RegsToPassVector &RegsToPass,
1974 CCValAssign &VA, CCValAssign &NextVA,
1975 SDValue &StackPtr,
1976 SmallVectorImpl<SDValue> &MemOpChains,
1977 bool IsTailCall,
1978 int SPDiff) const {
1979 SDValue fmrrd = DAG.getNode(Opcode: ARMISD::VMOVRRD, DL: dl,
1980 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N: Arg);
1981 unsigned id = Subtarget->isLittle() ? 0 : 1;
1982 RegsToPass.push_back(Elt: std::make_pair(x: VA.getLocReg(), y: fmrrd.getValue(R: id)));
1983
1984 if (NextVA.isRegLoc())
1985 RegsToPass.push_back(Elt: std::make_pair(x: NextVA.getLocReg(), y: fmrrd.getValue(R: 1-id)));
1986 else {
1987 assert(NextVA.isMemLoc());
1988 if (!StackPtr.getNode())
1989 StackPtr = DAG.getCopyFromReg(Chain, dl, Reg: ARM::SP,
1990 VT: getPointerTy(DL: DAG.getDataLayout()));
1991
1992 SDValue DstAddr;
1993 MachinePointerInfo DstInfo;
1994 std::tie(args&: DstAddr, args&: DstInfo) =
1995 computeAddrForCallArg(dl, DAG, VA: NextVA, StackPtr, IsTailCall, SPDiff);
1996 MemOpChains.push_back(
1997 Elt: DAG.getStore(Chain, dl, Val: fmrrd.getValue(R: 1 - id), Ptr: DstAddr, PtrInfo: DstInfo));
1998 }
1999}
2000
2001static bool canGuaranteeTCO(CallingConv::ID CC, bool GuaranteeTailCalls) {
2002 return (CC == CallingConv::Fast && GuaranteeTailCalls) ||
2003 CC == CallingConv::Tail || CC == CallingConv::SwiftTail;
2004}
2005
2006/// LowerCall - Lowering a call into a callseq_start <-
2007/// ARMISD:CALL <- callseq_end chain. Also add input and output parameter
2008/// nodes.
2009SDValue
2010ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
2011 SmallVectorImpl<SDValue> &InVals) const {
2012 SelectionDAG &DAG = CLI.DAG;
2013 SDLoc &dl = CLI.DL;
2014 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
2015 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
2016 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
2017 SDValue Chain = CLI.Chain;
2018 SDValue Callee = CLI.Callee;
2019 bool &isTailCall = CLI.IsTailCall;
2020 CallingConv::ID CallConv = CLI.CallConv;
2021 bool doesNotRet = CLI.DoesNotReturn;
2022 bool isVarArg = CLI.IsVarArg;
2023 const CallBase *CB = CLI.CB;
2024
2025 MachineFunction &MF = DAG.getMachineFunction();
2026 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
2027 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
2028 MachineFunction::CallSiteInfo CSInfo;
2029 bool isStructRet = (Outs.empty()) ? false : Outs[0].Flags.isSRet();
2030 bool isThisReturn = false;
2031 bool isCmseNSCall = false;
2032 bool isSibCall = false;
2033 bool PreferIndirect = false;
2034 bool GuardWithBTI = false;
2035
2036 // Analyze operands of the call, assigning locations to each operand.
2037 SmallVector<CCValAssign, 16> ArgLocs;
2038 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs,
2039 *DAG.getContext());
2040 CCInfo.AnalyzeCallOperands(Outs, Fn: CCAssignFnForCall(CC: CallConv, isVarArg));
2041
2042 // Lower 'returns_twice' calls to a pseudo-instruction.
2043 if (CLI.CB && CLI.CB->getAttributes().hasFnAttr(Kind: Attribute::ReturnsTwice) &&
2044 !Subtarget->noBTIAtReturnTwice())
2045 GuardWithBTI = AFI->branchTargetEnforcement();
2046
2047 // Set type id for call site info.
2048 setTypeIdForCallsiteInfo(CB, MF, CSInfo);
2049
2050 // Determine whether this is a non-secure function call.
2051 if (CLI.CB && CLI.CB->getAttributes().hasFnAttr(Kind: "cmse_nonsecure_call"))
2052 isCmseNSCall = true;
2053
2054 // Disable tail calls if they're not supported.
2055 if (!Subtarget->supportsTailCall())
2056 isTailCall = false;
2057
2058 // For both the non-secure calls and the returns from a CMSE entry function,
2059 // the function needs to do some extra work after the call, or before the
2060 // return, respectively, thus it cannot end with a tail call
2061 if (isCmseNSCall || AFI->isCmseNSEntryFunction())
2062 isTailCall = false;
2063
2064 if (isa<GlobalAddressSDNode>(Val: Callee)) {
2065 // If we're optimizing for minimum size and the function is called three or
2066 // more times in this block, we can improve codesize by calling indirectly
2067 // as BLXr has a 16-bit encoding.
2068 auto *GV = cast<GlobalAddressSDNode>(Val&: Callee)->getGlobal();
2069 if (CLI.CB) {
2070 auto *BB = CLI.CB->getParent();
2071 PreferIndirect = Subtarget->isThumb() && Subtarget->hasMinSize() &&
2072 count_if(Range: GV->users(), P: [&BB](const User *U) {
2073 return isa<Instruction>(Val: U) &&
2074 cast<Instruction>(Val: U)->getParent() == BB;
2075 }) > 2;
2076 }
2077 }
2078 if (isTailCall) {
2079 // Check if it's really possible to do a tail call.
2080 isTailCall =
2081 IsEligibleForTailCallOptimization(CLI, CCInfo, ArgLocs, isIndirect: PreferIndirect);
2082
2083 if (isTailCall && !getTargetMachine().Options.GuaranteedTailCallOpt &&
2084 CallConv != CallingConv::Tail && CallConv != CallingConv::SwiftTail)
2085 isSibCall = true;
2086
2087 // We don't support GuaranteedTailCallOpt for ARM, only automatically
2088 // detected sibcalls.
2089 if (isTailCall)
2090 ++NumTailCalls;
2091 }
2092
2093 if (!isTailCall && CLI.CB && CLI.CB->isMustTailCall())
2094 report_fatal_error(reason: "failed to perform tail call elimination on a call "
2095 "site marked musttail");
2096
2097 // Get a count of how many bytes are to be pushed on the stack.
2098 unsigned NumBytes = CCInfo.getStackSize();
2099
2100 // SPDiff is the byte offset of the call's argument area from the callee's.
2101 // Stores to callee stack arguments will be placed in FixedStackSlots offset
2102 // by this amount for a tail call. In a sibling call it must be 0 because the
2103 // caller will deallocate the entire stack and the callee still expects its
2104 // arguments to begin at SP+0. Completely unused for non-tail calls.
2105 int SPDiff = 0;
2106
2107 if (isTailCall && !isSibCall) {
2108 auto FuncInfo = MF.getInfo<ARMFunctionInfo>();
2109 unsigned NumReusableBytes = FuncInfo->getArgumentStackSize();
2110
2111 // Since callee will pop argument stack as a tail call, we must keep the
2112 // popped size 16-byte aligned.
2113 MaybeAlign StackAlign = DAG.getDataLayout().getStackAlignment();
2114 assert(StackAlign && "data layout string is missing stack alignment");
2115 NumBytes = alignTo(Size: NumBytes, A: *StackAlign);
2116
2117 // SPDiff will be negative if this tail call requires more space than we
2118 // would automatically have in our incoming argument space. Positive if we
2119 // can actually shrink the stack.
2120 SPDiff = NumReusableBytes - NumBytes;
2121
2122 // If this call requires more stack than we have available from
2123 // LowerFormalArguments, tell FrameLowering to reserve space for it.
2124 if (SPDiff < 0 && AFI->getArgRegsSaveSize() < (unsigned)-SPDiff)
2125 AFI->setArgRegsSaveSize(-SPDiff);
2126 }
2127
2128 if (isSibCall) {
2129 // For sibling tail calls, memory operands are available in our caller's stack.
2130 NumBytes = 0;
2131 } else {
2132 // Adjust the stack pointer for the new arguments...
2133 // These operations are automatically eliminated by the prolog/epilog pass
2134 Chain = DAG.getCALLSEQ_START(Chain, InSize: isTailCall ? 0 : NumBytes, OutSize: 0, DL: dl);
2135 }
2136
2137 SDValue StackPtr =
2138 DAG.getCopyFromReg(Chain, dl, Reg: ARM::SP, VT: getPointerTy(DL: DAG.getDataLayout()));
2139
2140 RegsToPassVector RegsToPass;
2141 SmallVector<SDValue, 8> MemOpChains;
2142
2143 // A musttail call in a variadic function must receive the caller's unnamed
2144 // register arguments unchanged. The forwarded set is disjoint from the named
2145 // arguments handled below.
2146 if (isVarArg && CLI.CB && CLI.CB->isMustTailCall()) {
2147 const auto &Forwards = AFI->getForwardedMustTailRegParms();
2148 for (const ForwardedRegister &F : Forwards) {
2149 SDValue Val = DAG.getCopyFromReg(Chain, dl, Reg: F.VReg, VT: F.VT);
2150 RegsToPass.push_back(Elt: std::make_pair(x: F.PReg, y&: Val));
2151 }
2152 }
2153
2154 // If we are doing a tail-call, any byval arguments will be written to stack
2155 // space which was used for incoming arguments. If any the values being used
2156 // are incoming byval arguments to this function, then they might be
2157 // overwritten by the stores of the outgoing arguments. To avoid this, we
2158 // need to make a temporary copy of them in local stack space, then copy back
2159 // to the argument area.
2160 DenseMap<unsigned, SDValue> ByValTemporaries;
2161 SDValue ByValTempChain;
2162 if (isTailCall) {
2163 SmallVector<SDValue, 8> ByValCopyChains;
2164 for (const CCValAssign &VA : ArgLocs) {
2165 unsigned ArgIdx = VA.getValNo();
2166 SDValue Src = OutVals[ArgIdx];
2167 ISD::ArgFlagsTy Flags = Outs[ArgIdx].Flags;
2168
2169 if (!Flags.isByVal())
2170 continue;
2171
2172 SDValue Dst;
2173 MachinePointerInfo DstInfo;
2174 std::tie(args&: Dst, args&: DstInfo) =
2175 computeAddrForCallArg(dl, DAG, VA, StackPtr: SDValue(), IsTailCall: true, SPDiff);
2176 ByValCopyKind Copy = ByValNeedsCopyForTailCall(DAG, Src, Dst, Flags);
2177
2178 if (Copy == NoCopy) {
2179 // If the argument is already at the correct offset on the stack
2180 // (because we are forwarding a byval argument from our caller), we
2181 // don't need any copying.
2182 continue;
2183 } else if (Copy == CopyOnce) {
2184 // If the argument is in our local stack frame, no other argument
2185 // preparation can clobber it, so we can copy it to the final location
2186 // later.
2187 ByValTemporaries[ArgIdx] = Src;
2188 } else {
2189 assert(Copy == CopyViaTemp && "unexpected enum value");
2190 // If we might be copying this argument from the outgoing argument
2191 // stack area, we need to copy via a temporary in the local stack
2192 // frame.
2193 int TempFrameIdx = MFI.CreateStackObject(
2194 Size: Flags.getByValSize(), Alignment: Flags.getNonZeroByValAlign(), isSpillSlot: false);
2195 SDValue Temp =
2196 DAG.getFrameIndex(FI: TempFrameIdx, VT: getPointerTy(DL: DAG.getDataLayout()));
2197
2198 SDValue SizeNode = DAG.getConstant(Val: Flags.getByValSize(), DL: dl, VT: MVT::i32);
2199 SDValue AlignNode =
2200 DAG.getConstant(Val: Flags.getNonZeroByValAlign().value(), DL: dl, VT: MVT::i32);
2201
2202 SDVTList VTs = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
2203 SDValue Ops[] = {Chain, Temp, Src, SizeNode, AlignNode};
2204 ByValCopyChains.push_back(
2205 Elt: DAG.getNode(Opcode: ARMISD::COPY_STRUCT_BYVAL, DL: dl, VTList: VTs, Ops));
2206 ByValTemporaries[ArgIdx] = Temp;
2207 }
2208 }
2209 if (!ByValCopyChains.empty())
2210 ByValTempChain =
2211 DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: ByValCopyChains);
2212 }
2213
2214 // During a tail call, stores to the argument area must happen after all of
2215 // the function's incoming arguments have been loaded because they may alias.
2216 // This is done by folding in a TokenFactor from LowerFormalArguments, but
2217 // there's no point in doing so repeatedly so this tracks whether that's
2218 // happened yet.
2219 bool AfterFormalArgLoads = false;
2220
2221 // Walk the register/memloc assignments, inserting copies/loads. In the case
2222 // of tail call optimization, arguments are handled later.
2223 for (unsigned i = 0, realArgIdx = 0, e = ArgLocs.size();
2224 i != e;
2225 ++i, ++realArgIdx) {
2226 CCValAssign &VA = ArgLocs[i];
2227 SDValue Arg = OutVals[realArgIdx];
2228 ISD::ArgFlagsTy Flags = Outs[realArgIdx].Flags;
2229 bool isByVal = Flags.isByVal();
2230
2231 // Promote the value if needed.
2232 switch (VA.getLocInfo()) {
2233 default: llvm_unreachable("Unknown loc info!");
2234 case CCValAssign::Full: break;
2235 case CCValAssign::SExt:
2236 Arg = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: Arg);
2237 break;
2238 case CCValAssign::ZExt:
2239 Arg = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: Arg);
2240 break;
2241 case CCValAssign::AExt:
2242 Arg = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: Arg);
2243 break;
2244 case CCValAssign::BCvt:
2245 Arg = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: VA.getLocVT(), Operand: Arg);
2246 break;
2247 }
2248
2249 if (isTailCall && VA.isMemLoc() && !AfterFormalArgLoads) {
2250 Chain = DAG.getStackArgumentTokenFactor(Chain);
2251 if (ByValTempChain) {
2252 // In case of large byval copies, re-using the stackframe for tail-calls
2253 // can lead to overwriting incoming arguments on the stack. Force
2254 // loading these stack arguments before the copy to avoid that.
2255 SmallVector<SDValue, 8> IncomingLoad;
2256 for (unsigned I = 0; I < OutVals.size(); ++I) {
2257 if (Outs[I].Flags.isByVal())
2258 continue;
2259
2260 SDValue OutVal = OutVals[I];
2261 LoadSDNode *OutLN = dyn_cast_or_null<LoadSDNode>(Val&: OutVal);
2262 if (!OutLN)
2263 continue;
2264
2265 FrameIndexSDNode *FIN =
2266 dyn_cast_or_null<FrameIndexSDNode>(Val: OutLN->getBasePtr());
2267 if (!FIN)
2268 continue;
2269
2270 if (!MFI.isFixedObjectIndex(ObjectIdx: FIN->getIndex()))
2271 continue;
2272
2273 for (const CCValAssign &VA : ArgLocs) {
2274 if (VA.isMemLoc())
2275 IncomingLoad.push_back(Elt: OutVal.getValue(R: 1));
2276 }
2277 }
2278
2279 // Update the chain to force loads for potentially clobbered argument
2280 // loads to happen before the byval copy.
2281 if (!IncomingLoad.empty()) {
2282 IncomingLoad.push_back(Elt: Chain);
2283 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: IncomingLoad);
2284 }
2285
2286 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Chain,
2287 N2: ByValTempChain);
2288 }
2289 AfterFormalArgLoads = true;
2290 }
2291
2292 // f16 arguments have their size extended to 4 bytes and passed as if they
2293 // had been copied to the LSBs of a 32-bit register.
2294 // For that, it's passed extended to i32 (soft ABI) or to f32 (hard ABI)
2295 if (VA.needsCustom() &&
2296 (VA.getValVT() == MVT::f16 || VA.getValVT() == MVT::bf16)) {
2297 Arg = MoveFromHPR(dl, DAG, LocVT: VA.getLocVT(), ValVT: VA.getValVT(), Val: Arg);
2298 } else {
2299 // f16 arguments could have been extended prior to argument lowering.
2300 // Mask them arguments if this is a CMSE nonsecure call.
2301 auto ArgVT = Outs[realArgIdx].ArgVT;
2302 if (isCmseNSCall && (ArgVT == MVT::f16)) {
2303 auto LocBits = VA.getLocVT().getSizeInBits();
2304 auto MaskValue = APInt::getLowBitsSet(numBits: LocBits, loBitsSet: ArgVT.getSizeInBits());
2305 SDValue Mask =
2306 DAG.getConstant(Val: MaskValue, DL: dl, VT: MVT::getIntegerVT(BitWidth: LocBits));
2307 Arg = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::getIntegerVT(BitWidth: LocBits), Operand: Arg);
2308 Arg = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::getIntegerVT(BitWidth: LocBits), N1: Arg, N2: Mask);
2309 Arg = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: VA.getLocVT(), Operand: Arg);
2310 }
2311 }
2312
2313 // f64 and v2f64 might be passed in i32 pairs and must be split into pieces
2314 if (VA.needsCustom() && VA.getLocVT() == MVT::v2f64) {
2315 SDValue Op0 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: MVT::f64, N1: Arg,
2316 N2: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
2317 SDValue Op1 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: MVT::f64, N1: Arg,
2318 N2: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
2319
2320 PassF64ArgInRegs(dl, DAG, Chain, Arg&: Op0, RegsToPass, VA, NextVA&: ArgLocs[++i],
2321 StackPtr, MemOpChains, IsTailCall: isTailCall, SPDiff);
2322
2323 VA = ArgLocs[++i]; // skip ahead to next loc
2324 if (VA.isRegLoc()) {
2325 PassF64ArgInRegs(dl, DAG, Chain, Arg&: Op1, RegsToPass, VA, NextVA&: ArgLocs[++i],
2326 StackPtr, MemOpChains, IsTailCall: isTailCall, SPDiff);
2327 } else {
2328 assert(VA.isMemLoc());
2329 SDValue DstAddr;
2330 MachinePointerInfo DstInfo;
2331 std::tie(args&: DstAddr, args&: DstInfo) =
2332 computeAddrForCallArg(dl, DAG, VA, StackPtr, IsTailCall: isTailCall, SPDiff);
2333 MemOpChains.push_back(Elt: DAG.getStore(Chain, dl, Val: Op1, Ptr: DstAddr, PtrInfo: DstInfo));
2334 }
2335 } else if (VA.needsCustom() && VA.getLocVT() == MVT::f64) {
2336 PassF64ArgInRegs(dl, DAG, Chain, Arg, RegsToPass, VA, NextVA&: ArgLocs[++i],
2337 StackPtr, MemOpChains, IsTailCall: isTailCall, SPDiff);
2338 } else if (VA.isRegLoc()) {
2339 if (realArgIdx == 0 && Flags.isReturned() && !Flags.isSwiftSelf() &&
2340 Outs[0].VT == MVT::i32) {
2341 assert(VA.getLocVT() == MVT::i32 &&
2342 "unexpected calling convention register assignment");
2343 assert(!Ins.empty() && Ins[0].VT == MVT::i32 &&
2344 "unexpected use of 'returned'");
2345 isThisReturn = true;
2346 }
2347 const TargetOptions &Options = DAG.getTarget().Options;
2348 if (Options.EmitCallSiteInfo)
2349 CSInfo.ArgRegPairs.emplace_back(Args: VA.getLocReg(), Args&: i);
2350 RegsToPass.push_back(Elt: std::make_pair(x: VA.getLocReg(), y&: Arg));
2351 } else if (isByVal) {
2352 assert(VA.isMemLoc());
2353 unsigned offset = 0;
2354
2355 // True if this byval aggregate will be split between registers
2356 // and memory.
2357 unsigned ByValArgsCount = CCInfo.getInRegsParamsCount();
2358 unsigned CurByValIdx = CCInfo.getInRegsParamsProcessed();
2359
2360 SDValue ByValSrc;
2361 bool NeedsStackCopy;
2362 if (auto It = ByValTemporaries.find(Val: realArgIdx);
2363 It != ByValTemporaries.end()) {
2364 ByValSrc = It->second;
2365 NeedsStackCopy = true;
2366 } else {
2367 ByValSrc = Arg;
2368 NeedsStackCopy = !isTailCall;
2369 }
2370
2371 // If part of the argument is in registers, load them.
2372 if (CurByValIdx < ByValArgsCount) {
2373 unsigned RegBegin, RegEnd;
2374 CCInfo.getInRegsParamInfo(InRegsParamRecordIndex: CurByValIdx, BeginReg&: RegBegin, EndReg&: RegEnd);
2375
2376 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
2377 unsigned int i, j;
2378 for (i = 0, j = RegBegin; j < RegEnd; i++, j++) {
2379 SDValue Const = DAG.getConstant(Val: 4*i, DL: dl, VT: MVT::i32);
2380 SDValue AddArg = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: ByValSrc, N2: Const);
2381 SDValue Load =
2382 DAG.getLoad(VT: PtrVT, dl, Chain, Ptr: AddArg, PtrInfo: MachinePointerInfo(),
2383 Alignment: DAG.InferPtrAlign(Ptr: AddArg));
2384 MemOpChains.push_back(Elt: Load.getValue(R: 1));
2385 RegsToPass.push_back(Elt: std::make_pair(x&: j, y&: Load));
2386 }
2387
2388 // If parameter size outsides register area, "offset" value
2389 // helps us to calculate stack slot for remained part properly.
2390 offset = RegEnd - RegBegin;
2391
2392 CCInfo.nextInRegsParam();
2393 }
2394
2395 // If the memory part of the argument isn't already in the correct place
2396 // (which can happen with tail calls), copy it into the argument area.
2397 if (NeedsStackCopy && Flags.getByValSize() > 4 * offset) {
2398 auto PtrVT = getPointerTy(DL: DAG.getDataLayout());
2399 SDValue Dst;
2400 MachinePointerInfo DstInfo;
2401 std::tie(args&: Dst, args&: DstInfo) =
2402 computeAddrForCallArg(dl, DAG, VA, StackPtr, IsTailCall: isTailCall, SPDiff);
2403 SDValue SrcOffset = DAG.getIntPtrConstant(Val: 4*offset, DL: dl);
2404 SDValue Src = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: ByValSrc, N2: SrcOffset);
2405 SDValue SizeNode = DAG.getConstant(Val: Flags.getByValSize() - 4*offset, DL: dl,
2406 VT: MVT::i32);
2407 SDValue AlignNode =
2408 DAG.getConstant(Val: Flags.getNonZeroByValAlign().value(), DL: dl, VT: MVT::i32);
2409
2410 SDVTList VTs = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
2411 SDValue Ops[] = { Chain, Dst, Src, SizeNode, AlignNode};
2412 MemOpChains.push_back(Elt: DAG.getNode(Opcode: ARMISD::COPY_STRUCT_BYVAL, DL: dl, VTList: VTs,
2413 Ops));
2414 }
2415 } else {
2416 assert(VA.isMemLoc());
2417 SDValue DstAddr;
2418 MachinePointerInfo DstInfo;
2419 std::tie(args&: DstAddr, args&: DstInfo) =
2420 computeAddrForCallArg(dl, DAG, VA, StackPtr, IsTailCall: isTailCall, SPDiff);
2421
2422 SDValue Store = DAG.getStore(Chain, dl, Val: Arg, Ptr: DstAddr, PtrInfo: DstInfo);
2423 MemOpChains.push_back(Elt: Store);
2424 }
2425 }
2426
2427 if (!MemOpChains.empty())
2428 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: MemOpChains);
2429
2430 // Build a sequence of copy-to-reg nodes chained together with token chain
2431 // and flag operands which copy the outgoing args into the appropriate regs.
2432 SDValue InGlue;
2433 for (const auto &[Reg, N] : RegsToPass) {
2434 Chain = DAG.getCopyToReg(Chain, dl, Reg, N, Glue: InGlue);
2435 InGlue = Chain.getValue(R: 1);
2436 }
2437
2438 // If the callee is a GlobalAddress/ExternalSymbol node (quite common, every
2439 // direct call is) turn it into a TargetGlobalAddress/TargetExternalSymbol
2440 // node so that legalize doesn't hack it.
2441 bool isDirect = false;
2442
2443 const TargetMachine &TM = getTargetMachine();
2444 const Triple &TT = TM.getTargetTriple();
2445 const GlobalValue *GVal = nullptr;
2446 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Val&: Callee))
2447 GVal = G->getGlobal();
2448 bool isStub = !TM.shouldAssumeDSOLocal(GV: GVal) && TT.isOSBinFormatMachO();
2449
2450 bool isARMFunc = !Subtarget->isThumb() || (isStub && !Subtarget->isMClass());
2451 bool isLocalARMFunc = false;
2452 auto PtrVt = getPointerTy(DL: DAG.getDataLayout());
2453
2454 if (Subtarget->genLongCalls()) {
2455 bool isPIC = isPositionIndependent() && !TT.isOSWindows();
2456 if (isPIC && Subtarget->genExecuteOnly())
2457 reportFatalUsageError(reason: "long-calls with execute-only and "
2458 "position-independent code is not supported");
2459 if (Subtarget->isROPI())
2460 reportFatalUsageError(reason: "long-calls with ROPI is not currently supported");
2461
2462 // Handle a global address or an external symbol. If it's not one of
2463 // those, the target's already in a register, so we don't need to do
2464 // anything extra.
2465 if (isa<GlobalAddressSDNode>(Val: Callee)) {
2466 if (Subtarget->genExecuteOnly()) {
2467 // Execute-only forbids constant pools in .text, so use movw/movt.
2468 // fPIC is not supported with execute-only.
2469 if (Subtarget->useMovt())
2470 ++NumMovwMovt;
2471 Callee = DAG.getNode(Opcode: ARMISD::Wrapper, DL: dl, VT: PtrVt,
2472 Operand: DAG.getTargetGlobalAddress(GV: GVal, DL: dl, VT: PtrVt));
2473 } else if (isPIC) {
2474 // PIC without execute-only: use GOT-based addressing.
2475 // DSO-local symbols use a plain PC-relative WrapperPIC;
2476 // non-DSO-local symbols additionally load the address from the GOT.
2477 SDValue G = DAG.getTargetGlobalAddress(
2478 GV: GVal, DL: dl, VT: PtrVt, offset: 0, TargetFlags: GVal->isDSOLocal() ? 0 : ARMII::MO_GOT);
2479 Callee = DAG.getNode(Opcode: ARMISD::WrapperPIC, DL: dl, VT: PtrVt, Operand: G);
2480 if (!GVal->isDSOLocal())
2481 Callee =
2482 DAG.getLoad(VT: PtrVt, dl, Chain: DAG.getEntryNode(), Ptr: Callee,
2483 PtrInfo: MachinePointerInfo::getGOT(MF&: DAG.getMachineFunction()));
2484 } else {
2485 // Neither execute-only nor PIC: load the address from a constant pool.
2486 unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
2487 ARMConstantPoolValue *CPV = ARMConstantPoolConstant::Create(
2488 C: GVal, ID: ARMPCLabelIndex, Kind: ARMCP::CPValue, PCAdj: 0);
2489
2490 // Get the address of the callee into a register
2491 SDValue Addr = DAG.getTargetConstantPool(C: CPV, VT: PtrVt, Align: Align(4));
2492 Addr = DAG.getNode(Opcode: ARMISD::Wrapper, DL: dl, VT: MVT::i32, Operand: Addr);
2493 Callee = DAG.getLoad(
2494 VT: PtrVt, dl, Chain: DAG.getEntryNode(), Ptr: Addr,
2495 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
2496 }
2497 } else if (ExternalSymbolSDNode *S=dyn_cast<ExternalSymbolSDNode>(Val&: Callee)) {
2498 const char *Sym = S->getSymbol();
2499
2500 if (Subtarget->genExecuteOnly()) {
2501 // Execute-only forbids constant pools in .text, so use movw/movt.
2502 // fPIC is not supported with execute-only.
2503 if (Subtarget->useMovt())
2504 ++NumMovwMovt;
2505 Callee = DAG.getNode(Opcode: ARMISD::Wrapper, DL: dl, VT: PtrVt,
2506 Operand: DAG.getTargetExternalSymbol(Sym, VT: PtrVt, TargetFlags: 0));
2507 } else if (isPIC) {
2508 // PIC without execute-only: load the symbol's address from the GOT via
2509 // a GOT_PREL constant pool entry consumed by a PICLDR.
2510 unsigned PCAdj = Subtarget->isThumb() ? 4 : 8;
2511 unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
2512 ARMConstantPoolValue *CPV = ARMConstantPoolSymbol::Create(
2513 C&: *DAG.getContext(), s: Sym, ID: ARMPCLabelIndex, PCAdj, Modifier: ARMCP::GOT_PREL,
2514 /*AddCurrentAddress=*/true);
2515 SDValue CPAddr = DAG.getTargetConstantPool(C: CPV, VT: PtrVt, Align: Align(4));
2516 CPAddr = DAG.getNode(Opcode: ARMISD::Wrapper, DL: dl, VT: MVT::i32, Operand: CPAddr);
2517 SDValue GOTOffset = DAG.getLoad(
2518 VT: PtrVt, dl, Chain: DAG.getEntryNode(), Ptr: CPAddr,
2519 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
2520 SDValue PICLabel = DAG.getConstant(Val: ARMPCLabelIndex, DL: dl, VT: MVT::i32);
2521 Callee = DAG.getNode(Opcode: ARMISD::PIC_ADD, DL: dl, VT: PtrVt, N1: GOTOffset, N2: PICLabel);
2522 Callee =
2523 DAG.getLoad(VT: PtrVt, dl, Chain: DAG.getEntryNode(), Ptr: Callee,
2524 PtrInfo: MachinePointerInfo::getGOT(MF&: DAG.getMachineFunction()));
2525 } else {
2526 // Neither execute-only nor PIC: load the address from a constant pool.
2527 unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
2528 ARMConstantPoolValue *CPV = ARMConstantPoolSymbol::Create(
2529 C&: *DAG.getContext(), s: Sym, ID: ARMPCLabelIndex, PCAdj: 0);
2530
2531 // Get the address of the callee into a register
2532 SDValue Addr = DAG.getTargetConstantPool(C: CPV, VT: PtrVt, Align: Align(4));
2533 Addr = DAG.getNode(Opcode: ARMISD::Wrapper, DL: dl, VT: MVT::i32, Operand: Addr);
2534 Callee = DAG.getLoad(
2535 VT: PtrVt, dl, Chain: DAG.getEntryNode(), Ptr: Addr,
2536 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
2537 }
2538 }
2539 } else if (isa<GlobalAddressSDNode>(Val: Callee)) {
2540 if (!PreferIndirect) {
2541 isDirect = true;
2542 bool isDef = GVal->isStrongDefinitionForLinker();
2543
2544 // ARM call to a local ARM function is predicable.
2545 isLocalARMFunc = !Subtarget->isThumb() && (isDef || !ARMInterworking);
2546 // tBX takes a register source operand.
2547 if (isStub && Subtarget->isThumb1Only() && !Subtarget->hasV5TOps()) {
2548 assert(TT.isOSBinFormatMachO() && "WrapperPIC use on non-MachO?");
2549 Callee = DAG.getNode(
2550 Opcode: ARMISD::WrapperPIC, DL: dl, VT: PtrVt,
2551 Operand: DAG.getTargetGlobalAddress(GV: GVal, DL: dl, VT: PtrVt, offset: 0, TargetFlags: ARMII::MO_NONLAZY));
2552 Callee = DAG.getLoad(
2553 VT: PtrVt, dl, Chain: DAG.getEntryNode(), Ptr: Callee,
2554 PtrInfo: MachinePointerInfo::getGOT(MF&: DAG.getMachineFunction()), Alignment: MaybeAlign(),
2555 MMOFlags: MachineMemOperand::MODereferenceable |
2556 MachineMemOperand::MOInvariant);
2557 } else if (Subtarget->isTargetCOFF()) {
2558 assert(Subtarget->isTargetWindows() &&
2559 "Windows is the only supported COFF target");
2560 unsigned TargetFlags = ARMII::MO_NO_FLAG;
2561 if (GVal->hasDLLImportStorageClass())
2562 TargetFlags = ARMII::MO_DLLIMPORT;
2563 else if (!TM.shouldAssumeDSOLocal(GV: GVal))
2564 TargetFlags = ARMII::MO_COFFSTUB;
2565 Callee = DAG.getTargetGlobalAddress(GV: GVal, DL: dl, VT: PtrVt, /*offset=*/0,
2566 TargetFlags);
2567 if (TargetFlags & (ARMII::MO_DLLIMPORT | ARMII::MO_COFFSTUB))
2568 Callee =
2569 DAG.getLoad(VT: PtrVt, dl, Chain: DAG.getEntryNode(),
2570 Ptr: DAG.getNode(Opcode: ARMISD::Wrapper, DL: dl, VT: PtrVt, Operand: Callee),
2571 PtrInfo: MachinePointerInfo::getGOT(MF&: DAG.getMachineFunction()));
2572 } else {
2573 Callee = DAG.getTargetGlobalAddress(GV: GVal, DL: dl, VT: PtrVt, offset: 0, TargetFlags: 0);
2574 }
2575 }
2576 } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Val&: Callee)) {
2577 isDirect = true;
2578 // tBX takes a register source operand.
2579 const char *Sym = S->getSymbol();
2580 if (isARMFunc && Subtarget->isThumb1Only() && !Subtarget->hasV5TOps()) {
2581 unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
2582 ARMConstantPoolValue *CPV =
2583 ARMConstantPoolSymbol::Create(C&: *DAG.getContext(), s: Sym,
2584 ID: ARMPCLabelIndex, PCAdj: 4);
2585 SDValue CPAddr = DAG.getTargetConstantPool(C: CPV, VT: PtrVt, Align: Align(4));
2586 CPAddr = DAG.getNode(Opcode: ARMISD::Wrapper, DL: dl, VT: MVT::i32, Operand: CPAddr);
2587 Callee = DAG.getLoad(
2588 VT: PtrVt, dl, Chain: DAG.getEntryNode(), Ptr: CPAddr,
2589 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
2590 SDValue PICLabel = DAG.getConstant(Val: ARMPCLabelIndex, DL: dl, VT: MVT::i32);
2591 Callee = DAG.getNode(Opcode: ARMISD::PIC_ADD, DL: dl, VT: PtrVt, N1: Callee, N2: PICLabel);
2592 } else {
2593 Callee = DAG.getTargetExternalSymbol(Sym, VT: PtrVt, TargetFlags: 0);
2594 }
2595 }
2596
2597 if (isCmseNSCall) {
2598 assert(!isARMFunc && !isDirect &&
2599 "Cannot handle call to ARM function or direct call");
2600 if (NumBytes > 0) {
2601 DAG.getContext()->diagnose(
2602 DI: DiagnosticInfoUnsupported(DAG.getMachineFunction().getFunction(),
2603 "call to non-secure function would require "
2604 "passing arguments on stack",
2605 dl.getDebugLoc()));
2606 }
2607 if (isStructRet) {
2608 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
2609 DAG.getMachineFunction().getFunction(),
2610 "call to non-secure function would return value through pointer",
2611 dl.getDebugLoc()));
2612 }
2613 }
2614
2615 // FIXME: handle tail calls differently.
2616 unsigned CallOpc;
2617 if (Subtarget->isThumb()) {
2618 if (GuardWithBTI)
2619 CallOpc = ARMISD::t2CALL_BTI;
2620 else if (isCmseNSCall)
2621 CallOpc = ARMISD::tSECALL;
2622 else if ((!isDirect || isARMFunc) && !Subtarget->hasV5TOps())
2623 CallOpc = ARMISD::CALL_NOLINK;
2624 else
2625 CallOpc = ARMISD::CALL;
2626 } else {
2627 if (!isDirect && !Subtarget->hasV5TOps())
2628 CallOpc = ARMISD::CALL_NOLINK;
2629 else if (doesNotRet && isDirect && Subtarget->hasRetAddrStack() &&
2630 // Emit regular call when code size is the priority
2631 !Subtarget->hasMinSize())
2632 // "mov lr, pc; b _foo" to avoid confusing the RSP
2633 CallOpc = ARMISD::CALL_NOLINK;
2634 else
2635 CallOpc = isLocalARMFunc ? ARMISD::CALL_PRED : ARMISD::CALL;
2636 }
2637
2638 // We don't usually want to end the call-sequence here because we would tidy
2639 // the frame up *after* the call, however in the ABI-changing tail-call case
2640 // we've carefully laid out the parameters so that when sp is reset they'll be
2641 // in the correct location.
2642 if (isTailCall && !isSibCall) {
2643 Chain = DAG.getCALLSEQ_END(Chain, Size1: 0, Size2: 0, Glue: InGlue, DL: dl);
2644 InGlue = Chain.getValue(R: 1);
2645 }
2646
2647 std::vector<SDValue> Ops;
2648 Ops.push_back(x: Chain);
2649 Ops.push_back(x: Callee);
2650
2651 if (isTailCall) {
2652 Ops.push_back(x: DAG.getSignedTargetConstant(Val: SPDiff, DL: dl, VT: MVT::i32));
2653 }
2654
2655 // Add argument registers to the end of the list so that they are known live
2656 // into the call.
2657 for (const auto &[Reg, N] : RegsToPass)
2658 Ops.push_back(x: DAG.getRegister(Reg, VT: N.getValueType()));
2659
2660 // Add a register mask operand representing the call-preserved registers.
2661 const uint32_t *Mask;
2662 const ARMBaseRegisterInfo *ARI = Subtarget->getRegisterInfo();
2663 if (isThisReturn) {
2664 // For 'this' returns, use the R0-preserving mask if applicable
2665 Mask = ARI->getThisReturnPreservedMask(MF, CallConv);
2666 if (!Mask) {
2667 // Set isThisReturn to false if the calling convention is not one that
2668 // allows 'returned' to be modeled in this way, so LowerCallResult does
2669 // not try to pass 'this' straight through
2670 isThisReturn = false;
2671 Mask = ARI->getCallPreservedMask(MF, CallConv);
2672 }
2673 } else
2674 Mask = ARI->getCallPreservedMask(MF, CallConv);
2675
2676 assert(Mask && "Missing call preserved mask for calling convention");
2677 Ops.push_back(x: DAG.getRegisterMask(RegMask: Mask));
2678
2679 if (InGlue.getNode())
2680 Ops.push_back(x: InGlue);
2681
2682 if (isTailCall) {
2683 MF.getFrameInfo().setHasTailCall();
2684 SDValue Ret = DAG.getNode(Opcode: ARMISD::TC_RETURN, DL: dl, VT: MVT::Other, Ops);
2685 if (CLI.CFIType)
2686 Ret.getNode()->setCFIType(CLI.CFIType->getZExtValue());
2687 DAG.addNoMergeSiteInfo(Node: Ret.getNode(), NoMerge: CLI.NoMerge);
2688 DAG.addCallSiteInfo(Node: Ret.getNode(), CallInfo: std::move(CSInfo));
2689 return Ret;
2690 }
2691
2692 // Returns a chain and a flag for retval copy to use.
2693 Chain = DAG.getNode(Opcode: CallOpc, DL: dl, ResultTys: {MVT::Other, MVT::Glue}, Ops);
2694 if (CLI.CFIType)
2695 Chain.getNode()->setCFIType(CLI.CFIType->getZExtValue());
2696 DAG.addNoMergeSiteInfo(Node: Chain.getNode(), NoMerge: CLI.NoMerge);
2697 InGlue = Chain.getValue(R: 1);
2698 DAG.addCallSiteInfo(Node: Chain.getNode(), CallInfo: std::move(CSInfo));
2699
2700 // If we're guaranteeing tail-calls will be honoured, the callee must
2701 // pop its own argument stack on return. But this call is *not* a tail call so
2702 // we need to undo that after it returns to restore the status-quo.
2703 bool TailCallOpt = getTargetMachine().Options.GuaranteedTailCallOpt;
2704 uint64_t CalleePopBytes =
2705 canGuaranteeTCO(CC: CallConv, GuaranteeTailCalls: TailCallOpt) ? alignTo(Value: NumBytes, Align: 16) : -1U;
2706
2707 Chain = DAG.getCALLSEQ_END(Chain, Size1: NumBytes, Size2: CalleePopBytes, Glue: InGlue, DL: dl);
2708 if (!Ins.empty())
2709 InGlue = Chain.getValue(R: 1);
2710
2711 // Handle result values, copying them out of physregs into vregs that we
2712 // return.
2713 return LowerCallResult(Chain, InGlue, CallConv, isVarArg, Ins, dl, DAG,
2714 InVals, isThisReturn,
2715 ThisVal: isThisReturn ? OutVals[0] : SDValue(), isCmseNSCall);
2716}
2717
2718/// HandleByVal - Every parameter *after* a byval parameter is passed
2719/// on the stack. Remember the next parameter register to allocate,
2720/// and then confiscate the rest of the parameter registers to insure
2721/// this.
2722void ARMTargetLowering::HandleByVal(CCState *State, unsigned &Size,
2723 Align Alignment) const {
2724 // Byval (as with any stack) slots are always at least 4 byte aligned.
2725 Alignment = std::max(a: Alignment, b: Align(4));
2726
2727 MCRegister Reg = State->AllocateReg(Regs: GPRArgRegs);
2728 if (!Reg)
2729 return;
2730
2731 unsigned AlignInRegs = Alignment.value() / 4;
2732 unsigned Waste = (ARM::R4 - Reg) % AlignInRegs;
2733 for (unsigned i = 0; i < Waste; ++i)
2734 Reg = State->AllocateReg(Regs: GPRArgRegs);
2735
2736 if (!Reg)
2737 return;
2738
2739 unsigned Excess = 4 * (ARM::R4 - Reg);
2740
2741 // Special case when NSAA != SP and parameter size greater than size of
2742 // all remained GPR regs. In that case we can't split parameter, we must
2743 // send it to stack. We also must set NCRN to R4, so waste all
2744 // remained registers.
2745 const unsigned NSAAOffset = State->getStackSize();
2746 if (NSAAOffset != 0 && Size > Excess) {
2747 while (State->AllocateReg(Regs: GPRArgRegs))
2748 ;
2749 return;
2750 }
2751
2752 // First register for byval parameter is the first register that wasn't
2753 // allocated before this method call, so it would be "reg".
2754 // If parameter is small enough to be saved in range [reg, r4), then
2755 // the end (first after last) register would be reg + param-size-in-regs,
2756 // else parameter would be splitted between registers and stack,
2757 // end register would be r4 in this case.
2758 unsigned ByValRegBegin = Reg;
2759 unsigned ByValRegEnd = std::min<unsigned>(a: Reg + Size / 4, b: ARM::R4);
2760 State->addInRegsParamInfo(RegBegin: ByValRegBegin, RegEnd: ByValRegEnd);
2761 // Note, first register is allocated in the beginning of function already,
2762 // allocate remained amount of registers we need.
2763 for (unsigned i = Reg + 1; i != ByValRegEnd; ++i)
2764 State->AllocateReg(Regs: GPRArgRegs);
2765 // A byval parameter that is split between registers and memory needs its
2766 // size truncated here.
2767 // In the case where the entire structure fits in registers, we set the
2768 // size in memory to zero.
2769 Size = std::max<int>(a: Size - Excess, b: 0);
2770}
2771
2772/// IsEligibleForTailCallOptimization - Check whether the call is eligible
2773/// for tail call optimization. Targets which want to do tail call
2774/// optimization should implement this function. Note that this function also
2775/// processes musttail calls, so when this function returns false on a valid
2776/// musttail call, a fatal backend error occurs.
2777bool ARMTargetLowering::IsEligibleForTailCallOptimization(
2778 TargetLowering::CallLoweringInfo &CLI, CCState &CCInfo,
2779 SmallVectorImpl<CCValAssign> &ArgLocs, const bool isIndirect) const {
2780 CallingConv::ID CalleeCC = CLI.CallConv;
2781 SDValue Callee = CLI.Callee;
2782 bool isVarArg = CLI.IsVarArg;
2783 const SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
2784 const SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
2785 const SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
2786 const SelectionDAG &DAG = CLI.DAG;
2787 MachineFunction &MF = DAG.getMachineFunction();
2788 const Function &CallerF = MF.getFunction();
2789 CallingConv::ID CallerCC = CallerF.getCallingConv();
2790
2791 assert(Subtarget->supportsTailCall());
2792
2793 // Indirect tail-calls require a register to hold the target address. That
2794 // register must be:
2795 // * Allocatable (i.e. r0-r7 if the target is Thumb1).
2796 // * Not callee-saved, so must be one of r0-r3 or r12.
2797 // * Not used to hold an argument to the tail-called function, which might be
2798 // in r0-r3.
2799 // * Not used to hold the return address authentication code, which is in r12
2800 // if enabled.
2801 // Sometimes, no register matches all of these conditions, so we can't do a
2802 // tail-call.
2803 if (!isa<GlobalAddressSDNode>(Val: Callee.getNode()) || isIndirect) {
2804 SmallSet<MCPhysReg, 5> AddressRegisters = {ARM::R0, ARM::R1, ARM::R2,
2805 ARM::R3};
2806 if (!(Subtarget->isThumb1Only() ||
2807 MF.getInfo<ARMFunctionInfo>()->shouldSignReturnAddress(SpillsLR: true)))
2808 AddressRegisters.insert(V: ARM::R12);
2809 for (const CCValAssign &AL : ArgLocs)
2810 if (AL.isRegLoc())
2811 AddressRegisters.erase(V: AL.getLocReg());
2812 if (AddressRegisters.empty()) {
2813 LLVM_DEBUG(dbgs() << "false (no reg to hold function pointer)\n");
2814 return false;
2815 }
2816 }
2817
2818 // Look for obvious safe cases to perform tail call optimization that do not
2819 // require ABI changes. This is what gcc calls sibcall.
2820
2821 // Exception-handling functions need a special set of instructions to indicate
2822 // a return to the hardware. Tail-calling another function would probably
2823 // break this.
2824 if (CallerF.hasFnAttribute(Kind: "interrupt")) {
2825 LLVM_DEBUG(dbgs() << "false (interrupt attribute)\n");
2826 return false;
2827 }
2828
2829 if (canGuaranteeTCO(CC: CalleeCC,
2830 GuaranteeTailCalls: getTargetMachine().Options.GuaranteedTailCallOpt)) {
2831 LLVM_DEBUG(dbgs() << (CalleeCC == CallerCC ? "true" : "false")
2832 << " (guaranteed tail-call CC)\n");
2833 return CalleeCC == CallerCC;
2834 }
2835
2836 // Also avoid sibcall optimization if either caller or callee uses struct
2837 // return semantics.
2838 bool isCalleeStructRet = Outs.empty() ? false : Outs[0].Flags.isSRet();
2839 bool isCallerStructRet = MF.getFunction().hasStructRetAttr();
2840 if (isCalleeStructRet != isCallerStructRet) {
2841 LLVM_DEBUG(dbgs() << "false (struct-ret)\n");
2842 return false;
2843 }
2844
2845 // Externally-defined functions with weak linkage should not be
2846 // tail-called on ARM when the OS does not support dynamic
2847 // pre-emption of symbols, as the AAELF spec requires normal calls
2848 // to undefined weak functions to be replaced with a NOP or jump to the
2849 // next instruction. The behaviour of branch instructions in this
2850 // situation (as used for tail calls) is implementation-defined, so we
2851 // cannot rely on the linker replacing the tail call with a return.
2852 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Val&: Callee)) {
2853 const GlobalValue *GV = G->getGlobal();
2854 const Triple &TT = GV->getParent()->getTargetTriple();
2855 if (GV->hasExternalWeakLinkage() &&
2856 (!TT.isOSWindows() || TT.isOSBinFormatELF() ||
2857 TT.isOSBinFormatMachO())) {
2858 LLVM_DEBUG(dbgs() << "false (external weak linkage)\n");
2859 return false;
2860 }
2861 }
2862
2863 // Check that the call results are passed in the same way.
2864 LLVMContext &C = *DAG.getContext();
2865 if (!CCState::resultsCompatible(
2866 CalleeCC: getEffectiveCallingConv(CC: CalleeCC, isVarArg),
2867 CallerCC: getEffectiveCallingConv(CC: CallerCC, isVarArg: CallerF.isVarArg()), MF, C, Ins,
2868 CalleeFn: CCAssignFnForReturn(CC: CalleeCC, isVarArg),
2869 CallerFn: CCAssignFnForReturn(CC: CallerCC, isVarArg: CallerF.isVarArg()))) {
2870 LLVM_DEBUG(dbgs() << "false (incompatible results)\n");
2871 return false;
2872 }
2873 // The callee has to preserve all registers the caller needs to preserve.
2874 const ARMBaseRegisterInfo *TRI = Subtarget->getRegisterInfo();
2875 const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
2876 if (CalleeCC != CallerCC) {
2877 const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
2878 if (!TRI->regmaskSubsetEqual(mask0: CallerPreserved, mask1: CalleePreserved)) {
2879 LLVM_DEBUG(dbgs() << "false (not all registers preserved)\n");
2880 return false;
2881 }
2882 }
2883
2884 // If Caller's vararg argument has been split between registers and stack, do
2885 // not perform tail call, since part of the argument is in caller's local
2886 // frame.
2887 const ARMFunctionInfo *AFI_Caller = MF.getInfo<ARMFunctionInfo>();
2888 if (CLI.IsVarArg && AFI_Caller->getArgRegsSaveSize()) {
2889 LLVM_DEBUG(dbgs() << "false (arg reg save area)\n");
2890 return false;
2891 }
2892
2893 // If the callee takes no arguments then go on to check the results of the
2894 // call.
2895 const MachineRegisterInfo &MRI = MF.getRegInfo();
2896 if (!parametersInCSRMatch(MRI, CallerPreservedMask: CallerPreserved, ArgLocs, OutVals)) {
2897 LLVM_DEBUG(dbgs() << "false (parameters in CSRs do not match)\n");
2898 return false;
2899 }
2900
2901 // If the stack arguments for this call do not fit into our own save area then
2902 // the call cannot be made tail.
2903 if (CCInfo.getStackSize() > AFI_Caller->getArgumentStackSize())
2904 return false;
2905
2906 LLVM_DEBUG(dbgs() << "true\n");
2907 return true;
2908}
2909
2910bool
2911ARMTargetLowering::CanLowerReturn(CallingConv::ID CallConv,
2912 MachineFunction &MF, bool isVarArg,
2913 const SmallVectorImpl<ISD::OutputArg> &Outs,
2914 LLVMContext &Context, const Type *RetTy) const {
2915 SmallVector<CCValAssign, 16> RVLocs;
2916 CCState CCInfo(CallConv, isVarArg, MF, RVLocs, Context);
2917 return CCInfo.CheckReturn(Outs, Fn: CCAssignFnForReturn(CC: CallConv, isVarArg));
2918}
2919
2920static SDValue LowerInterruptReturn(SmallVectorImpl<SDValue> &RetOps,
2921 const SDLoc &DL, SelectionDAG &DAG) {
2922 const MachineFunction &MF = DAG.getMachineFunction();
2923 const Function &F = MF.getFunction();
2924
2925 StringRef IntKind = F.getFnAttribute(Kind: "interrupt").getValueAsString();
2926
2927 // See ARM ARM v7 B1.8.3. On exception entry LR is set to a possibly offset
2928 // version of the "preferred return address". These offsets affect the return
2929 // instruction if this is a return from PL1 without hypervisor extensions.
2930 // IRQ/FIQ: +4 "subs pc, lr, #4"
2931 // SWI: 0 "subs pc, lr, #0"
2932 // ABORT: +4 "subs pc, lr, #4"
2933 // UNDEF: +4/+2 "subs pc, lr, #0"
2934 // UNDEF varies depending on where the exception came from ARM or Thumb
2935 // mode. Alongside GCC, we throw our hands up in disgust and pretend it's 0.
2936
2937 int64_t LROffset;
2938 if (IntKind == "" || IntKind == "IRQ" || IntKind == "FIQ" ||
2939 IntKind == "ABORT")
2940 LROffset = 4;
2941 else if (IntKind == "SWI" || IntKind == "UNDEF")
2942 LROffset = 0;
2943 else
2944 report_fatal_error(reason: "Unsupported interrupt attribute. If present, value "
2945 "must be one of: IRQ, FIQ, SWI, ABORT or UNDEF");
2946
2947 RetOps.insert(I: RetOps.begin() + 1,
2948 Elt: DAG.getConstant(Val: LROffset, DL, VT: MVT::i32, isTarget: false));
2949
2950 return DAG.getNode(Opcode: ARMISD::INTRET_GLUE, DL, VT: MVT::Other, Ops: RetOps);
2951}
2952
2953SDValue
2954ARMTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
2955 bool isVarArg,
2956 const SmallVectorImpl<ISD::OutputArg> &Outs,
2957 const SmallVectorImpl<SDValue> &OutVals,
2958 const SDLoc &dl, SelectionDAG &DAG) const {
2959 // CCValAssign - represent the assignment of the return value to a location.
2960 SmallVector<CCValAssign, 16> RVLocs;
2961
2962 // CCState - Info about the registers and stack slots.
2963 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
2964 *DAG.getContext());
2965
2966 // Analyze outgoing return values.
2967 CCInfo.AnalyzeReturn(Outs, Fn: CCAssignFnForReturn(CC: CallConv, isVarArg));
2968
2969 SDValue Glue;
2970 SmallVector<SDValue, 4> RetOps;
2971 RetOps.push_back(Elt: Chain); // Operand #0 = Chain (updated below)
2972 bool isLittleEndian = Subtarget->isLittle();
2973
2974 MachineFunction &MF = DAG.getMachineFunction();
2975 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
2976 AFI->setReturnRegsCount(RVLocs.size());
2977
2978 // Report error if cmse entry function returns structure through first ptr arg.
2979 if (AFI->isCmseNSEntryFunction() && MF.getFunction().hasStructRetAttr()) {
2980 // Note: using an empty SDLoc(), as the first line of the function is a
2981 // better place to report than the last line.
2982 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
2983 DAG.getMachineFunction().getFunction(),
2984 "secure entry function would return value through pointer",
2985 SDLoc().getDebugLoc()));
2986 }
2987
2988 // Copy the result values into the output registers.
2989 for (unsigned i = 0, realRVLocIdx = 0;
2990 i != RVLocs.size();
2991 ++i, ++realRVLocIdx) {
2992 CCValAssign &VA = RVLocs[i];
2993 assert(VA.isRegLoc() && "Can only return in registers!");
2994
2995 SDValue Arg = OutVals[realRVLocIdx];
2996 bool ReturnF16 = false;
2997
2998 if (Subtarget->hasFullFP16() && Subtarget->isTargetHardFloat()) {
2999 // Half-precision return values can be returned like this:
3000 //
3001 // t11 f16 = fadd ...
3002 // t12: i16 = bitcast t11
3003 // t13: i32 = zero_extend t12
3004 // t14: f32 = bitcast t13 <~~~~~~~ Arg
3005 //
3006 // to avoid code generation for bitcasts, we simply set Arg to the node
3007 // that produces the f16 value, t11 in this case.
3008 //
3009 if (Arg.getValueType() == MVT::f32 && Arg.getOpcode() == ISD::BITCAST) {
3010 SDValue ZE = Arg.getOperand(i: 0);
3011 if (ZE.getOpcode() == ISD::ZERO_EXTEND && ZE.getValueType() == MVT::i32) {
3012 SDValue BC = ZE.getOperand(i: 0);
3013 if (BC.getOpcode() == ISD::BITCAST && BC.getValueType() == MVT::i16) {
3014 Arg = BC.getOperand(i: 0);
3015 ReturnF16 = true;
3016 }
3017 }
3018 }
3019 }
3020
3021 switch (VA.getLocInfo()) {
3022 default: llvm_unreachable("Unknown loc info!");
3023 case CCValAssign::Full: break;
3024 case CCValAssign::BCvt:
3025 if (!ReturnF16)
3026 Arg = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: VA.getLocVT(), Operand: Arg);
3027 break;
3028 }
3029
3030 // Mask f16 arguments if this is a CMSE nonsecure entry.
3031 auto RetVT = Outs[realRVLocIdx].ArgVT;
3032 if (AFI->isCmseNSEntryFunction() && (RetVT == MVT::f16)) {
3033 if (VA.needsCustom() && VA.getValVT() == MVT::f16) {
3034 Arg = MoveFromHPR(dl, DAG, LocVT: VA.getLocVT(), ValVT: VA.getValVT(), Val: Arg);
3035 } else {
3036 auto LocBits = VA.getLocVT().getSizeInBits();
3037 auto MaskValue = APInt::getLowBitsSet(numBits: LocBits, loBitsSet: RetVT.getSizeInBits());
3038 SDValue Mask =
3039 DAG.getConstant(Val: MaskValue, DL: dl, VT: MVT::getIntegerVT(BitWidth: LocBits));
3040 Arg = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::getIntegerVT(BitWidth: LocBits), Operand: Arg);
3041 Arg = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::getIntegerVT(BitWidth: LocBits), N1: Arg, N2: Mask);
3042 Arg = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: VA.getLocVT(), Operand: Arg);
3043 }
3044 }
3045
3046 if (VA.needsCustom() &&
3047 (VA.getLocVT() == MVT::v2f64 || VA.getLocVT() == MVT::f64)) {
3048 if (VA.getLocVT() == MVT::v2f64) {
3049 // Extract the first half and return it in two registers.
3050 SDValue Half = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: MVT::f64, N1: Arg,
3051 N2: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
3052 SDValue HalfGPRs = DAG.getNode(Opcode: ARMISD::VMOVRRD, DL: dl,
3053 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N: Half);
3054
3055 Chain =
3056 DAG.getCopyToReg(Chain, dl, Reg: VA.getLocReg(),
3057 N: HalfGPRs.getValue(R: isLittleEndian ? 0 : 1), Glue);
3058 Glue = Chain.getValue(R: 1);
3059 RetOps.push_back(Elt: DAG.getRegister(Reg: VA.getLocReg(), VT: VA.getLocVT()));
3060 VA = RVLocs[++i]; // skip ahead to next loc
3061 Chain =
3062 DAG.getCopyToReg(Chain, dl, Reg: VA.getLocReg(),
3063 N: HalfGPRs.getValue(R: isLittleEndian ? 1 : 0), Glue);
3064 Glue = Chain.getValue(R: 1);
3065 RetOps.push_back(Elt: DAG.getRegister(Reg: VA.getLocReg(), VT: VA.getLocVT()));
3066 VA = RVLocs[++i]; // skip ahead to next loc
3067
3068 // Extract the 2nd half and fall through to handle it as an f64 value.
3069 Arg = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: MVT::f64, N1: Arg,
3070 N2: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
3071 }
3072 // Legalize ret f64 -> ret 2 x i32. We always have fmrrd if f64 is
3073 // available.
3074 SDValue fmrrd = DAG.getNode(Opcode: ARMISD::VMOVRRD, DL: dl,
3075 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N: Arg);
3076 Chain = DAG.getCopyToReg(Chain, dl, Reg: VA.getLocReg(),
3077 N: fmrrd.getValue(R: isLittleEndian ? 0 : 1), Glue);
3078 Glue = Chain.getValue(R: 1);
3079 RetOps.push_back(Elt: DAG.getRegister(Reg: VA.getLocReg(), VT: VA.getLocVT()));
3080 VA = RVLocs[++i]; // skip ahead to next loc
3081 Chain = DAG.getCopyToReg(Chain, dl, Reg: VA.getLocReg(),
3082 N: fmrrd.getValue(R: isLittleEndian ? 1 : 0), Glue);
3083 } else
3084 Chain = DAG.getCopyToReg(Chain, dl, Reg: VA.getLocReg(), N: Arg, Glue);
3085
3086 // Guarantee that all emitted copies are
3087 // stuck together, avoiding something bad.
3088 Glue = Chain.getValue(R: 1);
3089 RetOps.push_back(Elt: DAG.getRegister(
3090 Reg: VA.getLocReg(), VT: ReturnF16 ? Arg.getValueType() : VA.getLocVT()));
3091 }
3092 const ARMBaseRegisterInfo *TRI = Subtarget->getRegisterInfo();
3093 const MCPhysReg *I =
3094 TRI->getCalleeSavedRegsViaCopy(MF: &DAG.getMachineFunction());
3095 if (I) {
3096 for (; *I; ++I) {
3097 if (ARM::GPRRegClass.contains(Reg: *I))
3098 RetOps.push_back(Elt: DAG.getRegister(Reg: *I, VT: MVT::i32));
3099 else if (ARM::DPRRegClass.contains(Reg: *I))
3100 RetOps.push_back(Elt: DAG.getRegister(Reg: *I, VT: MVT::getFloatingPointVT(BitWidth: 64)));
3101 else
3102 llvm_unreachable("Unexpected register class in CSRsViaCopy!");
3103 }
3104 }
3105
3106 // Update chain and glue.
3107 RetOps[0] = Chain;
3108 if (Glue.getNode())
3109 RetOps.push_back(Elt: Glue);
3110
3111 // CPUs which aren't M-class use a special sequence to return from
3112 // exceptions (roughly, any instruction setting pc and cpsr simultaneously,
3113 // though we use "subs pc, lr, #N").
3114 //
3115 // M-class CPUs actually use a normal return sequence with a special
3116 // (hardware-provided) value in LR, so the normal code path works.
3117 if (DAG.getMachineFunction().getFunction().hasFnAttribute(Kind: "interrupt") &&
3118 !Subtarget->isMClass()) {
3119 if (Subtarget->isThumb1Only())
3120 report_fatal_error(reason: "interrupt attribute is not supported in Thumb1");
3121 return LowerInterruptReturn(RetOps, DL: dl, DAG);
3122 }
3123
3124 unsigned RetNode =
3125 AFI->isCmseNSEntryFunction() ? ARMISD::SERET_GLUE : ARMISD::RET_GLUE;
3126 return DAG.getNode(Opcode: RetNode, DL: dl, VT: MVT::Other, Ops: RetOps);
3127}
3128
3129bool ARMTargetLowering::isUsedByReturnOnly(SDNode *N, SDValue &Chain) const {
3130 if (N->getNumValues() != 1)
3131 return false;
3132 if (!N->hasNUsesOfValue(NUses: 1, Value: 0))
3133 return false;
3134
3135 SDValue TCChain = Chain;
3136 SDNode *Copy = *N->user_begin();
3137 if (Copy->getOpcode() == ISD::CopyToReg) {
3138 // If the copy has a glue operand, we conservatively assume it isn't safe to
3139 // perform a tail call.
3140 if (Copy->getOperand(Num: Copy->getNumOperands()-1).getValueType() == MVT::Glue)
3141 return false;
3142 TCChain = Copy->getOperand(Num: 0);
3143 } else if (Copy->getOpcode() == ARMISD::VMOVRRD) {
3144 SDNode *VMov = Copy;
3145 // f64 returned in a pair of GPRs.
3146 SmallPtrSet<SDNode*, 2> Copies;
3147 for (SDNode *U : VMov->users()) {
3148 if (U->getOpcode() != ISD::CopyToReg)
3149 return false;
3150 Copies.insert(Ptr: U);
3151 }
3152 if (Copies.size() > 2)
3153 return false;
3154
3155 for (SDNode *U : VMov->users()) {
3156 SDValue UseChain = U->getOperand(Num: 0);
3157 if (Copies.count(Ptr: UseChain.getNode()))
3158 // Second CopyToReg
3159 Copy = U;
3160 else {
3161 // We are at the top of this chain.
3162 // If the copy has a glue operand, we conservatively assume it
3163 // isn't safe to perform a tail call.
3164 if (U->getOperand(Num: U->getNumOperands() - 1).getValueType() == MVT::Glue)
3165 return false;
3166 // First CopyToReg
3167 TCChain = UseChain;
3168 }
3169 }
3170 } else if (Copy->getOpcode() == ISD::BITCAST) {
3171 // f32 returned in a single GPR.
3172 if (!Copy->hasOneUse())
3173 return false;
3174 Copy = *Copy->user_begin();
3175 if (Copy->getOpcode() != ISD::CopyToReg || !Copy->hasNUsesOfValue(NUses: 1, Value: 0))
3176 return false;
3177 // If the copy has a glue operand, we conservatively assume it isn't safe to
3178 // perform a tail call.
3179 if (Copy->getOperand(Num: Copy->getNumOperands()-1).getValueType() == MVT::Glue)
3180 return false;
3181 TCChain = Copy->getOperand(Num: 0);
3182 } else {
3183 return false;
3184 }
3185
3186 bool HasRet = false;
3187 for (const SDNode *U : Copy->users()) {
3188 if (U->getOpcode() != ARMISD::RET_GLUE &&
3189 U->getOpcode() != ARMISD::INTRET_GLUE)
3190 return false;
3191 HasRet = true;
3192 }
3193
3194 if (!HasRet)
3195 return false;
3196
3197 Chain = TCChain;
3198 return true;
3199}
3200
3201bool ARMTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
3202 if (!Subtarget->supportsTailCall())
3203 return false;
3204
3205 if (!CI->isTailCall())
3206 return false;
3207
3208 return true;
3209}
3210
3211// Trying to write a 64 bit value so need to split into two 32 bit values first,
3212// and pass the lower and high parts through.
3213static SDValue LowerWRITE_REGISTER(SDValue Op, SelectionDAG &DAG) {
3214 SDLoc DL(Op);
3215 SDValue WriteValue = Op->getOperand(Num: 2);
3216
3217 // This function is only supposed to be called for i64 type argument.
3218 assert(WriteValue.getValueType() == MVT::i64
3219 && "LowerWRITE_REGISTER called for non-i64 type argument.");
3220
3221 SDValue Lo, Hi;
3222 std::tie(args&: Lo, args&: Hi) = DAG.SplitScalar(N: WriteValue, DL, LoVT: MVT::i32, HiVT: MVT::i32);
3223 SDValue Ops[] = { Op->getOperand(Num: 0), Op->getOperand(Num: 1), Lo, Hi };
3224 return DAG.getNode(Opcode: ISD::WRITE_REGISTER, DL, VT: MVT::Other, Ops);
3225}
3226
3227// ConstantPool, JumpTable, GlobalAddress, and ExternalSymbol are lowered as
3228// their target counterpart wrapped in the ARMISD::Wrapper node. Suppose N is
3229// one of the above mentioned nodes. It has to be wrapped because otherwise
3230// Select(N) returns N. So the raw TargetGlobalAddress nodes, etc. can only
3231// be used to form addressing mode. These wrapped nodes will be selected
3232// into MOVi.
3233SDValue ARMTargetLowering::LowerConstantPool(SDValue Op,
3234 SelectionDAG &DAG) const {
3235 EVT PtrVT = Op.getValueType();
3236 // FIXME there is no actual debug info here
3237 SDLoc dl(Op);
3238 ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Val&: Op);
3239 SDValue Res;
3240
3241 // When generating execute-only code Constant Pools must be promoted to the
3242 // global data section. It's a bit ugly that we can't share them across basic
3243 // blocks, but this way we guarantee that execute-only behaves correct with
3244 // position-independent addressing modes.
3245 if (Subtarget->genExecuteOnly()) {
3246 auto AFI = DAG.getMachineFunction().getInfo<ARMFunctionInfo>();
3247 auto *T = CP->getType();
3248 auto C = const_cast<Constant*>(CP->getConstVal());
3249 auto M = DAG.getMachineFunction().getFunction().getParent();
3250 auto GV = new GlobalVariable(
3251 *M, T, /*isConstant=*/true, GlobalVariable::InternalLinkage, C,
3252 Twine(DAG.getDataLayout().getInternalSymbolPrefix()) + "CP" +
3253 Twine(DAG.getMachineFunction().getFunctionNumber()) + "_" +
3254 Twine(AFI->createPICLabelUId()));
3255 SDValue GA = DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT);
3256 return LowerGlobalAddress(Op: GA, DAG);
3257 }
3258
3259 // The 16-bit ADR instruction can only encode offsets that are multiples of 4,
3260 // so we need to align to at least 4 bytes when we don't have 32-bit ADR.
3261 Align CPAlign = CP->getAlign();
3262 if (Subtarget->isThumb1Only())
3263 CPAlign = std::max(a: CPAlign, b: Align(4));
3264 if (CP->isMachineConstantPoolEntry())
3265 Res =
3266 DAG.getTargetConstantPool(C: CP->getMachineCPVal(), VT: PtrVT, Align: CPAlign);
3267 else
3268 Res = DAG.getTargetConstantPool(C: CP->getConstVal(), VT: PtrVT, Align: CPAlign);
3269 return DAG.getNode(Opcode: ARMISD::Wrapper, DL: dl, VT: MVT::i32, Operand: Res);
3270}
3271
3272unsigned ARMTargetLowering::getJumpTableEncoding() const {
3273 // If we don't have a 32-bit pc-relative branch instruction then the jump
3274 // table consists of block addresses. Usually this is inline, but for
3275 // execute-only it must be placed out-of-line.
3276 if (Subtarget->genExecuteOnly() && !Subtarget->hasV8MBaselineOps())
3277 return MachineJumpTableInfo::EK_BlockAddress;
3278 return MachineJumpTableInfo::EK_Inline;
3279}
3280
3281SDValue ARMTargetLowering::LowerBlockAddress(SDValue Op,
3282 SelectionDAG &DAG) const {
3283 MachineFunction &MF = DAG.getMachineFunction();
3284 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
3285 unsigned ARMPCLabelIndex = 0;
3286 SDLoc DL(Op);
3287 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
3288 const BlockAddress *BA = cast<BlockAddressSDNode>(Val&: Op)->getBlockAddress();
3289 SDValue CPAddr;
3290 bool IsPositionIndependent = isPositionIndependent() || Subtarget->isROPI();
3291 if (!IsPositionIndependent) {
3292 CPAddr = DAG.getTargetConstantPool(C: BA, VT: PtrVT, Align: Align(4));
3293 } else {
3294 unsigned PCAdj = Subtarget->isThumb() ? 4 : 8;
3295 ARMPCLabelIndex = AFI->createPICLabelUId();
3296 ARMConstantPoolValue *CPV =
3297 ARMConstantPoolConstant::Create(C: BA, ID: ARMPCLabelIndex,
3298 Kind: ARMCP::CPBlockAddress, PCAdj);
3299 CPAddr = DAG.getTargetConstantPool(C: CPV, VT: PtrVT, Align: Align(4));
3300 }
3301 CPAddr = DAG.getNode(Opcode: ARMISD::Wrapper, DL, VT: PtrVT, Operand: CPAddr);
3302 SDValue Result = DAG.getLoad(
3303 VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: CPAddr,
3304 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
3305 if (!IsPositionIndependent)
3306 return Result;
3307 SDValue PICLabel = DAG.getConstant(Val: ARMPCLabelIndex, DL, VT: MVT::i32);
3308 return DAG.getNode(Opcode: ARMISD::PIC_ADD, DL, VT: PtrVT, N1: Result, N2: PICLabel);
3309}
3310
3311/// Convert a TLS address reference into the correct sequence of loads
3312/// and calls to compute the variable's address for Darwin, and return an
3313/// SDValue containing the final node.
3314
3315/// Darwin only has one TLS scheme which must be capable of dealing with the
3316/// fully general situation, in the worst case. This means:
3317/// + "extern __thread" declaration.
3318/// + Defined in a possibly unknown dynamic library.
3319///
3320/// The general system is that each __thread variable has a [3 x i32] descriptor
3321/// which contains information used by the runtime to calculate the address. The
3322/// only part of this the compiler needs to know about is the first word, which
3323/// contains a function pointer that must be called with the address of the
3324/// entire descriptor in "r0".
3325///
3326/// Since this descriptor may be in a different unit, in general access must
3327/// proceed along the usual ARM rules. A common sequence to produce is:
3328///
3329/// movw rT1, :lower16:_var$non_lazy_ptr
3330/// movt rT1, :upper16:_var$non_lazy_ptr
3331/// ldr r0, [rT1]
3332/// ldr rT2, [r0]
3333/// blx rT2
3334/// [...address now in r0...]
3335SDValue
3336ARMTargetLowering::LowerGlobalTLSAddressDarwin(SDValue Op,
3337 SelectionDAG &DAG) const {
3338 assert(getTargetMachine().getTargetTriple().isOSDarwin() &&
3339 "This function expects a Darwin target");
3340 SDLoc DL(Op);
3341
3342 // First step is to get the address of the actua global symbol. This is where
3343 // the TLS descriptor lives.
3344 SDValue DescAddr = LowerGlobalAddressDarwin(Op, DAG);
3345
3346 // The first entry in the descriptor is a function pointer that we must call
3347 // to obtain the address of the variable.
3348 SDValue Chain = DAG.getEntryNode();
3349 SDValue FuncTLVGet = DAG.getLoad(
3350 VT: MVT::i32, dl: DL, Chain, Ptr: DescAddr,
3351 PtrInfo: MachinePointerInfo::getGOT(MF&: DAG.getMachineFunction()), Alignment: Align(4),
3352 MMOFlags: MachineMemOperand::MONonTemporal | MachineMemOperand::MODereferenceable |
3353 MachineMemOperand::MOInvariant);
3354 Chain = FuncTLVGet.getValue(R: 1);
3355
3356 MachineFunction &F = DAG.getMachineFunction();
3357 MachineFrameInfo &MFI = F.getFrameInfo();
3358 MFI.setAdjustsStack(true);
3359
3360 // TLS calls preserve all registers except those that absolutely must be
3361 // trashed: R0 (it takes an argument), LR (it's a call) and CPSR (let's not be
3362 // silly).
3363 auto TRI =
3364 getTargetMachine().getSubtargetImpl(F.getFunction())->getRegisterInfo();
3365 auto ARI = static_cast<const ARMRegisterInfo *>(TRI);
3366 const uint32_t *Mask = ARI->getTLSCallPreservedMask(MF: DAG.getMachineFunction());
3367
3368 // Finally, we can make the call. This is just a degenerate version of a
3369 // normal AArch64 call node: r0 takes the address of the descriptor, and
3370 // returns the address of the variable in this thread.
3371 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: ARM::R0, N: DescAddr, Glue: SDValue());
3372 Chain =
3373 DAG.getNode(Opcode: ARMISD::CALL, DL, VTList: DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue),
3374 N1: Chain, N2: FuncTLVGet, N3: DAG.getRegister(Reg: ARM::R0, VT: MVT::i32),
3375 N4: DAG.getRegisterMask(RegMask: Mask), N5: Chain.getValue(R: 1));
3376 return DAG.getCopyFromReg(Chain, dl: DL, Reg: ARM::R0, VT: MVT::i32, Glue: Chain.getValue(R: 1));
3377}
3378
3379SDValue
3380ARMTargetLowering::LowerGlobalTLSAddressWindows(SDValue Op,
3381 SelectionDAG &DAG) const {
3382 assert(getTargetMachine().getTargetTriple().isOSWindows() &&
3383 "Windows specific TLS lowering");
3384
3385 SDValue Chain = DAG.getEntryNode();
3386 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
3387 SDLoc DL(Op);
3388
3389 // Load the current TEB (thread environment block)
3390 SDValue Ops[] = {Chain,
3391 DAG.getTargetConstant(Val: Intrinsic::arm_mrc, DL, VT: MVT::i32),
3392 DAG.getTargetConstant(Val: 15, DL, VT: MVT::i32),
3393 DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32),
3394 DAG.getTargetConstant(Val: 13, DL, VT: MVT::i32),
3395 DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32),
3396 DAG.getTargetConstant(Val: 2, DL, VT: MVT::i32)};
3397 SDValue CurrentTEB = DAG.getNode(Opcode: ISD::INTRINSIC_W_CHAIN, DL,
3398 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other), Ops);
3399
3400 SDValue TEB = CurrentTEB.getValue(R: 0);
3401 Chain = CurrentTEB.getValue(R: 1);
3402
3403 // Load the ThreadLocalStoragePointer from the TEB
3404 // A pointer to the TLS array is located at offset 0x2c from the TEB.
3405 SDValue TLSArray =
3406 DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: TEB, N2: DAG.getIntPtrConstant(Val: 0x2c, DL));
3407 TLSArray = DAG.getLoad(VT: PtrVT, dl: DL, Chain, Ptr: TLSArray, PtrInfo: MachinePointerInfo());
3408
3409 // The pointer to the thread's TLS data area is at the TLS Index scaled by 4
3410 // offset into the TLSArray.
3411
3412 // Load the TLS index from the C runtime
3413 SDValue TLSIndex =
3414 DAG.getTargetExternalSymbol(Sym: "_tls_index", VT: PtrVT, TargetFlags: ARMII::MO_NO_FLAG);
3415 TLSIndex = DAG.getNode(Opcode: ARMISD::Wrapper, DL, VT: PtrVT, Operand: TLSIndex);
3416 TLSIndex = DAG.getLoad(VT: PtrVT, dl: DL, Chain, Ptr: TLSIndex, PtrInfo: MachinePointerInfo());
3417
3418 SDValue Slot = DAG.getNode(Opcode: ISD::SHL, DL, VT: PtrVT, N1: TLSIndex,
3419 N2: DAG.getConstant(Val: 2, DL, VT: MVT::i32));
3420 SDValue TLS = DAG.getLoad(VT: PtrVT, dl: DL, Chain,
3421 Ptr: DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: TLSArray, N2: Slot),
3422 PtrInfo: MachinePointerInfo());
3423
3424 // Get the offset of the start of the .tls section (section base)
3425 const auto *GA = cast<GlobalAddressSDNode>(Val&: Op);
3426 auto *CPV = ARMConstantPoolConstant::Create(GV: GA->getGlobal(), Modifier: ARMCP::SECREL);
3427 SDValue Offset = DAG.getLoad(
3428 VT: PtrVT, dl: DL, Chain,
3429 Ptr: DAG.getNode(Opcode: ARMISD::Wrapper, DL, VT: MVT::i32,
3430 Operand: DAG.getTargetConstantPool(C: CPV, VT: PtrVT, Align: Align(4))),
3431 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
3432
3433 return DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: TLS, N2: Offset);
3434}
3435
3436// Lower ISD::GlobalTLSAddress using the "general dynamic" model
3437SDValue
3438ARMTargetLowering::LowerToTLSGeneralDynamicModel(GlobalAddressSDNode *GA,
3439 SelectionDAG &DAG) const {
3440 SDLoc dl(GA);
3441 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
3442 unsigned char PCAdj = Subtarget->isThumb() ? 4 : 8;
3443 MachineFunction &MF = DAG.getMachineFunction();
3444 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
3445 unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
3446 ARMConstantPoolValue *CPV =
3447 ARMConstantPoolConstant::Create(C: GA->getGlobal(), ID: ARMPCLabelIndex,
3448 Kind: ARMCP::CPValue, PCAdj, Modifier: ARMCP::TLSGD, AddCurrentAddress: true);
3449 SDValue Argument = DAG.getTargetConstantPool(C: CPV, VT: PtrVT, Align: Align(4));
3450 Argument = DAG.getNode(Opcode: ARMISD::Wrapper, DL: dl, VT: MVT::i32, Operand: Argument);
3451 Argument = DAG.getLoad(
3452 VT: PtrVT, dl, Chain: DAG.getEntryNode(), Ptr: Argument,
3453 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
3454 SDValue Chain = Argument.getValue(R: 1);
3455
3456 SDValue PICLabel = DAG.getConstant(Val: ARMPCLabelIndex, DL: dl, VT: MVT::i32);
3457 Argument = DAG.getNode(Opcode: ARMISD::PIC_ADD, DL: dl, VT: PtrVT, N1: Argument, N2: PICLabel);
3458
3459 // call __tls_get_addr.
3460 ArgListTy Args;
3461 Args.emplace_back(args&: Argument, args: Type::getInt32Ty(C&: *DAG.getContext()));
3462
3463 // FIXME: is there useful debug info available here?
3464 TargetLowering::CallLoweringInfo CLI(DAG);
3465 CLI.setDebugLoc(dl).setChain(Chain).setLibCallee(
3466 CC: CallingConv::C, ResultType: Type::getInt32Ty(C&: *DAG.getContext()),
3467 Target: DAG.getExternalSymbol(Sym: "__tls_get_addr", VT: PtrVT), ArgsList: std::move(Args));
3468
3469 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
3470 return CallResult.first;
3471}
3472
3473// Lower ISD::GlobalTLSAddress using the "initial exec" or
3474// "local exec" model.
3475SDValue
3476ARMTargetLowering::LowerToTLSExecModels(GlobalAddressSDNode *GA,
3477 SelectionDAG &DAG,
3478 TLSModel::Model model) const {
3479 const GlobalValue *GV = GA->getGlobal();
3480 SDLoc dl(GA);
3481 SDValue Offset;
3482 SDValue Chain = DAG.getEntryNode();
3483 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
3484 // Get the Thread Pointer
3485 SDValue ThreadPointer = DAG.getNode(Opcode: ARMISD::THREAD_POINTER, DL: dl, VT: PtrVT);
3486
3487 if (model == TLSModel::InitialExec) {
3488 MachineFunction &MF = DAG.getMachineFunction();
3489 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
3490 unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
3491 // Initial exec model.
3492 unsigned char PCAdj = Subtarget->isThumb() ? 4 : 8;
3493 ARMConstantPoolValue *CPV =
3494 ARMConstantPoolConstant::Create(C: GA->getGlobal(), ID: ARMPCLabelIndex,
3495 Kind: ARMCP::CPValue, PCAdj, Modifier: ARMCP::GOTTPOFF,
3496 AddCurrentAddress: true);
3497 Offset = DAG.getTargetConstantPool(C: CPV, VT: PtrVT, Align: Align(4));
3498 Offset = DAG.getNode(Opcode: ARMISD::Wrapper, DL: dl, VT: MVT::i32, Operand: Offset);
3499 Offset = DAG.getLoad(
3500 VT: PtrVT, dl, Chain, Ptr: Offset,
3501 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
3502 Chain = Offset.getValue(R: 1);
3503
3504 SDValue PICLabel = DAG.getConstant(Val: ARMPCLabelIndex, DL: dl, VT: MVT::i32);
3505 Offset = DAG.getNode(Opcode: ARMISD::PIC_ADD, DL: dl, VT: PtrVT, N1: Offset, N2: PICLabel);
3506
3507 Offset = DAG.getLoad(
3508 VT: PtrVT, dl, Chain, Ptr: Offset,
3509 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
3510 } else {
3511 // local exec model
3512 assert(model == TLSModel::LocalExec);
3513 ARMConstantPoolValue *CPV =
3514 ARMConstantPoolConstant::Create(GV, Modifier: ARMCP::TPOFF);
3515 Offset = DAG.getTargetConstantPool(C: CPV, VT: PtrVT, Align: Align(4));
3516 Offset = DAG.getNode(Opcode: ARMISD::Wrapper, DL: dl, VT: MVT::i32, Operand: Offset);
3517 Offset = DAG.getLoad(
3518 VT: PtrVT, dl, Chain, Ptr: Offset,
3519 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
3520 }
3521
3522 // The address of the thread local variable is the add of the thread
3523 // pointer with the offset of the variable.
3524 return DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: ThreadPointer, N2: Offset);
3525}
3526
3527SDValue
3528ARMTargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const {
3529 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Val&: Op);
3530 if (DAG.getTarget().useEmulatedTLS())
3531 return LowerToTLSEmulatedModel(GA, DAG);
3532
3533 const Triple &TT = getTargetMachine().getTargetTriple();
3534 if (TT.isOSDarwin())
3535 return LowerGlobalTLSAddressDarwin(Op, DAG);
3536
3537 if (TT.isOSWindows())
3538 return LowerGlobalTLSAddressWindows(Op, DAG);
3539
3540 // TODO: implement the "local dynamic" model
3541 assert(TT.isOSBinFormatELF() && "Only ELF implemented here");
3542 TLSModel::Model model = getTargetMachine().getTLSModel(GV: GA->getGlobal());
3543
3544 switch (model) {
3545 case TLSModel::GeneralDynamic:
3546 case TLSModel::LocalDynamic:
3547 return LowerToTLSGeneralDynamicModel(GA, DAG);
3548 case TLSModel::InitialExec:
3549 case TLSModel::LocalExec:
3550 return LowerToTLSExecModels(GA, DAG, model);
3551 }
3552 llvm_unreachable("bogus TLS model");
3553}
3554
3555/// Return true if all users of V are within function F, looking through
3556/// ConstantExprs.
3557static bool allUsersAreInFunction(const Value *V, const Function *F) {
3558 SmallVector<const User*,4> Worklist(V->users());
3559 while (!Worklist.empty()) {
3560 auto *U = Worklist.pop_back_val();
3561 if (isa<ConstantExpr>(Val: U)) {
3562 append_range(C&: Worklist, R: U->users());
3563 continue;
3564 }
3565
3566 auto *I = dyn_cast<Instruction>(Val: U);
3567 if (!I || I->getParent()->getParent() != F)
3568 return false;
3569 }
3570 return true;
3571}
3572
3573static SDValue promoteToConstantPool(const ARMTargetLowering *TLI,
3574 const GlobalValue *GV, SelectionDAG &DAG,
3575 EVT PtrVT, const SDLoc &dl) {
3576 // If we're creating a pool entry for a constant global with unnamed address,
3577 // and the global is small enough, we can emit it inline into the constant pool
3578 // to save ourselves an indirection.
3579 //
3580 // This is a win if the constant is only used in one function (so it doesn't
3581 // need to be duplicated) or duplicating the constant wouldn't increase code
3582 // size (implying the constant is no larger than 4 bytes).
3583 const Function &F = DAG.getMachineFunction().getFunction();
3584
3585 // We rely on this decision to inline being idempotent and unrelated to the
3586 // use-site. We know that if we inline a variable at one use site, we'll
3587 // inline it elsewhere too (and reuse the constant pool entry). Fast-isel
3588 // doesn't know about this optimization, so bail out if it's enabled else
3589 // we could decide to inline here (and thus never emit the GV) but require
3590 // the GV from fast-isel generated code.
3591 if (!EnableConstpoolPromotion ||
3592 DAG.getMachineFunction().getTarget().Options.EnableFastISel)
3593 return SDValue();
3594
3595 auto *GVar = dyn_cast<GlobalVariable>(Val: GV);
3596 if (!GVar || !GVar->hasInitializer() ||
3597 !GVar->isConstant() || !GVar->hasGlobalUnnamedAddr() ||
3598 !GVar->hasLocalLinkage())
3599 return SDValue();
3600
3601 // If we inline a value that contains relocations, we move the relocations
3602 // from .data to .text. This is not allowed in position-independent code.
3603 auto *Init = GVar->getInitializer();
3604 if ((TLI->isPositionIndependent() || TLI->getSubtarget()->isROPI()) &&
3605 Init->needsDynamicRelocation())
3606 return SDValue();
3607
3608 // The constant islands pass can only really deal with alignment requests
3609 // <= 4 bytes and cannot pad constants itself. Therefore we cannot promote
3610 // any type wanting greater alignment requirements than 4 bytes. We also
3611 // can only promote constants that are multiples of 4 bytes in size or
3612 // are paddable to a multiple of 4. Currently we only try and pad constants
3613 // that are strings for simplicity.
3614 auto *CDAInit = dyn_cast<ConstantDataArray>(Val: Init);
3615 unsigned Size = DAG.getDataLayout().getTypeAllocSize(Ty: Init->getType());
3616 Align PrefAlign = DAG.getDataLayout().getPreferredAlign(GV: GVar);
3617 unsigned RequiredPadding = 4 - (Size % 4);
3618 bool PaddingPossible =
3619 RequiredPadding == 4 || (CDAInit && CDAInit->isString());
3620 if (!PaddingPossible || PrefAlign > 4 || Size > ConstpoolPromotionMaxSize ||
3621 Size == 0)
3622 return SDValue();
3623
3624 unsigned PaddedSize = Size + ((RequiredPadding == 4) ? 0 : RequiredPadding);
3625 MachineFunction &MF = DAG.getMachineFunction();
3626 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
3627
3628 // We can't bloat the constant pool too much, else the ConstantIslands pass
3629 // may fail to converge. If we haven't promoted this global yet (it may have
3630 // multiple uses), and promoting it would increase the constant pool size (Sz
3631 // > 4), ensure we have space to do so up to MaxTotal.
3632 if (!AFI->getGlobalsPromotedToConstantPool().count(Ptr: GVar) && Size > 4)
3633 if (AFI->getPromotedConstpoolIncrease() + PaddedSize - 4 >=
3634 ConstpoolPromotionMaxTotal)
3635 return SDValue();
3636
3637 // This is only valid if all users are in a single function; we can't clone
3638 // the constant in general. The LLVM IR unnamed_addr allows merging
3639 // constants, but not cloning them.
3640 //
3641 // We could potentially allow cloning if we could prove all uses of the
3642 // constant in the current function don't care about the address, like
3643 // printf format strings. But that isn't implemented for now.
3644 if (!allUsersAreInFunction(V: GVar, F: &F))
3645 return SDValue();
3646
3647 // We're going to inline this global. Pad it out if needed.
3648 if (RequiredPadding != 4) {
3649 StringRef S = CDAInit->getAsString();
3650
3651 SmallVector<uint8_t,16> V(S.size());
3652 std::copy(first: S.bytes_begin(), last: S.bytes_end(), result: V.begin());
3653 while (RequiredPadding--)
3654 V.push_back(Elt: 0);
3655 Init = ConstantDataArray::get(Context&: *DAG.getContext(), Elts&: V);
3656 }
3657
3658 auto CPVal = ARMConstantPoolConstant::Create(GV: GVar, Initializer: Init);
3659 SDValue CPAddr = DAG.getTargetConstantPool(C: CPVal, VT: PtrVT, Align: Align(4));
3660 if (!AFI->getGlobalsPromotedToConstantPool().count(Ptr: GVar)) {
3661 AFI->markGlobalAsPromotedToConstantPool(GV: GVar);
3662 AFI->setPromotedConstpoolIncrease(AFI->getPromotedConstpoolIncrease() +
3663 PaddedSize - 4);
3664 }
3665 ++NumConstpoolPromoted;
3666 return DAG.getNode(Opcode: ARMISD::Wrapper, DL: dl, VT: MVT::i32, Operand: CPAddr);
3667}
3668
3669bool ARMTargetLowering::isReadOnly(const GlobalValue *GV) const {
3670 if (const GlobalAlias *GA = dyn_cast<GlobalAlias>(Val: GV))
3671 if (!(GV = GA->getAliaseeObject()))
3672 return false;
3673 if (const auto *V = dyn_cast<GlobalVariable>(Val: GV))
3674 return V->isConstant();
3675 return isa<Function>(Val: GV);
3676}
3677
3678SDValue ARMTargetLowering::LowerGlobalAddress(SDValue Op,
3679 SelectionDAG &DAG) const {
3680 switch (Subtarget->getTargetTriple().getObjectFormat()) {
3681 default: llvm_unreachable("unknown object format");
3682 case Triple::COFF:
3683 return LowerGlobalAddressWindows(Op, DAG);
3684 case Triple::ELF:
3685 return LowerGlobalAddressELF(Op, DAG);
3686 case Triple::MachO:
3687 return LowerGlobalAddressDarwin(Op, DAG);
3688 }
3689}
3690
3691SDValue ARMTargetLowering::LowerGlobalAddressELF(SDValue Op,
3692 SelectionDAG &DAG) const {
3693 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
3694 SDLoc dl(Op);
3695 const GlobalValue *GV = cast<GlobalAddressSDNode>(Val&: Op)->getGlobal();
3696 bool IsRO = isReadOnly(GV);
3697
3698 // promoteToConstantPool only if not generating XO text section
3699 if (GV->isDSOLocal() && !Subtarget->genExecuteOnly())
3700 if (SDValue V = promoteToConstantPool(TLI: this, GV, DAG, PtrVT, dl))
3701 return V;
3702
3703 if (isPositionIndependent()) {
3704 SDValue G = DAG.getTargetGlobalAddress(
3705 GV, DL: dl, VT: PtrVT, offset: 0, TargetFlags: GV->isDSOLocal() ? 0 : ARMII::MO_GOT);
3706 SDValue Result = DAG.getNode(Opcode: ARMISD::WrapperPIC, DL: dl, VT: PtrVT, Operand: G);
3707 if (!GV->isDSOLocal())
3708 Result =
3709 DAG.getLoad(VT: PtrVT, dl, Chain: DAG.getEntryNode(), Ptr: Result,
3710 PtrInfo: MachinePointerInfo::getGOT(MF&: DAG.getMachineFunction()));
3711 return Result;
3712 } else if (Subtarget->isROPI() && IsRO) {
3713 // PC-relative.
3714 SDValue G = DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT);
3715 SDValue Result = DAG.getNode(Opcode: ARMISD::WrapperPIC, DL: dl, VT: PtrVT, Operand: G);
3716 return Result;
3717 } else if (Subtarget->isRWPI() && !IsRO) {
3718 // SB-relative.
3719 SDValue RelAddr;
3720 if (Subtarget->useMovt()) {
3721 ++NumMovwMovt;
3722 SDValue G = DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: 0, TargetFlags: ARMII::MO_SBREL);
3723 RelAddr = DAG.getNode(Opcode: ARMISD::Wrapper, DL: dl, VT: PtrVT, Operand: G);
3724 } else { // use literal pool for address constant
3725 ARMConstantPoolValue *CPV =
3726 ARMConstantPoolConstant::Create(GV, Modifier: ARMCP::SBREL);
3727 SDValue CPAddr = DAG.getTargetConstantPool(C: CPV, VT: PtrVT, Align: Align(4));
3728 CPAddr = DAG.getNode(Opcode: ARMISD::Wrapper, DL: dl, VT: MVT::i32, Operand: CPAddr);
3729 RelAddr = DAG.getLoad(
3730 VT: PtrVT, dl, Chain: DAG.getEntryNode(), Ptr: CPAddr,
3731 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
3732 }
3733 SDValue SB = DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl, Reg: ARM::R9, VT: PtrVT);
3734 SDValue Result = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: SB, N2: RelAddr);
3735 return Result;
3736 }
3737
3738 // If we have T2 ops, we can materialize the address directly via movt/movw
3739 // pair. This is always cheaper. If need to generate Execute Only code, and we
3740 // only have Thumb1 available, we can't use a constant pool and are forced to
3741 // use immediate relocations.
3742 if (Subtarget->useMovt() || Subtarget->genExecuteOnly()) {
3743 if (Subtarget->useMovt())
3744 ++NumMovwMovt;
3745 // FIXME: Once remat is capable of dealing with instructions with register
3746 // operands, expand this into two nodes.
3747 return DAG.getNode(Opcode: ARMISD::Wrapper, DL: dl, VT: PtrVT,
3748 Operand: DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT));
3749 } else {
3750 SDValue CPAddr = DAG.getTargetConstantPool(C: GV, VT: PtrVT, Align: Align(4));
3751 CPAddr = DAG.getNode(Opcode: ARMISD::Wrapper, DL: dl, VT: MVT::i32, Operand: CPAddr);
3752 return DAG.getLoad(
3753 VT: PtrVT, dl, Chain: DAG.getEntryNode(), Ptr: CPAddr,
3754 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
3755 }
3756}
3757
3758SDValue ARMTargetLowering::LowerGlobalAddressDarwin(SDValue Op,
3759 SelectionDAG &DAG) const {
3760 assert(!Subtarget->isROPI() && !Subtarget->isRWPI() &&
3761 "ROPI/RWPI not currently supported for Darwin");
3762 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
3763 SDLoc dl(Op);
3764 const GlobalValue *GV = cast<GlobalAddressSDNode>(Val&: Op)->getGlobal();
3765
3766 if (Subtarget->useMovt())
3767 ++NumMovwMovt;
3768
3769 // FIXME: Once remat is capable of dealing with instructions with register
3770 // operands, expand this into multiple nodes
3771 unsigned Wrapper =
3772 isPositionIndependent() ? ARMISD::WrapperPIC : ARMISD::Wrapper;
3773
3774 SDValue G = DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: 0, TargetFlags: ARMII::MO_NONLAZY);
3775 SDValue Result = DAG.getNode(Opcode: Wrapper, DL: dl, VT: PtrVT, Operand: G);
3776
3777 if (Subtarget->isGVIndirectSymbol(GV))
3778 Result = DAG.getLoad(VT: PtrVT, dl, Chain: DAG.getEntryNode(), Ptr: Result,
3779 PtrInfo: MachinePointerInfo::getGOT(MF&: DAG.getMachineFunction()));
3780 return Result;
3781}
3782
3783SDValue ARMTargetLowering::LowerGlobalAddressWindows(SDValue Op,
3784 SelectionDAG &DAG) const {
3785 assert(getTargetMachine().getTargetTriple().isOSWindows() &&
3786 "non-Windows COFF is not supported");
3787 assert(Subtarget->useMovt() &&
3788 "Windows on ARM expects to use movw/movt");
3789 assert(!Subtarget->isROPI() && !Subtarget->isRWPI() &&
3790 "ROPI/RWPI not currently supported for Windows");
3791
3792 const TargetMachine &TM = getTargetMachine();
3793 const GlobalValue *GV = cast<GlobalAddressSDNode>(Val&: Op)->getGlobal();
3794 ARMII::TOF TargetFlags = ARMII::MO_NO_FLAG;
3795 if (GV->hasDLLImportStorageClass())
3796 TargetFlags = ARMII::MO_DLLIMPORT;
3797 else if (!TM.shouldAssumeDSOLocal(GV))
3798 TargetFlags = ARMII::MO_COFFSTUB;
3799 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
3800 SDValue Result;
3801 SDLoc DL(Op);
3802
3803 ++NumMovwMovt;
3804
3805 // FIXME: Once remat is capable of dealing with instructions with register
3806 // operands, expand this into two nodes.
3807 Result = DAG.getNode(Opcode: ARMISD::Wrapper, DL, VT: PtrVT,
3808 Operand: DAG.getTargetGlobalAddress(GV, DL, VT: PtrVT, /*offset=*/0,
3809 TargetFlags));
3810 if (TargetFlags & (ARMII::MO_DLLIMPORT | ARMII::MO_COFFSTUB))
3811 Result = DAG.getLoad(VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: Result,
3812 PtrInfo: MachinePointerInfo::getGOT(MF&: DAG.getMachineFunction()));
3813 return Result;
3814}
3815
3816SDValue
3817ARMTargetLowering::LowerEH_SJLJ_SETJMP(SDValue Op, SelectionDAG &DAG) const {
3818 SDLoc dl(Op);
3819 SDValue Val = DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32);
3820 return DAG.getNode(Opcode: ARMISD::EH_SJLJ_SETJMP, DL: dl,
3821 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other), N1: Op.getOperand(i: 0),
3822 N2: Op.getOperand(i: 1), N3: Val);
3823}
3824
3825SDValue
3826ARMTargetLowering::LowerEH_SJLJ_LONGJMP(SDValue Op, SelectionDAG &DAG) const {
3827 SDLoc dl(Op);
3828 return DAG.getNode(Opcode: ARMISD::EH_SJLJ_LONGJMP, DL: dl, VT: MVT::Other, N1: Op.getOperand(i: 0),
3829 N2: Op.getOperand(i: 1), N3: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
3830}
3831
3832SDValue ARMTargetLowering::LowerEH_SJLJ_SETUP_DISPATCH(SDValue Op,
3833 SelectionDAG &DAG) const {
3834 SDLoc dl(Op);
3835 return DAG.getNode(Opcode: ARMISD::EH_SJLJ_SETUP_DISPATCH, DL: dl, VT: MVT::Other,
3836 Operand: Op.getOperand(i: 0));
3837}
3838
3839SDValue ARMTargetLowering::LowerINTRINSIC_VOID(
3840 SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget) const {
3841 unsigned IntNo =
3842 Op.getConstantOperandVal(i: Op.getOperand(i: 0).getValueType() == MVT::Other);
3843 switch (IntNo) {
3844 default:
3845 return SDValue(); // Don't custom lower most intrinsics.
3846 case Intrinsic::arm_gnu_eabi_mcount: {
3847 MachineFunction &MF = DAG.getMachineFunction();
3848 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
3849 SDLoc dl(Op);
3850 SDValue Chain = Op.getOperand(i: 0);
3851 // call "\01__gnu_mcount_nc"
3852 const ARMBaseRegisterInfo *ARI = Subtarget->getRegisterInfo();
3853 const uint32_t *Mask =
3854 ARI->getCallPreservedMask(MF: DAG.getMachineFunction(), CallingConv::C);
3855 assert(Mask && "Missing call preserved mask for calling convention");
3856 // Mark LR an implicit live-in.
3857 Register Reg = MF.addLiveIn(PReg: ARM::LR, RC: getRegClassFor(VT: MVT::i32));
3858 SDValue ReturnAddress =
3859 DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl, Reg, VT: PtrVT);
3860 constexpr EVT ResultTys[] = {MVT::Other, MVT::Glue};
3861 SDValue Callee =
3862 DAG.getTargetExternalSymbol(Sym: "\01__gnu_mcount_nc", VT: PtrVT, TargetFlags: 0);
3863 SDValue RegisterMask = DAG.getRegisterMask(RegMask: Mask);
3864 if (Subtarget->isThumb())
3865 return SDValue(
3866 DAG.getMachineNode(
3867 Opcode: ARM::tBL_PUSHLR, dl, ResultTys,
3868 Ops: {ReturnAddress, DAG.getTargetConstant(Val: ARMCC::AL, DL: dl, VT: PtrVT),
3869 DAG.getRegister(Reg: 0, VT: PtrVT), Callee, RegisterMask, Chain}),
3870 0);
3871 return SDValue(
3872 DAG.getMachineNode(Opcode: ARM::BL_PUSHLR, dl, ResultTys,
3873 Ops: {ReturnAddress, Callee, RegisterMask, Chain}),
3874 0);
3875 }
3876 }
3877}
3878
3879SDValue
3880ARMTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG,
3881 const ARMSubtarget *Subtarget) const {
3882 unsigned IntNo = Op.getConstantOperandVal(i: 0);
3883 SDLoc dl(Op);
3884 switch (IntNo) {
3885 default: return SDValue(); // Don't custom lower most intrinsics.
3886 case Intrinsic::localaddress: {
3887 const MachineFunction &MF = DAG.getMachineFunction();
3888 const auto *RegInfo = Subtarget->getRegisterInfo();
3889 unsigned Reg = RegInfo->getLocalAddressRegister(MF);
3890 return DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl, Reg,
3891 VT: Op.getSimpleValueType());
3892 }
3893 case Intrinsic::eh_recoverfp: {
3894 SDValue FnOp = Op.getOperand(i: 1);
3895 GlobalAddressSDNode *GSD = dyn_cast<GlobalAddressSDNode>(Val&: FnOp);
3896 auto *Fn = dyn_cast_or_null<Function>(Val: GSD ? GSD->getGlobal() : nullptr);
3897 if (!Fn)
3898 report_fatal_error(
3899 reason: "llvm.eh.recoverfp must take a function as the first argument");
3900 const auto *RegInfo = Subtarget->getRegisterInfo();
3901 Register BaseReg = RegInfo->getBaseRegister();
3902 MachineFunction &MF = DAG.getMachineFunction();
3903 MachineBasicBlock &MBB = *MF.begin();
3904 if (!MBB.isLiveIn(Reg: BaseReg))
3905 MBB.addLiveIn(PhysReg: BaseReg);
3906 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
3907 return DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl, Reg: BaseReg, VT: PtrVT);
3908 }
3909 case Intrinsic::thread_pointer: {
3910 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
3911 return DAG.getNode(Opcode: ARMISD::THREAD_POINTER, DL: dl, VT: PtrVT);
3912 }
3913 case Intrinsic::arm_cls: {
3914 // Note: arm_cls and arm_cls64 intrinsics are expanded directly here
3915 // in LowerINTRINSIC_WO_CHAIN since there's no native scalar CLS
3916 // instruction.
3917 const SDValue &Operand = Op.getOperand(i: 1);
3918 const EVT VTy = Op.getValueType();
3919 return DAG.getNode(Opcode: ISD::CTLS, DL: dl, VT: VTy, Operand);
3920 }
3921 case Intrinsic::arm_cls64: {
3922 // arm_cls64 returns i32 but takes i64 input.
3923 // Use ISD::CTLS for i64 and truncate the result.
3924 SDValue CTLS64 = DAG.getNode(Opcode: ISD::CTLS, DL: dl, VT: MVT::i64, Operand: Op.getOperand(i: 1));
3925 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i32, Operand: CTLS64);
3926 }
3927 case Intrinsic::arm_neon_vcls:
3928 case Intrinsic::arm_mve_vcls: {
3929 // Lower vector CLS intrinsics to ISD::CTLS.
3930 // Vector CTLS is Legal when NEON/MVE is available (set elsewhere).
3931 const EVT VTy = Op.getValueType();
3932 return DAG.getNode(Opcode: ISD::CTLS, DL: dl, VT: VTy, Operand: Op.getOperand(i: 1));
3933 }
3934 case Intrinsic::eh_sjlj_lsda: {
3935 MachineFunction &MF = DAG.getMachineFunction();
3936 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
3937 unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
3938 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
3939 SDValue CPAddr;
3940 bool IsPositionIndependent = isPositionIndependent();
3941 unsigned PCAdj = IsPositionIndependent ? (Subtarget->isThumb() ? 4 : 8) : 0;
3942 ARMConstantPoolValue *CPV =
3943 ARMConstantPoolConstant::Create(C: &MF.getFunction(), ID: ARMPCLabelIndex,
3944 Kind: ARMCP::CPLSDA, PCAdj);
3945 CPAddr = DAG.getTargetConstantPool(C: CPV, VT: PtrVT, Align: Align(4));
3946 CPAddr = DAG.getNode(Opcode: ARMISD::Wrapper, DL: dl, VT: MVT::i32, Operand: CPAddr);
3947 SDValue Result = DAG.getLoad(
3948 VT: PtrVT, dl, Chain: DAG.getEntryNode(), Ptr: CPAddr,
3949 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
3950
3951 if (IsPositionIndependent) {
3952 SDValue PICLabel = DAG.getConstant(Val: ARMPCLabelIndex, DL: dl, VT: MVT::i32);
3953 Result = DAG.getNode(Opcode: ARMISD::PIC_ADD, DL: dl, VT: PtrVT, N1: Result, N2: PICLabel);
3954 }
3955 return Result;
3956 }
3957 case Intrinsic::arm_neon_vabs:
3958 return DAG.getNode(Opcode: ISD::ABS, DL: SDLoc(Op), VT: Op.getValueType(),
3959 Operand: Op.getOperand(i: 1));
3960 case Intrinsic::arm_neon_vabds:
3961 if (Op.getValueType().isInteger())
3962 return DAG.getNode(Opcode: ISD::ABDS, DL: SDLoc(Op), VT: Op.getValueType(),
3963 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
3964 return SDValue();
3965 case Intrinsic::arm_neon_vabdu:
3966 return DAG.getNode(Opcode: ISD::ABDU, DL: SDLoc(Op), VT: Op.getValueType(),
3967 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
3968 case Intrinsic::arm_neon_vmulls:
3969 case Intrinsic::arm_neon_vmullu: {
3970 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmulls)
3971 ? ARMISD::VMULLs : ARMISD::VMULLu;
3972 return DAG.getNode(Opcode: NewOpc, DL: SDLoc(Op), VT: Op.getValueType(),
3973 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
3974 }
3975 case Intrinsic::arm_neon_vminnm:
3976 case Intrinsic::arm_neon_vmaxnm: {
3977 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vminnm)
3978 ? ISD::FMINNUM : ISD::FMAXNUM;
3979 return DAG.getNode(Opcode: NewOpc, DL: SDLoc(Op), VT: Op.getValueType(),
3980 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
3981 }
3982 case Intrinsic::arm_neon_vminu:
3983 case Intrinsic::arm_neon_vmaxu: {
3984 if (Op.getValueType().isFloatingPoint())
3985 return SDValue();
3986 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vminu)
3987 ? ISD::UMIN : ISD::UMAX;
3988 return DAG.getNode(Opcode: NewOpc, DL: SDLoc(Op), VT: Op.getValueType(),
3989 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
3990 }
3991 case Intrinsic::arm_neon_vmins:
3992 case Intrinsic::arm_neon_vmaxs: {
3993 // v{min,max}s is overloaded between signed integers and floats.
3994 if (!Op.getValueType().isFloatingPoint()) {
3995 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmins)
3996 ? ISD::SMIN : ISD::SMAX;
3997 return DAG.getNode(Opcode: NewOpc, DL: SDLoc(Op), VT: Op.getValueType(),
3998 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
3999 }
4000 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmins)
4001 ? ISD::FMINIMUM : ISD::FMAXIMUM;
4002 return DAG.getNode(Opcode: NewOpc, DL: SDLoc(Op), VT: Op.getValueType(),
4003 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
4004 }
4005 case Intrinsic::arm_neon_vtbl1:
4006 return DAG.getNode(Opcode: ARMISD::VTBL1, DL: SDLoc(Op), VT: Op.getValueType(),
4007 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
4008 case Intrinsic::arm_neon_vtbl2:
4009 return DAG.getNode(Opcode: ARMISD::VTBL2, DL: SDLoc(Op), VT: Op.getValueType(),
4010 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
4011 case Intrinsic::arm_mve_pred_i2v:
4012 case Intrinsic::arm_mve_pred_v2i:
4013 return DAG.getNode(Opcode: ARMISD::PREDICATE_CAST, DL: SDLoc(Op), VT: Op.getValueType(),
4014 Operand: Op.getOperand(i: 1));
4015 case Intrinsic::arm_mve_vreinterpretq:
4016 return DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: SDLoc(Op), VT: Op.getValueType(),
4017 Operand: Op.getOperand(i: 1));
4018 case Intrinsic::arm_mve_lsll:
4019 return DAG.getNode(Opcode: ARMISD::LSLL, DL: SDLoc(Op), VTList: Op->getVTList(),
4020 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
4021 case Intrinsic::arm_mve_asrl:
4022 return DAG.getNode(Opcode: ARMISD::ASRL, DL: SDLoc(Op), VTList: Op->getVTList(),
4023 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
4024 case Intrinsic::arm_mve_vsli:
4025 return DAG.getNode(Opcode: ARMISD::VSLIIMM, DL: SDLoc(Op), VTList: Op->getVTList(),
4026 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
4027 case Intrinsic::arm_mve_vsri:
4028 return DAG.getNode(Opcode: ARMISD::VSRIIMM, DL: SDLoc(Op), VTList: Op->getVTList(),
4029 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
4030 }
4031}
4032
4033static SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG,
4034 const ARMSubtarget *Subtarget) {
4035 SDLoc dl(Op);
4036 auto SSID = static_cast<SyncScope::ID>(Op.getConstantOperandVal(i: 2));
4037 if (SSID == SyncScope::SingleThread)
4038 return Op;
4039
4040 if (!Subtarget->hasDataBarrier()) {
4041 // Some ARMv6 cpus can support data barriers with an mcr instruction.
4042 // Thumb1 and pre-v6 ARM mode use a libcall instead and should never get
4043 // here.
4044 assert(Subtarget->hasV6Ops() && !Subtarget->isThumb() &&
4045 "Unexpected ISD::ATOMIC_FENCE encountered. Should be libcall!");
4046 return DAG.getNode(Opcode: ARMISD::MEMBARRIER_MCR, DL: dl, VT: MVT::Other, N1: Op.getOperand(i: 0),
4047 N2: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
4048 }
4049
4050 AtomicOrdering Ord =
4051 static_cast<AtomicOrdering>(Op.getConstantOperandVal(i: 1));
4052 ARM_MB::MemBOpt Domain = ARM_MB::ISH;
4053 if (Subtarget->isMClass()) {
4054 // Only a full system barrier exists in the M-class architectures.
4055 Domain = ARM_MB::SY;
4056 } else if (Subtarget->preferISHSTBarriers() &&
4057 Ord == AtomicOrdering::Release) {
4058 // Swift happens to implement ISHST barriers in a way that's compatible with
4059 // Release semantics but weaker than ISH so we'd be fools not to use
4060 // it. Beware: other processors probably don't!
4061 Domain = ARM_MB::ISHST;
4062 }
4063
4064 return DAG.getNode(Opcode: ISD::INTRINSIC_VOID, DL: dl, VT: MVT::Other, N1: Op.getOperand(i: 0),
4065 N2: DAG.getConstant(Val: Intrinsic::arm_dmb, DL: dl, VT: MVT::i32),
4066 N3: DAG.getConstant(Val: Domain, DL: dl, VT: MVT::i32));
4067}
4068
4069static SDValue LowerPREFETCH(SDValue Op, SelectionDAG &DAG,
4070 const ARMSubtarget *Subtarget) {
4071 // ARM pre v5TE and Thumb1 does not have preload instructions.
4072 if (!(Subtarget->isThumb2() ||
4073 (!Subtarget->isThumb1Only() && Subtarget->hasV5TEOps())))
4074 // Just preserve the chain.
4075 return Op.getOperand(i: 0);
4076
4077 SDLoc dl(Op);
4078 unsigned isRead = ~Op.getConstantOperandVal(i: 2) & 1;
4079 if (!isRead &&
4080 (!Subtarget->hasV7Ops() || !Subtarget->hasMPExtension()))
4081 // ARMv7 with MP extension has PLDW.
4082 return Op.getOperand(i: 0);
4083
4084 unsigned isData = Op.getConstantOperandVal(i: 4);
4085 if (Subtarget->isThumb()) {
4086 // Invert the bits.
4087 isRead = ~isRead & 1;
4088 isData = ~isData & 1;
4089 }
4090
4091 return DAG.getNode(Opcode: ARMISD::PRELOAD, DL: dl, VT: MVT::Other, N1: Op.getOperand(i: 0),
4092 N2: Op.getOperand(i: 1), N3: DAG.getConstant(Val: isRead, DL: dl, VT: MVT::i32),
4093 N4: DAG.getConstant(Val: isData, DL: dl, VT: MVT::i32));
4094}
4095
4096static SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) {
4097 MachineFunction &MF = DAG.getMachineFunction();
4098 ARMFunctionInfo *FuncInfo = MF.getInfo<ARMFunctionInfo>();
4099
4100 // vastart just stores the address of the VarArgsFrameIndex slot into the
4101 // memory location argument.
4102 SDLoc dl(Op);
4103 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DL: DAG.getDataLayout());
4104 SDValue FR = DAG.getFrameIndex(FI: FuncInfo->getVarArgsFrameIndex(), VT: PtrVT);
4105 const Value *SV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 2))->getValue();
4106 return DAG.getStore(Chain: Op.getOperand(i: 0), dl, Val: FR, Ptr: Op.getOperand(i: 1),
4107 PtrInfo: MachinePointerInfo(SV));
4108}
4109
4110SDValue ARMTargetLowering::GetF64FormalArgument(CCValAssign &VA,
4111 CCValAssign &NextVA,
4112 SDValue &Root,
4113 SelectionDAG &DAG,
4114 const SDLoc &dl) const {
4115 MachineFunction &MF = DAG.getMachineFunction();
4116 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
4117
4118 const TargetRegisterClass *RC;
4119 if (AFI->isThumb1OnlyFunction())
4120 RC = &ARM::tGPRRegClass;
4121 else
4122 RC = &ARM::GPRRegClass;
4123
4124 // Transform the arguments stored in physical registers into virtual ones.
4125 Register Reg = MF.addLiveIn(PReg: VA.getLocReg(), RC);
4126 SDValue ArgValue = DAG.getCopyFromReg(Chain: Root, dl, Reg, VT: MVT::i32);
4127
4128 SDValue ArgValue2;
4129 if (NextVA.isMemLoc()) {
4130 MachineFrameInfo &MFI = MF.getFrameInfo();
4131 int FI = MFI.CreateFixedObject(Size: 4, SPOffset: NextVA.getLocMemOffset(), IsImmutable: true);
4132
4133 // Create load node to retrieve arguments from the stack.
4134 SDValue FIN = DAG.getFrameIndex(FI, VT: getPointerTy(DL: DAG.getDataLayout()));
4135 ArgValue2 = DAG.getLoad(
4136 VT: MVT::i32, dl, Chain: Root, Ptr: FIN,
4137 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI));
4138 } else {
4139 Reg = MF.addLiveIn(PReg: NextVA.getLocReg(), RC);
4140 ArgValue2 = DAG.getCopyFromReg(Chain: Root, dl, Reg, VT: MVT::i32);
4141 }
4142 if (!Subtarget->isLittle())
4143 std::swap (a&: ArgValue, b&: ArgValue2);
4144 return DAG.getNode(Opcode: ARMISD::VMOVDRR, DL: dl, VT: MVT::f64, N1: ArgValue, N2: ArgValue2);
4145}
4146
4147// The remaining GPRs hold either the beginning of variable-argument
4148// data, or the beginning of an aggregate passed by value (usually
4149// byval). Either way, we allocate stack slots adjacent to the data
4150// provided by our caller, and store the unallocated registers there.
4151// If this is a variadic function, the va_list pointer will begin with
4152// these values; otherwise, this reassembles a (byval) structure that
4153// was split between registers and memory.
4154// Return: The frame index registers were stored into.
4155int ARMTargetLowering::StoreByValRegs(CCState &CCInfo, SelectionDAG &DAG,
4156 const SDLoc &dl, SDValue &Chain,
4157 const Value *OrigArg,
4158 unsigned InRegsParamRecordIdx,
4159 int ArgOffset, unsigned ArgSize) const {
4160 // Currently, two use-cases possible:
4161 // Case #1. Non-var-args function, and we meet first byval parameter.
4162 // Setup first unallocated register as first byval register;
4163 // eat all remained registers
4164 // (these two actions are performed by HandleByVal method).
4165 // Then, here, we initialize stack frame with
4166 // "store-reg" instructions.
4167 // Case #2. Var-args function, that doesn't contain byval parameters.
4168 // The same: eat all remained unallocated registers,
4169 // initialize stack frame.
4170
4171 MachineFunction &MF = DAG.getMachineFunction();
4172 MachineFrameInfo &MFI = MF.getFrameInfo();
4173 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
4174 unsigned RBegin, REnd;
4175 if (InRegsParamRecordIdx < CCInfo.getInRegsParamsCount()) {
4176 CCInfo.getInRegsParamInfo(InRegsParamRecordIndex: InRegsParamRecordIdx, BeginReg&: RBegin, EndReg&: REnd);
4177 } else {
4178 unsigned RBeginIdx = CCInfo.getFirstUnallocated(Regs: GPRArgRegs);
4179 RBegin = RBeginIdx == 4 ? (unsigned)ARM::R4 : GPRArgRegs[RBeginIdx];
4180 REnd = ARM::R4;
4181 }
4182
4183 if (REnd != RBegin)
4184 ArgOffset = -4 * (ARM::R4 - RBegin);
4185
4186 auto PtrVT = getPointerTy(DL: DAG.getDataLayout());
4187 int FrameIndex = MFI.CreateFixedObject(Size: ArgSize, SPOffset: ArgOffset, IsImmutable: false);
4188 SDValue FIN = DAG.getFrameIndex(FI: FrameIndex, VT: PtrVT);
4189
4190 SmallVector<SDValue, 4> MemOps;
4191 const TargetRegisterClass *RC =
4192 AFI->isThumb1OnlyFunction() ? &ARM::tGPRRegClass : &ARM::GPRRegClass;
4193
4194 for (unsigned Reg = RBegin, i = 0; Reg < REnd; ++Reg, ++i) {
4195 Register VReg = MF.addLiveIn(PReg: Reg, RC);
4196 SDValue Val = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: MVT::i32);
4197 SDValue Store = DAG.getStore(Chain: Val.getValue(R: 1), dl, Val, Ptr: FIN,
4198 PtrInfo: MachinePointerInfo(OrigArg, 4 * i));
4199 MemOps.push_back(Elt: Store);
4200 FIN = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: FIN, N2: DAG.getConstant(Val: 4, DL: dl, VT: PtrVT));
4201 }
4202
4203 if (!MemOps.empty())
4204 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: MemOps);
4205 return FrameIndex;
4206}
4207
4208// Setup stack frame, the va_list pointer will start from.
4209void ARMTargetLowering::VarArgStyleRegisters(CCState &CCInfo, SelectionDAG &DAG,
4210 const SDLoc &dl, SDValue &Chain,
4211 unsigned ArgOffset,
4212 unsigned TotalArgRegsSaveSize,
4213 bool ForceMutable) const {
4214 MachineFunction &MF = DAG.getMachineFunction();
4215 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
4216
4217 // Try to store any remaining integer argument regs
4218 // to their spots on the stack so that they may be loaded by dereferencing
4219 // the result of va_next.
4220 // If there is no regs to be stored, just point address after last
4221 // argument passed via stack.
4222 int FrameIndex = StoreByValRegs(
4223 CCInfo, DAG, dl, Chain, OrigArg: nullptr, InRegsParamRecordIdx: CCInfo.getInRegsParamsCount(),
4224 ArgOffset: CCInfo.getStackSize(), ArgSize: std::max(a: 4U, b: TotalArgRegsSaveSize));
4225 AFI->setVarArgsFrameIndex(FrameIndex);
4226}
4227
4228bool ARMTargetLowering::splitValueIntoRegisterParts(
4229 SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
4230 unsigned NumParts, MVT PartVT, std::optional<CallingConv::ID> CC) const {
4231 EVT ValueVT = Val.getValueType();
4232 if ((ValueVT == MVT::f16 || ValueVT == MVT::bf16) && PartVT == MVT::f32) {
4233 unsigned ValueBits = ValueVT.getSizeInBits();
4234 unsigned PartBits = PartVT.getSizeInBits();
4235 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::getIntegerVT(BitWidth: ValueBits), Operand: Val);
4236 Val = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::getIntegerVT(BitWidth: PartBits), Operand: Val);
4237 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: PartVT, Operand: Val);
4238 Parts[0] = Val;
4239 return true;
4240 }
4241 return false;
4242}
4243
4244SDValue ARMTargetLowering::joinRegisterPartsIntoValue(
4245 SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts,
4246 MVT PartVT, EVT ValueVT, std::optional<CallingConv::ID> CC) const {
4247 if ((ValueVT == MVT::f16 || ValueVT == MVT::bf16) && PartVT == MVT::f32) {
4248 unsigned ValueBits = ValueVT.getSizeInBits();
4249 unsigned PartBits = PartVT.getSizeInBits();
4250 SDValue Val = Parts[0];
4251
4252 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::getIntegerVT(BitWidth: PartBits), Operand: Val);
4253 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::getIntegerVT(BitWidth: ValueBits), Operand: Val);
4254 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ValueVT, Operand: Val);
4255 return Val;
4256 }
4257 return SDValue();
4258}
4259
4260SDValue ARMTargetLowering::LowerFormalArguments(
4261 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
4262 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
4263 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
4264 MachineFunction &MF = DAG.getMachineFunction();
4265 MachineFrameInfo &MFI = MF.getFrameInfo();
4266
4267 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
4268
4269 // Assign locations to all of the incoming arguments.
4270 SmallVector<CCValAssign, 16> ArgLocs;
4271 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs,
4272 *DAG.getContext());
4273 CCInfo.AnalyzeFormalArguments(Ins, Fn: CCAssignFnForCall(CC: CallConv, isVarArg));
4274
4275 Function::const_arg_iterator CurOrigArg = MF.getFunction().arg_begin();
4276 unsigned CurArgIdx = 0;
4277
4278 // Initially ArgRegsSaveSize is zero.
4279 // Then we increase this value each time we meet byval parameter.
4280 // We also increase this value in case of varargs function.
4281 AFI->setArgRegsSaveSize(0);
4282
4283 // Calculate the amount of stack space that we need to allocate to store
4284 // byval and variadic arguments that are passed in registers.
4285 // We need to know this before we allocate the first byval or variadic
4286 // argument, as they will be allocated a stack slot below the CFA (Canonical
4287 // Frame Address, the stack pointer at entry to the function).
4288 unsigned ArgRegBegin = ARM::R4;
4289 for (const CCValAssign &VA : ArgLocs) {
4290 if (CCInfo.getInRegsParamsProcessed() >= CCInfo.getInRegsParamsCount())
4291 break;
4292
4293 unsigned Index = VA.getValNo();
4294 ISD::ArgFlagsTy Flags = Ins[Index].Flags;
4295 if (!Flags.isByVal())
4296 continue;
4297
4298 assert(VA.isMemLoc() && "unexpected byval pointer in reg");
4299 unsigned RBegin, REnd;
4300 CCInfo.getInRegsParamInfo(InRegsParamRecordIndex: CCInfo.getInRegsParamsProcessed(), BeginReg&: RBegin, EndReg&: REnd);
4301 ArgRegBegin = std::min(a: ArgRegBegin, b: RBegin);
4302
4303 CCInfo.nextInRegsParam();
4304 }
4305 CCInfo.rewindByValRegsInfo();
4306
4307 int lastInsIndex = -1;
4308 if (isVarArg && MFI.hasVAStart()) {
4309 unsigned RegIdx = CCInfo.getFirstUnallocated(Regs: GPRArgRegs);
4310 if (RegIdx != std::size(GPRArgRegs))
4311 ArgRegBegin = std::min(a: ArgRegBegin, b: (unsigned)GPRArgRegs[RegIdx]);
4312 }
4313
4314 unsigned TotalArgRegsSaveSize = 4 * (ARM::R4 - ArgRegBegin);
4315 AFI->setArgRegsSaveSize(TotalArgRegsSaveSize);
4316 auto PtrVT = getPointerTy(DL: DAG.getDataLayout());
4317
4318 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
4319 CCValAssign &VA = ArgLocs[i];
4320 if (Ins[VA.getValNo()].isOrigArg()) {
4321 std::advance(i&: CurOrigArg,
4322 n: Ins[VA.getValNo()].getOrigArgIndex() - CurArgIdx);
4323 CurArgIdx = Ins[VA.getValNo()].getOrigArgIndex();
4324 }
4325 // Arguments stored in registers.
4326 if (VA.isRegLoc()) {
4327 EVT RegVT = VA.getLocVT();
4328 SDValue ArgValue;
4329
4330 if (VA.needsCustom() && VA.getLocVT() == MVT::v2f64) {
4331 // f64 and vector types are split up into multiple registers or
4332 // combinations of registers and stack slots.
4333 SDValue ArgValue1 =
4334 GetF64FormalArgument(VA, NextVA&: ArgLocs[++i], Root&: Chain, DAG, dl);
4335 VA = ArgLocs[++i]; // skip ahead to next loc
4336 SDValue ArgValue2;
4337 if (VA.isMemLoc()) {
4338 int FI = MFI.CreateFixedObject(Size: 8, SPOffset: VA.getLocMemOffset(), IsImmutable: true);
4339 SDValue FIN = DAG.getFrameIndex(FI, VT: PtrVT);
4340 ArgValue2 = DAG.getLoad(
4341 VT: MVT::f64, dl, Chain, Ptr: FIN,
4342 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI));
4343 } else {
4344 ArgValue2 = GetF64FormalArgument(VA, NextVA&: ArgLocs[++i], Root&: Chain, DAG, dl);
4345 }
4346 ArgValue = DAG.getNode(Opcode: ISD::UNDEF, DL: dl, VT: MVT::v2f64);
4347 ArgValue = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: MVT::v2f64, N1: ArgValue,
4348 N2: ArgValue1, N3: DAG.getIntPtrConstant(Val: 0, DL: dl));
4349 ArgValue = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: MVT::v2f64, N1: ArgValue,
4350 N2: ArgValue2, N3: DAG.getIntPtrConstant(Val: 1, DL: dl));
4351 } else if (VA.needsCustom() && VA.getLocVT() == MVT::f64) {
4352 ArgValue = GetF64FormalArgument(VA, NextVA&: ArgLocs[++i], Root&: Chain, DAG, dl);
4353 } else {
4354 const TargetRegisterClass *RC;
4355
4356 if (RegVT == MVT::f16 || RegVT == MVT::bf16)
4357 RC = &ARM::HPRRegClass;
4358 else if (RegVT == MVT::f32)
4359 RC = &ARM::SPRRegClass;
4360 else if (RegVT == MVT::f64 || RegVT == MVT::v4f16 ||
4361 RegVT == MVT::v4bf16)
4362 RC = &ARM::DPRRegClass;
4363 else if (RegVT == MVT::v2f64 || RegVT == MVT::v8f16 ||
4364 RegVT == MVT::v8bf16)
4365 RC = &ARM::QPRRegClass;
4366 else if (RegVT == MVT::i32)
4367 RC = AFI->isThumb1OnlyFunction() ? &ARM::tGPRRegClass
4368 : &ARM::GPRRegClass;
4369 else
4370 llvm_unreachable("RegVT not supported by FORMAL_ARGUMENTS Lowering");
4371
4372 // Transform the arguments in physical registers into virtual ones.
4373 Register Reg = MF.addLiveIn(PReg: VA.getLocReg(), RC);
4374 ArgValue = DAG.getCopyFromReg(Chain, dl, Reg, VT: RegVT);
4375
4376 // If this value is passed in r0 and has the returned attribute (e.g.
4377 // C++ 'structors), record this fact for later use.
4378 if (VA.getLocReg() == ARM::R0 && Ins[VA.getValNo()].Flags.isReturned()) {
4379 AFI->setPreservesR0();
4380 }
4381 }
4382
4383 // If this is an 8 or 16-bit value, it is really passed promoted
4384 // to 32 bits. Insert an assert[sz]ext to capture this, then
4385 // truncate to the right size.
4386 switch (VA.getLocInfo()) {
4387 default: llvm_unreachable("Unknown loc info!");
4388 case CCValAssign::Full: break;
4389 case CCValAssign::BCvt:
4390 ArgValue = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: VA.getValVT(), Operand: ArgValue);
4391 break;
4392 }
4393
4394 // f16 arguments have their size extended to 4 bytes and passed as if they
4395 // had been copied to the LSBs of a 32-bit register.
4396 // For that, it's passed extended to i32 (soft ABI) or to f32 (hard ABI)
4397 if (VA.needsCustom() &&
4398 (VA.getValVT() == MVT::f16 || VA.getValVT() == MVT::bf16))
4399 ArgValue = MoveToHPR(dl, DAG, LocVT: VA.getLocVT(), ValVT: VA.getValVT(), Val: ArgValue);
4400
4401 // On CMSE Entry Functions, formal integer arguments whose bitwidth is
4402 // less than 32 bits must be sign- or zero-extended in the callee for
4403 // security reasons. Although the ABI mandates an extension done by the
4404 // caller, the latter cannot be trusted to follow the rules of the ABI.
4405 const ISD::InputArg &Arg = Ins[VA.getValNo()];
4406 if (AFI->isCmseNSEntryFunction() && Arg.ArgVT.isScalarInteger() &&
4407 RegVT.isScalarInteger() && Arg.ArgVT.bitsLT(VT: MVT::i32))
4408 ArgValue = handleCMSEValue(Value: ArgValue, Arg, DAG, DL: dl);
4409
4410 InVals.push_back(Elt: ArgValue);
4411 } else { // VA.isRegLoc()
4412 // Only arguments passed on the stack should make it here.
4413 assert(VA.isMemLoc());
4414 assert(VA.getValVT() != MVT::i64 && "i64 should already be lowered");
4415
4416 int index = VA.getValNo();
4417
4418 // Some Ins[] entries become multiple ArgLoc[] entries.
4419 // Process them only once.
4420 if (index != lastInsIndex)
4421 {
4422 ISD::ArgFlagsTy Flags = Ins[index].Flags;
4423 // FIXME: For now, all byval parameter objects are marked mutable.
4424 // This can be changed with more analysis.
4425 // In case of tail call optimization mark all arguments mutable.
4426 // Since they could be overwritten by lowering of arguments in case of
4427 // a tail call.
4428 if (Flags.isByVal()) {
4429 assert(Ins[index].isOrigArg() &&
4430 "Byval arguments cannot be implicit");
4431 unsigned CurByValIndex = CCInfo.getInRegsParamsProcessed();
4432
4433 int FrameIndex = StoreByValRegs(
4434 CCInfo, DAG, dl, Chain, OrigArg: &*CurOrigArg, InRegsParamRecordIdx: CurByValIndex,
4435 ArgOffset: VA.getLocMemOffset(), ArgSize: Flags.getByValSize());
4436 InVals.push_back(Elt: DAG.getFrameIndex(FI: FrameIndex, VT: PtrVT));
4437 CCInfo.nextInRegsParam();
4438 } else if (VA.needsCustom() && (VA.getValVT() == MVT::f16 ||
4439 VA.getValVT() == MVT::bf16)) {
4440 // f16 and bf16 values are passed in the least-significant half of
4441 // a 4 byte stack slot. This is done as-if the extension was done
4442 // in a 32-bit register, so the actual bytes used for the value
4443 // differ between little and big endian.
4444 assert(VA.getLocVT().getSizeInBits() == 32);
4445 unsigned FIOffset = VA.getLocMemOffset();
4446 int FI = MFI.CreateFixedObject(Size: VA.getLocVT().getSizeInBits() / 8,
4447 SPOffset: FIOffset, IsImmutable: true);
4448
4449 SDValue Addr = DAG.getFrameIndex(FI, VT: PtrVT);
4450 if (DAG.getDataLayout().isBigEndian())
4451 Addr = DAG.getObjectPtrOffset(SL: dl, Ptr: Addr, Offset: TypeSize::getFixed(ExactSize: 2));
4452
4453 InVals.push_back(Elt: DAG.getLoad(VT: VA.getValVT(), dl, Chain, Ptr: Addr,
4454 PtrInfo: MachinePointerInfo::getFixedStack(
4455 MF&: DAG.getMachineFunction(), FI)));
4456
4457 } else {
4458 unsigned FIOffset = VA.getLocMemOffset();
4459 int FI = MFI.CreateFixedObject(Size: VA.getLocVT().getSizeInBits()/8,
4460 SPOffset: FIOffset, IsImmutable: true);
4461
4462 // Create load nodes to retrieve arguments from the stack.
4463 SDValue FIN = DAG.getFrameIndex(FI, VT: PtrVT);
4464 InVals.push_back(Elt: DAG.getLoad(VT: VA.getValVT(), dl, Chain, Ptr: FIN,
4465 PtrInfo: MachinePointerInfo::getFixedStack(
4466 MF&: DAG.getMachineFunction(), FI)));
4467 }
4468 lastInsIndex = index;
4469 }
4470 }
4471 }
4472
4473 // varargs
4474 if (isVarArg && MFI.hasVAStart()) {
4475 VarArgStyleRegisters(CCInfo, DAG, dl, Chain, ArgOffset: CCInfo.getStackSize(),
4476 TotalArgRegsSaveSize);
4477 if (AFI->isCmseNSEntryFunction()) {
4478 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
4479 DAG.getMachineFunction().getFunction(),
4480 "secure entry function must not be variadic", dl.getDebugLoc()));
4481 }
4482 }
4483
4484 // A variadic function that forwards through a musttail call needs its unnamed
4485 // register arguments modelled as live, so the allocator does not use one as
4486 // scratch. This must run after the va_list save area above, which consumes
4487 // the same registers, and is deliberately *not* predicated on hasVAStart():
4488 // a forwarding thunk never calls va_start.
4489 if (isVarArg && MFI.hasMustTailInVarArgFunc()) {
4490 // getRemainingRegParmsForType() walks the assign function until an argument
4491 // lands in memory, so the convention must have a stack fallback. GHC and
4492 // the Windows CFGuard check stub assign to registers only and would run off
4493 // the end of their tables; neither supports variadic arguments anyway.
4494 CallingConv::ID EffCC = getEffectiveCallingConv(CC: CallConv, isVarArg);
4495 if (EffCC != CallingConv::GHC && EffCC != CallingConv::CFGuard_Check) {
4496 // Only the core registers can hold an unnamed argument:
4497 // getEffectiveCallingConv() degrades every VFP convention to ARM_AAPCS
4498 // when isVarArg is set.
4499 SmallVector<MVT, 1> RegParmTypes;
4500 RegParmTypes.push_back(Elt: MVT::i32);
4501 SmallVectorImpl<ForwardedRegister> &Forwards =
4502 AFI->getForwardedMustTailRegParms();
4503 CCInfo.analyzeMustTailForwardedRegisters(
4504 Forwards, RegParmTypes, Fn: CCAssignFnForCall(CC: CallConv, isVarArg));
4505 }
4506 }
4507
4508 unsigned StackArgSize = CCInfo.getStackSize();
4509 bool TailCallOpt = MF.getTarget().Options.GuaranteedTailCallOpt;
4510 if (canGuaranteeTCO(CC: CallConv, GuaranteeTailCalls: TailCallOpt)) {
4511 // The only way to guarantee a tail call is if the callee restores its
4512 // argument area, but it must also keep the stack aligned when doing so.
4513 MaybeAlign StackAlign = DAG.getDataLayout().getStackAlignment();
4514 assert(StackAlign && "data layout string is missing stack alignment");
4515 StackArgSize = alignTo(Size: StackArgSize, A: *StackAlign);
4516
4517 AFI->setArgumentStackToRestore(StackArgSize);
4518 }
4519 AFI->setArgumentStackSize(StackArgSize);
4520
4521 if (CCInfo.getStackSize() > 0 && AFI->isCmseNSEntryFunction()) {
4522 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
4523 DAG.getMachineFunction().getFunction(),
4524 "secure entry function requires arguments on stack", dl.getDebugLoc()));
4525 }
4526
4527 return Chain;
4528}
4529
4530/// isFloatingPointZero - Return true if this is +0.0.
4531static bool isFloatingPointZero(SDValue Op) {
4532 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Val&: Op))
4533 return CFP->getValueAPF().isPosZero();
4534 else if (ISD::isEXTLoad(N: Op.getNode()) || ISD::isNON_EXTLoad(N: Op.getNode())) {
4535 // Maybe this has already been legalized into the constant pool?
4536 if (Op.getOperand(i: 1).getOpcode() == ARMISD::Wrapper) {
4537 SDValue WrapperOp = Op.getOperand(i: 1).getOperand(i: 0);
4538 if (ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(Val&: WrapperOp))
4539 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(Val: CP->getConstVal()))
4540 return CFP->getValueAPF().isPosZero();
4541 }
4542 } else if (Op->getOpcode() == ISD::BITCAST &&
4543 Op->getValueType(ResNo: 0) == MVT::f64) {
4544 // Handle (ISD::BITCAST (ARMISD::VMOVIMM (ISD::TargetConstant 0)) MVT::f64)
4545 // created by LowerConstantFP().
4546 SDValue BitcastOp = Op->getOperand(Num: 0);
4547 if (BitcastOp->getOpcode() == ARMISD::VMOVIMM &&
4548 isNullConstant(V: BitcastOp->getOperand(Num: 0)))
4549 return true;
4550 }
4551 return false;
4552}
4553
4554static bool isSafeSignedCMN(SDValue Op, SelectionDAG &DAG) {
4555 // 0 - INT_MIN sign wraps, so no signed wrap means cmn is safe.
4556 if (Op->getFlags().hasNoSignedWrap())
4557 return true;
4558
4559 // We can still figure out if the second operand is safe to use
4560 // in a CMN instruction by checking if it is known to be not the minimum
4561 // signed value. If it is not, then we can safely use CMN.
4562 // Note: We can eventually remove this check and simply rely on
4563 // Op->getFlags().hasNoSignedWrap() once SelectionDAG/ISelLowering
4564 // consistently sets them appropriately when making said nodes.
4565
4566 KnownBits KnownSrc = DAG.computeKnownBits(Op: Op.getOperand(i: 1));
4567 return !KnownSrc.getSignedMinValue().isMinSignedValue();
4568}
4569
4570static bool isCMN(SDValue Op, ISD::CondCode CC, SelectionDAG &DAG) {
4571 return Op.getOpcode() == ISD::SUB && isNullConstant(V: Op.getOperand(i: 0)) &&
4572 (isIntEqualitySetCC(Code: CC) ||
4573 (isUnsignedIntSetCC(Code: CC) && DAG.isKnownNeverZero(Op: Op.getOperand(i: 1))) ||
4574 (isSignedIntSetCC(Code: CC) && isSafeSignedCMN(Op, DAG)));
4575}
4576
4577/// Returns how profitable it is to fold a comparison's operand's shift and/or
4578/// extension operations into the comparison instruction's second operand
4579/// (so_reg_imm / so_reg_reg for ARM, t2_so_reg for Thumb-2).
4580static unsigned getCmpOperandFoldingProfit(SDValue Op, const ARMSubtarget &ST) {
4581 // Thumb-1 CMP does not support shifted second operands.
4582 if (ST.isThumb1Only() || !Op.hasOneUse())
4583 return 0;
4584
4585 unsigned Opc = Op.getOpcode();
4586 if (Opc == ISD::SHL || Opc == ISD::SRL || Opc == ISD::SRA) {
4587 if (auto *ShiftAmt = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1)))
4588 return ShiftAmt->getZExtValue() <= 31 ? 1 : 0;
4589 // Register-controlled shift: only ARM-mode CMP/CMN (so_reg_reg) supports
4590 // this; Thumb-2 t2_so_reg requires an immediate shift amount.
4591 return ST.isThumb() ? 0 : 1;
4592 }
4593
4594 if (Opc == ISD::ROTR) {
4595 // Rotr constants will be normalized via mod 32, or & 31,
4596 // so we do not have to bounds check.
4597 if (isa<ConstantSDNode>(Val: Op.getOperand(i: 1)))
4598 return 1;
4599 return ST.isThumb() ? 0 : 1;
4600 }
4601
4602 return 0;
4603}
4604
4605// Return a cheaper value with the same zero/nonzero result as a low-mask AND.
4606static SDValue getShiftedLowMaskZeroTest(SDValue Value, SelectionDAG &DAG,
4607 const SDLoc &DL) {
4608 if (Value.getValueType() != MVT::i32 || Value.getOpcode() != ISD::AND ||
4609 !Value.hasOneUse())
4610 return SDValue();
4611
4612 auto *Mask = dyn_cast<ConstantSDNode>(Val: Value.getOperand(i: 1));
4613 if (!Mask)
4614 return SDValue();
4615 uint32_t MaskValue = Mask->getZExtValue();
4616 // Keep the existing single-bit lowering and UXTB/UXTH for byte/halfword
4617 // masks, and avoid a zero or no-op shift.
4618 if (!isMask_32(Value: MaskValue) || MaskValue == 1 || MaskValue == 0xff ||
4619 MaskValue == 0xffff || MaskValue == ~0U)
4620 return SDValue();
4621
4622 return DAG.getNode(
4623 Opcode: ISD::SHL, DL, VT: MVT::i32, N1: Value.getOperand(i: 0),
4624 N2: DAG.getConstant(Val: llvm::countl_zero(Val: MaskValue), DL, VT: MVT::i32));
4625}
4626
4627/// Returns appropriate ARM CMP (cmp) and corresponding condition code for
4628/// the given operands.
4629SDValue ARMTargetLowering::getARMCmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
4630 SDValue &ARMcc, SelectionDAG &DAG,
4631 const SDLoc &dl) const {
4632 // For a zero test, replace a low-mask AND feeding an OR with a shift. ARM
4633 // and Thumb-2 can fold the shift into the OR. Require single uses to avoid
4634 // duplicating live work.
4635 if (!Subtarget->isThumb1Only() && isIntEqualitySetCC(Code: CC) &&
4636 isNullConstant(V: RHS) && LHS.getValueType() == MVT::i32 &&
4637 LHS.getOpcode() == ISD::OR && LHS.hasOneUse()) {
4638 for (unsigned I = 0; I != 2; ++I) {
4639 if (SDValue Shift =
4640 getShiftedLowMaskZeroTest(Value: LHS.getOperand(i: I), DAG, DL: dl)) {
4641 LHS = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: MVT::i32, N1: LHS.getOperand(i: I ^ 1), N2: Shift);
4642 break;
4643 }
4644 }
4645 }
4646
4647 if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(Val: RHS.getNode())) {
4648 unsigned C = RHSC->getZExtValue();
4649 if (!isLegalICmpImmediate(Imm: (int32_t)C)) {
4650 // Constant does not fit, try adjusting it by one.
4651 switch (CC) {
4652 default: break;
4653 case ISD::SETLT:
4654 case ISD::SETGE:
4655 if (C != 0x80000000 && isLegalICmpImmediate(Imm: C-1)) {
4656 CC = (CC == ISD::SETLT) ? ISD::SETLE : ISD::SETGT;
4657 RHS = DAG.getConstant(Val: C - 1, DL: dl, VT: MVT::i32);
4658 }
4659 break;
4660 case ISD::SETULT:
4661 case ISD::SETUGE:
4662 if (C != 0 && isLegalICmpImmediate(Imm: C-1)) {
4663 CC = (CC == ISD::SETULT) ? ISD::SETULE : ISD::SETUGT;
4664 RHS = DAG.getConstant(Val: C - 1, DL: dl, VT: MVT::i32);
4665 }
4666 break;
4667 case ISD::SETLE:
4668 case ISD::SETGT:
4669 if (C != 0x7fffffff && isLegalICmpImmediate(Imm: C+1)) {
4670 CC = (CC == ISD::SETLE) ? ISD::SETLT : ISD::SETGE;
4671 RHS = DAG.getConstant(Val: C + 1, DL: dl, VT: MVT::i32);
4672 }
4673 break;
4674 case ISD::SETULE:
4675 case ISD::SETUGT:
4676 if (C != 0xffffffff && isLegalICmpImmediate(Imm: C+1)) {
4677 CC = (CC == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE;
4678 RHS = DAG.getConstant(Val: C + 1, DL: dl, VT: MVT::i32);
4679 }
4680 break;
4681 }
4682 }
4683 }
4684
4685 // Thumb1 has very limited immediate modes, so turning an "and" into a
4686 // shift can save multiple instructions.
4687 //
4688 // If we have (x & C1), and C1 is an appropriate mask, we can transform it
4689 // into "((x << n) >> n)". But that isn't necessarily profitable on its
4690 // own. If it's the operand to an unsigned comparison with an immediate,
4691 // we can eliminate one of the shifts: we transform
4692 // "((x << n) >> n) == C2" to "(x << n) == (C2 << n)".
4693 //
4694 // We avoid transforming cases which aren't profitable due to encoding
4695 // details:
4696 //
4697 // 1. C2 fits into the immediate field of a cmp, and the transformed version
4698 // would not; in that case, we're essentially trading one immediate load for
4699 // another.
4700 // 2. C1 is 255 or 65535, so we can use uxtb or uxth.
4701 // 3. C2 is zero; we have other code for this special case.
4702 //
4703 // FIXME: Figure out profitability for Thumb2; we usually can't save an
4704 // instruction, since the AND is always one instruction anyway, but we could
4705 // use narrow instructions in some cases.
4706 if (Subtarget->isThumb1Only() && LHS->getOpcode() == ISD::AND &&
4707 LHS->hasOneUse() && isa<ConstantSDNode>(Val: LHS.getOperand(i: 1)) &&
4708 LHS.getValueType() == MVT::i32 && isa<ConstantSDNode>(Val: RHS) &&
4709 !isSignedIntSetCC(Code: CC)) {
4710 unsigned Mask = LHS.getConstantOperandVal(i: 1);
4711 auto *RHSC = cast<ConstantSDNode>(Val: RHS.getNode());
4712 uint64_t RHSV = RHSC->getZExtValue();
4713 if (isMask_32(Value: Mask) && (RHSV & ~Mask) == 0 && Mask != 255 && Mask != 65535) {
4714 unsigned ShiftBits = llvm::countl_zero(Val: Mask);
4715 if (RHSV && (RHSV > 255 || (RHSV << ShiftBits) <= 255)) {
4716 SDValue ShiftAmt = DAG.getConstant(Val: ShiftBits, DL: dl, VT: MVT::i32);
4717 LHS = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: MVT::i32, N1: LHS.getOperand(i: 0), N2: ShiftAmt);
4718 RHS = DAG.getConstant(Val: RHSV << ShiftBits, DL: dl, VT: MVT::i32);
4719 }
4720 }
4721 }
4722
4723 // The specific comparison "(x<<c) > 0x80000000U" can be optimized to a
4724 // single "lsls x, c+1". The shift sets the "C" and "Z" flags the same
4725 // way a cmp would.
4726 // FIXME: Add support for ARM/Thumb2; this would need isel patterns, and
4727 // some tweaks to the heuristics for the previous and->shift transform.
4728 // FIXME: Optimize cases where the LHS isn't a shift.
4729 if (Subtarget->isThumb1Only() && LHS->getOpcode() == ISD::SHL &&
4730 isa<ConstantSDNode>(Val: RHS) && RHS->getAsZExtVal() == 0x80000000U &&
4731 CC == ISD::SETUGT && isa<ConstantSDNode>(Val: LHS.getOperand(i: 1)) &&
4732 LHS.getConstantOperandVal(i: 1) < 31) {
4733 unsigned ShiftAmt = LHS.getConstantOperandVal(i: 1) + 1;
4734 SDValue Shift =
4735 DAG.getNode(Opcode: ARMISD::LSLS, DL: dl, VTList: DAG.getVTList(VT1: MVT::i32, VT2: FlagsVT),
4736 N1: LHS.getOperand(i: 0), N2: DAG.getConstant(Val: ShiftAmt, DL: dl, VT: MVT::i32));
4737 ARMcc = DAG.getConstant(Val: ARMCC::HI, DL: dl, VT: MVT::i32);
4738 return Shift.getValue(R: 1);
4739 }
4740
4741 ARMCC::CondCodes CondCode = IntCCToARMCC(CC);
4742
4743 unsigned CompareType;
4744 switch (CondCode) {
4745 default:
4746 CompareType = ARMISD::CMP;
4747 break;
4748 case ARMCC::EQ:
4749 case ARMCC::NE:
4750 // Uses only Z Flag
4751 CompareType = ARMISD::CMPZ;
4752 break;
4753 }
4754
4755 // TODO: Remove CMPZ check once we generalize and remove the CMPZ enum from
4756 // the codebase.
4757
4758 // TODO: When we have a solution to the vselect predicate not allowing pl/mi
4759 // all the time, allow those cases to be cmn too no matter what.
4760 if (CompareType != ARMISD::CMPZ && isCMN(Op: RHS, CC, DAG)) {
4761 CompareType = ARMISD::CMN;
4762 RHS = RHS.getOperand(i: 1);
4763 } else if (CompareType != ARMISD::CMPZ && isCMN(Op: LHS, CC, DAG)) {
4764 CompareType = ARMISD::CMN;
4765 LHS = LHS.getOperand(i: 1);
4766 CondCode = IntCCToARMCC(CC: ISD::getSetCCSwappedOperands(Operation: CC));
4767 }
4768
4769 // Prefer folding shifts / CMN into the cmp/cmn second operand (so_reg /
4770 // t2_so_reg). When both sides compete, pick the higher
4771 // getCmpOperandFoldingProfit. Only when RHS is not a legal icmp
4772 // immediate: otherwise keep the canonical (reg, imm) form.
4773 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val: RHS.getNode());
4774 if (!C || !isLegalICmpImmediate(Imm: C->getSExtValue())) {
4775 if (getCmpOperandFoldingProfit(Op: LHS, ST: *Subtarget) >
4776 getCmpOperandFoldingProfit(Op: RHS, ST: *Subtarget)) {
4777 std::swap(a&: LHS, b&: RHS);
4778 if (CompareType == ARMISD::CMP)
4779 CondCode = ARMCC::getSwappedCondition(CC: CondCode);
4780 }
4781 }
4782
4783 // If the RHS is a constant zero then the V (overflow) flag will never be
4784 // set. This can allow us to simplify GE to PL or LT to MI, which can be
4785 // simpler for other passes (like the peephole optimiser) to deal with.
4786 if (isNullConstant(V: RHS)) {
4787 switch (CondCode) {
4788 default:
4789 break;
4790 case ARMCC::GE:
4791 CondCode = ARMCC::PL;
4792 break;
4793 case ARMCC::LT:
4794 CondCode = ARMCC::MI;
4795 break;
4796 }
4797 }
4798
4799 ARMcc = DAG.getConstant(Val: CondCode, DL: dl, VT: MVT::i32);
4800 return DAG.getNode(Opcode: CompareType, DL: dl, VT: FlagsVT, N1: LHS, N2: RHS);
4801}
4802
4803/// Returns a appropriate VFP CMP (fcmp{s|d}+fmstat) for the given operands.
4804SDValue ARMTargetLowering::getVFPCmp(SDValue LHS, SDValue RHS,
4805 SelectionDAG &DAG, const SDLoc &dl,
4806 bool Signaling) const {
4807 assert(Subtarget->hasFP64() || RHS.getValueType() != MVT::f64);
4808 SDValue Flags;
4809 if (!isFloatingPointZero(Op: RHS))
4810 Flags = DAG.getNode(Opcode: Signaling ? ARMISD::CMPFPE : ARMISD::CMPFP, DL: dl, VT: FlagsVT,
4811 N1: LHS, N2: RHS);
4812 else
4813 Flags = DAG.getNode(Opcode: Signaling ? ARMISD::CMPFPEw0 : ARMISD::CMPFPw0, DL: dl,
4814 VT: FlagsVT, Operand: LHS);
4815 return DAG.getNode(Opcode: ARMISD::FMSTAT, DL: dl, VT: FlagsVT, Operand: Flags);
4816}
4817
4818// This function returns three things: the arithmetic computation itself
4819// (Value), a comparison (OverflowCmp), and a condition code (ARMcc). The
4820// comparison and the condition code define the case in which the arithmetic
4821// computation *does not* overflow.
4822std::pair<SDValue, SDValue>
4823ARMTargetLowering::getARMXALUOOp(SDValue Op, SelectionDAG &DAG,
4824 SDValue &ARMcc) const {
4825 assert(Op.getValueType() == MVT::i32 && "Unsupported value type");
4826
4827 SDValue Value, OverflowCmp;
4828 SDValue LHS = Op.getOperand(i: 0);
4829 SDValue RHS = Op.getOperand(i: 1);
4830 SDLoc dl(Op);
4831
4832 // FIXME: We are currently always generating CMPs because we don't support
4833 // generating CMN through the backend. This is not as good as the natural
4834 // CMP case because it causes a register dependency and cannot be folded
4835 // later.
4836
4837 switch (Op.getOpcode()) {
4838 default:
4839 llvm_unreachable("Unknown overflow instruction!");
4840 case ISD::SADDO:
4841 ARMcc = DAG.getConstant(Val: ARMCC::VC, DL: dl, VT: MVT::i32);
4842 Value = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: Op.getValueType(), N1: LHS, N2: RHS);
4843 OverflowCmp = DAG.getNode(Opcode: ARMISD::CMP, DL: dl, VT: FlagsVT, N1: Value, N2: LHS);
4844 break;
4845 case ISD::UADDO:
4846 ARMcc = DAG.getConstant(Val: ARMCC::HS, DL: dl, VT: MVT::i32);
4847 // We use ADDC here to correspond to its use in LowerALUO.
4848 // We do not use it in the USUBO case as Value may not be used.
4849 Value = DAG.getNode(Opcode: ARMISD::ADDC, DL: dl,
4850 VTList: DAG.getVTList(VT1: Op.getValueType(), VT2: MVT::i32), N1: LHS, N2: RHS)
4851 .getValue(R: 0);
4852 OverflowCmp = DAG.getNode(Opcode: ARMISD::CMP, DL: dl, VT: FlagsVT, N1: Value, N2: LHS);
4853 break;
4854 case ISD::SSUBO:
4855 ARMcc = DAG.getConstant(Val: ARMCC::VC, DL: dl, VT: MVT::i32);
4856 Value = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: Op.getValueType(), N1: LHS, N2: RHS);
4857 OverflowCmp = DAG.getNode(Opcode: ARMISD::CMP, DL: dl, VT: FlagsVT, N1: LHS, N2: RHS);
4858 break;
4859 case ISD::USUBO:
4860 ARMcc = DAG.getConstant(Val: ARMCC::HS, DL: dl, VT: MVT::i32);
4861 Value = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: Op.getValueType(), N1: LHS, N2: RHS);
4862 OverflowCmp = DAG.getNode(Opcode: ARMISD::CMP, DL: dl, VT: FlagsVT, N1: LHS, N2: RHS);
4863 break;
4864 case ISD::UMULO:
4865 // We generate a UMUL_LOHI and then check if the high word is 0.
4866 ARMcc = DAG.getConstant(Val: ARMCC::EQ, DL: dl, VT: MVT::i32);
4867 Value = DAG.getNode(Opcode: ISD::UMUL_LOHI, DL: dl,
4868 VTList: DAG.getVTList(VT1: Op.getValueType(), VT2: Op.getValueType()),
4869 N1: LHS, N2: RHS);
4870 OverflowCmp = DAG.getNode(Opcode: ARMISD::CMPZ, DL: dl, VT: FlagsVT, N1: Value.getValue(R: 1),
4871 N2: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
4872 Value = Value.getValue(R: 0); // We only want the low 32 bits for the result.
4873 break;
4874 case ISD::SMULO:
4875 // We generate a SMUL_LOHI and then check if all the bits of the high word
4876 // are the same as the sign bit of the low word.
4877 ARMcc = DAG.getConstant(Val: ARMCC::EQ, DL: dl, VT: MVT::i32);
4878 Value = DAG.getNode(Opcode: ISD::SMUL_LOHI, DL: dl,
4879 VTList: DAG.getVTList(VT1: Op.getValueType(), VT2: Op.getValueType()),
4880 N1: LHS, N2: RHS);
4881 OverflowCmp = DAG.getNode(Opcode: ARMISD::CMPZ, DL: dl, VT: FlagsVT, N1: Value.getValue(R: 1),
4882 N2: DAG.getNode(Opcode: ISD::SRA, DL: dl, VT: Op.getValueType(),
4883 N1: Value.getValue(R: 0),
4884 N2: DAG.getConstant(Val: 31, DL: dl, VT: MVT::i32)));
4885 Value = Value.getValue(R: 0); // We only want the low 32 bits for the result.
4886 break;
4887 } // switch (...)
4888
4889 return std::make_pair(x&: Value, y&: OverflowCmp);
4890}
4891
4892static SDValue valueToCarryFlag(SDValue Value, SelectionDAG &DAG, bool Invert) {
4893 SDLoc DL(Value);
4894 EVT VT = Value.getValueType();
4895
4896 if (Invert)
4897 Value = DAG.getNode(Opcode: ISD::SUB, DL, VT: MVT::i32,
4898 N1: DAG.getConstant(Val: 1, DL, VT: MVT::i32), N2: Value);
4899
4900 SDValue Cmp = DAG.getNode(Opcode: ARMISD::SUBC, DL, VTList: DAG.getVTList(VT1: VT, VT2: MVT::i32),
4901 N1: Value, N2: DAG.getConstant(Val: 1, DL, VT));
4902 return Cmp.getValue(R: 1);
4903}
4904
4905static SDValue carryFlagToValue(SDValue Flags, EVT VT, SelectionDAG &DAG,
4906 bool Invert) {
4907 SDLoc DL(Flags);
4908
4909 if (Invert) {
4910 // Convert flags to boolean with ADDE 0,0,Carry then compute 1 - bool.
4911 SDValue BoolCarry = DAG.getNode(
4912 Opcode: ARMISD::ADDE, DL, VTList: DAG.getVTList(VT1: VT, VT2: MVT::i32),
4913 N1: DAG.getConstant(Val: 0, DL, VT), N2: DAG.getConstant(Val: 0, DL, VT), N3: Flags);
4914 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: DAG.getConstant(Val: 1, DL, VT), N2: BoolCarry);
4915 }
4916
4917 // Now convert the carry flag into a boolean carry. We do this
4918 // using ARMISD::ADDE 0, 0, Carry
4919 return DAG.getNode(Opcode: ARMISD::ADDE, DL, VTList: DAG.getVTList(VT1: VT, VT2: MVT::i32),
4920 N1: DAG.getConstant(Val: 0, DL, VT), N2: DAG.getConstant(Val: 0, DL, VT),
4921 N3: Flags);
4922}
4923
4924// Value is 1 if 'V' bit is 1, else 0
4925static SDValue overflowFlagToValue(SDValue Flags, EVT VT, SelectionDAG &DAG) {
4926 SDLoc DL(Flags);
4927 SDValue Zero = DAG.getConstant(Val: 0, DL, VT);
4928 SDValue One = DAG.getConstant(Val: 1, DL, VT);
4929 SDValue ARMcc = DAG.getConstant(Val: ARMCC::VS, DL, VT: MVT::i32);
4930 return DAG.getNode(Opcode: ARMISD::CMOV, DL, VT, N1: Zero, N2: One, N3: ARMcc, N4: Flags);
4931}
4932
4933SDValue ARMTargetLowering::LowerALUO(SDValue Op, SelectionDAG &DAG) const {
4934 // Let legalize expand this if it isn't a legal type yet.
4935 if (!isTypeLegal(VT: Op.getValueType()))
4936 return SDValue();
4937
4938 SDValue LHS = Op.getOperand(i: 0);
4939 SDValue RHS = Op.getOperand(i: 1);
4940 SDLoc dl(Op);
4941
4942 EVT VT = Op.getValueType();
4943 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: MVT::i32);
4944 SDValue Value;
4945 SDValue Overflow;
4946 switch (Op.getOpcode()) {
4947 case ISD::UADDO:
4948 Value = DAG.getNode(Opcode: ARMISD::ADDC, DL: dl, VTList: VTs, N1: LHS, N2: RHS);
4949 // Convert the carry flag into a boolean value.
4950 Overflow = carryFlagToValue(Flags: Value.getValue(R: 1), VT, DAG, Invert: false);
4951 break;
4952 case ISD::USUBO:
4953 Value = DAG.getNode(Opcode: ARMISD::SUBC, DL: dl, VTList: VTs, N1: LHS, N2: RHS);
4954 // Convert the carry flag into a boolean value.
4955 Overflow = carryFlagToValue(Flags: Value.getValue(R: 1), VT, DAG, Invert: true);
4956 break;
4957 default: {
4958 // Handle other operations with getARMXALUOOp
4959 SDValue OverflowCmp, ARMcc;
4960 std::tie(args&: Value, args&: OverflowCmp) = getARMXALUOOp(Op, DAG, ARMcc);
4961 // We use 0 and 1 as false and true values.
4962 // ARMcc represents the "no overflow" condition (e.g., VC for signed ops).
4963 // CMOV operand order is (FalseVal, TrueVal), so we put 1 in FalseVal
4964 // position to get Overflow=1 when the "no overflow" condition is false.
4965 Overflow =
4966 DAG.getNode(Opcode: ARMISD::CMOV, DL: dl, VT: MVT::i32,
4967 N1: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32), // FalseVal: overflow
4968 N2: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32), // TrueVal: no overflow
4969 N3: ARMcc, N4: OverflowCmp);
4970 break;
4971 }
4972 }
4973
4974 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: dl, VTList: VTs, N1: Value, N2: Overflow);
4975}
4976
4977static SDValue LowerADDSUBSAT(SDValue Op, SelectionDAG &DAG,
4978 const ARMSubtarget *Subtarget) {
4979 EVT VT = Op.getValueType();
4980 if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP() || Subtarget->isThumb1Only())
4981 return SDValue();
4982 if (!VT.isSimple())
4983 return SDValue();
4984
4985 unsigned NewOpcode;
4986 switch (VT.getSimpleVT().SimpleTy) {
4987 default:
4988 return SDValue();
4989 case MVT::i8:
4990 switch (Op->getOpcode()) {
4991 case ISD::UADDSAT:
4992 NewOpcode = ARMISD::UQADD8b;
4993 break;
4994 case ISD::SADDSAT:
4995 NewOpcode = ARMISD::QADD8b;
4996 break;
4997 case ISD::USUBSAT:
4998 NewOpcode = ARMISD::UQSUB8b;
4999 break;
5000 case ISD::SSUBSAT:
5001 NewOpcode = ARMISD::QSUB8b;
5002 break;
5003 }
5004 break;
5005 case MVT::i16:
5006 switch (Op->getOpcode()) {
5007 case ISD::UADDSAT:
5008 NewOpcode = ARMISD::UQADD16b;
5009 break;
5010 case ISD::SADDSAT:
5011 NewOpcode = ARMISD::QADD16b;
5012 break;
5013 case ISD::USUBSAT:
5014 NewOpcode = ARMISD::UQSUB16b;
5015 break;
5016 case ISD::SSUBSAT:
5017 NewOpcode = ARMISD::QSUB16b;
5018 break;
5019 }
5020 break;
5021 }
5022
5023 SDLoc dl(Op);
5024 SDValue Add =
5025 DAG.getNode(Opcode: NewOpcode, DL: dl, VT: MVT::i32,
5026 N1: DAG.getSExtOrTrunc(Op: Op->getOperand(Num: 0), DL: dl, VT: MVT::i32),
5027 N2: DAG.getSExtOrTrunc(Op: Op->getOperand(Num: 1), DL: dl, VT: MVT::i32));
5028 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: Add);
5029}
5030
5031SDValue ARMTargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
5032 SDValue Cond = Op.getOperand(i: 0);
5033 SDValue SelectTrue = Op.getOperand(i: 1);
5034 SDValue SelectFalse = Op.getOperand(i: 2);
5035 SDLoc dl(Op);
5036 unsigned Opc = Cond.getOpcode();
5037
5038 if (Cond.getResNo() == 1 &&
5039 (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO ||
5040 Opc == ISD::USUBO)) {
5041 if (!isTypeLegal(VT: Cond->getValueType(ResNo: 0)))
5042 return SDValue();
5043
5044 SDValue Value, OverflowCmp;
5045 SDValue ARMcc;
5046 std::tie(args&: Value, args&: OverflowCmp) = getARMXALUOOp(Op: Cond, DAG, ARMcc);
5047 EVT VT = Op.getValueType();
5048
5049 return getCMOV(dl, VT, FalseVal: SelectTrue, TrueVal: SelectFalse, ARMcc, Flags: OverflowCmp, DAG);
5050 }
5051
5052 // Convert:
5053 //
5054 // (select (cmov 1, 0, cond), t, f) -> (cmov t, f, cond)
5055 // (select (cmov 0, 1, cond), t, f) -> (cmov f, t, cond)
5056 //
5057 if (Cond.getOpcode() == ARMISD::CMOV && Cond.hasOneUse()) {
5058 const ConstantSDNode *CMOVTrue =
5059 dyn_cast<ConstantSDNode>(Val: Cond.getOperand(i: 0));
5060 const ConstantSDNode *CMOVFalse =
5061 dyn_cast<ConstantSDNode>(Val: Cond.getOperand(i: 1));
5062
5063 if (CMOVTrue && CMOVFalse) {
5064 unsigned CMOVTrueVal = CMOVTrue->getZExtValue();
5065 unsigned CMOVFalseVal = CMOVFalse->getZExtValue();
5066
5067 SDValue True;
5068 SDValue False;
5069 if (CMOVTrueVal == 1 && CMOVFalseVal == 0) {
5070 True = SelectTrue;
5071 False = SelectFalse;
5072 } else if (CMOVTrueVal == 0 && CMOVFalseVal == 1) {
5073 True = SelectFalse;
5074 False = SelectTrue;
5075 }
5076
5077 if (True.getNode() && False.getNode())
5078 return getCMOV(dl, VT: Op.getValueType(), FalseVal: True, TrueVal: False, ARMcc: Cond.getOperand(i: 2),
5079 Flags: Cond.getOperand(i: 3), DAG);
5080 }
5081 }
5082
5083 return DAG.getSelectCC(DL: dl, LHS: Cond,
5084 RHS: DAG.getConstant(Val: 0, DL: dl, VT: Cond.getValueType()),
5085 True: SelectTrue, False: SelectFalse, Cond: ISD::SETNE);
5086}
5087
5088static void checkVSELConstraints(ISD::CondCode CC, ARMCC::CondCodes &CondCode,
5089 bool &swpCmpOps, bool &swpVselOps) {
5090 // Start by selecting the GE condition code for opcodes that return true for
5091 // 'equality'
5092 if (CC == ISD::SETUGE || CC == ISD::SETOGE || CC == ISD::SETOLE ||
5093 CC == ISD::SETULE || CC == ISD::SETGE || CC == ISD::SETLE)
5094 CondCode = ARMCC::GE;
5095
5096 // and GT for opcodes that return false for 'equality'.
5097 else if (CC == ISD::SETUGT || CC == ISD::SETOGT || CC == ISD::SETOLT ||
5098 CC == ISD::SETULT || CC == ISD::SETGT || CC == ISD::SETLT)
5099 CondCode = ARMCC::GT;
5100
5101 // Since we are constrained to GE/GT, if the opcode contains 'less', we need
5102 // to swap the compare operands.
5103 if (CC == ISD::SETOLE || CC == ISD::SETULE || CC == ISD::SETOLT ||
5104 CC == ISD::SETULT || CC == ISD::SETLE || CC == ISD::SETLT)
5105 swpCmpOps = true;
5106
5107 // Both GT and GE are ordered comparisons, and return false for 'unordered'.
5108 // If we have an unordered opcode, we need to swap the operands to the VSEL
5109 // instruction (effectively negating the condition).
5110 //
5111 // This also has the effect of swapping which one of 'less' or 'greater'
5112 // returns true, so we also swap the compare operands. It also switches
5113 // whether we return true for 'equality', so we compensate by picking the
5114 // opposite condition code to our original choice.
5115 if (CC == ISD::SETULE || CC == ISD::SETULT || CC == ISD::SETUGE ||
5116 CC == ISD::SETUGT) {
5117 swpCmpOps = !swpCmpOps;
5118 swpVselOps = !swpVselOps;
5119 CondCode = CondCode == ARMCC::GT ? ARMCC::GE : ARMCC::GT;
5120 }
5121
5122 // 'ordered' is 'anything but unordered', so use the VS condition code and
5123 // swap the VSEL operands.
5124 if (CC == ISD::SETO) {
5125 CondCode = ARMCC::VS;
5126 swpVselOps = true;
5127 }
5128
5129 // 'unordered or not equal' is 'anything but equal', so use the EQ condition
5130 // code and swap the VSEL operands. Also do this if we don't care about the
5131 // unordered case.
5132 if (CC == ISD::SETUNE || CC == ISD::SETNE) {
5133 CondCode = ARMCC::EQ;
5134 swpVselOps = true;
5135 }
5136}
5137
5138SDValue ARMTargetLowering::getCMOV(const SDLoc &dl, EVT VT, SDValue FalseVal,
5139 SDValue TrueVal, SDValue ARMcc,
5140 SDValue Flags, SelectionDAG &DAG) const {
5141 if (!Subtarget->hasFP64() && VT == MVT::f64) {
5142 FalseVal = DAG.getNode(Opcode: ARMISD::VMOVRRD, DL: dl,
5143 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N: FalseVal);
5144 TrueVal = DAG.getNode(Opcode: ARMISD::VMOVRRD, DL: dl,
5145 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N: TrueVal);
5146
5147 SDValue TrueLow = TrueVal.getValue(R: 0);
5148 SDValue TrueHigh = TrueVal.getValue(R: 1);
5149 SDValue FalseLow = FalseVal.getValue(R: 0);
5150 SDValue FalseHigh = FalseVal.getValue(R: 1);
5151
5152 SDValue Low = DAG.getNode(Opcode: ARMISD::CMOV, DL: dl, VT: MVT::i32, N1: FalseLow, N2: TrueLow,
5153 N3: ARMcc, N4: Flags);
5154 SDValue High = DAG.getNode(Opcode: ARMISD::CMOV, DL: dl, VT: MVT::i32, N1: FalseHigh, N2: TrueHigh,
5155 N3: ARMcc, N4: Flags);
5156
5157 return DAG.getNode(Opcode: ARMISD::VMOVDRR, DL: dl, VT: MVT::f64, N1: Low, N2: High);
5158 }
5159 return DAG.getNode(Opcode: ARMISD::CMOV, DL: dl, VT, N1: FalseVal, N2: TrueVal, N3: ARMcc, N4: Flags);
5160}
5161
5162static bool isGTorGE(ISD::CondCode CC) {
5163 return CC == ISD::SETGT || CC == ISD::SETGE;
5164}
5165
5166static bool isLTorLE(ISD::CondCode CC) {
5167 return CC == ISD::SETLT || CC == ISD::SETLE;
5168}
5169
5170// See if a conditional (LHS CC RHS ? TrueVal : FalseVal) is lower-saturating.
5171// All of these conditions (and their <= and >= counterparts) will do:
5172// x < k ? k : x
5173// x > k ? x : k
5174// k < x ? x : k
5175// k > x ? k : x
5176static bool isLowerSaturate(const SDValue LHS, const SDValue RHS,
5177 const SDValue TrueVal, const SDValue FalseVal,
5178 const ISD::CondCode CC, const SDValue K) {
5179 return (isGTorGE(CC) &&
5180 ((K == LHS && K == TrueVal) || (K == RHS && K == FalseVal))) ||
5181 (isLTorLE(CC) &&
5182 ((K == RHS && K == TrueVal) || (K == LHS && K == FalseVal)));
5183}
5184
5185// Check if two chained conditionals could be converted into SSAT or USAT.
5186//
5187// SSAT can replace a set of two conditional selectors that bound a number to an
5188// interval of type [k, ~k] when k + 1 is a power of 2. Here are some examples:
5189//
5190// x < -k ? -k : (x > k ? k : x)
5191// x < -k ? -k : (x < k ? x : k)
5192// x > -k ? (x > k ? k : x) : -k
5193// x < k ? (x < -k ? -k : x) : k
5194// etc.
5195//
5196// LLVM canonicalizes these to either a min(max()) or a max(min())
5197// pattern. This function tries to match one of these and will return a SSAT
5198// node if successful.
5199//
5200// USAT works similarly to SSAT but bounds on the interval [0, k] where k + 1
5201// is a power of 2.
5202static SDValue LowerSaturatingConditional(SDValue Op, SelectionDAG &DAG) {
5203 EVT VT = Op.getValueType();
5204 SDValue V1 = Op.getOperand(i: 0);
5205 SDValue K1 = Op.getOperand(i: 1);
5206 SDValue TrueVal1 = Op.getOperand(i: 2);
5207 SDValue FalseVal1 = Op.getOperand(i: 3);
5208 ISD::CondCode CC1 = Op.getOperandCondCode(i: 4);
5209
5210 const SDValue Op2 = isa<ConstantSDNode>(Val: TrueVal1) ? FalseVal1 : TrueVal1;
5211 if (Op2.getOpcode() != ISD::SELECT_CC)
5212 return SDValue();
5213
5214 SDValue V2 = Op2.getOperand(i: 0);
5215 SDValue K2 = Op2.getOperand(i: 1);
5216 SDValue TrueVal2 = Op2.getOperand(i: 2);
5217 SDValue FalseVal2 = Op2.getOperand(i: 3);
5218 ISD::CondCode CC2 = Op2.getOperandCondCode(i: 4);
5219
5220 SDValue V1Tmp = V1;
5221 SDValue V2Tmp = V2;
5222
5223 // Check that the registers and the constants match a max(min()) or min(max())
5224 // pattern
5225 if (V1Tmp != TrueVal1 || V2Tmp != TrueVal2 || K1 != FalseVal1 ||
5226 K2 != FalseVal2 ||
5227 !((isGTorGE(CC: CC1) && isLTorLE(CC: CC2)) || (isLTorLE(CC: CC1) && isGTorGE(CC: CC2))))
5228 return SDValue();
5229
5230 // Check that the constant in the lower-bound check is
5231 // the opposite of the constant in the upper-bound check
5232 // in 1's complement.
5233 if (!isa<ConstantSDNode>(Val: K1) || !isa<ConstantSDNode>(Val: K2))
5234 return SDValue();
5235
5236 int64_t Val1 = cast<ConstantSDNode>(Val&: K1)->getSExtValue();
5237 int64_t Val2 = cast<ConstantSDNode>(Val&: K2)->getSExtValue();
5238 int64_t PosVal = std::max(a: Val1, b: Val2);
5239 int64_t NegVal = std::min(a: Val1, b: Val2);
5240
5241 if (!((Val1 > Val2 && isLTorLE(CC: CC1)) || (Val1 < Val2 && isLTorLE(CC: CC2))) ||
5242 !isPowerOf2_64(Value: PosVal + 1))
5243 return SDValue();
5244
5245 // Handle the difference between USAT (unsigned) and SSAT (signed)
5246 // saturation
5247 // At this point, PosVal is guaranteed to be positive
5248 uint64_t K = PosVal;
5249 SDLoc dl(Op);
5250 if (Val1 == ~Val2)
5251 return DAG.getNode(Opcode: ARMISD::SSAT, DL: dl, VT, N1: V2Tmp,
5252 N2: DAG.getConstant(Val: llvm::countr_one(Value: K), DL: dl, VT));
5253 if (NegVal == 0)
5254 return DAG.getNode(Opcode: ARMISD::USAT, DL: dl, VT, N1: V2Tmp,
5255 N2: DAG.getConstant(Val: llvm::countr_one(Value: K), DL: dl, VT));
5256
5257 return SDValue();
5258}
5259
5260// Check if a condition of the type x < k ? k : x can be converted into a
5261// bit operation instead of conditional moves.
5262// Currently this is allowed given:
5263// - The conditions and values match up
5264// - k is 0 or -1 (all ones)
5265// This function will not check the last condition, thats up to the caller
5266// It returns true if the transformation can be made, and in such case
5267// returns x in V, and k in SatK.
5268static bool isLowerSaturatingConditional(const SDValue &Op, SDValue &V,
5269 SDValue &SatK)
5270{
5271 SDValue LHS = Op.getOperand(i: 0);
5272 SDValue RHS = Op.getOperand(i: 1);
5273 ISD::CondCode CC = Op.getOperandCondCode(i: 4);
5274 SDValue TrueVal = Op.getOperand(i: 2);
5275 SDValue FalseVal = Op.getOperand(i: 3);
5276
5277 SDValue *K = isa<ConstantSDNode>(Val: LHS) ? &LHS : isa<ConstantSDNode>(Val: RHS)
5278 ? &RHS
5279 : nullptr;
5280
5281 // No constant operation in comparison, early out
5282 if (!K)
5283 return false;
5284
5285 SDValue KTmp = isa<ConstantSDNode>(Val: TrueVal) ? TrueVal : FalseVal;
5286 V = (KTmp == TrueVal) ? FalseVal : TrueVal;
5287 SDValue VTmp = (K && *K == LHS) ? RHS : LHS;
5288
5289 // If the constant on left and right side, or variable on left and right,
5290 // does not match, early out
5291 if (*K != KTmp || V != VTmp)
5292 return false;
5293
5294 if (isLowerSaturate(LHS, RHS, TrueVal, FalseVal, CC, K: *K)) {
5295 SatK = *K;
5296 return true;
5297 }
5298
5299 return false;
5300}
5301
5302bool ARMTargetLowering::isUnsupportedFloatingType(EVT VT) const {
5303 if (VT == MVT::f32)
5304 return !Subtarget->hasVFP2Base();
5305 if (VT == MVT::f64)
5306 return !Subtarget->hasFP64();
5307 if (VT == MVT::f16)
5308 return !Subtarget->hasFullFP16();
5309 return false;
5310}
5311
5312static SDValue matchCSET(unsigned &Opcode, bool &InvertCond, SDValue TrueVal,
5313 SDValue FalseVal, const ARMSubtarget *Subtarget) {
5314 ConstantSDNode *CFVal = dyn_cast<ConstantSDNode>(Val&: FalseVal);
5315 ConstantSDNode *CTVal = dyn_cast<ConstantSDNode>(Val&: TrueVal);
5316 if (!CFVal || !CTVal || !Subtarget->hasV8_1MMainlineOps())
5317 return SDValue();
5318
5319 unsigned TVal = CTVal->getZExtValue();
5320 unsigned FVal = CFVal->getZExtValue();
5321
5322 Opcode = 0;
5323 InvertCond = false;
5324 if (TVal == ~FVal) {
5325 Opcode = ARMISD::CSINV;
5326 } else if (TVal == ~FVal + 1) {
5327 Opcode = ARMISD::CSNEG;
5328 } else if (TVal + 1 == FVal) {
5329 Opcode = ARMISD::CSINC;
5330 } else if (TVal == FVal + 1) {
5331 Opcode = ARMISD::CSINC;
5332 std::swap(a&: TrueVal, b&: FalseVal);
5333 std::swap(a&: TVal, b&: FVal);
5334 InvertCond = !InvertCond;
5335 } else {
5336 return SDValue();
5337 }
5338
5339 // If one of the constants is cheaper than another, materialise the
5340 // cheaper one and let the csel generate the other.
5341 if (Opcode != ARMISD::CSINC &&
5342 HasLowerConstantMaterializationCost(Val1: FVal, Val2: TVal, Subtarget)) {
5343 std::swap(a&: TrueVal, b&: FalseVal);
5344 std::swap(a&: TVal, b&: FVal);
5345 InvertCond = !InvertCond;
5346 }
5347
5348 // Attempt to use ZR checking TVal is 0, possibly inverting the condition
5349 // to get there. CSINC not is invertable like the other two (~(~a) == a,
5350 // -(-a) == a, but (a+1)+1 != a).
5351 if (FVal == 0 && Opcode != ARMISD::CSINC) {
5352 std::swap(a&: TrueVal, b&: FalseVal);
5353 std::swap(a&: TVal, b&: FVal);
5354 InvertCond = !InvertCond;
5355 }
5356
5357 return TrueVal;
5358}
5359
5360SDValue ARMTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
5361 EVT VT = Op.getValueType();
5362 SDLoc dl(Op);
5363
5364 // Try to convert two saturating conditional selects into a single SSAT
5365 if ((!Subtarget->isThumb() && Subtarget->hasV6Ops()) || Subtarget->isThumb2())
5366 if (SDValue SatValue = LowerSaturatingConditional(Op, DAG))
5367 return SatValue;
5368
5369 // Try to convert expressions of the form x < k ? k : x (and similar forms)
5370 // into more efficient bit operations, which is possible when k is 0 or -1
5371 // On ARM and Thumb-2 which have flexible operand 2 this will result in
5372 // single instructions. On Thumb the shift and the bit operation will be two
5373 // instructions.
5374 // Only allow this transformation on full-width (32-bit) operations
5375 SDValue LowerSatConstant;
5376 SDValue SatValue;
5377 if (VT == MVT::i32 &&
5378 isLowerSaturatingConditional(Op, V&: SatValue, SatK&: LowerSatConstant)) {
5379 SDValue ShiftV = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: SatValue,
5380 N2: DAG.getConstant(Val: 31, DL: dl, VT));
5381 if (isNullConstant(V: LowerSatConstant)) {
5382 SDValue NotShiftV = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: ShiftV,
5383 N2: DAG.getAllOnesConstant(DL: dl, VT));
5384 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: SatValue, N2: NotShiftV);
5385 } else if (isAllOnesConstant(V: LowerSatConstant))
5386 return DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: SatValue, N2: ShiftV);
5387 }
5388
5389 SDValue LHS = Op.getOperand(i: 0);
5390 SDValue RHS = Op.getOperand(i: 1);
5391 ISD::CondCode CC = Op.getOperandCondCode(i: 4);
5392 SDValue TrueVal = Op.getOperand(i: 2);
5393 SDValue FalseVal = Op.getOperand(i: 3);
5394 ConstantSDNode *CFVal = dyn_cast<ConstantSDNode>(Val&: FalseVal);
5395 ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(Val&: RHS);
5396 if (Op.getValueType().isInteger()) {
5397
5398 // Check for SMAX(lhs, 0) and SMIN(lhs, 0) patterns.
5399 // (SELECT_CC setgt, lhs, 0, lhs, 0) -> (BIC lhs, (SRA lhs, typesize-1))
5400 // (SELECT_CC setlt, lhs, 0, lhs, 0) -> (AND lhs, (SRA lhs, typesize-1))
5401 // Both require less instructions than compare and conditional select.
5402 if ((CC == ISD::SETGT || CC == ISD::SETLT) && LHS == TrueVal && RHSC &&
5403 RHSC->isZero() && CFVal && CFVal->isZero() &&
5404 LHS.getValueType() == RHS.getValueType()) {
5405 EVT VT = LHS.getValueType();
5406 SDValue Shift =
5407 DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: LHS,
5408 N2: DAG.getConstant(Val: VT.getSizeInBits() - 1, DL: dl, VT));
5409
5410 if (CC == ISD::SETGT)
5411 Shift = DAG.getNOT(DL: dl, Val: Shift, VT);
5412
5413 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: LHS, N2: Shift);
5414 }
5415
5416 // (SELECT_CC setlt, x, 0, 1, 0) -> SRL(x, bw-1)
5417 if (CC == ISD::SETLT && isNullConstant(V: RHS) && isOneConstant(V: TrueVal) &&
5418 isNullConstant(V: FalseVal) && LHS.getValueType() == VT)
5419 return DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: LHS,
5420 N2: DAG.getConstant(Val: VT.getSizeInBits() - 1, DL: dl, VT));
5421 }
5422
5423 if (LHS.getValueType() == MVT::i32) {
5424 unsigned Opcode;
5425 bool InvertCond;
5426 if (SDValue Op =
5427 matchCSET(Opcode, InvertCond, TrueVal, FalseVal, Subtarget)) {
5428 if (InvertCond)
5429 CC = ISD::getSetCCInverse(Operation: CC, Type: LHS.getValueType());
5430
5431 SDValue ARMcc;
5432 SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMcc, DAG, dl);
5433 EVT VT = Op.getValueType();
5434 return DAG.getNode(Opcode, DL: dl, VT, N1: Op, N2: Op, N3: ARMcc, N4: Cmp);
5435 }
5436 }
5437
5438 if (isUnsupportedFloatingType(VT: LHS.getValueType())) {
5439 softenSetCCOperands(DAG, VT: LHS.getValueType(), NewLHS&: LHS, NewRHS&: RHS, CCCode&: CC, DL: dl, OldLHS: LHS, OldRHS: RHS);
5440
5441 // If softenSetCCOperands only returned one value, we should compare it to
5442 // zero.
5443 if (!RHS.getNode()) {
5444 RHS = DAG.getConstant(Val: 0, DL: dl, VT: LHS.getValueType());
5445 CC = ISD::SETNE;
5446 }
5447 }
5448
5449 if (LHS.getValueType() == MVT::i32) {
5450 // Try to generate VSEL on ARMv8.
5451 // The VSEL instruction can't use all the usual ARM condition
5452 // codes: it only has two bits to select the condition code, so it's
5453 // constrained to use only GE, GT, VS and EQ.
5454 //
5455 // To implement all the various ISD::SETXXX opcodes, we sometimes need to
5456 // swap the operands of the previous compare instruction (effectively
5457 // inverting the compare condition, swapping 'less' and 'greater') and
5458 // sometimes need to swap the operands to the VSEL (which inverts the
5459 // condition in the sense of firing whenever the previous condition didn't)
5460 if (Subtarget->hasFPARMv8Base() && (TrueVal.getValueType() == MVT::f16 ||
5461 TrueVal.getValueType() == MVT::f32 ||
5462 TrueVal.getValueType() == MVT::f64)) {
5463 ARMCC::CondCodes CondCode = IntCCToARMCC(CC);
5464 if (CondCode == ARMCC::LT || CondCode == ARMCC::LE ||
5465 CondCode == ARMCC::VC || CondCode == ARMCC::NE) {
5466 CC = ISD::getSetCCInverse(Operation: CC, Type: LHS.getValueType());
5467 std::swap(a&: TrueVal, b&: FalseVal);
5468 }
5469 }
5470
5471 SDValue ARMcc;
5472 SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMcc, DAG, dl);
5473 // Choose GE over PL, which vsel does now support
5474 if (ARMcc->getAsZExtVal() == ARMCC::PL)
5475 ARMcc = DAG.getConstant(Val: ARMCC::GE, DL: dl, VT: MVT::i32);
5476 return getCMOV(dl, VT, FalseVal, TrueVal, ARMcc, Flags: Cmp, DAG);
5477 }
5478
5479 ARMCC::CondCodes CondCode, CondCode2;
5480 FPCCToARMCC(CC, CondCode, CondCode2);
5481
5482 // Normalize the fp compare. If RHS is zero we prefer to keep it there so we
5483 // match CMPFPw0 instead of CMPFP, though we don't do this for f16 because we
5484 // must use VSEL (limited condition codes), due to not having conditional f16
5485 // moves.
5486 if (Subtarget->hasFPARMv8Base() &&
5487 !(isFloatingPointZero(Op: RHS) && TrueVal.getValueType() != MVT::f16) &&
5488 (TrueVal.getValueType() == MVT::f16 ||
5489 TrueVal.getValueType() == MVT::f32 ||
5490 TrueVal.getValueType() == MVT::f64)) {
5491 bool swpCmpOps = false;
5492 bool swpVselOps = false;
5493 checkVSELConstraints(CC, CondCode, swpCmpOps, swpVselOps);
5494
5495 if (CondCode == ARMCC::GT || CondCode == ARMCC::GE ||
5496 CondCode == ARMCC::VS || CondCode == ARMCC::EQ) {
5497 if (swpCmpOps)
5498 std::swap(a&: LHS, b&: RHS);
5499 if (swpVselOps)
5500 std::swap(a&: TrueVal, b&: FalseVal);
5501 }
5502 }
5503
5504 SDValue ARMcc = DAG.getConstant(Val: CondCode, DL: dl, VT: MVT::i32);
5505 SDValue Cmp = getVFPCmp(LHS, RHS, DAG, dl);
5506 SDValue Result = getCMOV(dl, VT, FalseVal, TrueVal, ARMcc, Flags: Cmp, DAG);
5507 if (CondCode2 != ARMCC::AL) {
5508 SDValue ARMcc2 = DAG.getConstant(Val: CondCode2, DL: dl, VT: MVT::i32);
5509 Result = getCMOV(dl, VT, FalseVal: Result, TrueVal, ARMcc: ARMcc2, Flags: Cmp, DAG);
5510 }
5511 return Result;
5512}
5513
5514/// canChangeToInt - Given the fp compare operand, return true if it is suitable
5515/// to morph to an integer compare sequence.
5516static bool canChangeToInt(SDValue Op, bool &SeenZero,
5517 const ARMSubtarget *Subtarget) {
5518 SDNode *N = Op.getNode();
5519 if (!N->hasOneUse())
5520 // Otherwise it requires moving the value from fp to integer registers.
5521 return false;
5522 if (!N->getNumValues())
5523 return false;
5524 EVT VT = Op.getValueType();
5525 if (VT != MVT::f32 && !Subtarget->isFPBrccSlow())
5526 // f32 case is generally profitable. f64 case only makes sense when vcmpe +
5527 // vmrs are very slow, e.g. cortex-a8.
5528 return false;
5529
5530 if (isFloatingPointZero(Op)) {
5531 SeenZero = true;
5532 return true;
5533 }
5534 return ISD::isNormalLoad(N);
5535}
5536
5537static SDValue bitcastf32Toi32(SDValue Op, SelectionDAG &DAG) {
5538 if (isFloatingPointZero(Op))
5539 return DAG.getConstant(Val: 0, DL: SDLoc(Op), VT: MVT::i32);
5540
5541 if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(Val&: Op))
5542 return DAG.getLoad(VT: MVT::i32, dl: SDLoc(Op), Chain: Ld->getChain(), Ptr: Ld->getBasePtr(),
5543 PtrInfo: Ld->getPointerInfo(), Alignment: Ld->getAlign(),
5544 MMOFlags: Ld->getMemOperand()->getFlags());
5545
5546 llvm_unreachable("Unknown VFP cmp argument!");
5547}
5548
5549static void expandf64Toi32(SDValue Op, SelectionDAG &DAG,
5550 SDValue &RetVal1, SDValue &RetVal2) {
5551 SDLoc dl(Op);
5552
5553 if (isFloatingPointZero(Op)) {
5554 RetVal1 = DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32);
5555 RetVal2 = DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32);
5556 return;
5557 }
5558
5559 if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(Val&: Op)) {
5560 SDValue Ptr = Ld->getBasePtr();
5561 RetVal1 =
5562 DAG.getLoad(VT: MVT::i32, dl, Chain: Ld->getChain(), Ptr, PtrInfo: Ld->getPointerInfo(),
5563 Alignment: Ld->getAlign(), MMOFlags: Ld->getMemOperand()->getFlags());
5564
5565 EVT PtrType = Ptr.getValueType();
5566 SDValue NewPtr = DAG.getNode(Opcode: ISD::ADD, DL: dl,
5567 VT: PtrType, N1: Ptr, N2: DAG.getConstant(Val: 4, DL: dl, VT: PtrType));
5568 RetVal2 = DAG.getLoad(VT: MVT::i32, dl, Chain: Ld->getChain(), Ptr: NewPtr,
5569 PtrInfo: Ld->getPointerInfo().getWithOffset(O: 4),
5570 Alignment: commonAlignment(A: Ld->getAlign(), Offset: 4),
5571 MMOFlags: Ld->getMemOperand()->getFlags());
5572 return;
5573 }
5574
5575 llvm_unreachable("Unknown VFP cmp argument!");
5576}
5577
5578/// OptimizeVFPBrcond - With nnan and without daz, it's legal to optimize some
5579/// f32 and even f64 comparisons to integer ones.
5580SDValue
5581ARMTargetLowering::OptimizeVFPBrcond(SDValue Op, SelectionDAG &DAG) const {
5582 SDValue Chain = Op.getOperand(i: 0);
5583 ISD::CondCode CC = Op.getOperandCondCode(i: 1);
5584 SDValue LHS = Op.getOperand(i: 2);
5585 SDValue RHS = Op.getOperand(i: 3);
5586 SDValue Dest = Op.getOperand(i: 4);
5587 SDLoc dl(Op);
5588
5589 bool LHSSeenZero = false;
5590 bool LHSOk = canChangeToInt(Op: LHS, SeenZero&: LHSSeenZero, Subtarget);
5591 bool RHSSeenZero = false;
5592 bool RHSOk = canChangeToInt(Op: RHS, SeenZero&: RHSSeenZero, Subtarget);
5593 if (LHSOk && RHSOk && (LHSSeenZero || RHSSeenZero)) {
5594 // If unsafe fp math optimization is enabled and there are no other uses of
5595 // the CMP operands, and the condition code is EQ or NE, we can optimize it
5596 // to an integer comparison.
5597 if (CC == ISD::SETOEQ)
5598 CC = ISD::SETEQ;
5599 else if (CC == ISD::SETUNE)
5600 CC = ISD::SETNE;
5601
5602 SDValue Mask = DAG.getConstant(Val: 0x7fffffff, DL: dl, VT: MVT::i32);
5603 SDValue ARMcc;
5604 if (LHS.getValueType() == MVT::f32) {
5605 LHS = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32,
5606 N1: bitcastf32Toi32(Op: LHS, DAG), N2: Mask);
5607 RHS = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32,
5608 N1: bitcastf32Toi32(Op: RHS, DAG), N2: Mask);
5609 SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMcc, DAG, dl);
5610 return DAG.getNode(Opcode: ARMISD::BRCOND, DL: dl, VT: MVT::Other, N1: Chain, N2: Dest, N3: ARMcc,
5611 N4: Cmp);
5612 }
5613
5614 SDValue LHS1, LHS2;
5615 SDValue RHS1, RHS2;
5616 expandf64Toi32(Op: LHS, DAG, RetVal1&: LHS1, RetVal2&: LHS2);
5617 expandf64Toi32(Op: RHS, DAG, RetVal1&: RHS1, RetVal2&: RHS2);
5618 LHS2 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32, N1: LHS2, N2: Mask);
5619 RHS2 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32, N1: RHS2, N2: Mask);
5620 ARMCC::CondCodes CondCode = IntCCToARMCC(CC);
5621 ARMcc = DAG.getConstant(Val: CondCode, DL: dl, VT: MVT::i32);
5622 SDValue Ops[] = { Chain, ARMcc, LHS1, LHS2, RHS1, RHS2, Dest };
5623 return DAG.getNode(Opcode: ARMISD::BCC_i64, DL: dl, VT: MVT::Other, Ops);
5624 }
5625
5626 return SDValue();
5627}
5628
5629// Generate CMP + CMOV for integer abs.
5630SDValue ARMTargetLowering::LowerABS(SDValue Op, SelectionDAG &DAG) const {
5631 SDLoc DL(Op);
5632
5633 SDValue Neg = DAG.getNegative(Val: Op.getOperand(i: 0), DL, VT: MVT::i32);
5634
5635 // Generate CMP & CMOV.
5636 SDValue Cmp = DAG.getNode(Opcode: ARMISD::CMP, DL, VT: FlagsVT, N1: Op.getOperand(i: 0),
5637 N2: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
5638 return DAG.getNode(Opcode: ARMISD::CMOV, DL, VT: MVT::i32, N1: Op.getOperand(i: 0), N2: Neg,
5639 N3: DAG.getConstant(Val: ARMCC::MI, DL, VT: MVT::i32), N4: Cmp);
5640}
5641
5642static SDValue getInvertedARMCondCode(SDValue ARMcc, SelectionDAG &DAG) {
5643 ARMCC::CondCodes CondCode =
5644 (ARMCC::CondCodes)cast<ConstantSDNode>(Val&: ARMcc)->getZExtValue();
5645 CondCode = ARMCC::getOppositeCondition(CC: CondCode);
5646 return DAG.getConstant(Val: CondCode, DL: SDLoc(ARMcc), VT: MVT::i32);
5647}
5648
5649SDValue ARMTargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
5650 SDValue Chain = Op.getOperand(i: 0);
5651 SDValue Cond = Op.getOperand(i: 1);
5652 SDValue Dest = Op.getOperand(i: 2);
5653 SDLoc dl(Op);
5654
5655 // Optimize {s|u}{add|sub|mul}.with.overflow feeding into a branch
5656 // instruction.
5657 unsigned Opc = Cond.getOpcode();
5658 bool OptimizeMul = (Opc == ISD::SMULO || Opc == ISD::UMULO) &&
5659 !Subtarget->isThumb1Only();
5660 if (Cond.getResNo() == 1 &&
5661 (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO ||
5662 Opc == ISD::USUBO || OptimizeMul)) {
5663 // Only lower legal XALUO ops.
5664 if (!isTypeLegal(VT: Cond->getValueType(ResNo: 0)))
5665 return SDValue();
5666
5667 // The actual operation with overflow check.
5668 SDValue Value, OverflowCmp;
5669 SDValue ARMcc;
5670 std::tie(args&: Value, args&: OverflowCmp) = getARMXALUOOp(Op: Cond, DAG, ARMcc);
5671
5672 // Reverse the condition code.
5673 ARMcc = getInvertedARMCondCode(ARMcc, DAG);
5674
5675 return DAG.getNode(Opcode: ARMISD::BRCOND, DL: dl, VT: MVT::Other, N1: Chain, N2: Dest, N3: ARMcc,
5676 N4: OverflowCmp);
5677 }
5678
5679 return SDValue();
5680}
5681
5682SDValue ARMTargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
5683 SDValue Chain = Op.getOperand(i: 0);
5684 ISD::CondCode CC = Op.getOperandCondCode(i: 1);
5685 SDValue LHS = Op.getOperand(i: 2);
5686 SDValue RHS = Op.getOperand(i: 3);
5687 SDValue Dest = Op.getOperand(i: 4);
5688 SDLoc dl(Op);
5689
5690 if (isUnsupportedFloatingType(VT: LHS.getValueType())) {
5691 softenSetCCOperands(DAG, VT: LHS.getValueType(), NewLHS&: LHS, NewRHS&: RHS, CCCode&: CC, DL: dl, OldLHS: LHS, OldRHS: RHS);
5692
5693 // If softenSetCCOperands only returned one value, we should compare it to
5694 // zero.
5695 if (!RHS.getNode()) {
5696 RHS = DAG.getConstant(Val: 0, DL: dl, VT: LHS.getValueType());
5697 CC = ISD::SETNE;
5698 }
5699 }
5700
5701 // Optimize {s|u}{add|sub|mul}.with.overflow feeding into a branch
5702 // instruction.
5703 unsigned Opc = LHS.getOpcode();
5704 bool OptimizeMul = (Opc == ISD::SMULO || Opc == ISD::UMULO) &&
5705 !Subtarget->isThumb1Only();
5706 if (LHS.getResNo() == 1 && (isOneConstant(V: RHS) || isNullConstant(V: RHS)) &&
5707 (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO ||
5708 Opc == ISD::USUBO || OptimizeMul) &&
5709 (CC == ISD::SETEQ || CC == ISD::SETNE)) {
5710 // Only lower legal XALUO ops.
5711 if (!isTypeLegal(VT: LHS->getValueType(ResNo: 0)))
5712 return SDValue();
5713
5714 // The actual operation with overflow check.
5715 SDValue Value, OverflowCmp;
5716 SDValue ARMcc;
5717 std::tie(args&: Value, args&: OverflowCmp) = getARMXALUOOp(Op: LHS.getValue(R: 0), DAG, ARMcc);
5718
5719 if ((CC == ISD::SETNE) != isOneConstant(V: RHS)) {
5720 // Reverse the condition code.
5721 ARMcc = getInvertedARMCondCode(ARMcc, DAG);
5722 }
5723
5724 return DAG.getNode(Opcode: ARMISD::BRCOND, DL: dl, VT: MVT::Other, N1: Chain, N2: Dest, N3: ARMcc,
5725 N4: OverflowCmp);
5726 }
5727
5728 if (LHS.getValueType() == MVT::i32) {
5729 SDValue ARMcc;
5730 SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMcc, DAG, dl);
5731 return DAG.getNode(Opcode: ARMISD::BRCOND, DL: dl, VT: MVT::Other, N1: Chain, N2: Dest, N3: ARMcc, N4: Cmp);
5732 }
5733
5734 SDNodeFlags Flags = Op->getFlags();
5735 if (Flags.hasNoNaNs() &&
5736 DAG.getDenormalMode(VT: MVT::f32) == DenormalMode::getIEEE() &&
5737 DAG.getDenormalMode(VT: MVT::f64) == DenormalMode::getIEEE() &&
5738 (CC == ISD::SETEQ || CC == ISD::SETOEQ || CC == ISD::SETNE ||
5739 CC == ISD::SETUNE)) {
5740 if (SDValue Result = OptimizeVFPBrcond(Op, DAG))
5741 return Result;
5742 }
5743
5744 ARMCC::CondCodes CondCode, CondCode2;
5745 FPCCToARMCC(CC, CondCode, CondCode2);
5746
5747 SDValue ARMcc = DAG.getConstant(Val: CondCode, DL: dl, VT: MVT::i32);
5748 SDValue Cmp = getVFPCmp(LHS, RHS, DAG, dl);
5749 SDValue Ops[] = {Chain, Dest, ARMcc, Cmp};
5750 SDValue Res = DAG.getNode(Opcode: ARMISD::BRCOND, DL: dl, VT: MVT::Other, Ops);
5751 if (CondCode2 != ARMCC::AL) {
5752 ARMcc = DAG.getConstant(Val: CondCode2, DL: dl, VT: MVT::i32);
5753 SDValue Ops[] = {Res, Dest, ARMcc, Cmp};
5754 Res = DAG.getNode(Opcode: ARMISD::BRCOND, DL: dl, VT: MVT::Other, Ops);
5755 }
5756 return Res;
5757}
5758
5759SDValue ARMTargetLowering::LowerBR_JT(SDValue Op, SelectionDAG &DAG) const {
5760 SDValue Chain = Op.getOperand(i: 0);
5761 SDValue Table = Op.getOperand(i: 1);
5762 SDValue Index = Op.getOperand(i: 2);
5763 SDLoc dl(Op);
5764
5765 EVT PTy = getPointerTy(DL: DAG.getDataLayout());
5766 JumpTableSDNode *JT = cast<JumpTableSDNode>(Val&: Table);
5767 SDValue JTI = DAG.getTargetJumpTable(JTI: JT->getIndex(), VT: PTy);
5768 Table = DAG.getNode(Opcode: ARMISD::WrapperJT, DL: dl, VT: MVT::i32, Operand: JTI);
5769 Index = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: PTy, N1: Index, N2: DAG.getConstant(Val: 4, DL: dl, VT: PTy));
5770 SDValue Addr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PTy, N1: Table, N2: Index);
5771 if (Subtarget->isThumb2() || (Subtarget->hasV8MBaselineOps() && Subtarget->isThumb())) {
5772 // Thumb2 and ARMv8-M use a two-level jump. That is, it jumps into the jump table
5773 // which does another jump to the destination. This also makes it easier
5774 // to translate it to TBB / TBH later (Thumb2 only).
5775 // FIXME: This might not work if the function is extremely large.
5776 return DAG.getNode(Opcode: ARMISD::BR2_JT, DL: dl, VT: MVT::Other, N1: Chain,
5777 N2: Addr, N3: Op.getOperand(i: 2), N4: JTI);
5778 }
5779 if (isPositionIndependent() || Subtarget->isROPI()) {
5780 Addr =
5781 DAG.getLoad(VT: (EVT)MVT::i32, dl, Chain, Ptr: Addr,
5782 PtrInfo: MachinePointerInfo::getJumpTable(MF&: DAG.getMachineFunction()));
5783 Chain = Addr.getValue(R: 1);
5784 Addr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PTy, N1: Table, N2: Addr);
5785 return DAG.getNode(Opcode: ARMISD::BR_JT, DL: dl, VT: MVT::Other, N1: Chain, N2: Addr, N3: JTI);
5786 } else {
5787 Addr =
5788 DAG.getLoad(VT: PTy, dl, Chain, Ptr: Addr,
5789 PtrInfo: MachinePointerInfo::getJumpTable(MF&: DAG.getMachineFunction()));
5790 Chain = Addr.getValue(R: 1);
5791 return DAG.getNode(Opcode: ARMISD::BR_JT, DL: dl, VT: MVT::Other, N1: Chain, N2: Addr, N3: JTI);
5792 }
5793}
5794
5795static SDValue LowerVectorFP_TO_INT(SDValue Op, SelectionDAG &DAG) {
5796 EVT VT = Op.getValueType();
5797 SDLoc dl(Op);
5798
5799 if (Op.getValueType().getVectorElementType() == MVT::i32) {
5800 if (Op.getOperand(i: 0).getValueType().getVectorElementType() == MVT::f32)
5801 return Op;
5802 return DAG.UnrollVectorOp(N: Op.getNode());
5803 }
5804
5805 const bool HasFullFP16 = DAG.getSubtarget<ARMSubtarget>().hasFullFP16();
5806
5807 EVT NewTy;
5808 const EVT OpTy = Op.getOperand(i: 0).getValueType();
5809 if (OpTy == MVT::v4f32)
5810 NewTy = MVT::v4i32;
5811 else if (OpTy == MVT::v4f16 && HasFullFP16)
5812 NewTy = MVT::v4i16;
5813 else if (OpTy == MVT::v8f16 && HasFullFP16)
5814 NewTy = MVT::v8i16;
5815 else
5816 llvm_unreachable("Invalid type for custom lowering!");
5817
5818 if (VT != MVT::v4i16 && VT != MVT::v8i16)
5819 return DAG.UnrollVectorOp(N: Op.getNode());
5820
5821 Op = DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT: NewTy, Operand: Op.getOperand(i: 0));
5822 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: Op);
5823}
5824
5825SDValue ARMTargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const {
5826 EVT VT = Op.getValueType();
5827 if (VT.isVector())
5828 return LowerVectorFP_TO_INT(Op, DAG);
5829
5830 bool IsStrict = Op->isStrictFPOpcode();
5831 SDValue SrcVal = Op.getOperand(i: IsStrict ? 1 : 0);
5832
5833 if (isUnsupportedFloatingType(VT: SrcVal.getValueType())) {
5834 RTLIB::Libcall LC;
5835 if (Op.getOpcode() == ISD::FP_TO_SINT ||
5836 Op.getOpcode() == ISD::STRICT_FP_TO_SINT)
5837 LC = RTLIB::getFPTOSINT(OpVT: SrcVal.getValueType(),
5838 RetVT: Op.getValueType());
5839 else
5840 LC = RTLIB::getFPTOUINT(OpVT: SrcVal.getValueType(),
5841 RetVT: Op.getValueType());
5842 SDLoc Loc(Op);
5843 MakeLibCallOptions CallOptions;
5844 SDValue Chain = IsStrict ? Op.getOperand(i: 0) : SDValue();
5845 SDValue Result;
5846 std::tie(args&: Result, args&: Chain) = makeLibCall(DAG, LC, RetVT: Op.getValueType(), Ops: SrcVal,
5847 CallOptions, dl: Loc, Chain);
5848 return IsStrict ? DAG.getMergeValues(Ops: {Result, Chain}, dl: Loc) : Result;
5849 }
5850
5851 return Op;
5852}
5853
5854static SDValue LowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG,
5855 const ARMSubtarget *Subtarget) {
5856 EVT VT = Op.getValueType();
5857 EVT ToVT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
5858 EVT FromVT = Op.getOperand(i: 0).getValueType();
5859
5860 if (VT == MVT::i32 && ToVT == MVT::i32 && FromVT == MVT::f32)
5861 return Op;
5862 if (VT == MVT::i32 && ToVT == MVT::i32 && FromVT == MVT::f64 &&
5863 Subtarget->hasFP64())
5864 return Op;
5865 if (VT == MVT::i32 && ToVT == MVT::i32 && FromVT == MVT::f16 &&
5866 Subtarget->hasFullFP16())
5867 return Op;
5868 if (VT == MVT::v4i32 && ToVT == MVT::i32 && FromVT == MVT::v4f32 &&
5869 Subtarget->hasMVEFloatOps())
5870 return Op;
5871 if (VT == MVT::v8i16 && ToVT == MVT::i16 && FromVT == MVT::v8f16 &&
5872 Subtarget->hasMVEFloatOps())
5873 return Op;
5874
5875 if (FromVT != MVT::v4f32 && FromVT != MVT::v8f16)
5876 return SDValue();
5877
5878 SDLoc DL(Op);
5879 bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT_SAT;
5880 unsigned BW = ToVT.getScalarSizeInBits() - IsSigned;
5881 SDValue CVT = DAG.getNode(Opcode: Op.getOpcode(), DL, VT, N1: Op.getOperand(i: 0),
5882 N2: DAG.getValueType(VT.getScalarType()));
5883 SDValue Max = DAG.getNode(Opcode: IsSigned ? ISD::SMIN : ISD::UMIN, DL, VT, N1: CVT,
5884 N2: DAG.getConstant(Val: (1 << BW) - 1, DL, VT));
5885 if (IsSigned)
5886 Max = DAG.getNode(Opcode: ISD::SMAX, DL, VT, N1: Max,
5887 N2: DAG.getSignedConstant(Val: -(1 << BW), DL, VT));
5888 return Max;
5889}
5890
5891static SDValue LowerVectorINT_TO_FP(SDValue Op, SelectionDAG &DAG) {
5892 EVT VT = Op.getValueType();
5893 SDLoc dl(Op);
5894
5895 if (Op.getOperand(i: 0).getValueType().getVectorElementType() == MVT::i32) {
5896 if (VT.getVectorElementType() == MVT::f32)
5897 return Op;
5898 return DAG.UnrollVectorOp(N: Op.getNode());
5899 }
5900
5901 assert((Op.getOperand(0).getValueType() == MVT::v4i16 ||
5902 Op.getOperand(0).getValueType() == MVT::v8i16) &&
5903 "Invalid type for custom lowering!");
5904
5905 const bool HasFullFP16 = DAG.getSubtarget<ARMSubtarget>().hasFullFP16();
5906
5907 EVT DestVecType;
5908 if (VT == MVT::v4f32)
5909 DestVecType = MVT::v4i32;
5910 else if (VT == MVT::v4f16 && HasFullFP16)
5911 DestVecType = MVT::v4i16;
5912 else if (VT == MVT::v8f16 && HasFullFP16)
5913 DestVecType = MVT::v8i16;
5914 else
5915 return DAG.UnrollVectorOp(N: Op.getNode());
5916
5917 unsigned CastOpc;
5918 unsigned Opc;
5919 switch (Op.getOpcode()) {
5920 default: llvm_unreachable("Invalid opcode!");
5921 case ISD::SINT_TO_FP:
5922 CastOpc = ISD::SIGN_EXTEND;
5923 Opc = ISD::SINT_TO_FP;
5924 break;
5925 case ISD::UINT_TO_FP:
5926 CastOpc = ISD::ZERO_EXTEND;
5927 Opc = ISD::UINT_TO_FP;
5928 break;
5929 }
5930
5931 Op = DAG.getNode(Opcode: CastOpc, DL: dl, VT: DestVecType, Operand: Op.getOperand(i: 0));
5932 return DAG.getNode(Opcode: Opc, DL: dl, VT, Operand: Op);
5933}
5934
5935SDValue ARMTargetLowering::LowerINT_TO_FP(SDValue Op, SelectionDAG &DAG) const {
5936 EVT VT = Op.getValueType();
5937 if (VT.isVector())
5938 return LowerVectorINT_TO_FP(Op, DAG);
5939
5940 bool IsStrict = Op->isStrictFPOpcode();
5941 SDValue SrcVal = Op.getOperand(i: IsStrict ? 1 : 0);
5942
5943 if (isUnsupportedFloatingType(VT)) {
5944 RTLIB::Libcall LC;
5945 if (Op.getOpcode() == ISD::SINT_TO_FP ||
5946 Op.getOpcode() == ISD::STRICT_SINT_TO_FP)
5947 LC = RTLIB::getSINTTOFP(OpVT: SrcVal.getValueType(), RetVT: Op.getValueType());
5948 else
5949 LC = RTLIB::getUINTTOFP(OpVT: SrcVal.getValueType(), RetVT: Op.getValueType());
5950 SDLoc Loc(Op);
5951 MakeLibCallOptions CallOptions;
5952 SDValue Chain = IsStrict ? Op.getOperand(i: 0) : SDValue();
5953 SDValue Result;
5954 std::tie(args&: Result, args&: Chain) = makeLibCall(DAG, LC, RetVT: Op.getValueType(), Ops: SrcVal,
5955 CallOptions, dl: Loc, Chain);
5956 return IsStrict ? DAG.getMergeValues(Ops: {Result, Chain}, dl: Loc) : Result;
5957 }
5958
5959 return Op;
5960}
5961
5962SDValue ARMTargetLowering::LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const {
5963 // Implement fcopysign with a fabs and a conditional fneg.
5964 SDValue Tmp0 = Op.getOperand(i: 0);
5965 SDValue Tmp1 = Op.getOperand(i: 1);
5966 SDLoc dl(Op);
5967 EVT VT = Op.getValueType();
5968 EVT SrcVT = Tmp1.getValueType();
5969 bool InGPR = Tmp0.getOpcode() == ISD::BITCAST ||
5970 Tmp0.getOpcode() == ARMISD::VMOVDRR;
5971 bool UseNEON = !InGPR && Subtarget->hasNEON();
5972
5973 if (UseNEON) {
5974 // Use VBSL to copy the sign bit.
5975 unsigned EncodedVal = ARM_AM::createVMOVModImm(OpCmode: 0x6, Val: 0x80);
5976 SDValue Mask = DAG.getNode(Opcode: ARMISD::VMOVIMM, DL: dl, VT: MVT::v2i32,
5977 Operand: DAG.getTargetConstant(Val: EncodedVal, DL: dl, VT: MVT::i32));
5978 EVT OpVT = (VT == MVT::f32) ? MVT::v2i32 : MVT::v1i64;
5979 if (VT == MVT::f64)
5980 Mask = DAG.getNode(Opcode: ARMISD::VSHLIMM, DL: dl, VT: OpVT,
5981 N1: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: OpVT, Operand: Mask),
5982 N2: DAG.getConstant(Val: 32, DL: dl, VT: MVT::i32));
5983 else /*if (VT == MVT::f32)*/
5984 Tmp0 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: MVT::v2f32, Operand: Tmp0);
5985 if (SrcVT == MVT::f32) {
5986 Tmp1 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: MVT::v2f32, Operand: Tmp1);
5987 if (VT == MVT::f64)
5988 Tmp1 = DAG.getNode(Opcode: ARMISD::VSHLIMM, DL: dl, VT: OpVT,
5989 N1: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: OpVT, Operand: Tmp1),
5990 N2: DAG.getConstant(Val: 32, DL: dl, VT: MVT::i32));
5991 } else if (VT == MVT::f32)
5992 Tmp1 = DAG.getNode(Opcode: ARMISD::VSHRuIMM, DL: dl, VT: MVT::v1i64,
5993 N1: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v1i64, Operand: Tmp1),
5994 N2: DAG.getConstant(Val: 32, DL: dl, VT: MVT::i32));
5995 Tmp0 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: OpVT, Operand: Tmp0);
5996 Tmp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: OpVT, Operand: Tmp1);
5997
5998 SDValue AllOnes = DAG.getTargetConstant(Val: ARM_AM::createVMOVModImm(OpCmode: 0xe, Val: 0xff),
5999 DL: dl, VT: MVT::i32);
6000 AllOnes = DAG.getNode(Opcode: ARMISD::VMOVIMM, DL: dl, VT: MVT::v8i8, Operand: AllOnes);
6001 SDValue MaskNot = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: OpVT, N1: Mask,
6002 N2: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: OpVT, Operand: AllOnes));
6003
6004 SDValue Res = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: OpVT,
6005 N1: DAG.getNode(Opcode: ISD::AND, DL: dl, VT: OpVT, N1: Tmp1, N2: Mask),
6006 N2: DAG.getNode(Opcode: ISD::AND, DL: dl, VT: OpVT, N1: Tmp0, N2: MaskNot));
6007 if (VT == MVT::f32) {
6008 Res = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v2f32, Operand: Res);
6009 Res = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: MVT::f32, N1: Res,
6010 N2: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
6011 } else {
6012 Res = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::f64, Operand: Res);
6013 }
6014
6015 return Res;
6016 }
6017
6018 // Bitcast operand 1 to i32.
6019 if (SrcVT == MVT::f64)
6020 Tmp1 = DAG.getNode(Opcode: ARMISD::VMOVRRD, DL: dl, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
6021 N: Tmp1).getValue(R: 1);
6022 Tmp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: Tmp1);
6023
6024 // Or in the signbit with integer operations.
6025 SDValue Mask1 = DAG.getConstant(Val: 0x80000000, DL: dl, VT: MVT::i32);
6026 SDValue Mask2 = DAG.getConstant(Val: 0x7fffffff, DL: dl, VT: MVT::i32);
6027 Tmp1 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32, N1: Tmp1, N2: Mask1);
6028 if (VT == MVT::f32) {
6029 Tmp0 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32,
6030 N1: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: Tmp0), N2: Mask2);
6031 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::f32,
6032 Operand: DAG.getNode(Opcode: ISD::OR, DL: dl, VT: MVT::i32, N1: Tmp0, N2: Tmp1));
6033 }
6034
6035 // f64: Or the high part with signbit and then combine two parts.
6036 Tmp0 = DAG.getNode(Opcode: ARMISD::VMOVRRD, DL: dl, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
6037 N: Tmp0);
6038 SDValue Lo = Tmp0.getValue(R: 0);
6039 SDValue Hi = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32, N1: Tmp0.getValue(R: 1), N2: Mask2);
6040 Hi = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: MVT::i32, N1: Hi, N2: Tmp1);
6041 return DAG.getNode(Opcode: ARMISD::VMOVDRR, DL: dl, VT: MVT::f64, N1: Lo, N2: Hi);
6042}
6043
6044SDValue ARMTargetLowering::LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const{
6045 MachineFunction &MF = DAG.getMachineFunction();
6046 MachineFrameInfo &MFI = MF.getFrameInfo();
6047 MFI.setReturnAddressIsTaken(true);
6048
6049 EVT VT = Op.getValueType();
6050 SDLoc dl(Op);
6051 unsigned Depth = Op.getConstantOperandVal(i: 0);
6052 if (Depth) {
6053 SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
6054 SDValue Offset = DAG.getConstant(Val: 4, DL: dl, VT: MVT::i32);
6055 return DAG.getLoad(VT, dl, Chain: DAG.getEntryNode(),
6056 Ptr: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: FrameAddr, N2: Offset),
6057 PtrInfo: MachinePointerInfo());
6058 }
6059
6060 // Return LR, which contains the return address. Mark it an implicit live-in.
6061 Register Reg = MF.addLiveIn(PReg: ARM::LR, RC: getRegClassFor(VT: MVT::i32));
6062 return DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl, Reg, VT);
6063}
6064
6065SDValue ARMTargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const {
6066 const ARMBaseRegisterInfo &ARI =
6067 *static_cast<const ARMBaseRegisterInfo*>(RegInfo);
6068 MachineFunction &MF = DAG.getMachineFunction();
6069 MachineFrameInfo &MFI = MF.getFrameInfo();
6070 MFI.setFrameAddressIsTaken(true);
6071
6072 EVT VT = Op.getValueType();
6073 SDLoc dl(Op); // FIXME probably not meaningful
6074 unsigned Depth = Op.getConstantOperandVal(i: 0);
6075 Register FrameReg = ARI.getFrameRegister(MF);
6076 SDValue FrameAddr = DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl, Reg: FrameReg, VT);
6077 while (Depth--)
6078 FrameAddr = DAG.getLoad(VT, dl, Chain: DAG.getEntryNode(), Ptr: FrameAddr,
6079 PtrInfo: MachinePointerInfo());
6080 return FrameAddr;
6081}
6082
6083// FIXME? Maybe this could be a TableGen attribute on some registers and
6084// this table could be generated automatically from RegInfo.
6085Register ARMTargetLowering::getRegisterByName(const char* RegName, LLT VT,
6086 const MachineFunction &MF) const {
6087 return StringSwitch<Register>(RegName)
6088 .Case(S: "sp", Value: ARM::SP)
6089 .Default(Value: Register());
6090}
6091
6092// Result is 64 bit value so split into two 32 bit values and return as a
6093// pair of values.
6094static void ExpandREAD_REGISTER(SDNode *N, SmallVectorImpl<SDValue> &Results,
6095 SelectionDAG &DAG) {
6096 SDLoc DL(N);
6097
6098 // This function is only supposed to be called for i64 type destination.
6099 assert(N->getValueType(0) == MVT::i64
6100 && "ExpandREAD_REGISTER called for non-i64 type result.");
6101
6102 SDValue Read = DAG.getNode(Opcode: ISD::READ_REGISTER, DL,
6103 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32, VT3: MVT::Other),
6104 N1: N->getOperand(Num: 0),
6105 N2: N->getOperand(Num: 1));
6106
6107 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: MVT::i64, N1: Read.getValue(R: 0),
6108 N2: Read.getValue(R: 1)));
6109 Results.push_back(Elt: Read.getValue(R: 2)); // Chain
6110}
6111
6112/// \p BC is a bitcast that is about to be turned into a VMOVDRR.
6113/// When \p DstVT, the destination type of \p BC, is on the vector
6114/// register bank and the source of bitcast, \p Op, operates on the same bank,
6115/// it might be possible to combine them, such that everything stays on the
6116/// vector register bank.
6117/// \p return The node that would replace \p BT, if the combine
6118/// is possible.
6119static SDValue CombineVMOVDRRCandidateWithVecOp(const SDNode *BC,
6120 SelectionDAG &DAG) {
6121 SDValue Op = BC->getOperand(Num: 0);
6122 EVT DstVT = BC->getValueType(ResNo: 0);
6123
6124 // The only vector instruction that can produce a scalar (remember,
6125 // since the bitcast was about to be turned into VMOVDRR, the source
6126 // type is i64) from a vector is EXTRACT_VECTOR_ELT.
6127 // Moreover, we can do this combine only if there is one use.
6128 // Finally, if the destination type is not a vector, there is not
6129 // much point on forcing everything on the vector bank.
6130 if (!DstVT.isVector() || Op.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
6131 !Op.hasOneUse())
6132 return SDValue();
6133
6134 // If the index is not constant, we will introduce an additional
6135 // multiply that will stick.
6136 // Give up in that case.
6137 ConstantSDNode *Index = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1));
6138 if (!Index)
6139 return SDValue();
6140 unsigned DstNumElt = DstVT.getVectorNumElements();
6141
6142 // Compute the new index.
6143 const APInt &APIntIndex = Index->getAPIntValue();
6144 APInt NewIndex(APIntIndex.getBitWidth(), DstNumElt);
6145 NewIndex *= APIntIndex;
6146 // Check if the new constant index fits into i32.
6147 if (NewIndex.getBitWidth() > 32)
6148 return SDValue();
6149
6150 // vMTy bitcast(i64 extractelt vNi64 src, i32 index) ->
6151 // vMTy extractsubvector vNxMTy (bitcast vNi64 src), i32 index*M)
6152 SDLoc dl(Op);
6153 SDValue ExtractSrc = Op.getOperand(i: 0);
6154 EVT VecVT = EVT::getVectorVT(
6155 Context&: *DAG.getContext(), VT: DstVT.getScalarType(),
6156 NumElements: ExtractSrc.getValueType().getVectorNumElements() * DstNumElt);
6157 SDValue BitCast = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: VecVT, Operand: ExtractSrc);
6158 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: DstVT, N1: BitCast,
6159 N2: DAG.getConstant(Val: NewIndex.getZExtValue(), DL: dl, VT: MVT::i32));
6160}
6161
6162/// ExpandBITCAST - If the target supports VFP, this function is called to
6163/// expand a bit convert where either the source or destination type is i64 to
6164/// use a VMOVDRR or VMOVRRD node. This should not be done when the non-i64
6165/// operand type is illegal (e.g., v2f32 for a target that doesn't support
6166/// vectors), since the legalizer won't know what to do with that.
6167SDValue ARMTargetLowering::ExpandBITCAST(SDNode *N, SelectionDAG &DAG,
6168 const ARMSubtarget *Subtarget) const {
6169 SDLoc dl(N);
6170 SDValue Op = N->getOperand(Num: 0);
6171
6172 // This function is only supposed to be called for i16 and i64 types, either
6173 // as the source or destination of the bit convert.
6174 EVT SrcVT = Op.getValueType();
6175 EVT DstVT = N->getValueType(ResNo: 0);
6176
6177 if ((SrcVT == MVT::f16 || SrcVT == MVT::bf16) &&
6178 (DstVT == MVT::f16 || DstVT == MVT::bf16))
6179 return SDValue(N, 0);
6180
6181 if ((SrcVT == MVT::i16 || SrcVT == MVT::i32) &&
6182 (DstVT == MVT::f16 || DstVT == MVT::bf16)) {
6183 SDValue Ext = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SDLoc(N), VT: MVT::i32, Operand: Op);
6184 if (Subtarget->hasFullFP16())
6185 return DAG.getNode(Opcode: ARMISD::VMOVhr, DL: dl, VT: DstVT, Operand: Ext);
6186 return DAG.getNode(Opcode: ARMISD::VMOVsr, DL: dl, VT: DstVT, Operand: Ext);
6187 }
6188
6189 if ((DstVT == MVT::i16 || DstVT == MVT::i32) &&
6190 (SrcVT == MVT::f16 || SrcVT == MVT::bf16)) {
6191 if (Subtarget->hasFullFP16() && !Subtarget->hasBF16())
6192 Op = DAG.getBitcast(VT: MVT::f16, V: Op);
6193 return DAG.getNode(
6194 Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: DstVT,
6195 Operand: MoveFromHPR(dl: SDLoc(N), DAG, LocVT: MVT::i32, ValVT: SrcVT.getSimpleVT(), Val: Op));
6196 }
6197
6198 if (!(SrcVT == MVT::i64 || DstVT == MVT::i64))
6199 return SDValue();
6200
6201 // Turn i64->f64 into VMOVDRR.
6202 if (SrcVT == MVT::i64 && isTypeLegal(VT: DstVT)) {
6203 // Do not force values to GPRs (this is what VMOVDRR does for the inputs)
6204 // if we can combine the bitcast with its source.
6205 if (SDValue Val = CombineVMOVDRRCandidateWithVecOp(BC: N, DAG))
6206 return Val;
6207 SDValue Lo, Hi;
6208 std::tie(args&: Lo, args&: Hi) = DAG.SplitScalar(N: Op, DL: dl, LoVT: MVT::i32, HiVT: MVT::i32);
6209 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: DstVT,
6210 Operand: DAG.getNode(Opcode: ARMISD::VMOVDRR, DL: dl, VT: MVT::f64, N1: Lo, N2: Hi));
6211 }
6212
6213 // Turn f64->i64 into VMOVRRD.
6214 if (DstVT == MVT::i64 && isTypeLegal(VT: SrcVT)) {
6215 SDValue Cvt;
6216 if (DAG.getDataLayout().isBigEndian() && SrcVT.isVector() &&
6217 SrcVT.getVectorNumElements() > 1)
6218 Cvt = DAG.getNode(Opcode: ARMISD::VMOVRRD, DL: dl,
6219 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
6220 N: DAG.getNode(Opcode: ARMISD::VREV64, DL: dl, VT: SrcVT, Operand: Op));
6221 else
6222 Cvt = DAG.getNode(Opcode: ARMISD::VMOVRRD, DL: dl,
6223 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N: Op);
6224 // Merge the pieces into a single i64 value.
6225 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: MVT::i64, N1: Cvt, N2: Cvt.getValue(R: 1));
6226 }
6227
6228 return SDValue();
6229}
6230
6231/// getZeroVector - Returns a vector of specified type with all zero elements.
6232/// Zero vectors are used to represent vector negation and in those cases
6233/// will be implemented with the NEON VNEG instruction. However, VNEG does
6234/// not support i64 elements, so sometimes the zero vectors will need to be
6235/// explicitly constructed. Regardless, use a canonical VMOV to create the
6236/// zero vector.
6237static SDValue getZeroVector(EVT VT, SelectionDAG &DAG, const SDLoc &dl) {
6238 assert(VT.isVector() && "Expected a vector type");
6239 // The canonical modified immediate encoding of a zero vector is....0!
6240 SDValue EncodedVal = DAG.getTargetConstant(Val: 0, DL: dl, VT: MVT::i32);
6241 EVT VmovVT = VT.is128BitVector() ? MVT::v4i32 : MVT::v2i32;
6242 SDValue Vmov = DAG.getNode(Opcode: ARMISD::VMOVIMM, DL: dl, VT: VmovVT, Operand: EncodedVal);
6243 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: Vmov);
6244}
6245
6246/// LowerShiftRightParts - Lower SRA_PARTS, which returns two
6247/// i32 values and take a 2 x i32 value to shift plus a shift amount.
6248SDValue ARMTargetLowering::LowerShiftRightParts(SDValue Op,
6249 SelectionDAG &DAG) const {
6250 assert(Op.getNumOperands() == 3 && "Not a double-shift!");
6251 EVT VT = Op.getValueType();
6252 unsigned VTBits = VT.getSizeInBits();
6253 SDLoc dl(Op);
6254 SDValue ShOpLo = Op.getOperand(i: 0);
6255 SDValue ShOpHi = Op.getOperand(i: 1);
6256 SDValue ShAmt = Op.getOperand(i: 2);
6257 SDValue ARMcc;
6258 unsigned Opc = (Op.getOpcode() == ISD::SRA_PARTS) ? ISD::SRA : ISD::SRL;
6259
6260 assert(Op.getOpcode() == ISD::SRA_PARTS || Op.getOpcode() == ISD::SRL_PARTS);
6261
6262 SDValue RevShAmt = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32,
6263 N1: DAG.getConstant(Val: VTBits, DL: dl, VT: MVT::i32), N2: ShAmt);
6264 SDValue Tmp1 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: ShOpLo, N2: ShAmt);
6265 SDValue ExtraShAmt = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32, N1: ShAmt,
6266 N2: DAG.getConstant(Val: VTBits, DL: dl, VT: MVT::i32));
6267 SDValue Tmp2 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: ShOpHi, N2: RevShAmt);
6268 SDValue LoSmallShift = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp1, N2: Tmp2);
6269 SDValue LoBigShift = DAG.getNode(Opcode: Opc, DL: dl, VT, N1: ShOpHi, N2: ExtraShAmt);
6270 SDValue CmpLo = getARMCmp(LHS: ExtraShAmt, RHS: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32),
6271 CC: ISD::SETGE, ARMcc, DAG, dl);
6272 SDValue Lo =
6273 DAG.getNode(Opcode: ARMISD::CMOV, DL: dl, VT, N1: LoSmallShift, N2: LoBigShift, N3: ARMcc, N4: CmpLo);
6274
6275 SDValue HiSmallShift = DAG.getNode(Opcode: Opc, DL: dl, VT, N1: ShOpHi, N2: ShAmt);
6276 SDValue HiBigShift = Opc == ISD::SRA
6277 ? DAG.getNode(Opcode: Opc, DL: dl, VT, N1: ShOpHi,
6278 N2: DAG.getConstant(Val: VTBits - 1, DL: dl, VT))
6279 : DAG.getConstant(Val: 0, DL: dl, VT);
6280 SDValue CmpHi = getARMCmp(LHS: ExtraShAmt, RHS: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32),
6281 CC: ISD::SETGE, ARMcc, DAG, dl);
6282 SDValue Hi =
6283 DAG.getNode(Opcode: ARMISD::CMOV, DL: dl, VT, N1: HiSmallShift, N2: HiBigShift, N3: ARMcc, N4: CmpHi);
6284
6285 SDValue Ops[2] = { Lo, Hi };
6286 return DAG.getMergeValues(Ops, dl);
6287}
6288
6289/// LowerShiftLeftParts - Lower SHL_PARTS, which returns two
6290/// i32 values and take a 2 x i32 value to shift plus a shift amount.
6291SDValue ARMTargetLowering::LowerShiftLeftParts(SDValue Op,
6292 SelectionDAG &DAG) const {
6293 assert(Op.getNumOperands() == 3 && "Not a double-shift!");
6294 EVT VT = Op.getValueType();
6295 unsigned VTBits = VT.getSizeInBits();
6296 SDLoc dl(Op);
6297 SDValue ShOpLo = Op.getOperand(i: 0);
6298 SDValue ShOpHi = Op.getOperand(i: 1);
6299 SDValue ShAmt = Op.getOperand(i: 2);
6300 SDValue ARMcc;
6301
6302 assert(Op.getOpcode() == ISD::SHL_PARTS);
6303 SDValue RevShAmt = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32,
6304 N1: DAG.getConstant(Val: VTBits, DL: dl, VT: MVT::i32), N2: ShAmt);
6305 SDValue Tmp1 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: ShOpLo, N2: RevShAmt);
6306 SDValue Tmp2 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: ShOpHi, N2: ShAmt);
6307 SDValue HiSmallShift = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp1, N2: Tmp2);
6308
6309 SDValue ExtraShAmt = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32, N1: ShAmt,
6310 N2: DAG.getConstant(Val: VTBits, DL: dl, VT: MVT::i32));
6311 SDValue HiBigShift = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: ShOpLo, N2: ExtraShAmt);
6312 SDValue CmpHi = getARMCmp(LHS: ExtraShAmt, RHS: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32),
6313 CC: ISD::SETGE, ARMcc, DAG, dl);
6314 SDValue Hi =
6315 DAG.getNode(Opcode: ARMISD::CMOV, DL: dl, VT, N1: HiSmallShift, N2: HiBigShift, N3: ARMcc, N4: CmpHi);
6316
6317 SDValue CmpLo = getARMCmp(LHS: ExtraShAmt, RHS: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32),
6318 CC: ISD::SETGE, ARMcc, DAG, dl);
6319 SDValue LoSmallShift = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: ShOpLo, N2: ShAmt);
6320 SDValue Lo = DAG.getNode(Opcode: ARMISD::CMOV, DL: dl, VT, N1: LoSmallShift,
6321 N2: DAG.getConstant(Val: 0, DL: dl, VT), N3: ARMcc, N4: CmpLo);
6322
6323 SDValue Ops[2] = { Lo, Hi };
6324 return DAG.getMergeValues(Ops, dl);
6325}
6326
6327SDValue ARMTargetLowering::LowerGET_ROUNDING(SDValue Op,
6328 SelectionDAG &DAG) const {
6329 // The rounding mode is in bits 23:22 of the FPSCR.
6330 // The ARM rounding mode value to FLT_ROUNDS mapping is 0->1, 1->2, 2->3, 3->0
6331 // The formula we use to implement this is (((FPSCR + 1 << 22) >> 22) & 3)
6332 // so that the shift + and get folded into a bitfield extract.
6333 SDLoc dl(Op);
6334 SDValue Chain = Op.getOperand(i: 0);
6335 SDValue Ops[] = {Chain,
6336 DAG.getConstant(Val: Intrinsic::arm_get_fpscr, DL: dl, VT: MVT::i32)};
6337
6338 SDValue FPSCR =
6339 DAG.getNode(Opcode: ISD::INTRINSIC_W_CHAIN, DL: dl, ResultTys: {MVT::i32, MVT::Other}, Ops);
6340 Chain = FPSCR.getValue(R: 1);
6341 SDValue FltRounds = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::i32, N1: FPSCR,
6342 N2: DAG.getConstant(Val: 1U << 22, DL: dl, VT: MVT::i32));
6343 SDValue RMODE = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i32, N1: FltRounds,
6344 N2: DAG.getConstant(Val: 22, DL: dl, VT: MVT::i32));
6345 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32, N1: RMODE,
6346 N2: DAG.getConstant(Val: 3, DL: dl, VT: MVT::i32));
6347 return DAG.getMergeValues(Ops: {And, Chain}, dl);
6348}
6349
6350SDValue ARMTargetLowering::LowerSET_ROUNDING(SDValue Op,
6351 SelectionDAG &DAG) const {
6352 SDLoc DL(Op);
6353 SDValue Chain = Op->getOperand(Num: 0);
6354 SDValue RMValue = Op->getOperand(Num: 1);
6355
6356 // The rounding mode is in bits 23:22 of the FPSCR.
6357 // The llvm.set.rounding argument value to ARM rounding mode value mapping
6358 // is 0->3, 1->0, 2->1, 3->2. The formula we use to implement this is
6359 // ((arg - 1) & 3) << 22).
6360 //
6361 // It is expected that the argument of llvm.set.rounding is within the
6362 // segment [0, 3], so NearestTiesToAway (4) is not handled here. It is
6363 // responsibility of the code generated llvm.set.rounding to ensure this
6364 // condition.
6365
6366 // Calculate new value of FPSCR[23:22].
6367 RMValue = DAG.getNode(Opcode: ISD::SUB, DL, VT: MVT::i32, N1: RMValue,
6368 N2: DAG.getConstant(Val: 1, DL, VT: MVT::i32));
6369 RMValue = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: RMValue,
6370 N2: DAG.getConstant(Val: 0x3, DL, VT: MVT::i32));
6371 RMValue = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i32, N1: RMValue,
6372 N2: DAG.getConstant(Val: ARM::RoundingBitsPos, DL, VT: MVT::i32));
6373
6374 // Get current value of FPSCR.
6375 SDValue Ops[] = {Chain,
6376 DAG.getConstant(Val: Intrinsic::arm_get_fpscr, DL, VT: MVT::i32)};
6377 SDValue FPSCR =
6378 DAG.getNode(Opcode: ISD::INTRINSIC_W_CHAIN, DL, ResultTys: {MVT::i32, MVT::Other}, Ops);
6379 Chain = FPSCR.getValue(R: 1);
6380 FPSCR = FPSCR.getValue(R: 0);
6381
6382 // Put new rounding mode into FPSCR[23:22].
6383 const unsigned RMMask = ~(ARM::Rounding::rmMask << ARM::RoundingBitsPos);
6384 FPSCR = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: FPSCR,
6385 N2: DAG.getConstant(Val: RMMask, DL, VT: MVT::i32));
6386 FPSCR = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: FPSCR, N2: RMValue);
6387 SDValue Ops2[] = {
6388 Chain, DAG.getConstant(Val: Intrinsic::arm_set_fpscr, DL, VT: MVT::i32), FPSCR};
6389 return DAG.getNode(Opcode: ISD::INTRINSIC_VOID, DL, VT: MVT::Other, Ops: Ops2);
6390}
6391
6392SDValue ARMTargetLowering::LowerSET_FPMODE(SDValue Op,
6393 SelectionDAG &DAG) const {
6394 SDLoc DL(Op);
6395 SDValue Chain = Op->getOperand(Num: 0);
6396 SDValue Mode = Op->getOperand(Num: 1);
6397
6398 // Generate nodes to build:
6399 // FPSCR = (FPSCR & FPStatusBits) | (Mode & ~FPStatusBits)
6400 SDValue Ops[] = {Chain,
6401 DAG.getConstant(Val: Intrinsic::arm_get_fpscr, DL, VT: MVT::i32)};
6402 SDValue FPSCR =
6403 DAG.getNode(Opcode: ISD::INTRINSIC_W_CHAIN, DL, ResultTys: {MVT::i32, MVT::Other}, Ops);
6404 Chain = FPSCR.getValue(R: 1);
6405 FPSCR = FPSCR.getValue(R: 0);
6406
6407 SDValue FPSCRMasked =
6408 DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: FPSCR,
6409 N2: DAG.getConstant(Val: ARM::FPStatusBits, DL, VT: MVT::i32));
6410 SDValue InputMasked =
6411 DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: Mode,
6412 N2: DAG.getConstant(Val: ~ARM::FPStatusBits, DL, VT: MVT::i32));
6413 FPSCR = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: FPSCRMasked, N2: InputMasked);
6414
6415 SDValue Ops2[] = {
6416 Chain, DAG.getConstant(Val: Intrinsic::arm_set_fpscr, DL, VT: MVT::i32), FPSCR};
6417 return DAG.getNode(Opcode: ISD::INTRINSIC_VOID, DL, VT: MVT::Other, Ops: Ops2);
6418}
6419
6420SDValue ARMTargetLowering::LowerRESET_FPMODE(SDValue Op,
6421 SelectionDAG &DAG) const {
6422 SDLoc DL(Op);
6423 SDValue Chain = Op->getOperand(Num: 0);
6424
6425 // To get the default FP mode all control bits are cleared:
6426 // FPSCR = FPSCR & (FPStatusBits | FPReservedBits)
6427 SDValue Ops[] = {Chain,
6428 DAG.getConstant(Val: Intrinsic::arm_get_fpscr, DL, VT: MVT::i32)};
6429 SDValue FPSCR =
6430 DAG.getNode(Opcode: ISD::INTRINSIC_W_CHAIN, DL, ResultTys: {MVT::i32, MVT::Other}, Ops);
6431 Chain = FPSCR.getValue(R: 1);
6432 FPSCR = FPSCR.getValue(R: 0);
6433
6434 SDValue FPSCRMasked = DAG.getNode(
6435 Opcode: ISD::AND, DL, VT: MVT::i32, N1: FPSCR,
6436 N2: DAG.getConstant(Val: ARM::FPStatusBits | ARM::FPReservedBits, DL, VT: MVT::i32));
6437 SDValue Ops2[] = {Chain,
6438 DAG.getConstant(Val: Intrinsic::arm_set_fpscr, DL, VT: MVT::i32),
6439 FPSCRMasked};
6440 return DAG.getNode(Opcode: ISD::INTRINSIC_VOID, DL, VT: MVT::Other, Ops: Ops2);
6441}
6442
6443static SDValue LowerCTTZ(SDNode *N, SelectionDAG &DAG,
6444 const ARMSubtarget *ST) {
6445 SDLoc dl(N);
6446 EVT VT = N->getValueType(ResNo: 0);
6447 if (VT.isVector() && ST->hasNEON()) {
6448
6449 // Compute the least significant set bit: LSB = X & -X
6450 SDValue X = N->getOperand(Num: 0);
6451 SDValue NX = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: getZeroVector(VT, DAG, dl), N2: X);
6452 SDValue LSB = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: X, N2: NX);
6453
6454 EVT ElemTy = VT.getVectorElementType();
6455
6456 if (ElemTy == MVT::i8) {
6457 // Compute with: cttz(x) = ctpop(lsb - 1)
6458 SDValue One = DAG.getNode(Opcode: ARMISD::VMOVIMM, DL: dl, VT,
6459 Operand: DAG.getTargetConstant(Val: 1, DL: dl, VT: ElemTy));
6460 SDValue Bits = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: LSB, N2: One);
6461 return DAG.getNode(Opcode: ISD::CTPOP, DL: dl, VT, Operand: Bits);
6462 }
6463
6464 if ((ElemTy == MVT::i16 || ElemTy == MVT::i32) &&
6465 (N->getOpcode() == ISD::CTTZ_ZERO_POISON)) {
6466 // Compute with: cttz(x) = (width - 1) - ctlz(lsb), if x != 0
6467 unsigned NumBits = ElemTy.getSizeInBits();
6468 SDValue WidthMinus1 =
6469 DAG.getNode(Opcode: ARMISD::VMOVIMM, DL: dl, VT,
6470 Operand: DAG.getTargetConstant(Val: NumBits - 1, DL: dl, VT: ElemTy));
6471 SDValue CTLZ = DAG.getNode(Opcode: ISD::CTLZ, DL: dl, VT, Operand: LSB);
6472 return DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: WidthMinus1, N2: CTLZ);
6473 }
6474
6475 // Compute with: cttz(x) = ctpop(lsb - 1)
6476
6477 // Compute LSB - 1.
6478 SDValue Bits;
6479 if (ElemTy == MVT::i64) {
6480 // Load constant 0xffff'ffff'ffff'ffff to register.
6481 SDValue FF = DAG.getNode(Opcode: ARMISD::VMOVIMM, DL: dl, VT,
6482 Operand: DAG.getTargetConstant(Val: 0x1eff, DL: dl, VT: MVT::i32));
6483 Bits = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: LSB, N2: FF);
6484 } else {
6485 SDValue One = DAG.getNode(Opcode: ARMISD::VMOVIMM, DL: dl, VT,
6486 Operand: DAG.getTargetConstant(Val: 1, DL: dl, VT: ElemTy));
6487 Bits = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: LSB, N2: One);
6488 }
6489 return DAG.getNode(Opcode: ISD::CTPOP, DL: dl, VT, Operand: Bits);
6490 }
6491
6492 if (!ST->hasV6T2Ops())
6493 return SDValue();
6494
6495 SDValue rbit = DAG.getNode(Opcode: ISD::BITREVERSE, DL: dl, VT, Operand: N->getOperand(Num: 0));
6496 return DAG.getNode(Opcode: ISD::CTLZ, DL: dl, VT, Operand: rbit);
6497}
6498
6499static SDValue LowerCTPOP(SDNode *N, SelectionDAG &DAG,
6500 const ARMSubtarget *ST) {
6501 EVT VT = N->getValueType(ResNo: 0);
6502 SDLoc DL(N);
6503
6504 assert(ST->hasNEON() && "Custom ctpop lowering requires NEON.");
6505 assert((VT == MVT::v1i64 || VT == MVT::v2i64 || VT == MVT::v2i32 ||
6506 VT == MVT::v4i32 || VT == MVT::v4i16 || VT == MVT::v8i16) &&
6507 "Unexpected type for custom ctpop lowering");
6508
6509 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6510 EVT VT8Bit = VT.is64BitVector() ? MVT::v8i8 : MVT::v16i8;
6511 SDValue Res = DAG.getBitcast(VT: VT8Bit, V: N->getOperand(Num: 0));
6512 Res = DAG.getNode(Opcode: ISD::CTPOP, DL, VT: VT8Bit, Operand: Res);
6513
6514 // Widen v8i8/v16i8 CTPOP result to VT by repeatedly widening pairwise adds.
6515 unsigned EltSize = 8;
6516 unsigned NumElts = VT.is64BitVector() ? 8 : 16;
6517 while (EltSize != VT.getScalarSizeInBits()) {
6518 SmallVector<SDValue, 8> Ops;
6519 Ops.push_back(Elt: DAG.getConstant(Val: Intrinsic::arm_neon_vpaddlu, DL,
6520 VT: TLI.getPointerTy(DL: DAG.getDataLayout())));
6521 Ops.push_back(Elt: Res);
6522
6523 EltSize *= 2;
6524 NumElts /= 2;
6525 MVT WidenVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: EltSize), NumElements: NumElts);
6526 Res = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: WidenVT, Ops);
6527 }
6528
6529 return Res;
6530}
6531
6532/// Getvshiftimm - Check if this is a valid build_vector for the immediate
6533/// operand of a vector shift operation, where all the elements of the
6534/// build_vector must have the same constant integer value.
6535static bool getVShiftImm(SDValue Op, unsigned ElementBits, int64_t &Cnt) {
6536 // Ignore bit_converts.
6537 while (Op.getOpcode() == ISD::BITCAST)
6538 Op = Op.getOperand(i: 0);
6539 BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(Val: Op.getNode());
6540 APInt SplatBits, SplatUndef;
6541 unsigned SplatBitSize;
6542 bool HasAnyUndefs;
6543 if (!BVN ||
6544 !BVN->isConstantSplat(SplatValue&: SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs,
6545 MinSplatBits: ElementBits) ||
6546 SplatBitSize > ElementBits)
6547 return false;
6548 Cnt = SplatBits.getSExtValue();
6549 return true;
6550}
6551
6552/// isVShiftLImm - Check if this is a valid build_vector for the immediate
6553/// operand of a vector shift left operation. That value must be in the range:
6554/// 0 <= Value < ElementBits for a left shift; or
6555/// 0 <= Value <= ElementBits for a long left shift.
6556static bool isVShiftLImm(SDValue Op, EVT VT, bool isLong, int64_t &Cnt) {
6557 assert(VT.isVector() && "vector shift count is not a vector type");
6558 int64_t ElementBits = VT.getScalarSizeInBits();
6559 if (!getVShiftImm(Op, ElementBits, Cnt))
6560 return false;
6561 return (Cnt >= 0 && (isLong ? Cnt - 1 : Cnt) < ElementBits);
6562}
6563
6564/// isVShiftRImm - Check if this is a valid build_vector for the immediate
6565/// operand of a vector shift right operation. For a shift opcode, the value
6566/// is positive, but for an intrinsic the value count must be negative. The
6567/// absolute value must be in the range:
6568/// 1 <= |Value| <= ElementBits for a right shift; or
6569/// 1 <= |Value| <= ElementBits/2 for a narrow right shift.
6570static bool isVShiftRImm(SDValue Op, EVT VT, bool isNarrow, bool isIntrinsic,
6571 int64_t &Cnt) {
6572 assert(VT.isVector() && "vector shift count is not a vector type");
6573 int64_t ElementBits = VT.getScalarSizeInBits();
6574 if (!getVShiftImm(Op, ElementBits, Cnt))
6575 return false;
6576 if (!isIntrinsic)
6577 return (Cnt >= 1 && Cnt <= (isNarrow ? ElementBits / 2 : ElementBits));
6578 if (Cnt >= -(isNarrow ? ElementBits / 2 : ElementBits) && Cnt <= -1) {
6579 Cnt = -Cnt;
6580 return true;
6581 }
6582 return false;
6583}
6584
6585static SDValue LowerShift(SDNode *N, SelectionDAG &DAG,
6586 const ARMSubtarget *ST) {
6587 EVT VT = N->getValueType(ResNo: 0);
6588 SDLoc dl(N);
6589 int64_t Cnt;
6590
6591 if (!VT.isVector())
6592 return SDValue();
6593
6594 // We essentially have two forms here. Shift by an immediate and shift by a
6595 // vector register (there are also shift by a gpr, but that is just handled
6596 // with a tablegen pattern). We cannot easily match shift by an immediate in
6597 // tablegen so we do that here and generate a VSHLIMM/VSHRsIMM/VSHRuIMM.
6598 // For shifting by a vector, we don't have VSHR, only VSHL (which can be
6599 // signed or unsigned, and a negative shift indicates a shift right).
6600 if (N->getOpcode() == ISD::SHL) {
6601 if (isVShiftLImm(Op: N->getOperand(Num: 1), VT, isLong: false, Cnt))
6602 return DAG.getNode(Opcode: ARMISD::VSHLIMM, DL: dl, VT, N1: N->getOperand(Num: 0),
6603 N2: DAG.getConstant(Val: Cnt, DL: dl, VT: MVT::i32));
6604 return DAG.getNode(Opcode: ARMISD::VSHLu, DL: dl, VT, N1: N->getOperand(Num: 0),
6605 N2: N->getOperand(Num: 1));
6606 }
6607
6608 assert((N->getOpcode() == ISD::SRA || N->getOpcode() == ISD::SRL) &&
6609 "unexpected vector shift opcode");
6610
6611 if (isVShiftRImm(Op: N->getOperand(Num: 1), VT, isNarrow: false, isIntrinsic: false, Cnt)) {
6612 unsigned VShiftOpc =
6613 (N->getOpcode() == ISD::SRA ? ARMISD::VSHRsIMM : ARMISD::VSHRuIMM);
6614 return DAG.getNode(Opcode: VShiftOpc, DL: dl, VT, N1: N->getOperand(Num: 0),
6615 N2: DAG.getConstant(Val: Cnt, DL: dl, VT: MVT::i32));
6616 }
6617
6618 // Other right shifts we don't have operations for (we use a shift left by a
6619 // negative number).
6620 EVT ShiftVT = N->getOperand(Num: 1).getValueType();
6621 SDValue NegatedCount = DAG.getNode(
6622 Opcode: ISD::SUB, DL: dl, VT: ShiftVT, N1: getZeroVector(VT: ShiftVT, DAG, dl), N2: N->getOperand(Num: 1));
6623 unsigned VShiftOpc =
6624 (N->getOpcode() == ISD::SRA ? ARMISD::VSHLs : ARMISD::VSHLu);
6625 return DAG.getNode(Opcode: VShiftOpc, DL: dl, VT, N1: N->getOperand(Num: 0), N2: NegatedCount);
6626}
6627
6628static SDValue Expand64BitShift(SDNode *N, SelectionDAG &DAG,
6629 const ARMSubtarget *ST) {
6630 EVT VT = N->getValueType(ResNo: 0);
6631 SDLoc dl(N);
6632
6633 // We can get here for a node like i32 = ISD::SHL i32, i64
6634 if (VT != MVT::i64)
6635 return SDValue();
6636
6637 assert((N->getOpcode() == ISD::SRL || N->getOpcode() == ISD::SRA ||
6638 N->getOpcode() == ISD::SHL) &&
6639 "Unknown shift to lower!");
6640
6641 unsigned ShOpc = N->getOpcode();
6642 if (ST->hasMVEIntegerOps()) {
6643 SDValue ShAmt = N->getOperand(Num: 1);
6644 unsigned ShPartsOpc = ARMISD::LSLL;
6645 ConstantSDNode *Con = dyn_cast<ConstantSDNode>(Val&: ShAmt);
6646
6647 // If the shift amount is greater than 32 or has a greater bitwidth than 64
6648 // then do the default optimisation
6649 if ((!Con && ShAmt->getValueType(ResNo: 0).getSizeInBits() > 64) ||
6650 (Con && (Con->getAPIntValue() == 0 || Con->getAPIntValue().uge(RHS: 32))))
6651 return SDValue();
6652
6653 // Extract the lower 32 bits of the shift amount if it's not an i32
6654 if (ShAmt->getValueType(ResNo: 0) != MVT::i32)
6655 ShAmt = DAG.getZExtOrTrunc(Op: ShAmt, DL: dl, VT: MVT::i32);
6656
6657 if (ShOpc == ISD::SRL) {
6658 if (!Con)
6659 // There is no t2LSRLr instruction so negate and perform an lsll if the
6660 // shift amount is in a register, emulating a right shift.
6661 ShAmt = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32,
6662 N1: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32), N2: ShAmt);
6663 else
6664 // Else generate an lsrl on the immediate shift amount
6665 ShPartsOpc = ARMISD::LSRL;
6666 } else if (ShOpc == ISD::SRA)
6667 ShPartsOpc = ARMISD::ASRL;
6668
6669 // Split Lower/Upper 32 bits of the destination/source
6670 SDValue Lo, Hi;
6671 std::tie(args&: Lo, args&: Hi) =
6672 DAG.SplitScalar(N: N->getOperand(Num: 0), DL: dl, LoVT: MVT::i32, HiVT: MVT::i32);
6673 // Generate the shift operation as computed above
6674 Lo = DAG.getNode(Opcode: ShPartsOpc, DL: dl, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N1: Lo, N2: Hi,
6675 N3: ShAmt);
6676 // The upper 32 bits come from the second return value of lsll
6677 Hi = SDValue(Lo.getNode(), 1);
6678 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: MVT::i64, N1: Lo, N2: Hi);
6679 }
6680
6681 // We only lower SRA, SRL of 1 here, all others use generic lowering.
6682 if (!isOneConstant(V: N->getOperand(Num: 1)) || N->getOpcode() == ISD::SHL)
6683 return SDValue();
6684
6685 // If we are in thumb mode, we don't have RRX.
6686 if (ST->isThumb1Only())
6687 return SDValue();
6688
6689 // Okay, we have a 64-bit SRA or SRL of 1. Lower this to an RRX expr.
6690 SDValue Lo, Hi;
6691 std::tie(args&: Lo, args&: Hi) = DAG.SplitScalar(N: N->getOperand(Num: 0), DL: dl, LoVT: MVT::i32, HiVT: MVT::i32);
6692
6693 // First, build a LSRS1/ASRS1 op, which shifts the top part by one and
6694 // captures the shifted out bit into a carry flag.
6695 unsigned Opc = N->getOpcode() == ISD::SRL ? ARMISD::LSRS1 : ARMISD::ASRS1;
6696 Hi = DAG.getNode(Opcode: Opc, DL: dl, VTList: DAG.getVTList(VT1: MVT::i32, VT2: FlagsVT), N: Hi);
6697
6698 // The low part is an ARMISD::RRX operand, which shifts the carry in.
6699 Lo = DAG.getNode(Opcode: ARMISD::RRX, DL: dl, VT: MVT::i32, N1: Lo, N2: Hi.getValue(R: 1));
6700
6701 // Merge the pieces into a single i64 value.
6702 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: MVT::i64, N1: Lo, N2: Hi);
6703}
6704
6705static SDValue LowerVSETCC(SDValue Op, SelectionDAG &DAG,
6706 const ARMSubtarget *ST) {
6707 bool Invert = false;
6708 bool Swap = false;
6709 unsigned Opc = ARMCC::AL;
6710
6711 SDValue Op0 = Op.getOperand(i: 0);
6712 SDValue Op1 = Op.getOperand(i: 1);
6713 SDValue CC = Op.getOperand(i: 2);
6714 EVT VT = Op.getValueType();
6715 ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(Val&: CC)->get();
6716 SDLoc dl(Op);
6717
6718 EVT CmpVT;
6719 if (ST->hasNEON())
6720 CmpVT = Op0.getValueType().changeVectorElementTypeToInteger();
6721 else {
6722 assert(ST->hasMVEIntegerOps() &&
6723 "No hardware support for integer vector comparison!");
6724
6725 if (Op.getValueType().getVectorElementType() != MVT::i1)
6726 return SDValue();
6727
6728 // Make sure we expand floating point setcc to scalar if we do not have
6729 // mve.fp, so that we can handle them from there.
6730 if (Op0.getValueType().isFloatingPoint() && !ST->hasMVEFloatOps())
6731 return SDValue();
6732
6733 CmpVT = VT;
6734 }
6735
6736 if (Op0.getValueType().getVectorElementType() == MVT::i64 &&
6737 (SetCCOpcode == ISD::SETEQ || SetCCOpcode == ISD::SETNE)) {
6738 // Special-case integer 64-bit equality comparisons. They aren't legal,
6739 // but they can be lowered with a few vector instructions.
6740 unsigned CmpElements = CmpVT.getVectorNumElements() * 2;
6741 EVT SplitVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32, NumElements: CmpElements);
6742 SDValue CastOp0 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: SplitVT, Operand: Op0);
6743 SDValue CastOp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: SplitVT, Operand: Op1);
6744 SDValue Cmp = DAG.getNode(Opcode: ISD::SETCC, DL: dl, VT: SplitVT, N1: CastOp0, N2: CastOp1,
6745 N3: DAG.getCondCode(Cond: ISD::SETEQ));
6746 SDValue Reversed = DAG.getNode(Opcode: ARMISD::VREV64, DL: dl, VT: SplitVT, Operand: Cmp);
6747 SDValue Merged = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SplitVT, N1: Cmp, N2: Reversed);
6748 Merged = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: CmpVT, Operand: Merged);
6749 if (SetCCOpcode == ISD::SETNE)
6750 Merged = DAG.getNOT(DL: dl, Val: Merged, VT: CmpVT);
6751 Merged = DAG.getSExtOrTrunc(Op: Merged, DL: dl, VT);
6752 return Merged;
6753 }
6754
6755 if (CmpVT.getVectorElementType() == MVT::i64)
6756 // 64-bit comparisons are not legal in general.
6757 return SDValue();
6758
6759 if (Op1.getValueType().isFloatingPoint()) {
6760 switch (SetCCOpcode) {
6761 default: llvm_unreachable("Illegal FP comparison");
6762 case ISD::SETUNE:
6763 case ISD::SETNE:
6764 if (ST->hasMVEFloatOps()) {
6765 Opc = ARMCC::NE; break;
6766 } else {
6767 Invert = true; [[fallthrough]];
6768 }
6769 case ISD::SETOEQ:
6770 case ISD::SETEQ: Opc = ARMCC::EQ; break;
6771 case ISD::SETOLT:
6772 case ISD::SETLT: Swap = true; [[fallthrough]];
6773 case ISD::SETOGT:
6774 case ISD::SETGT: Opc = ARMCC::GT; break;
6775 case ISD::SETOLE:
6776 case ISD::SETLE: Swap = true; [[fallthrough]];
6777 case ISD::SETOGE:
6778 case ISD::SETGE: Opc = ARMCC::GE; break;
6779 case ISD::SETUGE: Swap = true; [[fallthrough]];
6780 case ISD::SETULE: Invert = true; Opc = ARMCC::GT; break;
6781 case ISD::SETUGT: Swap = true; [[fallthrough]];
6782 case ISD::SETULT: Invert = true; Opc = ARMCC::GE; break;
6783 case ISD::SETUEQ: Invert = true; [[fallthrough]];
6784 case ISD::SETONE: {
6785 // Expand this to (OLT | OGT).
6786 SDValue TmpOp0 = DAG.getNode(Opcode: ARMISD::VCMP, DL: dl, VT: CmpVT, N1: Op1, N2: Op0,
6787 N3: DAG.getConstant(Val: ARMCC::GT, DL: dl, VT: MVT::i32));
6788 SDValue TmpOp1 = DAG.getNode(Opcode: ARMISD::VCMP, DL: dl, VT: CmpVT, N1: Op0, N2: Op1,
6789 N3: DAG.getConstant(Val: ARMCC::GT, DL: dl, VT: MVT::i32));
6790 SDValue Result = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: CmpVT, N1: TmpOp0, N2: TmpOp1);
6791 if (Invert)
6792 Result = DAG.getNOT(DL: dl, Val: Result, VT);
6793 return Result;
6794 }
6795 case ISD::SETUO: Invert = true; [[fallthrough]];
6796 case ISD::SETO: {
6797 // Expand this to (OLT | OGE).
6798 SDValue TmpOp0 = DAG.getNode(Opcode: ARMISD::VCMP, DL: dl, VT: CmpVT, N1: Op1, N2: Op0,
6799 N3: DAG.getConstant(Val: ARMCC::GT, DL: dl, VT: MVT::i32));
6800 SDValue TmpOp1 = DAG.getNode(Opcode: ARMISD::VCMP, DL: dl, VT: CmpVT, N1: Op0, N2: Op1,
6801 N3: DAG.getConstant(Val: ARMCC::GE, DL: dl, VT: MVT::i32));
6802 SDValue Result = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: CmpVT, N1: TmpOp0, N2: TmpOp1);
6803 if (Invert)
6804 Result = DAG.getNOT(DL: dl, Val: Result, VT);
6805 return Result;
6806 }
6807 }
6808 } else {
6809 // Integer comparisons.
6810 switch (SetCCOpcode) {
6811 default: llvm_unreachable("Illegal integer comparison");
6812 case ISD::SETNE:
6813 if (ST->hasMVEIntegerOps()) {
6814 Opc = ARMCC::NE; break;
6815 } else {
6816 Invert = true; [[fallthrough]];
6817 }
6818 case ISD::SETEQ: Opc = ARMCC::EQ; break;
6819 case ISD::SETLT: Swap = true; [[fallthrough]];
6820 case ISD::SETGT: Opc = ARMCC::GT; break;
6821 case ISD::SETLE: Swap = true; [[fallthrough]];
6822 case ISD::SETGE: Opc = ARMCC::GE; break;
6823 case ISD::SETULT: Swap = true; [[fallthrough]];
6824 case ISD::SETUGT: Opc = ARMCC::HI; break;
6825 case ISD::SETULE: Swap = true; [[fallthrough]];
6826 case ISD::SETUGE: Opc = ARMCC::HS; break;
6827 }
6828
6829 // Detect VTST (Vector Test Bits) = icmp ne (and (op0, op1), zero).
6830 if (ST->hasNEON() && Opc == ARMCC::EQ) {
6831 SDValue AndOp;
6832 if (ISD::isBuildVectorAllZeros(V: Op1))
6833 AndOp = Op0;
6834 else if (ISD::isBuildVectorAllZeros(V: Op0))
6835 AndOp = Op1;
6836
6837 // Ignore bitconvert.
6838 if (AndOp.getNode() && AndOp.getOpcode() == ISD::BITCAST)
6839 AndOp = AndOp.getOperand(i: 0);
6840
6841 if (AndOp.getNode() && AndOp.getOpcode() == ISD::AND) {
6842 Op0 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: CmpVT, Operand: AndOp.getOperand(i: 0));
6843 Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: CmpVT, Operand: AndOp.getOperand(i: 1));
6844 SDValue Result = DAG.getNode(Opcode: ARMISD::VTST, DL: dl, VT: CmpVT, N1: Op0, N2: Op1);
6845 if (!Invert)
6846 Result = DAG.getNOT(DL: dl, Val: Result, VT);
6847 return Result;
6848 }
6849 }
6850 }
6851
6852 if (Swap)
6853 std::swap(a&: Op0, b&: Op1);
6854
6855 // If one of the operands is a constant vector zero, attempt to fold the
6856 // comparison to a specialized compare-against-zero form.
6857 if (ISD::isBuildVectorAllZeros(V: Op0) &&
6858 (Opc == ARMCC::GE || Opc == ARMCC::GT || Opc == ARMCC::EQ ||
6859 Opc == ARMCC::NE)) {
6860 if (Opc == ARMCC::GE)
6861 Opc = ARMCC::LE;
6862 else if (Opc == ARMCC::GT)
6863 Opc = ARMCC::LT;
6864 std::swap(a&: Op0, b&: Op1);
6865 }
6866
6867 SDValue Result;
6868 if (ISD::isBuildVectorAllZeros(V: Op1) &&
6869 (Opc == ARMCC::GE || Opc == ARMCC::GT || Opc == ARMCC::LE ||
6870 Opc == ARMCC::LT || Opc == ARMCC::NE || Opc == ARMCC::EQ))
6871 Result = DAG.getNode(Opcode: ARMISD::VCMPZ, DL: dl, VT: CmpVT, N1: Op0,
6872 N2: DAG.getConstant(Val: Opc, DL: dl, VT: MVT::i32));
6873 else
6874 Result = DAG.getNode(Opcode: ARMISD::VCMP, DL: dl, VT: CmpVT, N1: Op0, N2: Op1,
6875 N3: DAG.getConstant(Val: Opc, DL: dl, VT: MVT::i32));
6876
6877 Result = DAG.getSExtOrTrunc(Op: Result, DL: dl, VT);
6878
6879 if (Invert)
6880 Result = DAG.getNOT(DL: dl, Val: Result, VT);
6881
6882 return Result;
6883}
6884
6885static SDValue LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG) {
6886 SDValue LHS = Op.getOperand(i: 0);
6887 SDValue RHS = Op.getOperand(i: 1);
6888
6889 assert(LHS.getSimpleValueType().isInteger() && "SETCCCARRY is integer only.");
6890
6891 SDValue Carry = Op.getOperand(i: 2);
6892 SDValue Cond = Op.getOperand(i: 3);
6893 SDLoc DL(Op);
6894
6895 // ARMISD::SUBE expects a carry not a borrow like ISD::USUBO_CARRY so we
6896 // have to invert the carry first.
6897 SDValue InvCarry = valueToCarryFlag(Value: Carry, DAG, Invert: true);
6898
6899 SDVTList VTs = DAG.getVTList(VT1: LHS.getValueType(), VT2: MVT::i32);
6900 SDValue Cmp = DAG.getNode(Opcode: ARMISD::SUBE, DL, VTList: VTs, N1: LHS, N2: RHS, N3: InvCarry);
6901
6902 SDValue FVal = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
6903 SDValue TVal = DAG.getConstant(Val: 1, DL, VT: MVT::i32);
6904 SDValue ARMcc = DAG.getConstant(
6905 Val: IntCCToARMCC(CC: cast<CondCodeSDNode>(Val&: Cond)->get()), DL, VT: MVT::i32);
6906 return DAG.getNode(Opcode: ARMISD::CMOV, DL, VT: Op.getValueType(), N1: FVal, N2: TVal, N3: ARMcc,
6907 N4: Cmp.getValue(R: 1));
6908}
6909
6910/// isVMOVModifiedImm - Check if the specified splat value corresponds to a
6911/// valid vector constant for a NEON or MVE instruction with a "modified
6912/// immediate" operand (e.g., VMOV). If so, return the encoded value.
6913static SDValue isVMOVModifiedImm(uint64_t SplatBits, uint64_t SplatUndef,
6914 unsigned SplatBitSize, SelectionDAG &DAG,
6915 const SDLoc &dl, EVT &VT, EVT VectorVT,
6916 VMOVModImmType type) {
6917 unsigned OpCmode, Imm;
6918 bool is128Bits = VectorVT.is128BitVector();
6919
6920 // SplatBitSize is set to the smallest size that splats the vector, so a
6921 // zero vector will always have SplatBitSize == 8. However, NEON modified
6922 // immediate instructions others than VMOV do not support the 8-bit encoding
6923 // of a zero vector, and the default encoding of zero is supposed to be the
6924 // 32-bit version.
6925 if (SplatBits == 0)
6926 SplatBitSize = 32;
6927
6928 switch (SplatBitSize) {
6929 case 8:
6930 if (type != VMOVModImm)
6931 return SDValue();
6932 // Any 1-byte value is OK. Op=0, Cmode=1110.
6933 assert((SplatBits & ~0xff) == 0 && "one byte splat value is too big");
6934 OpCmode = 0xe;
6935 Imm = SplatBits;
6936 VT = is128Bits ? MVT::v16i8 : MVT::v8i8;
6937 break;
6938
6939 case 16:
6940 // NEON's 16-bit VMOV supports splat values where only one byte is nonzero.
6941 VT = is128Bits ? MVT::v8i16 : MVT::v4i16;
6942 if ((SplatBits & ~0xff) == 0) {
6943 // Value = 0x00nn: Op=x, Cmode=100x.
6944 OpCmode = 0x8;
6945 Imm = SplatBits;
6946 break;
6947 }
6948 if ((SplatBits & ~0xff00) == 0) {
6949 // Value = 0xnn00: Op=x, Cmode=101x.
6950 OpCmode = 0xa;
6951 Imm = SplatBits >> 8;
6952 break;
6953 }
6954 return SDValue();
6955
6956 case 32:
6957 // NEON's 32-bit VMOV supports splat values where:
6958 // * only one byte is nonzero, or
6959 // * the least significant byte is 0xff and the second byte is nonzero, or
6960 // * the least significant 2 bytes are 0xff and the third is nonzero.
6961 VT = is128Bits ? MVT::v4i32 : MVT::v2i32;
6962 if ((SplatBits & ~0xff) == 0) {
6963 // Value = 0x000000nn: Op=x, Cmode=000x.
6964 OpCmode = 0;
6965 Imm = SplatBits;
6966 break;
6967 }
6968 if ((SplatBits & ~0xff00) == 0) {
6969 // Value = 0x0000nn00: Op=x, Cmode=001x.
6970 OpCmode = 0x2;
6971 Imm = SplatBits >> 8;
6972 break;
6973 }
6974 if ((SplatBits & ~0xff0000) == 0) {
6975 // Value = 0x00nn0000: Op=x, Cmode=010x.
6976 OpCmode = 0x4;
6977 Imm = SplatBits >> 16;
6978 break;
6979 }
6980 if ((SplatBits & ~0xff000000) == 0) {
6981 // Value = 0xnn000000: Op=x, Cmode=011x.
6982 OpCmode = 0x6;
6983 Imm = SplatBits >> 24;
6984 break;
6985 }
6986
6987 // cmode == 0b1100 and cmode == 0b1101 are not supported for VORR or VBIC
6988 if (type == OtherModImm) return SDValue();
6989
6990 if ((SplatBits & ~0xffff) == 0 &&
6991 ((SplatBits | SplatUndef) & 0xff) == 0xff) {
6992 // Value = 0x0000nnff: Op=x, Cmode=1100.
6993 OpCmode = 0xc;
6994 Imm = SplatBits >> 8;
6995 break;
6996 }
6997
6998 // cmode == 0b1101 is not supported for MVE VMVN
6999 if (type == MVEVMVNModImm)
7000 return SDValue();
7001
7002 if ((SplatBits & ~0xffffff) == 0 &&
7003 ((SplatBits | SplatUndef) & 0xffff) == 0xffff) {
7004 // Value = 0x00nnffff: Op=x, Cmode=1101.
7005 OpCmode = 0xd;
7006 Imm = SplatBits >> 16;
7007 break;
7008 }
7009
7010 // Note: there are a few 32-bit splat values (specifically: 00ffff00,
7011 // ff000000, ff0000ff, and ffff00ff) that are valid for VMOV.I64 but not
7012 // VMOV.I32. A (very) minor optimization would be to replicate the value
7013 // and fall through here to test for a valid 64-bit splat. But, then the
7014 // caller would also need to check and handle the change in size.
7015 return SDValue();
7016
7017 case 64: {
7018 if (type != VMOVModImm)
7019 return SDValue();
7020 // NEON has a 64-bit VMOV splat where each byte is either 0 or 0xff.
7021 uint64_t BitMask = 0xff;
7022 unsigned ImmMask = 1;
7023 Imm = 0;
7024 for (int ByteNum = 0; ByteNum < 8; ++ByteNum) {
7025 if (((SplatBits | SplatUndef) & BitMask) == BitMask) {
7026 Imm |= ImmMask;
7027 } else if ((SplatBits & BitMask) != 0) {
7028 return SDValue();
7029 }
7030 BitMask <<= 8;
7031 ImmMask <<= 1;
7032 }
7033
7034 // Op=1, Cmode=1110.
7035 OpCmode = 0x1e;
7036 VT = is128Bits ? MVT::v2i64 : MVT::v1i64;
7037 break;
7038 }
7039
7040 default:
7041 llvm_unreachable("unexpected size for isVMOVModifiedImm");
7042 }
7043
7044 unsigned EncodedVal = ARM_AM::createVMOVModImm(OpCmode, Val: Imm);
7045 return DAG.getTargetConstant(Val: EncodedVal, DL: dl, VT: MVT::i32);
7046}
7047
7048SDValue ARMTargetLowering::LowerConstantFP(SDValue Op, SelectionDAG &DAG,
7049 const ARMSubtarget *ST) const {
7050 EVT VT = Op.getValueType();
7051 bool IsDouble = (VT == MVT::f64);
7052 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Val&: Op);
7053 const APFloat &FPVal = CFP->getValueAPF();
7054
7055 // Prevent floating-point constants from using literal loads
7056 // when execute-only is enabled.
7057 if (ST->genExecuteOnly()) {
7058 // We shouldn't trigger this for v6m execute-only
7059 assert((!ST->isThumb1Only() || ST->hasV8MBaselineOps()) &&
7060 "Unexpected architecture");
7061
7062 // If we can represent the constant as an immediate, don't lower it
7063 if (isFPImmLegal(Imm: FPVal, VT))
7064 return Op;
7065 // Otherwise, construct as integer, and move to float register
7066 APInt INTVal = FPVal.bitcastToAPInt();
7067 SDLoc DL(CFP);
7068 switch (VT.getSimpleVT().SimpleTy) {
7069 default:
7070 llvm_unreachable("Unknown floating point type!");
7071 break;
7072 case MVT::f64: {
7073 SDValue Lo = DAG.getConstant(Val: INTVal.trunc(width: 32), DL, VT: MVT::i32);
7074 SDValue Hi = DAG.getConstant(Val: INTVal.lshr(shiftAmt: 32).trunc(width: 32), DL, VT: MVT::i32);
7075 return DAG.getNode(Opcode: ARMISD::VMOVDRR, DL, VT: MVT::f64, N1: Lo, N2: Hi);
7076 }
7077 case MVT::f32:
7078 return DAG.getNode(Opcode: ARMISD::VMOVsr, DL, VT,
7079 Operand: DAG.getConstant(Val: INTVal, DL, VT: MVT::i32));
7080 }
7081 }
7082
7083 if (!ST->hasVFP3Base())
7084 return SDValue();
7085
7086 // Use the default (constant pool) lowering for double constants when we have
7087 // an SP-only FPU
7088 if (IsDouble && !Subtarget->hasFP64())
7089 return SDValue();
7090
7091 // Try splatting with a VMOV.f32...
7092 int ImmVal = IsDouble ? ARM_AM::getFP64Imm(FPImm: FPVal) : ARM_AM::getFP32Imm(FPImm: FPVal);
7093
7094 if (ImmVal != -1) {
7095 if (IsDouble || !ST->useNEONForSinglePrecisionFP()) {
7096 // We have code in place to select a valid ConstantFP already, no need to
7097 // do any mangling.
7098 return Op;
7099 }
7100
7101 // It's a float and we are trying to use NEON operations where
7102 // possible. Lower it to a splat followed by an extract.
7103 SDLoc DL(Op);
7104 SDValue NewVal = DAG.getTargetConstant(Val: ImmVal, DL, VT: MVT::i32);
7105 SDValue VecConstant = DAG.getNode(Opcode: ARMISD::VMOVFPIMM, DL, VT: MVT::v2f32,
7106 Operand: NewVal);
7107 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::f32, N1: VecConstant,
7108 N2: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
7109 }
7110
7111 // The rest of our options are NEON only, make sure that's allowed before
7112 // proceeding..
7113 if (!ST->hasNEON() || (!IsDouble && !ST->useNEONForSinglePrecisionFP()))
7114 return SDValue();
7115
7116 EVT VMovVT;
7117 uint64_t iVal = FPVal.bitcastToAPInt().getZExtValue();
7118
7119 // It wouldn't really be worth bothering for doubles except for one very
7120 // important value, which does happen to match: 0.0. So make sure we don't do
7121 // anything stupid.
7122 if (IsDouble && (iVal & 0xffffffff) != (iVal >> 32))
7123 return SDValue();
7124
7125 // Try a VMOV.i32 (FIXME: i8, i16, or i64 could work too).
7126 SDValue NewVal = isVMOVModifiedImm(SplatBits: iVal & 0xffffffffU, SplatUndef: 0, SplatBitSize: 32, DAG, dl: SDLoc(Op),
7127 VT&: VMovVT, VectorVT: VT, type: VMOVModImm);
7128 if (NewVal != SDValue()) {
7129 SDLoc DL(Op);
7130 SDValue VecConstant = DAG.getNode(Opcode: ARMISD::VMOVIMM, DL, VT: VMovVT,
7131 Operand: NewVal);
7132 if (IsDouble)
7133 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f64, Operand: VecConstant);
7134
7135 // It's a float: cast and extract a vector element.
7136 SDValue VecFConstant = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v2f32,
7137 Operand: VecConstant);
7138 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::f32, N1: VecFConstant,
7139 N2: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
7140 }
7141
7142 // Finally, try a VMVN.i32
7143 NewVal = isVMOVModifiedImm(SplatBits: ~iVal & 0xffffffffU, SplatUndef: 0, SplatBitSize: 32, DAG, dl: SDLoc(Op), VT&: VMovVT,
7144 VectorVT: VT, type: VMVNModImm);
7145 if (NewVal != SDValue()) {
7146 SDLoc DL(Op);
7147 SDValue VecConstant = DAG.getNode(Opcode: ARMISD::VMVNIMM, DL, VT: VMovVT, Operand: NewVal);
7148
7149 if (IsDouble)
7150 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f64, Operand: VecConstant);
7151
7152 // It's a float: cast and extract a vector element.
7153 SDValue VecFConstant = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v2f32,
7154 Operand: VecConstant);
7155 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::f32, N1: VecFConstant,
7156 N2: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
7157 }
7158
7159 return SDValue();
7160}
7161
7162// check if an VEXT instruction can handle the shuffle mask when the
7163// vector sources of the shuffle are the same.
7164static bool isSingletonVEXTMask(ArrayRef<int> M, EVT VT, unsigned &Imm) {
7165 unsigned NumElts = VT.getVectorNumElements();
7166
7167 // Assume that the first shuffle index is not UNDEF. Fail if it is.
7168 if (M[0] < 0)
7169 return false;
7170
7171 Imm = M[0];
7172
7173 // If this is a VEXT shuffle, the immediate value is the index of the first
7174 // element. The other shuffle indices must be the successive elements after
7175 // the first one.
7176 unsigned ExpectedElt = Imm;
7177 for (unsigned i = 1; i < NumElts; ++i) {
7178 // Increment the expected index. If it wraps around, just follow it
7179 // back to index zero and keep going.
7180 ++ExpectedElt;
7181 if (ExpectedElt == NumElts)
7182 ExpectedElt = 0;
7183
7184 if (M[i] < 0) continue; // ignore UNDEF indices
7185 if (ExpectedElt != static_cast<unsigned>(M[i]))
7186 return false;
7187 }
7188
7189 return true;
7190}
7191
7192static bool isVEXTMask(ArrayRef<int> M, EVT VT,
7193 bool &ReverseVEXT, unsigned &Imm) {
7194 unsigned NumElts = VT.getVectorNumElements();
7195 ReverseVEXT = false;
7196
7197 // Assume that the first shuffle index is not UNDEF. Fail if it is.
7198 if (M[0] < 0)
7199 return false;
7200
7201 Imm = M[0];
7202
7203 // If this is a VEXT shuffle, the immediate value is the index of the first
7204 // element. The other shuffle indices must be the successive elements after
7205 // the first one.
7206 unsigned ExpectedElt = Imm;
7207 for (unsigned i = 1; i < NumElts; ++i) {
7208 // Increment the expected index. If it wraps around, it may still be
7209 // a VEXT but the source vectors must be swapped.
7210 ExpectedElt += 1;
7211 if (ExpectedElt == NumElts * 2) {
7212 ExpectedElt = 0;
7213 ReverseVEXT = true;
7214 }
7215
7216 if (M[i] < 0) continue; // ignore UNDEF indices
7217 if (ExpectedElt != static_cast<unsigned>(M[i]))
7218 return false;
7219 }
7220
7221 // Adjust the index value if the source operands will be swapped.
7222 if (ReverseVEXT)
7223 Imm -= NumElts;
7224
7225 return true;
7226}
7227
7228static bool isVTBLMask(ArrayRef<int> M, EVT VT) {
7229 // We can handle <8 x i8> vector shuffles. If the index in the mask is out of
7230 // range, then 0 is placed into the resulting vector. So pretty much any mask
7231 // of 8 elements can work here.
7232 return VT == MVT::v8i8 && M.size() == 8;
7233}
7234
7235/// Check if \p ShuffleMask is a NEON two-result shuffle (VZIP, VUZP, VTRN),
7236/// and return the corresponding ARMISD opcode if it is, or 0 if it isn't.
7237static unsigned isNEONTwoResultShuffleMask(ArrayRef<int> ShuffleMask, EVT VT,
7238 unsigned &WhichResult,
7239 bool &isV_UNDEF) {
7240 isV_UNDEF = false;
7241 if (isVTRNMask(M: ShuffleMask, VT, WhichResult))
7242 return ARMISD::VTRN;
7243 if (isVUZPMask(M: ShuffleMask, VT, WhichResult))
7244 return ARMISD::VUZP;
7245 if (isVZIPMask(M: ShuffleMask, VT, WhichResult))
7246 return ARMISD::VZIP;
7247
7248 isV_UNDEF = true;
7249 if (isVTRN_v_undef_Mask(M: ShuffleMask, VT, WhichResult))
7250 return ARMISD::VTRN;
7251 if (isVUZP_v_undef_Mask(M: ShuffleMask, VT, WhichResult))
7252 return ARMISD::VUZP;
7253 if (isVZIP_v_undef_Mask(M: ShuffleMask, VT, WhichResult))
7254 return ARMISD::VZIP;
7255
7256 return 0;
7257}
7258
7259/// \return true if this is a reverse operation on an vector.
7260static bool isReverseMask(ArrayRef<int> M, EVT VT) {
7261 unsigned NumElts = VT.getVectorNumElements();
7262 // Make sure the mask has the right size.
7263 if (NumElts != M.size())
7264 return false;
7265
7266 // Look for <15, ..., 3, -1, 1, 0>.
7267 for (unsigned i = 0; i != NumElts; ++i)
7268 if (M[i] >= 0 && M[i] != (int) (NumElts - 1 - i))
7269 return false;
7270
7271 return true;
7272}
7273
7274static bool isTruncMask(ArrayRef<int> M, EVT VT, bool Top, bool SingleSource) {
7275 unsigned NumElts = VT.getVectorNumElements();
7276 // Make sure the mask has the right size.
7277 if (NumElts != M.size() || (VT != MVT::v8i16 && VT != MVT::v16i8))
7278 return false;
7279
7280 // Half-width truncation patterns (e.g. v4i32 -> v8i16):
7281 // !Top && SingleSource: <0, 2, 4, 6, 0, 2, 4, 6>
7282 // !Top && !SingleSource: <0, 2, 4, 6, 8, 10, 12, 14>
7283 // Top && SingleSource: <1, 3, 5, 7, 1, 3, 5, 7>
7284 // Top && !SingleSource: <1, 3, 5, 7, 9, 11, 13, 15>
7285 int Ofs = Top ? 1 : 0;
7286 int Upper = SingleSource ? 0 : NumElts;
7287 for (int i = 0, e = NumElts / 2; i != e; ++i) {
7288 if (M[i] >= 0 && M[i] != (i * 2) + Ofs)
7289 return false;
7290 if (M[i + e] >= 0 && M[i + e] != (i * 2) + Ofs + Upper)
7291 return false;
7292 }
7293 return true;
7294}
7295
7296static bool isVMOVNMask(ArrayRef<int> M, EVT VT, bool Top, bool SingleSource) {
7297 unsigned NumElts = VT.getVectorNumElements();
7298 // Make sure the mask has the right size.
7299 if (NumElts != M.size() || (VT != MVT::v8i16 && VT != MVT::v16i8))
7300 return false;
7301
7302 // If Top
7303 // Look for <0, N, 2, N+2, 4, N+4, ..>.
7304 // This inserts Input2 into Input1
7305 // else if not Top
7306 // Look for <0, N+1, 2, N+3, 4, N+5, ..>
7307 // This inserts Input1 into Input2
7308 unsigned Offset = Top ? 0 : 1;
7309 unsigned N = SingleSource ? 0 : NumElts;
7310 for (unsigned i = 0; i < NumElts; i += 2) {
7311 if (M[i] >= 0 && M[i] != (int)i)
7312 return false;
7313 if (M[i + 1] >= 0 && M[i + 1] != (int)(N + i + Offset))
7314 return false;
7315 }
7316
7317 return true;
7318}
7319
7320static bool isVMOVNTruncMask(ArrayRef<int> M, EVT ToVT, bool rev) {
7321 unsigned NumElts = ToVT.getVectorNumElements();
7322 if (NumElts != M.size())
7323 return false;
7324
7325 // Test if the Trunc can be convertible to a VMOVN with this shuffle. We are
7326 // looking for patterns of:
7327 // !rev: 0 N/2 1 N/2+1 2 N/2+2 ...
7328 // rev: N/2 0 N/2+1 1 N/2+2 2 ...
7329
7330 unsigned Off0 = rev ? NumElts / 2 : 0;
7331 unsigned Off1 = rev ? 0 : NumElts / 2;
7332 for (unsigned i = 0; i < NumElts; i += 2) {
7333 if (M[i] >= 0 && M[i] != (int)(Off0 + i / 2))
7334 return false;
7335 if (M[i + 1] >= 0 && M[i + 1] != (int)(Off1 + i / 2))
7336 return false;
7337 }
7338
7339 return true;
7340}
7341
7342// Reconstruct an MVE VCVT from a BuildVector of scalar fptrunc, all extracted
7343// from a pair of inputs. For example:
7344// BUILDVECTOR(FP_ROUND(EXTRACT_ELT(X, 0),
7345// FP_ROUND(EXTRACT_ELT(Y, 0),
7346// FP_ROUND(EXTRACT_ELT(X, 1),
7347// FP_ROUND(EXTRACT_ELT(Y, 1), ...)
7348static SDValue LowerBuildVectorOfFPTrunc(SDValue BV, SelectionDAG &DAG,
7349 const ARMSubtarget *ST) {
7350 assert(BV.getOpcode() == ISD::BUILD_VECTOR && "Unknown opcode!");
7351 if (!ST->hasMVEFloatOps())
7352 return SDValue();
7353
7354 SDLoc dl(BV);
7355 EVT VT = BV.getValueType();
7356 if (VT != MVT::v8f16)
7357 return SDValue();
7358
7359 // We are looking for a buildvector of fptrunc elements, where all the
7360 // elements are interleavingly extracted from two sources. Check the first two
7361 // items are valid enough and extract some info from them (they are checked
7362 // properly in the loop below).
7363 if (BV.getOperand(i: 0).getOpcode() != ISD::FP_ROUND ||
7364 BV.getOperand(i: 0).getOperand(i: 0).getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
7365 BV.getOperand(i: 0).getOperand(i: 0).getConstantOperandVal(i: 1) != 0)
7366 return SDValue();
7367 if (BV.getOperand(i: 1).getOpcode() != ISD::FP_ROUND ||
7368 BV.getOperand(i: 1).getOperand(i: 0).getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
7369 BV.getOperand(i: 1).getOperand(i: 0).getConstantOperandVal(i: 1) != 0)
7370 return SDValue();
7371 SDValue Op0 = BV.getOperand(i: 0).getOperand(i: 0).getOperand(i: 0);
7372 SDValue Op1 = BV.getOperand(i: 1).getOperand(i: 0).getOperand(i: 0);
7373 if (Op0.getValueType() != MVT::v4f32 || Op1.getValueType() != MVT::v4f32)
7374 return SDValue();
7375
7376 // Check all the values in the BuildVector line up with our expectations.
7377 for (unsigned i = 1; i < 4; i++) {
7378 auto Check = [](SDValue Trunc, SDValue Op, unsigned Idx) {
7379 return Trunc.getOpcode() == ISD::FP_ROUND &&
7380 Trunc.getOperand(i: 0).getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
7381 Trunc.getOperand(i: 0).getOperand(i: 0) == Op &&
7382 Trunc.getOperand(i: 0).getConstantOperandVal(i: 1) == Idx;
7383 };
7384 if (!Check(BV.getOperand(i: i * 2 + 0), Op0, i))
7385 return SDValue();
7386 if (!Check(BV.getOperand(i: i * 2 + 1), Op1, i))
7387 return SDValue();
7388 }
7389
7390 SDValue N1 = DAG.getNode(Opcode: ARMISD::VCVTN, DL: dl, VT, N1: DAG.getUNDEF(VT), N2: Op0,
7391 N3: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
7392 return DAG.getNode(Opcode: ARMISD::VCVTN, DL: dl, VT, N1, N2: Op1,
7393 N3: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
7394}
7395
7396// Reconstruct an MVE VCVT from a BuildVector of scalar fpext, all extracted
7397// from a single input on alternating lanes. For example:
7398// BUILDVECTOR(FP_ROUND(EXTRACT_ELT(X, 0),
7399// FP_ROUND(EXTRACT_ELT(X, 2),
7400// FP_ROUND(EXTRACT_ELT(X, 4), ...)
7401static SDValue LowerBuildVectorOfFPExt(SDValue BV, SelectionDAG &DAG,
7402 const ARMSubtarget *ST) {
7403 assert(BV.getOpcode() == ISD::BUILD_VECTOR && "Unknown opcode!");
7404 if (!ST->hasMVEFloatOps())
7405 return SDValue();
7406
7407 SDLoc dl(BV);
7408 EVT VT = BV.getValueType();
7409 if (VT != MVT::v4f32)
7410 return SDValue();
7411
7412 // We are looking for a buildvector of fptext elements, where all the
7413 // elements are alternating lanes from a single source. For example <0,2,4,6>
7414 // or <1,3,5,7>. Check the first two items are valid enough and extract some
7415 // info from them (they are checked properly in the loop below).
7416 if (BV.getOperand(i: 0).getOpcode() != ISD::FP_EXTEND ||
7417 BV.getOperand(i: 0).getOperand(i: 0).getOpcode() != ISD::EXTRACT_VECTOR_ELT)
7418 return SDValue();
7419 SDValue Op0 = BV.getOperand(i: 0).getOperand(i: 0).getOperand(i: 0);
7420 int Offset = BV.getOperand(i: 0).getOperand(i: 0).getConstantOperandVal(i: 1);
7421 if (Op0.getValueType() != MVT::v8f16 || (Offset != 0 && Offset != 1))
7422 return SDValue();
7423
7424 // Check all the values in the BuildVector line up with our expectations.
7425 for (unsigned i = 1; i < 4; i++) {
7426 auto Check = [](SDValue Trunc, SDValue Op, unsigned Idx) {
7427 return Trunc.getOpcode() == ISD::FP_EXTEND &&
7428 Trunc.getOperand(i: 0).getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
7429 Trunc.getOperand(i: 0).getOperand(i: 0) == Op &&
7430 Trunc.getOperand(i: 0).getConstantOperandVal(i: 1) == Idx;
7431 };
7432 if (!Check(BV.getOperand(i), Op0, 2 * i + Offset))
7433 return SDValue();
7434 }
7435
7436 return DAG.getNode(Opcode: ARMISD::VCVTL, DL: dl, VT, N1: Op0,
7437 N2: DAG.getConstant(Val: Offset, DL: dl, VT: MVT::i32));
7438}
7439
7440// If N is an integer constant that can be moved into a register in one
7441// instruction, return an SDValue of such a constant (will become a MOV
7442// instruction). Otherwise return null.
7443static SDValue IsSingleInstrConstant(SDValue N, SelectionDAG &DAG,
7444 const ARMSubtarget *ST, const SDLoc &dl) {
7445 uint64_t Val;
7446 if (!isa<ConstantSDNode>(Val: N))
7447 return SDValue();
7448 Val = N->getAsZExtVal();
7449
7450 if (ST->isThumb1Only()) {
7451 if (Val <= 255 || ~Val <= 255)
7452 return DAG.getConstant(Val, DL: dl, VT: MVT::i32);
7453 } else {
7454 if (ARM_AM::getSOImmVal(Arg: Val) != -1 || ARM_AM::getSOImmVal(Arg: ~Val) != -1)
7455 return DAG.getConstant(Val, DL: dl, VT: MVT::i32);
7456 }
7457 return SDValue();
7458}
7459
7460static SDValue LowerBUILD_VECTOR_i1(SDValue Op, SelectionDAG &DAG,
7461 const ARMSubtarget *ST) {
7462 SDLoc dl(Op);
7463 EVT VT = Op.getValueType();
7464
7465 assert(ST->hasMVEIntegerOps() && "LowerBUILD_VECTOR_i1 called without MVE!");
7466
7467 unsigned NumElts = VT.getVectorNumElements();
7468 unsigned BoolMask;
7469 unsigned BitsPerBool;
7470 if (NumElts == 2) {
7471 BitsPerBool = 8;
7472 BoolMask = 0xff;
7473 } else if (NumElts == 4) {
7474 BitsPerBool = 4;
7475 BoolMask = 0xf;
7476 } else if (NumElts == 8) {
7477 BitsPerBool = 2;
7478 BoolMask = 0x3;
7479 } else if (NumElts == 16) {
7480 BitsPerBool = 1;
7481 BoolMask = 0x1;
7482 } else
7483 return SDValue();
7484
7485 // If this is a single value copied into all lanes (a splat), we can just sign
7486 // extend that single value
7487 SDValue FirstOp = Op.getOperand(i: 0);
7488 if (!isa<ConstantSDNode>(Val: FirstOp) &&
7489 llvm::all_of(Range: llvm::drop_begin(RangeOrContainer: Op->ops()), P: [&FirstOp](const SDUse &U) {
7490 return U.get().isUndef() || U.get() == FirstOp;
7491 })) {
7492 SDValue Ext = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl, VT: MVT::i32, N1: FirstOp,
7493 N2: DAG.getValueType(MVT::i1));
7494 return DAG.getNode(Opcode: ARMISD::PREDICATE_CAST, DL: dl, VT: Op.getValueType(), Operand: Ext);
7495 }
7496
7497 // First create base with bits set where known
7498 unsigned Bits32 = 0;
7499 for (unsigned i = 0; i < NumElts; ++i) {
7500 SDValue V = Op.getOperand(i);
7501 if (!isa<ConstantSDNode>(Val: V) && !V.isUndef())
7502 continue;
7503 bool BitSet = V.isUndef() ? false : V->getAsZExtVal();
7504 if (BitSet)
7505 Bits32 |= BoolMask << (i * BitsPerBool);
7506 }
7507
7508 // Add in unknown nodes
7509 SDValue Base = DAG.getNode(Opcode: ARMISD::PREDICATE_CAST, DL: dl, VT,
7510 Operand: DAG.getConstant(Val: Bits32, DL: dl, VT: MVT::i32));
7511 for (unsigned i = 0; i < NumElts; ++i) {
7512 SDValue V = Op.getOperand(i);
7513 if (isa<ConstantSDNode>(Val: V) || V.isUndef())
7514 continue;
7515 Base = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT, N1: Base, N2: V,
7516 N3: DAG.getConstant(Val: i, DL: dl, VT: MVT::i32));
7517 }
7518
7519 return Base;
7520}
7521
7522static SDValue LowerBUILD_VECTORToVIDUP(SDValue Op, SelectionDAG &DAG,
7523 const ARMSubtarget *ST) {
7524 if (!ST->hasMVEIntegerOps())
7525 return SDValue();
7526
7527 // We are looking for a buildvector where each element is Op[0] + i*N
7528 EVT VT = Op.getValueType();
7529 SDValue Op0 = Op.getOperand(i: 0);
7530 unsigned NumElts = VT.getVectorNumElements();
7531
7532 // Get the increment value from operand 1
7533 SDValue Op1 = Op.getOperand(i: 1);
7534 if (Op1.getOpcode() != ISD::ADD || Op1.getOperand(i: 0) != Op0 ||
7535 !isa<ConstantSDNode>(Val: Op1.getOperand(i: 1)))
7536 return SDValue();
7537 unsigned N = Op1.getConstantOperandVal(i: 1);
7538 if (N != 1 && N != 2 && N != 4 && N != 8)
7539 return SDValue();
7540
7541 // Check that each other operand matches
7542 for (unsigned I = 2; I < NumElts; I++) {
7543 SDValue OpI = Op.getOperand(i: I);
7544 if (OpI.getOpcode() != ISD::ADD || OpI.getOperand(i: 0) != Op0 ||
7545 !isa<ConstantSDNode>(Val: OpI.getOperand(i: 1)) ||
7546 OpI.getConstantOperandVal(i: 1) != I * N)
7547 return SDValue();
7548 }
7549
7550 SDLoc DL(Op);
7551 return DAG.getNode(Opcode: ARMISD::VIDUP, DL, VTList: DAG.getVTList(VT1: VT, VT2: MVT::i32), N1: Op0,
7552 N2: DAG.getConstant(Val: N, DL, VT: MVT::i32));
7553}
7554
7555// Returns true if the operation N can be treated as qr instruction variant at
7556// operand Op.
7557static bool IsQRMVEInstruction(const SDNode *N, const SDNode *Op) {
7558 switch (N->getOpcode()) {
7559 case ISD::ADD:
7560 case ISD::MUL:
7561 case ISD::SADDSAT:
7562 case ISD::UADDSAT:
7563 case ISD::AVGFLOORS:
7564 case ISD::AVGFLOORU:
7565 return true;
7566 case ISD::SUB:
7567 case ISD::SSUBSAT:
7568 case ISD::USUBSAT:
7569 return N->getOperand(Num: 1).getNode() == Op;
7570 case ISD::INTRINSIC_WO_CHAIN:
7571 switch (N->getConstantOperandVal(Num: 0)) {
7572 case Intrinsic::arm_mve_add_predicated:
7573 case Intrinsic::arm_mve_mul_predicated:
7574 case Intrinsic::arm_mve_qadd_predicated:
7575 case Intrinsic::arm_mve_vhadd:
7576 case Intrinsic::arm_mve_hadd_predicated:
7577 case Intrinsic::arm_mve_vqdmulh:
7578 case Intrinsic::arm_mve_qdmulh_predicated:
7579 case Intrinsic::arm_mve_vqrdmulh:
7580 case Intrinsic::arm_mve_qrdmulh_predicated:
7581 case Intrinsic::arm_mve_vqdmull:
7582 case Intrinsic::arm_mve_vqdmull_predicated:
7583 return true;
7584 case Intrinsic::arm_mve_sub_predicated:
7585 case Intrinsic::arm_mve_qsub_predicated:
7586 case Intrinsic::arm_mve_vhsub:
7587 case Intrinsic::arm_mve_hsub_predicated:
7588 return N->getOperand(Num: 2).getNode() == Op;
7589 default:
7590 return false;
7591 }
7592 default:
7593 return false;
7594 }
7595}
7596
7597// If this is a case we can't handle, return null and let the default
7598// expansion code take care of it.
7599SDValue ARMTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
7600 const ARMSubtarget *ST) const {
7601 BuildVectorSDNode *BVN = cast<BuildVectorSDNode>(Val: Op.getNode());
7602 SDLoc dl(Op);
7603 EVT VT = Op.getValueType();
7604
7605 if (ST->hasMVEIntegerOps() && VT.getScalarSizeInBits() == 1)
7606 return LowerBUILD_VECTOR_i1(Op, DAG, ST);
7607
7608 if (SDValue R = LowerBUILD_VECTORToVIDUP(Op, DAG, ST))
7609 return R;
7610
7611 APInt SplatBits, SplatUndef;
7612 unsigned SplatBitSize;
7613 bool HasAnyUndefs;
7614 if (BVN->isConstantSplat(SplatValue&: SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
7615 if (SplatUndef.isAllOnes())
7616 return DAG.getUNDEF(VT);
7617
7618 // If all the users of this constant splat are qr instruction variants,
7619 // generate a vdup of the constant.
7620 if (ST->hasMVEIntegerOps() && VT.getScalarSizeInBits() == SplatBitSize &&
7621 (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32) &&
7622 all_of(Range: BVN->users(),
7623 P: [BVN](const SDNode *U) { return IsQRMVEInstruction(N: U, Op: BVN); })) {
7624 EVT DupVT = SplatBitSize == 32 ? MVT::v4i32
7625 : SplatBitSize == 16 ? MVT::v8i16
7626 : MVT::v16i8;
7627 SDValue Const = DAG.getConstant(Val: SplatBits.getZExtValue(), DL: dl, VT: MVT::i32);
7628 SDValue VDup = DAG.getNode(Opcode: ARMISD::VDUP, DL: dl, VT: DupVT, Operand: Const);
7629 return DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT, Operand: VDup);
7630 }
7631
7632 if ((ST->hasNEON() && SplatBitSize <= 64) ||
7633 (ST->hasMVEIntegerOps() && SplatBitSize <= 64)) {
7634 // Check if an immediate VMOV works.
7635 EVT VmovVT;
7636 SDValue Val =
7637 isVMOVModifiedImm(SplatBits: SplatBits.getZExtValue(), SplatUndef: SplatUndef.getZExtValue(),
7638 SplatBitSize, DAG, dl, VT&: VmovVT, VectorVT: VT, type: VMOVModImm);
7639
7640 if (Val.getNode()) {
7641 SDValue Vmov = DAG.getNode(Opcode: ARMISD::VMOVIMM, DL: dl, VT: VmovVT, Operand: Val);
7642 return DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT, Operand: Vmov);
7643 }
7644
7645 // Try an immediate VMVN.
7646 uint64_t NegatedImm = (~SplatBits).getZExtValue();
7647 Val = isVMOVModifiedImm(
7648 SplatBits: NegatedImm, SplatUndef: SplatUndef.getZExtValue(), SplatBitSize, DAG, dl, VT&: VmovVT,
7649 VectorVT: VT, type: ST->hasMVEIntegerOps() ? MVEVMVNModImm : VMVNModImm);
7650 if (Val.getNode()) {
7651 SDValue Vmov = DAG.getNode(Opcode: ARMISD::VMVNIMM, DL: dl, VT: VmovVT, Operand: Val);
7652 return DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT, Operand: Vmov);
7653 }
7654
7655 // Use vmov.f32 to materialize other v2f32 and v4f32 splats.
7656 if ((VT == MVT::v2f32 || VT == MVT::v4f32) && SplatBitSize == 32) {
7657 int ImmVal = ARM_AM::getFP32Imm(Imm: SplatBits);
7658 if (ImmVal != -1) {
7659 SDValue Val = DAG.getTargetConstant(Val: ImmVal, DL: dl, VT: MVT::i32);
7660 return DAG.getNode(Opcode: ARMISD::VMOVFPIMM, DL: dl, VT, Operand: Val);
7661 }
7662 }
7663
7664 // If we are under MVE, generate a VDUP(constant), bitcast to the original
7665 // type.
7666 if (ST->hasMVEIntegerOps() &&
7667 (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32)) {
7668 EVT DupVT = SplatBitSize == 32 ? MVT::v4i32
7669 : SplatBitSize == 16 ? MVT::v8i16
7670 : MVT::v16i8;
7671 SDValue Const = DAG.getConstant(Val: SplatBits.getZExtValue(), DL: dl, VT: MVT::i32);
7672 SDValue VDup = DAG.getNode(Opcode: ARMISD::VDUP, DL: dl, VT: DupVT, Operand: Const);
7673 return DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT, Operand: VDup);
7674 }
7675 }
7676 }
7677
7678 // Scan through the operands to see if only one value is used.
7679 //
7680 // As an optimisation, even if more than one value is used it may be more
7681 // profitable to splat with one value then change some lanes.
7682 //
7683 // Heuristically we decide to do this if the vector has a "dominant" value,
7684 // defined as splatted to more than half of the lanes.
7685 unsigned NumElts = VT.getVectorNumElements();
7686 bool isOnlyLowElement = true;
7687 bool usesOnlyOneValue = true;
7688 bool hasDominantValue = false;
7689 bool isConstant = true;
7690
7691 // Map of the number of times a particular SDValue appears in the
7692 // element list.
7693 DenseMap<SDValue, unsigned> ValueCounts;
7694 SDValue Value;
7695 for (unsigned i = 0; i < NumElts; ++i) {
7696 SDValue V = Op.getOperand(i);
7697 if (V.isUndef())
7698 continue;
7699 if (i > 0)
7700 isOnlyLowElement = false;
7701 if (!isa<ConstantFPSDNode>(Val: V) && !isa<ConstantSDNode>(Val: V))
7702 isConstant = false;
7703
7704 unsigned &Count = ValueCounts[V];
7705
7706 // Is this value dominant? (takes up more than half of the lanes)
7707 if (++Count > (NumElts / 2)) {
7708 hasDominantValue = true;
7709 Value = V;
7710 }
7711 }
7712 if (ValueCounts.size() != 1)
7713 usesOnlyOneValue = false;
7714 if (!Value.getNode() && !ValueCounts.empty())
7715 Value = ValueCounts.begin()->first;
7716
7717 if (ValueCounts.empty())
7718 return DAG.getUNDEF(VT);
7719
7720 // Loads are better lowered with insert_vector_elt/ARMISD::BUILD_VECTOR.
7721 // Keep going if we are hitting this case.
7722 if (isOnlyLowElement && !ISD::isNormalLoad(N: Value.getNode()) &&
7723 (VT != MVT::v8f16 || ST->hasFullFP16()))
7724 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT, Operand: Value);
7725
7726 unsigned EltSize = VT.getScalarSizeInBits();
7727
7728 // Use VDUP for non-constant splats. For f32 constant splats, reduce to
7729 // i32 and try again.
7730 if (hasDominantValue && EltSize <= 32) {
7731 if (!isConstant) {
7732 SDValue N;
7733
7734 // If we are VDUPing a value that comes directly from a vector, that will
7735 // cause an unnecessary move to and from a GPR, where instead we could
7736 // just use VDUPLANE. We can only do this if the lane being extracted
7737 // is at a constant index, as the VDUP from lane instructions only have
7738 // constant-index forms.
7739 ConstantSDNode *constIndex;
7740 if (Value->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
7741 (constIndex = dyn_cast<ConstantSDNode>(Val: Value->getOperand(Num: 1)))) {
7742 // We need to create a new undef vector to use for the VDUPLANE if the
7743 // size of the vector from which we get the value is different than the
7744 // size of the vector that we need to create. We will insert the element
7745 // such that the register coalescer will remove unnecessary copies.
7746 if (VT != Value->getOperand(Num: 0).getValueType()) {
7747 unsigned index = constIndex->getAPIntValue().getLimitedValue() %
7748 VT.getVectorNumElements();
7749 N = DAG.getNode(Opcode: ARMISD::VDUPLANE, DL: dl, VT,
7750 N1: DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT, N1: DAG.getUNDEF(VT),
7751 N2: Value, N3: DAG.getConstant(Val: index, DL: dl, VT: MVT::i32)),
7752 N2: DAG.getConstant(Val: index, DL: dl, VT: MVT::i32));
7753 } else
7754 N = DAG.getNode(Opcode: ARMISD::VDUPLANE, DL: dl, VT,
7755 N1: Value->getOperand(Num: 0), N2: Value->getOperand(Num: 1));
7756 } else
7757 N = DAG.getNode(Opcode: ARMISD::VDUP, DL: dl, VT, Operand: Value);
7758
7759 if (!usesOnlyOneValue) {
7760 // The dominant value was splatted as 'N', but we now have to insert
7761 // all differing elements.
7762 for (unsigned I = 0; I < NumElts; ++I) {
7763 if (Op.getOperand(i: I) == Value)
7764 continue;
7765 SmallVector<SDValue, 3> Ops;
7766 Ops.push_back(Elt: N);
7767 Ops.push_back(Elt: Op.getOperand(i: I));
7768 Ops.push_back(Elt: DAG.getConstant(Val: I, DL: dl, VT: MVT::i32));
7769 N = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT, Ops);
7770 }
7771 }
7772 return N;
7773 }
7774 if (VT.getVectorElementType().isFloatingPoint()) {
7775 SmallVector<SDValue, 8> Ops;
7776 MVT FVT = VT.getVectorElementType().getSimpleVT();
7777 assert(FVT == MVT::f32 || FVT == MVT::f16);
7778 MVT IVT = (FVT == MVT::f32) ? MVT::i32 : MVT::i16;
7779 for (unsigned i = 0; i < NumElts; ++i)
7780 Ops.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: IVT,
7781 Operand: Op.getOperand(i)));
7782 EVT VecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: IVT, NumElements: NumElts);
7783 SDValue Val = DAG.getBuildVector(VT: VecVT, DL: dl, Ops);
7784 Val = LowerBUILD_VECTOR(Op: Val, DAG, ST);
7785 if (Val.getNode())
7786 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: Val);
7787 }
7788 if (usesOnlyOneValue) {
7789 SDValue Val = IsSingleInstrConstant(N: Value, DAG, ST, dl);
7790 if (isConstant && Val.getNode())
7791 return DAG.getNode(Opcode: ARMISD::VDUP, DL: dl, VT, Operand: Val);
7792 }
7793 }
7794
7795 // If all elements are constants and the case above didn't get hit, fall back
7796 // to the default expansion, which will generate a load from the constant
7797 // pool.
7798 if (isConstant)
7799 return SDValue();
7800
7801 // Reconstruct the BUILDVECTOR to one of the legal shuffles (such as vext and
7802 // vmovn). Empirical tests suggest this is rarely worth it for vectors of
7803 // length <= 2.
7804 if (NumElts >= 4)
7805 if (SDValue shuffle = ReconstructShuffle(Op, DAG))
7806 return shuffle;
7807
7808 // Attempt to turn a buildvector of scalar fptrunc's or fpext's back into
7809 // VCVT's
7810 if (SDValue VCVT = LowerBuildVectorOfFPTrunc(BV: Op, DAG, ST: Subtarget))
7811 return VCVT;
7812 if (SDValue VCVT = LowerBuildVectorOfFPExt(BV: Op, DAG, ST: Subtarget))
7813 return VCVT;
7814
7815 if (ST->hasNEON() && VT.is128BitVector() && VT != MVT::v2f64 && VT != MVT::v4f32) {
7816 // If we haven't found an efficient lowering, try splitting a 128-bit vector
7817 // into two 64-bit vectors; we might discover a better way to lower it.
7818 SmallVector<SDValue, 64> Ops(Op->op_begin(), Op->op_begin() + NumElts);
7819 EVT ExtVT = VT.getVectorElementType();
7820 EVT HVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ExtVT, NumElements: NumElts / 2);
7821 SDValue Lower = DAG.getBuildVector(VT: HVT, DL: dl, Ops: ArrayRef(&Ops[0], NumElts / 2));
7822 if (Lower.getOpcode() == ISD::BUILD_VECTOR)
7823 Lower = LowerBUILD_VECTOR(Op: Lower, DAG, ST);
7824 SDValue Upper =
7825 DAG.getBuildVector(VT: HVT, DL: dl, Ops: ArrayRef(&Ops[NumElts / 2], NumElts / 2));
7826 if (Upper.getOpcode() == ISD::BUILD_VECTOR)
7827 Upper = LowerBUILD_VECTOR(Op: Upper, DAG, ST);
7828 if (Lower && Upper)
7829 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT, N1: Lower, N2: Upper);
7830 }
7831
7832 // Vectors with 32- or 64-bit elements can be built by directly assigning
7833 // the subregisters. Lower it to an ARMISD::BUILD_VECTOR so the operands
7834 // will be legalized.
7835 if (EltSize >= 32) {
7836 // Do the expansion with floating-point types, since that is what the VFP
7837 // registers are defined to use, and since i64 is not legal.
7838 EVT EltVT = EVT::getFloatingPointVT(BitWidth: EltSize);
7839 EVT VecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, NumElements: NumElts);
7840 SmallVector<SDValue, 8> Ops;
7841 for (unsigned i = 0; i < NumElts; ++i)
7842 Ops.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: EltVT, Operand: Op.getOperand(i)));
7843 SDValue Val = DAG.getNode(Opcode: ARMISD::BUILD_VECTOR, DL: dl, VT: VecVT, Ops);
7844 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: Val);
7845 }
7846
7847 // If all else fails, just use a sequence of INSERT_VECTOR_ELT when we
7848 // know the default expansion would otherwise fall back on something even
7849 // worse. For a vector with one or two non-undef values, that's
7850 // scalar_to_vector for the elements followed by a shuffle (provided the
7851 // shuffle is valid for the target) and materialization element by element
7852 // on the stack followed by a load for everything else.
7853 if ((!isConstant && !usesOnlyOneValue) ||
7854 (VT == MVT::v8f16 && !ST->hasFullFP16())) {
7855 SDValue Vec = DAG.getUNDEF(VT);
7856 for (unsigned i = 0 ; i < NumElts; ++i) {
7857 SDValue V = Op.getOperand(i);
7858 if (V.isUndef())
7859 continue;
7860 SDValue LaneIdx = DAG.getConstant(Val: i, DL: dl, VT: MVT::i32);
7861 Vec = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT, N1: Vec, N2: V, N3: LaneIdx);
7862 }
7863 return Vec;
7864 }
7865
7866 return SDValue();
7867}
7868
7869// Gather data to see if the operation can be modelled as a
7870// shuffle in combination with VEXTs.
7871SDValue ARMTargetLowering::ReconstructShuffle(SDValue Op,
7872 SelectionDAG &DAG) const {
7873 assert(Op.getOpcode() == ISD::BUILD_VECTOR && "Unknown opcode!");
7874 SDLoc dl(Op);
7875 EVT VT = Op.getValueType();
7876 unsigned NumElts = VT.getVectorNumElements();
7877
7878 struct ShuffleSourceInfo {
7879 SDValue Vec;
7880 unsigned MinElt = std::numeric_limits<unsigned>::max();
7881 unsigned MaxElt = 0;
7882
7883 // We may insert some combination of BITCASTs and VEXT nodes to force Vec to
7884 // be compatible with the shuffle we intend to construct. As a result
7885 // ShuffleVec will be some sliding window into the original Vec.
7886 SDValue ShuffleVec;
7887
7888 // Code should guarantee that element i in Vec starts at element "WindowBase
7889 // + i * WindowScale in ShuffleVec".
7890 int WindowBase = 0;
7891 int WindowScale = 1;
7892
7893 ShuffleSourceInfo(SDValue Vec) : Vec(Vec), ShuffleVec(Vec) {}
7894
7895 bool operator ==(SDValue OtherVec) { return Vec == OtherVec; }
7896 };
7897
7898 // First gather all vectors used as an immediate source for this BUILD_VECTOR
7899 // node.
7900 SmallVector<ShuffleSourceInfo, 2> Sources;
7901 for (unsigned i = 0; i < NumElts; ++i) {
7902 SDValue V = Op.getOperand(i);
7903 if (V.isUndef())
7904 continue;
7905 else if (V.getOpcode() != ISD::EXTRACT_VECTOR_ELT) {
7906 // A shuffle can only come from building a vector from various
7907 // elements of other vectors.
7908 return SDValue();
7909 } else if (!isa<ConstantSDNode>(Val: V.getOperand(i: 1))) {
7910 // Furthermore, shuffles require a constant mask, whereas extractelts
7911 // accept variable indices.
7912 return SDValue();
7913 }
7914
7915 // Add this element source to the list if it's not already there.
7916 SDValue SourceVec = V.getOperand(i: 0);
7917 auto Source = llvm::find(Range&: Sources, Val: SourceVec);
7918 if (Source == Sources.end())
7919 Source = Sources.insert(I: Sources.end(), Elt: ShuffleSourceInfo(SourceVec));
7920
7921 // Update the minimum and maximum lane number seen.
7922 unsigned EltNo = V.getConstantOperandVal(i: 1);
7923 Source->MinElt = std::min(a: Source->MinElt, b: EltNo);
7924 Source->MaxElt = std::max(a: Source->MaxElt, b: EltNo);
7925 }
7926
7927 // Currently only do something sane when at most two source vectors
7928 // are involved.
7929 if (Sources.size() > 2)
7930 return SDValue();
7931
7932 // Find out the smallest element size among result and two sources, and use
7933 // it as element size to build the shuffle_vector.
7934 EVT SmallestEltTy = VT.getVectorElementType();
7935 for (auto &Source : Sources) {
7936 EVT SrcEltTy = Source.Vec.getValueType().getVectorElementType();
7937 if (SrcEltTy.bitsLT(VT: SmallestEltTy))
7938 SmallestEltTy = SrcEltTy;
7939 }
7940 unsigned ResMultiplier =
7941 VT.getScalarSizeInBits() / SmallestEltTy.getSizeInBits();
7942 NumElts = VT.getSizeInBits() / SmallestEltTy.getSizeInBits();
7943 EVT ShuffleVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: SmallestEltTy, NumElements: NumElts);
7944
7945 // If the source vector is too wide or too narrow, we may nevertheless be able
7946 // to construct a compatible shuffle either by concatenating it with UNDEF or
7947 // extracting a suitable range of elements.
7948 for (auto &Src : Sources) {
7949 EVT SrcVT = Src.ShuffleVec.getValueType();
7950
7951 uint64_t SrcVTSize = SrcVT.getFixedSizeInBits();
7952 uint64_t VTSize = VT.getFixedSizeInBits();
7953 if (SrcVTSize == VTSize)
7954 continue;
7955
7956 // This stage of the search produces a source with the same element type as
7957 // the original, but with a total width matching the BUILD_VECTOR output.
7958 EVT EltVT = SrcVT.getVectorElementType();
7959 unsigned NumSrcElts = VTSize / EltVT.getFixedSizeInBits();
7960 EVT DestVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, NumElements: NumSrcElts);
7961
7962 if (SrcVTSize < VTSize) {
7963 if (2 * SrcVTSize != VTSize)
7964 return SDValue();
7965 // We can pad out the smaller vector for free, so if it's part of a
7966 // shuffle...
7967 Src.ShuffleVec =
7968 DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: DestVT, N1: Src.ShuffleVec,
7969 N2: DAG.getUNDEF(VT: Src.ShuffleVec.getValueType()));
7970 continue;
7971 }
7972
7973 if (SrcVTSize != 2 * VTSize)
7974 return SDValue();
7975
7976 if (Src.MaxElt - Src.MinElt >= NumSrcElts) {
7977 // Span too large for a VEXT to cope
7978 return SDValue();
7979 }
7980
7981 if (Src.MinElt >= NumSrcElts) {
7982 // The extraction can just take the second half
7983 Src.ShuffleVec =
7984 DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: DestVT, N1: Src.ShuffleVec,
7985 N2: DAG.getConstant(Val: NumSrcElts, DL: dl, VT: MVT::i32));
7986 Src.WindowBase = -NumSrcElts;
7987 } else if (Src.MaxElt < NumSrcElts) {
7988 // The extraction can just take the first half
7989 Src.ShuffleVec =
7990 DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: DestVT, N1: Src.ShuffleVec,
7991 N2: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
7992 } else {
7993 // An actual VEXT is needed
7994 SDValue VEXTSrc1 =
7995 DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: DestVT, N1: Src.ShuffleVec,
7996 N2: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
7997 SDValue VEXTSrc2 =
7998 DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: DestVT, N1: Src.ShuffleVec,
7999 N2: DAG.getConstant(Val: NumSrcElts, DL: dl, VT: MVT::i32));
8000
8001 Src.ShuffleVec = DAG.getNode(Opcode: ARMISD::VEXT, DL: dl, VT: DestVT, N1: VEXTSrc1,
8002 N2: VEXTSrc2,
8003 N3: DAG.getConstant(Val: Src.MinElt, DL: dl, VT: MVT::i32));
8004 Src.WindowBase = -Src.MinElt;
8005 }
8006 }
8007
8008 // Another possible incompatibility occurs from the vector element types. We
8009 // can fix this by bitcasting the source vectors to the same type we intend
8010 // for the shuffle.
8011 for (auto &Src : Sources) {
8012 EVT SrcEltTy = Src.ShuffleVec.getValueType().getVectorElementType();
8013 if (SrcEltTy == SmallestEltTy)
8014 continue;
8015 assert(ShuffleVT.getVectorElementType() == SmallestEltTy);
8016 Src.ShuffleVec = DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT: ShuffleVT, Operand: Src.ShuffleVec);
8017 Src.WindowScale = SrcEltTy.getSizeInBits() / SmallestEltTy.getSizeInBits();
8018 Src.WindowBase *= Src.WindowScale;
8019 }
8020
8021 // Final check before we try to actually produce a shuffle.
8022 LLVM_DEBUG({
8023 for (auto Src : Sources)
8024 assert(Src.ShuffleVec.getValueType() == ShuffleVT);
8025 });
8026
8027 // The stars all align, our next step is to produce the mask for the shuffle.
8028 SmallVector<int, 8> Mask(ShuffleVT.getVectorNumElements(), -1);
8029 int BitsPerShuffleLane = ShuffleVT.getScalarSizeInBits();
8030 for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) {
8031 SDValue Entry = Op.getOperand(i);
8032 if (Entry.isUndef())
8033 continue;
8034
8035 auto Src = llvm::find(Range&: Sources, Val: Entry.getOperand(i: 0));
8036 int EltNo = cast<ConstantSDNode>(Val: Entry.getOperand(i: 1))->getSExtValue();
8037
8038 // EXTRACT_VECTOR_ELT performs an implicit any_ext; BUILD_VECTOR an implicit
8039 // trunc. So only std::min(SrcBits, DestBits) actually get defined in this
8040 // segment.
8041 EVT OrigEltTy = Entry.getOperand(i: 0).getValueType().getVectorElementType();
8042 int BitsDefined = std::min(a: OrigEltTy.getScalarSizeInBits(),
8043 b: VT.getScalarSizeInBits());
8044 int LanesDefined = BitsDefined / BitsPerShuffleLane;
8045
8046 // This source is expected to fill ResMultiplier lanes of the final shuffle,
8047 // starting at the appropriate offset.
8048 int *LaneMask = &Mask[i * ResMultiplier];
8049
8050 int ExtractBase = EltNo * Src->WindowScale + Src->WindowBase;
8051 ExtractBase += NumElts * (Src - Sources.begin());
8052 for (int j = 0; j < LanesDefined; ++j)
8053 LaneMask[j] = ExtractBase + j;
8054 }
8055
8056
8057 // We can't handle more than two sources. This should have already
8058 // been checked before this point.
8059 assert(Sources.size() <= 2 && "Too many sources!");
8060
8061 SDValue ShuffleOps[] = { DAG.getUNDEF(VT: ShuffleVT), DAG.getUNDEF(VT: ShuffleVT) };
8062 for (unsigned i = 0; i < Sources.size(); ++i)
8063 ShuffleOps[i] = Sources[i].ShuffleVec;
8064
8065 SDValue Shuffle = buildLegalVectorShuffle(VT: ShuffleVT, DL: dl, N0: ShuffleOps[0],
8066 N1: ShuffleOps[1], Mask, DAG);
8067 if (!Shuffle)
8068 return SDValue();
8069 return DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT, Operand: Shuffle);
8070}
8071
8072enum ShuffleOpCodes {
8073 OP_COPY = 0, // Copy, used for things like <u,u,u,3> to say it is <0,1,2,3>
8074 OP_VREV,
8075 OP_VDUP0,
8076 OP_VDUP1,
8077 OP_VDUP2,
8078 OP_VDUP3,
8079 OP_VEXT1,
8080 OP_VEXT2,
8081 OP_VEXT3,
8082 OP_VUZPL, // VUZP, left result
8083 OP_VUZPR, // VUZP, right result
8084 OP_VZIPL, // VZIP, left result
8085 OP_VZIPR, // VZIP, right result
8086 OP_VTRNL, // VTRN, left result
8087 OP_VTRNR // VTRN, right result
8088};
8089
8090static bool isLegalMVEShuffleOp(unsigned PFEntry) {
8091 unsigned OpNum = (PFEntry >> 26) & 0x0F;
8092 switch (OpNum) {
8093 case OP_COPY:
8094 case OP_VREV:
8095 case OP_VDUP0:
8096 case OP_VDUP1:
8097 case OP_VDUP2:
8098 case OP_VDUP3:
8099 return true;
8100 }
8101 return false;
8102}
8103
8104/// isShuffleMaskLegal - Targets can use this to indicate that they only
8105/// support *some* VECTOR_SHUFFLE operations, those with specific masks.
8106/// By default, if a target supports the VECTOR_SHUFFLE node, all mask values
8107/// are assumed to be legal.
8108bool ARMTargetLowering::isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const {
8109 if (VT.getVectorNumElements() == 4 &&
8110 (VT.is128BitVector() || VT.is64BitVector())) {
8111 unsigned PFIndexes[4];
8112 for (unsigned i = 0; i != 4; ++i) {
8113 if (M[i] < 0)
8114 PFIndexes[i] = 8;
8115 else
8116 PFIndexes[i] = M[i];
8117 }
8118
8119 // Compute the index in the perfect shuffle table.
8120 unsigned PFTableIndex =
8121 PFIndexes[0]*9*9*9+PFIndexes[1]*9*9+PFIndexes[2]*9+PFIndexes[3];
8122 unsigned PFEntry = PerfectShuffleTable[PFTableIndex];
8123 unsigned Cost = (PFEntry >> 30);
8124
8125 if (Cost <= 4 && (Subtarget->hasNEON() || isLegalMVEShuffleOp(PFEntry)))
8126 return true;
8127 }
8128
8129 bool ReverseVEXT, isV_UNDEF;
8130 unsigned Imm, WhichResult;
8131
8132 unsigned EltSize = VT.getScalarSizeInBits();
8133 if (EltSize >= 32 ||
8134 ShuffleVectorSDNode::isSplatMask(Mask: M) ||
8135 ShuffleVectorInst::isIdentityMask(Mask: M, NumSrcElts: M.size()) ||
8136 isVREVMask(M, VT, BlockSize: 64) ||
8137 isVREVMask(M, VT, BlockSize: 32) ||
8138 isVREVMask(M, VT, BlockSize: 16))
8139 return true;
8140 else if (Subtarget->hasNEON() &&
8141 (isVEXTMask(M, VT, ReverseVEXT, Imm) ||
8142 isVTBLMask(M, VT) ||
8143 isNEONTwoResultShuffleMask(ShuffleMask: M, VT, WhichResult, isV_UNDEF)))
8144 return true;
8145 else if ((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) &&
8146 isReverseMask(M, VT))
8147 return true;
8148 else if (Subtarget->hasMVEIntegerOps() &&
8149 (isVMOVNMask(M, VT, Top: true, SingleSource: false) ||
8150 isVMOVNMask(M, VT, Top: false, SingleSource: false) || isVMOVNMask(M, VT, Top: true, SingleSource: true)))
8151 return true;
8152 else if (Subtarget->hasMVEIntegerOps() &&
8153 (isTruncMask(M, VT, Top: false, SingleSource: false) ||
8154 isTruncMask(M, VT, Top: false, SingleSource: true) ||
8155 isTruncMask(M, VT, Top: true, SingleSource: false) || isTruncMask(M, VT, Top: true, SingleSource: true)))
8156 return true;
8157 else
8158 return false;
8159}
8160
8161/// GeneratePerfectShuffle - Given an entry in the perfect-shuffle table, emit
8162/// the specified operations to build the shuffle.
8163static SDValue GeneratePerfectShuffle(unsigned PFEntry, SDValue LHS,
8164 SDValue RHS, SelectionDAG &DAG,
8165 const SDLoc &dl) {
8166 unsigned OpNum = (PFEntry >> 26) & 0x0F;
8167 unsigned LHSID = (PFEntry >> 13) & ((1 << 13)-1);
8168 unsigned RHSID = (PFEntry >> 0) & ((1 << 13)-1);
8169
8170 if (OpNum == OP_COPY) {
8171 if (LHSID == (1*9+2)*9+3) return LHS;
8172 assert(LHSID == ((4*9+5)*9+6)*9+7 && "Illegal OP_COPY!");
8173 return RHS;
8174 }
8175
8176 SDValue OpLHS, OpRHS;
8177 OpLHS = GeneratePerfectShuffle(PFEntry: PerfectShuffleTable[LHSID], LHS, RHS, DAG, dl);
8178 OpRHS = GeneratePerfectShuffle(PFEntry: PerfectShuffleTable[RHSID], LHS, RHS, DAG, dl);
8179 EVT VT = OpLHS.getValueType();
8180
8181 switch (OpNum) {
8182 default: llvm_unreachable("Unknown shuffle opcode!");
8183 case OP_VREV:
8184 // VREV divides the vector in half and swaps within the half.
8185 if (VT.getScalarSizeInBits() == 32)
8186 return DAG.getNode(Opcode: ARMISD::VREV64, DL: dl, VT, Operand: OpLHS);
8187 // vrev <4 x i16> -> VREV32
8188 if (VT.getScalarSizeInBits() == 16)
8189 return DAG.getNode(Opcode: ARMISD::VREV32, DL: dl, VT, Operand: OpLHS);
8190 // vrev <4 x i8> -> VREV16
8191 assert(VT.getScalarSizeInBits() == 8);
8192 return DAG.getNode(Opcode: ARMISD::VREV16, DL: dl, VT, Operand: OpLHS);
8193 case OP_VDUP0:
8194 case OP_VDUP1:
8195 case OP_VDUP2:
8196 case OP_VDUP3:
8197 return DAG.getNode(Opcode: ARMISD::VDUPLANE, DL: dl, VT,
8198 N1: OpLHS, N2: DAG.getConstant(Val: OpNum-OP_VDUP0, DL: dl, VT: MVT::i32));
8199 case OP_VEXT1:
8200 case OP_VEXT2:
8201 case OP_VEXT3:
8202 return DAG.getNode(Opcode: ARMISD::VEXT, DL: dl, VT,
8203 N1: OpLHS, N2: OpRHS,
8204 N3: DAG.getConstant(Val: OpNum - OP_VEXT1 + 1, DL: dl, VT: MVT::i32));
8205 case OP_VUZPL:
8206 case OP_VUZPR:
8207 return DAG.getNode(Opcode: ARMISD::VUZP, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: VT),
8208 N1: OpLHS, N2: OpRHS).getValue(R: OpNum-OP_VUZPL);
8209 case OP_VZIPL:
8210 case OP_VZIPR:
8211 return DAG.getNode(Opcode: ARMISD::VZIP, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: VT),
8212 N1: OpLHS, N2: OpRHS).getValue(R: OpNum-OP_VZIPL);
8213 case OP_VTRNL:
8214 case OP_VTRNR:
8215 return DAG.getNode(Opcode: ARMISD::VTRN, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: VT),
8216 N1: OpLHS, N2: OpRHS).getValue(R: OpNum-OP_VTRNL);
8217 }
8218}
8219
8220static SDValue LowerVECTOR_SHUFFLEv8i8(SDValue Op,
8221 ArrayRef<int> ShuffleMask,
8222 SelectionDAG &DAG) {
8223 // Check to see if we can use the VTBL instruction.
8224 SDValue V1 = Op.getOperand(i: 0);
8225 SDValue V2 = Op.getOperand(i: 1);
8226 SDLoc DL(Op);
8227
8228 SmallVector<SDValue, 8> VTBLMask;
8229 for (int I : ShuffleMask)
8230 VTBLMask.push_back(Elt: DAG.getSignedConstant(Val: I, DL, VT: MVT::i32));
8231
8232 if (V2.getNode()->isUndef())
8233 return DAG.getNode(Opcode: ARMISD::VTBL1, DL, VT: MVT::v8i8, N1: V1,
8234 N2: DAG.getBuildVector(VT: MVT::v8i8, DL, Ops: VTBLMask));
8235
8236 return DAG.getNode(Opcode: ARMISD::VTBL2, DL, VT: MVT::v8i8, N1: V1, N2: V2,
8237 N3: DAG.getBuildVector(VT: MVT::v8i8, DL, Ops: VTBLMask));
8238}
8239
8240static SDValue LowerReverse_VECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) {
8241 SDLoc DL(Op);
8242 EVT VT = Op.getValueType();
8243
8244 assert((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) &&
8245 "Expect an v8i16/v16i8 type");
8246 SDValue OpLHS = DAG.getNode(Opcode: ARMISD::VREV64, DL, VT, Operand: Op.getOperand(i: 0));
8247 // For a v16i8 type: After the VREV, we have got <7, ..., 0, 15, ..., 8>. Now,
8248 // extract the first 8 bytes into the top double word and the last 8 bytes
8249 // into the bottom double word, through a new vector shuffle that will be
8250 // turned into a VEXT on Neon, or a couple of VMOVDs on MVE.
8251 std::vector<int> NewMask;
8252 for (unsigned i = 0; i < VT.getVectorNumElements() / 2; i++)
8253 NewMask.push_back(x: VT.getVectorNumElements() / 2 + i);
8254 for (unsigned i = 0; i < VT.getVectorNumElements() / 2; i++)
8255 NewMask.push_back(x: i);
8256 return DAG.getVectorShuffle(VT, dl: DL, N1: OpLHS, N2: OpLHS, Mask: NewMask);
8257}
8258
8259static EVT getVectorTyFromPredicateVector(EVT VT) {
8260 switch (VT.getSimpleVT().SimpleTy) {
8261 case MVT::v2i1:
8262 return MVT::v2f64;
8263 case MVT::v4i1:
8264 return MVT::v4i32;
8265 case MVT::v8i1:
8266 return MVT::v8i16;
8267 case MVT::v16i1:
8268 return MVT::v16i8;
8269 default:
8270 llvm_unreachable("Unexpected vector predicate type");
8271 }
8272}
8273
8274static SDValue PromoteMVEPredVector(SDLoc dl, SDValue Pred, EVT VT,
8275 SelectionDAG &DAG) {
8276 // Converting from boolean predicates to integers involves creating a vector
8277 // of all ones or all zeroes and selecting the lanes based upon the real
8278 // predicate.
8279 SDValue AllOnes =
8280 DAG.getTargetConstant(Val: ARM_AM::createVMOVModImm(OpCmode: 0xe, Val: 0xff), DL: dl, VT: MVT::i32);
8281 AllOnes = DAG.getNode(Opcode: ARMISD::VMOVIMM, DL: dl, VT: MVT::v16i8, Operand: AllOnes);
8282
8283 SDValue AllZeroes =
8284 DAG.getTargetConstant(Val: ARM_AM::createVMOVModImm(OpCmode: 0xe, Val: 0x0), DL: dl, VT: MVT::i32);
8285 AllZeroes = DAG.getNode(Opcode: ARMISD::VMOVIMM, DL: dl, VT: MVT::v16i8, Operand: AllZeroes);
8286
8287 // Get full vector type from predicate type
8288 EVT NewVT = getVectorTyFromPredicateVector(VT);
8289
8290 SDValue RecastV1;
8291 // If the real predicate is an v8i1 or v4i1 (not v16i1) then we need to recast
8292 // this to a v16i1. This cannot be done with an ordinary bitcast because the
8293 // sizes are not the same. We have to use a MVE specific PREDICATE_CAST node,
8294 // since we know in hardware the sizes are really the same.
8295 if (VT != MVT::v16i1)
8296 RecastV1 = DAG.getNode(Opcode: ARMISD::PREDICATE_CAST, DL: dl, VT: MVT::v16i1, Operand: Pred);
8297 else
8298 RecastV1 = Pred;
8299
8300 // Select either all ones or zeroes depending upon the real predicate bits.
8301 SDValue PredAsVector =
8302 DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: MVT::v16i8, N1: RecastV1, N2: AllOnes, N3: AllZeroes);
8303
8304 // Recast our new predicate-as-integer v16i8 vector into something
8305 // appropriate for the shuffle, i.e. v4i32 for a real v4i1 predicate.
8306 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVT, Operand: PredAsVector);
8307}
8308
8309static SDValue LowerVECTOR_SHUFFLE_i1(SDValue Op, SelectionDAG &DAG,
8310 const ARMSubtarget *ST) {
8311 EVT VT = Op.getValueType();
8312 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val: Op.getNode());
8313 ArrayRef<int> ShuffleMask = SVN->getMask();
8314
8315 assert(ST->hasMVEIntegerOps() &&
8316 "No support for vector shuffle of boolean predicates");
8317
8318 SDValue V1 = Op.getOperand(i: 0);
8319 SDValue V2 = Op.getOperand(i: 1);
8320 SDLoc dl(Op);
8321 if (isReverseMask(M: ShuffleMask, VT)) {
8322 SDValue cast = DAG.getNode(Opcode: ARMISD::PREDICATE_CAST, DL: dl, VT: MVT::i32, Operand: V1);
8323 SDValue rbit = DAG.getNode(Opcode: ISD::BITREVERSE, DL: dl, VT: MVT::i32, Operand: cast);
8324 SDValue srl = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i32, N1: rbit,
8325 N2: DAG.getConstant(Val: 16, DL: dl, VT: MVT::i32));
8326 return DAG.getNode(Opcode: ARMISD::PREDICATE_CAST, DL: dl, VT, Operand: srl);
8327 }
8328
8329 // Until we can come up with optimised cases for every single vector
8330 // shuffle in existence we have chosen the least painful strategy. This is
8331 // to essentially promote the boolean predicate to a 8-bit integer, where
8332 // each predicate represents a byte. Then we fall back on a normal integer
8333 // vector shuffle and convert the result back into a predicate vector. In
8334 // many cases the generated code might be even better than scalar code
8335 // operating on bits. Just imagine trying to shuffle 8 arbitrary 2-bit
8336 // fields in a register into 8 other arbitrary 2-bit fields!
8337 SDValue PredAsVector1 = PromoteMVEPredVector(dl, Pred: V1, VT, DAG);
8338 EVT NewVT = PredAsVector1.getValueType();
8339 SDValue PredAsVector2 = V2.isUndef() ? DAG.getUNDEF(VT: NewVT)
8340 : PromoteMVEPredVector(dl, Pred: V2, VT, DAG);
8341 assert(PredAsVector2.getValueType() == NewVT &&
8342 "Expected identical vector type in expanded i1 shuffle!");
8343
8344 // Do the shuffle!
8345 SDValue Shuffled = DAG.getVectorShuffle(VT: NewVT, dl, N1: PredAsVector1,
8346 N2: PredAsVector2, Mask: ShuffleMask);
8347
8348 // Now return the result of comparing the shuffled vector with zero,
8349 // which will generate a real predicate, i.e. v4i1, v8i1 or v16i1. For a v2i1
8350 // we convert to a v4i1 compare to fill in the two halves of the i64 as i32s.
8351 if (VT == MVT::v2i1) {
8352 SDValue BC = DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT: MVT::v4i32, Operand: Shuffled);
8353 SDValue Cmp = DAG.getNode(Opcode: ARMISD::VCMPZ, DL: dl, VT: MVT::v4i1, N1: BC,
8354 N2: DAG.getConstant(Val: ARMCC::NE, DL: dl, VT: MVT::i32));
8355 return DAG.getNode(Opcode: ARMISD::PREDICATE_CAST, DL: dl, VT: MVT::v2i1, Operand: Cmp);
8356 }
8357 return DAG.getNode(Opcode: ARMISD::VCMPZ, DL: dl, VT, N1: Shuffled,
8358 N2: DAG.getConstant(Val: ARMCC::NE, DL: dl, VT: MVT::i32));
8359}
8360
8361static SDValue LowerVECTOR_SHUFFLEUsingMovs(SDValue Op,
8362 ArrayRef<int> ShuffleMask,
8363 SelectionDAG &DAG) {
8364 // Attempt to lower the vector shuffle using as many whole register movs as
8365 // possible. This is useful for types smaller than 32bits, which would
8366 // often otherwise become a series for grp movs.
8367 SDLoc dl(Op);
8368 EVT VT = Op.getValueType();
8369 if (VT.getScalarSizeInBits() >= 32)
8370 return SDValue();
8371
8372 assert((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) &&
8373 "Unexpected vector type");
8374 int NumElts = VT.getVectorNumElements();
8375 int QuarterSize = NumElts / 4;
8376 // The four final parts of the vector, as i32's
8377 SDValue Parts[4];
8378
8379 // Look for full lane vmovs like <0,1,2,3> or <u,5,6,7> etc, (but not
8380 // <u,u,u,u>), returning the vmov lane index
8381 auto getMovIdx = [](ArrayRef<int> ShuffleMask, int Start, int Length) {
8382 // Detect which mov lane this would be from the first non-undef element.
8383 int MovIdx = -1;
8384 for (int i = 0; i < Length; i++) {
8385 if (ShuffleMask[Start + i] >= 0) {
8386 if (ShuffleMask[Start + i] % Length != i)
8387 return -1;
8388 MovIdx = ShuffleMask[Start + i] / Length;
8389 break;
8390 }
8391 }
8392 // If all items are undef, leave this for other combines
8393 if (MovIdx == -1)
8394 return -1;
8395 // Check the remaining values are the correct part of the same mov
8396 for (int i = 1; i < Length; i++) {
8397 if (ShuffleMask[Start + i] >= 0 &&
8398 (ShuffleMask[Start + i] / Length != MovIdx ||
8399 ShuffleMask[Start + i] % Length != i))
8400 return -1;
8401 }
8402 return MovIdx;
8403 };
8404
8405 for (int Part = 0; Part < 4; ++Part) {
8406 // Does this part look like a mov
8407 int Elt = getMovIdx(ShuffleMask, Part * QuarterSize, QuarterSize);
8408 if (Elt != -1) {
8409 SDValue Input = Op->getOperand(Num: 0);
8410 if (Elt >= 4) {
8411 Input = Op->getOperand(Num: 1);
8412 Elt -= 4;
8413 }
8414 SDValue BitCast = DAG.getBitcast(VT: MVT::v4f32, V: Input);
8415 Parts[Part] = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: MVT::f32, N1: BitCast,
8416 N2: DAG.getConstant(Val: Elt, DL: dl, VT: MVT::i32));
8417 }
8418 }
8419
8420 // Nothing interesting found, just return
8421 if (!Parts[0] && !Parts[1] && !Parts[2] && !Parts[3])
8422 return SDValue();
8423
8424 // The other parts need to be built with the old shuffle vector, cast to a
8425 // v4i32 and extract_vector_elts
8426 if (!Parts[0] || !Parts[1] || !Parts[2] || !Parts[3]) {
8427 SmallVector<int, 16> NewShuffleMask;
8428 for (int Part = 0; Part < 4; ++Part)
8429 for (int i = 0; i < QuarterSize; i++)
8430 NewShuffleMask.push_back(
8431 Elt: Parts[Part] ? -1 : ShuffleMask[Part * QuarterSize + i]);
8432 SDValue NewShuffle = DAG.getVectorShuffle(
8433 VT, dl, N1: Op->getOperand(Num: 0), N2: Op->getOperand(Num: 1), Mask: NewShuffleMask);
8434 SDValue BitCast = DAG.getBitcast(VT: MVT::v4f32, V: NewShuffle);
8435
8436 for (int Part = 0; Part < 4; ++Part)
8437 if (!Parts[Part])
8438 Parts[Part] = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: MVT::f32,
8439 N1: BitCast, N2: DAG.getConstant(Val: Part, DL: dl, VT: MVT::i32));
8440 }
8441 // Build a vector out of the various parts and bitcast it back to the original
8442 // type.
8443 SDValue NewVec = DAG.getNode(Opcode: ARMISD::BUILD_VECTOR, DL: dl, VT: MVT::v4f32, Ops: Parts);
8444 return DAG.getBitcast(VT, V: NewVec);
8445}
8446
8447static SDValue LowerVECTOR_SHUFFLEUsingOneOff(SDValue Op,
8448 ArrayRef<int> ShuffleMask,
8449 SelectionDAG &DAG) {
8450 SDValue V1 = Op.getOperand(i: 0);
8451 SDValue V2 = Op.getOperand(i: 1);
8452 EVT VT = Op.getValueType();
8453 unsigned NumElts = VT.getVectorNumElements();
8454
8455 // An One-Off Identity mask is one that is mostly an identity mask from as
8456 // single source but contains a single element out-of-place, either from a
8457 // different vector or from another position in the same vector. As opposed to
8458 // lowering this via a ARMISD::BUILD_VECTOR we can generate an extract/insert
8459 // pair directly.
8460 auto isOneOffIdentityMask = [](ArrayRef<int> Mask, EVT VT, int BaseOffset,
8461 int &OffElement) {
8462 OffElement = -1;
8463 int NonUndef = 0;
8464 for (int i = 0, NumMaskElts = Mask.size(); i < NumMaskElts; ++i) {
8465 if (Mask[i] == -1)
8466 continue;
8467 NonUndef++;
8468 if (Mask[i] != i + BaseOffset) {
8469 if (OffElement == -1)
8470 OffElement = i;
8471 else
8472 return false;
8473 }
8474 }
8475 return NonUndef > 2 && OffElement != -1;
8476 };
8477 int OffElement;
8478 SDValue VInput;
8479 if (isOneOffIdentityMask(ShuffleMask, VT, 0, OffElement))
8480 VInput = V1;
8481 else if (isOneOffIdentityMask(ShuffleMask, VT, NumElts, OffElement))
8482 VInput = V2;
8483 else
8484 return SDValue();
8485
8486 SDLoc dl(Op);
8487 EVT SVT = VT.getScalarType() == MVT::i8 || VT.getScalarType() == MVT::i16
8488 ? MVT::i32
8489 : VT.getScalarType();
8490 SDValue Elt = DAG.getNode(
8491 Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: SVT,
8492 N1: ShuffleMask[OffElement] < (int)NumElts ? V1 : V2,
8493 N2: DAG.getVectorIdxConstant(Val: ShuffleMask[OffElement] % NumElts, DL: dl));
8494 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT, N1: VInput, N2: Elt,
8495 N3: DAG.getVectorIdxConstant(Val: OffElement % NumElts, DL: dl));
8496}
8497
8498static SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG,
8499 const ARMSubtarget *ST) {
8500 SDValue V1 = Op.getOperand(i: 0);
8501 SDValue V2 = Op.getOperand(i: 1);
8502 SDLoc dl(Op);
8503 EVT VT = Op.getValueType();
8504 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val: Op.getNode());
8505 unsigned EltSize = VT.getScalarSizeInBits();
8506
8507 if (ST->hasMVEIntegerOps() && EltSize == 1)
8508 return LowerVECTOR_SHUFFLE_i1(Op, DAG, ST);
8509
8510 // Convert shuffles that are directly supported on NEON to target-specific
8511 // DAG nodes, instead of keeping them as shuffles and matching them again
8512 // during code selection. This is more efficient and avoids the possibility
8513 // of inconsistencies between legalization and selection.
8514 // FIXME: floating-point vectors should be canonicalized to integer vectors
8515 // of the same time so that they get CSEd properly.
8516 ArrayRef<int> ShuffleMask = SVN->getMask();
8517
8518 if (EltSize <= 32) {
8519 if (SVN->isSplat()) {
8520 int Lane = SVN->getSplatIndex();
8521 // If this is undef splat, generate it via "just" vdup, if possible.
8522 if (Lane == -1) Lane = 0;
8523
8524 // Test if V1 is a SCALAR_TO_VECTOR.
8525 if (Lane == 0 && V1.getOpcode() == ISD::SCALAR_TO_VECTOR) {
8526 return DAG.getNode(Opcode: ARMISD::VDUP, DL: dl, VT, Operand: V1.getOperand(i: 0));
8527 }
8528 // Test if V1 is a BUILD_VECTOR which is equivalent to a SCALAR_TO_VECTOR
8529 // (and probably will turn into a SCALAR_TO_VECTOR once legalization
8530 // reaches it).
8531 if (Lane == 0 && V1.getOpcode() == ISD::BUILD_VECTOR &&
8532 !isa<ConstantSDNode>(Val: V1.getOperand(i: 0))) {
8533 bool IsScalarToVector = true;
8534 for (unsigned i = 1, e = V1.getNumOperands(); i != e; ++i)
8535 if (!V1.getOperand(i).isUndef()) {
8536 IsScalarToVector = false;
8537 break;
8538 }
8539 if (IsScalarToVector)
8540 return DAG.getNode(Opcode: ARMISD::VDUP, DL: dl, VT, Operand: V1.getOperand(i: 0));
8541 }
8542 return DAG.getNode(Opcode: ARMISD::VDUPLANE, DL: dl, VT, N1: V1,
8543 N2: DAG.getConstant(Val: Lane, DL: dl, VT: MVT::i32));
8544 }
8545
8546 bool ReverseVEXT = false;
8547 unsigned Imm = 0;
8548 if (ST->hasNEON() && isVEXTMask(M: ShuffleMask, VT, ReverseVEXT, Imm)) {
8549 if (ReverseVEXT)
8550 std::swap(a&: V1, b&: V2);
8551 return DAG.getNode(Opcode: ARMISD::VEXT, DL: dl, VT, N1: V1, N2: V2,
8552 N3: DAG.getConstant(Val: Imm, DL: dl, VT: MVT::i32));
8553 }
8554
8555 if (isVREVMask(M: ShuffleMask, VT, BlockSize: 64))
8556 return DAG.getNode(Opcode: ARMISD::VREV64, DL: dl, VT, Operand: V1);
8557 if (isVREVMask(M: ShuffleMask, VT, BlockSize: 32))
8558 return DAG.getNode(Opcode: ARMISD::VREV32, DL: dl, VT, Operand: V1);
8559 if (isVREVMask(M: ShuffleMask, VT, BlockSize: 16))
8560 return DAG.getNode(Opcode: ARMISD::VREV16, DL: dl, VT, Operand: V1);
8561
8562 if (ST->hasNEON() && V2->isUndef() && isSingletonVEXTMask(M: ShuffleMask, VT, Imm)) {
8563 return DAG.getNode(Opcode: ARMISD::VEXT, DL: dl, VT, N1: V1, N2: V1,
8564 N3: DAG.getConstant(Val: Imm, DL: dl, VT: MVT::i32));
8565 }
8566
8567 // Check for Neon shuffles that modify both input vectors in place.
8568 // If both results are used, i.e., if there are two shuffles with the same
8569 // source operands and with masks corresponding to both results of one of
8570 // these operations, DAG memoization will ensure that a single node is
8571 // used for both shuffles.
8572 unsigned WhichResult = 0;
8573 bool isV_UNDEF = false;
8574 if (ST->hasNEON()) {
8575 if (unsigned ShuffleOpc = isNEONTwoResultShuffleMask(
8576 ShuffleMask, VT, WhichResult, isV_UNDEF)) {
8577 if (isV_UNDEF)
8578 V2 = V1;
8579 return DAG.getNode(Opcode: ShuffleOpc, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: VT), N1: V1, N2: V2)
8580 .getValue(R: WhichResult);
8581 }
8582 }
8583 if (ST->hasMVEIntegerOps()) {
8584 if (isVMOVNMask(M: ShuffleMask, VT, Top: false, SingleSource: false))
8585 return DAG.getNode(Opcode: ARMISD::VMOVN, DL: dl, VT, N1: V2, N2: V1,
8586 N3: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
8587 if (isVMOVNMask(M: ShuffleMask, VT, Top: true, SingleSource: false))
8588 return DAG.getNode(Opcode: ARMISD::VMOVN, DL: dl, VT, N1: V1, N2: V2,
8589 N3: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
8590 if (isVMOVNMask(M: ShuffleMask, VT, Top: true, SingleSource: true))
8591 return DAG.getNode(Opcode: ARMISD::VMOVN, DL: dl, VT, N1: V1, N2: V1,
8592 N3: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
8593 }
8594
8595 // Also check for these shuffles through CONCAT_VECTORS: we canonicalize
8596 // shuffles that produce a result larger than their operands with:
8597 // shuffle(concat(v1, undef), concat(v2, undef))
8598 // ->
8599 // shuffle(concat(v1, v2), undef)
8600 // because we can access quad vectors (see PerformVECTOR_SHUFFLECombine).
8601 //
8602 // This is useful in the general case, but there are special cases where
8603 // native shuffles produce larger results: the two-result ops.
8604 //
8605 // Look through the concat when lowering them:
8606 // shuffle(concat(v1, v2), undef)
8607 // ->
8608 // concat(VZIP(v1, v2):0, :1)
8609 //
8610 if (ST->hasNEON() && V1->getOpcode() == ISD::CONCAT_VECTORS && V2->isUndef()) {
8611 SDValue SubV1 = V1->getOperand(Num: 0);
8612 SDValue SubV2 = V1->getOperand(Num: 1);
8613 EVT SubVT = SubV1.getValueType();
8614
8615 // We expect these to have been canonicalized to -1.
8616 assert(llvm::all_of(ShuffleMask, [&](int i) {
8617 return i < (int)VT.getVectorNumElements();
8618 }) && "Unexpected shuffle index into UNDEF operand!");
8619
8620 if (unsigned ShuffleOpc = isNEONTwoResultShuffleMask(
8621 ShuffleMask, VT: SubVT, WhichResult, isV_UNDEF)) {
8622 if (isV_UNDEF)
8623 SubV2 = SubV1;
8624 assert((WhichResult == 0) &&
8625 "In-place shuffle of concat can only have one result!");
8626 SDValue Res = DAG.getNode(Opcode: ShuffleOpc, DL: dl, VTList: DAG.getVTList(VT1: SubVT, VT2: SubVT),
8627 N1: SubV1, N2: SubV2);
8628 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT, N1: Res.getValue(R: 0),
8629 N2: Res.getValue(R: 1));
8630 }
8631 }
8632 }
8633
8634 if (ST->hasMVEIntegerOps() && EltSize <= 32 &&
8635 (ST->hasFullFP16() || VT != MVT::v8f16)) {
8636 if (SDValue V = LowerVECTOR_SHUFFLEUsingOneOff(Op, ShuffleMask, DAG))
8637 return V;
8638
8639 for (bool Top : {false, true}) {
8640 for (bool SingleSource : {false, true}) {
8641 if (isTruncMask(M: ShuffleMask, VT, Top, SingleSource)) {
8642 MVT FromSVT = MVT::getIntegerVT(BitWidth: EltSize * 2);
8643 MVT FromVT = MVT::getVectorVT(VT: FromSVT, NumElements: ShuffleMask.size() / 2);
8644 SDValue Lo = DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT: FromVT, Operand: V1);
8645 SDValue Hi = DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT: FromVT,
8646 Operand: SingleSource ? V1 : V2);
8647 if (Top) {
8648 SDValue Amt = DAG.getConstant(Val: EltSize, DL: dl, VT: FromVT);
8649 Lo = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: FromVT, N1: Lo, N2: Amt);
8650 Hi = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: FromVT, N1: Hi, N2: Amt);
8651 }
8652 return DAG.getNode(Opcode: ARMISD::MVETRUNC, DL: dl, VT, N1: Lo, N2: Hi);
8653 }
8654 }
8655 }
8656 }
8657
8658 // If the shuffle is not directly supported and it has 4 elements, use
8659 // the PerfectShuffle-generated table to synthesize it from other shuffles.
8660 unsigned NumElts = VT.getVectorNumElements();
8661 if (NumElts == 4) {
8662 unsigned PFIndexes[4];
8663 for (unsigned i = 0; i != 4; ++i) {
8664 if (ShuffleMask[i] < 0)
8665 PFIndexes[i] = 8;
8666 else
8667 PFIndexes[i] = ShuffleMask[i];
8668 }
8669
8670 // Compute the index in the perfect shuffle table.
8671 unsigned PFTableIndex =
8672 PFIndexes[0]*9*9*9+PFIndexes[1]*9*9+PFIndexes[2]*9+PFIndexes[3];
8673 unsigned PFEntry = PerfectShuffleTable[PFTableIndex];
8674 unsigned Cost = (PFEntry >> 30);
8675
8676 if (Cost <= 4) {
8677 if (ST->hasNEON())
8678 return GeneratePerfectShuffle(PFEntry, LHS: V1, RHS: V2, DAG, dl);
8679 else if (isLegalMVEShuffleOp(PFEntry)) {
8680 unsigned LHSID = (PFEntry >> 13) & ((1 << 13)-1);
8681 unsigned RHSID = (PFEntry >> 0) & ((1 << 13)-1);
8682 unsigned PFEntryLHS = PerfectShuffleTable[LHSID];
8683 unsigned PFEntryRHS = PerfectShuffleTable[RHSID];
8684 if (isLegalMVEShuffleOp(PFEntry: PFEntryLHS) && isLegalMVEShuffleOp(PFEntry: PFEntryRHS))
8685 return GeneratePerfectShuffle(PFEntry, LHS: V1, RHS: V2, DAG, dl);
8686 }
8687 }
8688 }
8689
8690 // Implement shuffles with 32- or 64-bit elements as ARMISD::BUILD_VECTORs.
8691 if (EltSize >= 32) {
8692 // Do the expansion with floating-point types, since that is what the VFP
8693 // registers are defined to use, and since i64 is not legal.
8694 EVT EltVT = EVT::getFloatingPointVT(BitWidth: EltSize);
8695 EVT VecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, NumElements: NumElts);
8696 V1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: VecVT, Operand: V1);
8697 V2 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: VecVT, Operand: V2);
8698 SmallVector<SDValue, 8> Ops;
8699 for (unsigned i = 0; i < NumElts; ++i) {
8700 if (ShuffleMask[i] < 0)
8701 Ops.push_back(Elt: DAG.getUNDEF(VT: EltVT));
8702 else
8703 Ops.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT,
8704 N1: ShuffleMask[i] < (int)NumElts ? V1 : V2,
8705 N2: DAG.getConstant(Val: ShuffleMask[i] & (NumElts-1),
8706 DL: dl, VT: MVT::i32)));
8707 }
8708 SDValue Val = DAG.getNode(Opcode: ARMISD::BUILD_VECTOR, DL: dl, VT: VecVT, Ops);
8709 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: Val);
8710 }
8711
8712 if ((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) &&
8713 isReverseMask(M: ShuffleMask, VT))
8714 return LowerReverse_VECTOR_SHUFFLE(Op, DAG);
8715
8716 if (ST->hasNEON() && VT == MVT::v8i8)
8717 if (SDValue NewOp = LowerVECTOR_SHUFFLEv8i8(Op, ShuffleMask, DAG))
8718 return NewOp;
8719
8720 if (ST->hasMVEIntegerOps())
8721 if (SDValue NewOp = LowerVECTOR_SHUFFLEUsingMovs(Op, ShuffleMask, DAG))
8722 return NewOp;
8723
8724 // Lower v8f16 via v8i16 to avoid invalid f16 nodes.
8725 if (VT == MVT::v8f16 && !ST->hasFullFP16()) {
8726 SDValue BC0 =
8727 DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT: MVT::v8i16, Operand: Op.getOperand(i: 0));
8728 SDValue BC1 =
8729 DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT: MVT::v8i16, Operand: Op.getOperand(i: 1));
8730 SDValue Shuf = DAG.getVectorShuffle(VT: MVT::v8i16, dl, N1: BC0, N2: BC1, Mask: ShuffleMask);
8731 return DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT, Operand: Shuf);
8732 }
8733
8734 return SDValue();
8735}
8736
8737static SDValue LowerINSERT_VECTOR_ELT_i1(SDValue Op, SelectionDAG &DAG,
8738 const ARMSubtarget *ST) {
8739 EVT VecVT = Op.getOperand(i: 0).getValueType();
8740 SDLoc dl(Op);
8741
8742 assert(ST->hasMVEIntegerOps() &&
8743 "LowerINSERT_VECTOR_ELT_i1 called without MVE!");
8744
8745 SDValue Conv =
8746 DAG.getNode(Opcode: ARMISD::PREDICATE_CAST, DL: dl, VT: MVT::i32, Operand: Op->getOperand(Num: 0));
8747 unsigned Lane = Op.getConstantOperandVal(i: 2);
8748 unsigned LaneWidth =
8749 getVectorTyFromPredicateVector(VT: VecVT).getScalarSizeInBits() / 8;
8750 unsigned Mask = ((1 << LaneWidth) - 1) << Lane * LaneWidth;
8751 SDValue Ext = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl, VT: MVT::i32,
8752 N1: Op.getOperand(i: 1), N2: DAG.getValueType(MVT::i1));
8753 SDValue BFI = DAG.getNode(Opcode: ARMISD::BFI, DL: dl, VT: MVT::i32, N1: Conv, N2: Ext,
8754 N3: DAG.getConstant(Val: ~Mask, DL: dl, VT: MVT::i32));
8755 return DAG.getNode(Opcode: ARMISD::PREDICATE_CAST, DL: dl, VT: Op.getValueType(), Operand: BFI);
8756}
8757
8758SDValue ARMTargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
8759 SelectionDAG &DAG) const {
8760 // INSERT_VECTOR_ELT is legal only for immediate indexes.
8761 SDValue Lane = Op.getOperand(i: 2);
8762 if (!isa<ConstantSDNode>(Val: Lane))
8763 return SDValue();
8764
8765 SDValue Elt = Op.getOperand(i: 1);
8766 EVT EltVT = Elt.getValueType();
8767
8768 if (Subtarget->hasMVEIntegerOps() &&
8769 Op.getValueType().getScalarSizeInBits() == 1)
8770 return LowerINSERT_VECTOR_ELT_i1(Op, DAG, ST: Subtarget);
8771
8772 if (getTypeAction(Context&: *DAG.getContext(), VT: EltVT) ==
8773 TargetLowering::TypeSoftPromoteHalf) {
8774 // INSERT_VECTOR_ELT doesn't want f16 operands promoting to f32,
8775 // but the type system will try to do that if we don't intervene.
8776 // Reinterpret any such vector-element insertion as one with the
8777 // corresponding integer types.
8778
8779 SDLoc dl(Op);
8780
8781 EVT IEltVT = MVT::getIntegerVT(BitWidth: EltVT.getScalarSizeInBits());
8782 assert(getTypeAction(*DAG.getContext(), IEltVT) !=
8783 TargetLowering::TypeSoftPromoteHalf);
8784
8785 SDValue VecIn = Op.getOperand(i: 0);
8786 EVT VecVT = VecIn.getValueType();
8787 EVT IVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: IEltVT,
8788 NumElements: VecVT.getVectorNumElements());
8789
8790 SDValue IElt = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: IEltVT, Operand: Elt);
8791 SDValue IVecIn = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: IVecVT, Operand: VecIn);
8792 SDValue IVecOut = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: IVecVT,
8793 N1: IVecIn, N2: IElt, N3: Lane);
8794 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: VecVT, Operand: IVecOut);
8795 }
8796
8797 return Op;
8798}
8799
8800static SDValue LowerEXTRACT_VECTOR_ELT_i1(SDValue Op, SelectionDAG &DAG,
8801 const ARMSubtarget *ST) {
8802 EVT VecVT = Op.getOperand(i: 0).getValueType();
8803 SDLoc dl(Op);
8804
8805 assert(ST->hasMVEIntegerOps() &&
8806 "LowerINSERT_VECTOR_ELT_i1 called without MVE!");
8807
8808 SDValue Conv =
8809 DAG.getNode(Opcode: ARMISD::PREDICATE_CAST, DL: dl, VT: MVT::i32, Operand: Op->getOperand(Num: 0));
8810 unsigned Lane = Op.getConstantOperandVal(i: 1);
8811 unsigned LaneWidth =
8812 getVectorTyFromPredicateVector(VT: VecVT).getScalarSizeInBits() / 8;
8813 SDValue Shift = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i32, N1: Conv,
8814 N2: DAG.getConstant(Val: Lane * LaneWidth, DL: dl, VT: MVT::i32));
8815 return Shift;
8816}
8817
8818static SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG,
8819 const ARMSubtarget *ST) {
8820 // EXTRACT_VECTOR_ELT is legal only for immediate indexes.
8821 SDValue Lane = Op.getOperand(i: 1);
8822 if (!isa<ConstantSDNode>(Val: Lane))
8823 return SDValue();
8824
8825 SDValue Vec = Op.getOperand(i: 0);
8826 EVT VT = Vec.getValueType();
8827
8828 if (ST->hasMVEIntegerOps() && VT.getScalarSizeInBits() == 1)
8829 return LowerEXTRACT_VECTOR_ELT_i1(Op, DAG, ST);
8830
8831 if (Op.getValueType() == MVT::i32 && Vec.getScalarValueSizeInBits() < 32) {
8832 SDLoc dl(Op);
8833 return DAG.getNode(Opcode: ARMISD::VGETLANEu, DL: dl, VT: MVT::i32, N1: Vec, N2: Lane);
8834 }
8835
8836 return Op;
8837}
8838
8839static SDValue LowerCONCAT_VECTORS_i1(SDValue Op, SelectionDAG &DAG,
8840 const ARMSubtarget *ST) {
8841 SDLoc dl(Op);
8842 assert(Op.getValueType().getScalarSizeInBits() == 1 &&
8843 "Unexpected custom CONCAT_VECTORS lowering");
8844 assert(isPowerOf2_32(Op.getNumOperands()) &&
8845 "Unexpected custom CONCAT_VECTORS lowering");
8846 assert(ST->hasMVEIntegerOps() &&
8847 "CONCAT_VECTORS lowering only supported for MVE");
8848
8849 auto ConcatPair = [&](SDValue V1, SDValue V2) {
8850 EVT Op1VT = V1.getValueType();
8851 EVT Op2VT = V2.getValueType();
8852 assert(Op1VT == Op2VT && "Operand types don't match!");
8853 assert((Op1VT == MVT::v2i1 || Op1VT == MVT::v4i1 || Op1VT == MVT::v8i1) &&
8854 "Unexpected i1 concat operations!");
8855 EVT VT = Op1VT.getDoubleNumVectorElementsVT(Context&: *DAG.getContext());
8856
8857 SDValue NewV1 = PromoteMVEPredVector(dl, Pred: V1, VT: Op1VT, DAG);
8858 SDValue NewV2 = PromoteMVEPredVector(dl, Pred: V2, VT: Op2VT, DAG);
8859
8860 // We now have Op1 + Op2 promoted to vectors of integers, where v8i1 gets
8861 // promoted to v8i16, etc.
8862 MVT ElType =
8863 getVectorTyFromPredicateVector(VT).getScalarType().getSimpleVT();
8864 unsigned NumElts = 2 * Op1VT.getVectorNumElements();
8865
8866 EVT ConcatVT = MVT::getVectorVT(VT: ElType, NumElements: NumElts);
8867 if (Op1VT == MVT::v4i1 || Op1VT == MVT::v8i1) {
8868 // Use MVETRUNC to truncate the combined NewV1::NewV2 into the smaller
8869 // ConcatVT.
8870 SDValue ConVec =
8871 DAG.getNode(Opcode: ARMISD::MVETRUNC, DL: dl, VT: ConcatVT, N1: NewV1, N2: NewV2);
8872 return DAG.getNode(Opcode: ARMISD::VCMPZ, DL: dl, VT, N1: ConVec,
8873 N2: DAG.getConstant(Val: ARMCC::NE, DL: dl, VT: MVT::i32));
8874 }
8875
8876 // Extract the vector elements from Op1 and Op2 one by one and truncate them
8877 // to be the right size for the destination. For example, if Op1 is v4i1
8878 // then the promoted vector is v4i32. The result of concatenation gives a
8879 // v8i1, which when promoted is v8i16. That means each i32 element from Op1
8880 // needs truncating to i16 and inserting in the result.
8881 auto ExtractInto = [&DAG, &dl](SDValue NewV, SDValue ConVec, unsigned &j) {
8882 EVT NewVT = NewV.getValueType();
8883 EVT ConcatVT = ConVec.getValueType();
8884 unsigned ExtScale = 1;
8885 if (NewVT == MVT::v2f64) {
8886 NewV = DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT: MVT::v4i32, Operand: NewV);
8887 ExtScale = 2;
8888 }
8889 for (unsigned i = 0, e = NewVT.getVectorNumElements(); i < e; i++, j++) {
8890 SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: MVT::i32, N1: NewV,
8891 N2: DAG.getIntPtrConstant(Val: i * ExtScale, DL: dl));
8892 ConVec = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: ConcatVT, N1: ConVec, N2: Elt,
8893 N3: DAG.getConstant(Val: j, DL: dl, VT: MVT::i32));
8894 }
8895 return ConVec;
8896 };
8897 unsigned j = 0;
8898 SDValue ConVec = DAG.getNode(Opcode: ISD::UNDEF, DL: dl, VT: ConcatVT);
8899 ConVec = ExtractInto(NewV1, ConVec, j);
8900 ConVec = ExtractInto(NewV2, ConVec, j);
8901
8902 // Now return the result of comparing the subvector with zero, which will
8903 // generate a real predicate, i.e. v4i1, v8i1 or v16i1.
8904 return DAG.getNode(Opcode: ARMISD::VCMPZ, DL: dl, VT, N1: ConVec,
8905 N2: DAG.getConstant(Val: ARMCC::NE, DL: dl, VT: MVT::i32));
8906 };
8907
8908 // Concat each pair of subvectors and pack into the lower half of the array.
8909 SmallVector<SDValue> ConcatOps(Op->ops());
8910 while (ConcatOps.size() > 1) {
8911 for (unsigned I = 0, E = ConcatOps.size(); I != E; I += 2) {
8912 SDValue V1 = ConcatOps[I];
8913 SDValue V2 = ConcatOps[I + 1];
8914 ConcatOps[I / 2] = ConcatPair(V1, V2);
8915 }
8916 ConcatOps.resize(N: ConcatOps.size() / 2);
8917 }
8918 return ConcatOps[0];
8919}
8920
8921static SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG,
8922 const ARMSubtarget *ST) {
8923 EVT VT = Op->getValueType(ResNo: 0);
8924 if (ST->hasMVEIntegerOps() && VT.getScalarSizeInBits() == 1)
8925 return LowerCONCAT_VECTORS_i1(Op, DAG, ST);
8926
8927 // The only time a CONCAT_VECTORS operation can have legal types is when
8928 // two 64-bit vectors are concatenated to a 128-bit vector.
8929 assert(Op.getValueType().is128BitVector() && Op.getNumOperands() == 2 &&
8930 "unexpected CONCAT_VECTORS");
8931 SDLoc dl(Op);
8932 SDValue Val = DAG.getUNDEF(VT: MVT::v2f64);
8933 SDValue Op0 = Op.getOperand(i: 0);
8934 SDValue Op1 = Op.getOperand(i: 1);
8935 if (!Op0.isUndef())
8936 Val = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: MVT::v2f64, N1: Val,
8937 N2: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::f64, Operand: Op0),
8938 N3: DAG.getIntPtrConstant(Val: 0, DL: dl));
8939 if (!Op1.isUndef())
8940 Val = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: MVT::v2f64, N1: Val,
8941 N2: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::f64, Operand: Op1),
8942 N3: DAG.getIntPtrConstant(Val: 1, DL: dl));
8943 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: Op.getValueType(), Operand: Val);
8944}
8945
8946static SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, SelectionDAG &DAG,
8947 const ARMSubtarget *ST) {
8948 SDValue V1 = Op.getOperand(i: 0);
8949 SDValue V2 = Op.getOperand(i: 1);
8950 SDLoc dl(Op);
8951 EVT VT = Op.getValueType();
8952 EVT Op1VT = V1.getValueType();
8953 unsigned NumElts = VT.getVectorNumElements();
8954 unsigned Index = V2->getAsZExtVal();
8955
8956 assert(VT.getScalarSizeInBits() == 1 &&
8957 "Unexpected custom EXTRACT_SUBVECTOR lowering");
8958 assert(ST->hasMVEIntegerOps() &&
8959 "EXTRACT_SUBVECTOR lowering only supported for MVE");
8960
8961 SDValue NewV1 = PromoteMVEPredVector(dl, Pred: V1, VT: Op1VT, DAG);
8962
8963 // We now have Op1 promoted to a vector of integers, where v8i1 gets
8964 // promoted to v8i16, etc.
8965
8966 MVT ElType = getVectorTyFromPredicateVector(VT).getScalarType().getSimpleVT();
8967
8968 if (NumElts == 2) {
8969 EVT SubVT = MVT::v4i32;
8970 SDValue SubVec = DAG.getNode(Opcode: ISD::UNDEF, DL: dl, VT: SubVT);
8971 for (unsigned i = Index, j = 0; i < (Index + NumElts); i++, j += 2) {
8972 SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: MVT::i32, N1: NewV1,
8973 N2: DAG.getIntPtrConstant(Val: i, DL: dl));
8974 SubVec = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: SubVT, N1: SubVec, N2: Elt,
8975 N3: DAG.getConstant(Val: j, DL: dl, VT: MVT::i32));
8976 SubVec = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: SubVT, N1: SubVec, N2: Elt,
8977 N3: DAG.getConstant(Val: j + 1, DL: dl, VT: MVT::i32));
8978 }
8979 SDValue Cmp = DAG.getNode(Opcode: ARMISD::VCMPZ, DL: dl, VT: MVT::v4i1, N1: SubVec,
8980 N2: DAG.getConstant(Val: ARMCC::NE, DL: dl, VT: MVT::i32));
8981 return DAG.getNode(Opcode: ARMISD::PREDICATE_CAST, DL: dl, VT: MVT::v2i1, Operand: Cmp);
8982 }
8983
8984 EVT SubVT = MVT::getVectorVT(VT: ElType, NumElements: NumElts);
8985 SDValue SubVec = DAG.getNode(Opcode: ISD::UNDEF, DL: dl, VT: SubVT);
8986 for (unsigned i = Index, j = 0; i < (Index + NumElts); i++, j++) {
8987 SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: MVT::i32, N1: NewV1,
8988 N2: DAG.getIntPtrConstant(Val: i, DL: dl));
8989 SubVec = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: SubVT, N1: SubVec, N2: Elt,
8990 N3: DAG.getConstant(Val: j, DL: dl, VT: MVT::i32));
8991 }
8992
8993 // Now return the result of comparing the subvector with zero,
8994 // which will generate a real predicate, i.e. v4i1, v8i1 or v16i1.
8995 return DAG.getNode(Opcode: ARMISD::VCMPZ, DL: dl, VT, N1: SubVec,
8996 N2: DAG.getConstant(Val: ARMCC::NE, DL: dl, VT: MVT::i32));
8997}
8998
8999// Turn a truncate into a predicate (an i1 vector) into icmp(and(x, 1), 0).
9000static SDValue LowerTruncatei1(SDNode *N, SelectionDAG &DAG,
9001 const ARMSubtarget *ST) {
9002 assert(ST->hasMVEIntegerOps() && "Expected MVE!");
9003 EVT VT = N->getValueType(ResNo: 0);
9004 assert((VT == MVT::v16i1 || VT == MVT::v8i1 || VT == MVT::v4i1) &&
9005 "Expected a vector i1 type!");
9006 SDValue Op = N->getOperand(Num: 0);
9007 EVT FromVT = Op.getValueType();
9008 SDLoc DL(N);
9009
9010 SDValue And =
9011 DAG.getNode(Opcode: ISD::AND, DL, VT: FromVT, N1: Op, N2: DAG.getConstant(Val: 1, DL, VT: FromVT));
9012 return DAG.getNode(Opcode: ISD::SETCC, DL, VT, N1: And, N2: DAG.getConstant(Val: 0, DL, VT: FromVT),
9013 N3: DAG.getCondCode(Cond: ISD::SETNE));
9014}
9015
9016static SDValue LowerTruncate(SDNode *N, SelectionDAG &DAG,
9017 const ARMSubtarget *Subtarget) {
9018 if (!Subtarget->hasMVEIntegerOps())
9019 return SDValue();
9020
9021 EVT ToVT = N->getValueType(ResNo: 0);
9022 if (ToVT.getScalarType() == MVT::i1)
9023 return LowerTruncatei1(N, DAG, ST: Subtarget);
9024
9025 // MVE does not have a single instruction to perform the truncation of a v4i32
9026 // into the lower half of a v8i16, in the same way that a NEON vmovn would.
9027 // Most of the instructions in MVE follow the 'Beats' system, where moving
9028 // values from different lanes is usually something that the instructions
9029 // avoid.
9030 //
9031 // Instead it has top/bottom instructions such as VMOVLT/B and VMOVNT/B,
9032 // which take a the top/bottom half of a larger lane and extend it (or do the
9033 // opposite, truncating into the top/bottom lane from a larger lane). Note
9034 // that because of the way we widen lanes, a v4i16 is really a v4i32 using the
9035 // bottom 16bits from each vector lane. This works really well with T/B
9036 // instructions, but that doesn't extend to v8i32->v8i16 where the lanes need
9037 // to move order.
9038 //
9039 // But truncates and sext/zext are always going to be fairly common from llvm.
9040 // We have several options for how to deal with them:
9041 // - Wherever possible combine them into an instruction that makes them
9042 // "free". This includes loads/stores, which can perform the trunc as part
9043 // of the memory operation. Or certain shuffles that can be turned into
9044 // VMOVN/VMOVL.
9045 // - Lane Interleaving to transform blocks surrounded by ext/trunc. So
9046 // trunc(mul(sext(a), sext(b))) may become
9047 // VMOVNT(VMUL(VMOVLB(a), VMOVLB(b)), VMUL(VMOVLT(a), VMOVLT(b))). (Which in
9048 // this case can use VMULL). This is performed in the
9049 // MVELaneInterleavingPass.
9050 // - Otherwise we have an option. By default we would expand the
9051 // zext/sext/trunc into a series of lane extract/inserts going via GPR
9052 // registers. One for each vector lane in the vector. This can obviously be
9053 // very expensive.
9054 // - The other option is to use the fact that loads/store can extend/truncate
9055 // to turn a trunc into two truncating stack stores and a stack reload. This
9056 // becomes 3 back-to-back memory operations, but at least that is less than
9057 // all the insert/extracts.
9058 //
9059 // In order to do the last, we convert certain trunc's into MVETRUNC, which
9060 // are either optimized where they can be, or eventually lowered into stack
9061 // stores/loads. This prevents us from splitting a v8i16 trunc into two stores
9062 // two early, where other instructions would be better, and stops us from
9063 // having to reconstruct multiple buildvector shuffles into loads/stores.
9064 if (ToVT != MVT::v8i16 && ToVT != MVT::v16i8)
9065 return SDValue();
9066 EVT FromVT = N->getOperand(Num: 0).getValueType();
9067 if (FromVT != MVT::v8i32 && FromVT != MVT::v16i16)
9068 return SDValue();
9069
9070 SDValue Lo, Hi;
9071 std::tie(args&: Lo, args&: Hi) = DAG.SplitVectorOperand(N, OpNo: 0);
9072 SDLoc DL(N);
9073 return DAG.getNode(Opcode: ARMISD::MVETRUNC, DL, VT: ToVT, N1: Lo, N2: Hi);
9074}
9075
9076static SDValue LowerVectorExtend(SDNode *N, SelectionDAG &DAG,
9077 const ARMSubtarget *Subtarget) {
9078 if (!Subtarget->hasMVEIntegerOps())
9079 return SDValue();
9080
9081 // See LowerTruncate above for an explanation of MVEEXT/MVETRUNC.
9082
9083 EVT ToVT = N->getValueType(ResNo: 0);
9084 if (ToVT != MVT::v16i32 && ToVT != MVT::v8i32 && ToVT != MVT::v16i16)
9085 return SDValue();
9086 SDValue Op = N->getOperand(Num: 0);
9087 EVT FromVT = Op.getValueType();
9088 if (FromVT != MVT::v8i16 && FromVT != MVT::v16i8)
9089 return SDValue();
9090
9091 SDLoc DL(N);
9092 EVT ExtVT = ToVT.getHalfNumVectorElementsVT(Context&: *DAG.getContext());
9093 if (ToVT.getScalarType() == MVT::i32 && FromVT.getScalarType() == MVT::i8)
9094 ExtVT = MVT::v8i16;
9095
9096 unsigned Opcode =
9097 N->getOpcode() == ISD::SIGN_EXTEND ? ARMISD::MVESEXT : ARMISD::MVEZEXT;
9098 SDValue Ext = DAG.getNode(Opcode, DL, VTList: DAG.getVTList(VT1: ExtVT, VT2: ExtVT), N: Op);
9099 SDValue Ext1 = Ext.getValue(R: 1);
9100
9101 if (ToVT.getScalarType() == MVT::i32 && FromVT.getScalarType() == MVT::i8) {
9102 Ext = DAG.getNode(Opcode: N->getOpcode(), DL, VT: MVT::v8i32, Operand: Ext);
9103 Ext1 = DAG.getNode(Opcode: N->getOpcode(), DL, VT: MVT::v8i32, Operand: Ext1);
9104 }
9105
9106 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: ToVT, N1: Ext, N2: Ext1);
9107}
9108
9109/// isExtendedBUILD_VECTOR - Check if N is a constant BUILD_VECTOR where each
9110/// element has been zero/sign-extended, depending on the isSigned parameter,
9111/// from an integer type half its size.
9112static bool isExtendedBUILD_VECTOR(SDNode *N, SelectionDAG &DAG,
9113 bool isSigned) {
9114 // A v2i64 BUILD_VECTOR will have been legalized to a BITCAST from v4i32.
9115 EVT VT = N->getValueType(ResNo: 0);
9116 if (VT == MVT::v2i64 && N->getOpcode() == ISD::BITCAST) {
9117 SDNode *BVN = N->getOperand(Num: 0).getNode();
9118 if (BVN->getValueType(ResNo: 0) != MVT::v4i32 ||
9119 BVN->getOpcode() != ISD::BUILD_VECTOR)
9120 return false;
9121 unsigned LoElt = DAG.getDataLayout().isBigEndian() ? 1 : 0;
9122 unsigned HiElt = 1 - LoElt;
9123 ConstantSDNode *Lo0 = dyn_cast<ConstantSDNode>(Val: BVN->getOperand(Num: LoElt));
9124 ConstantSDNode *Hi0 = dyn_cast<ConstantSDNode>(Val: BVN->getOperand(Num: HiElt));
9125 ConstantSDNode *Lo1 = dyn_cast<ConstantSDNode>(Val: BVN->getOperand(Num: LoElt+2));
9126 ConstantSDNode *Hi1 = dyn_cast<ConstantSDNode>(Val: BVN->getOperand(Num: HiElt+2));
9127 if (!Lo0 || !Hi0 || !Lo1 || !Hi1)
9128 return false;
9129 if (isSigned) {
9130 if (Hi0->getSExtValue() == Lo0->getSExtValue() >> 32 &&
9131 Hi1->getSExtValue() == Lo1->getSExtValue() >> 32)
9132 return true;
9133 } else {
9134 if (Hi0->isZero() && Hi1->isZero())
9135 return true;
9136 }
9137 return false;
9138 }
9139
9140 if (N->getOpcode() != ISD::BUILD_VECTOR)
9141 return false;
9142
9143 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
9144 SDNode *Elt = N->getOperand(Num: i).getNode();
9145 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val: Elt)) {
9146 unsigned EltSize = VT.getScalarSizeInBits();
9147 unsigned HalfSize = EltSize / 2;
9148 if (isSigned) {
9149 if (!isIntN(N: HalfSize, x: C->getSExtValue()))
9150 return false;
9151 } else {
9152 if (!isUIntN(N: HalfSize, x: C->getZExtValue()))
9153 return false;
9154 }
9155 continue;
9156 }
9157 return false;
9158 }
9159
9160 return true;
9161}
9162
9163/// isSignExtended - Check if a node is a vector value that is sign-extended
9164/// or a constant BUILD_VECTOR with sign-extended elements.
9165static bool isSignExtended(SDNode *N, SelectionDAG &DAG) {
9166 if (N->getOpcode() == ISD::SIGN_EXTEND || ISD::isSEXTLoad(N))
9167 return true;
9168 if (isExtendedBUILD_VECTOR(N, DAG, isSigned: true))
9169 return true;
9170 return false;
9171}
9172
9173/// isZeroExtended - Check if a node is a vector value that is zero-extended (or
9174/// any-extended) or a constant BUILD_VECTOR with zero-extended elements.
9175static bool isZeroExtended(SDNode *N, SelectionDAG &DAG) {
9176 if (N->getOpcode() == ISD::ZERO_EXTEND || N->getOpcode() == ISD::ANY_EXTEND ||
9177 ISD::isZEXTLoad(N))
9178 return true;
9179 if (isExtendedBUILD_VECTOR(N, DAG, isSigned: false))
9180 return true;
9181 return false;
9182}
9183
9184static EVT getExtensionTo64Bits(const EVT &OrigVT) {
9185 if (OrigVT.getSizeInBits() >= 64)
9186 return OrigVT;
9187
9188 assert(OrigVT.isSimple() && "Expecting a simple value type");
9189
9190 MVT::SimpleValueType OrigSimpleTy = OrigVT.getSimpleVT().SimpleTy;
9191 switch (OrigSimpleTy) {
9192 default: llvm_unreachable("Unexpected Vector Type");
9193 case MVT::v2i8:
9194 case MVT::v2i16:
9195 return MVT::v2i32;
9196 case MVT::v4i8:
9197 return MVT::v4i16;
9198 }
9199}
9200
9201/// AddRequiredExtensionForVMULL - Add a sign/zero extension to extend the total
9202/// value size to 64 bits. We need a 64-bit D register as an operand to VMULL.
9203/// We insert the required extension here to get the vector to fill a D register.
9204static SDValue AddRequiredExtensionForVMULL(SDValue N, SelectionDAG &DAG,
9205 const EVT &OrigTy,
9206 const EVT &ExtTy,
9207 unsigned ExtOpcode) {
9208 // The vector originally had a size of OrigTy. It was then extended to ExtTy.
9209 // We expect the ExtTy to be 128-bits total. If the OrigTy is less than
9210 // 64-bits we need to insert a new extension so that it will be 64-bits.
9211 assert(ExtTy.is128BitVector() && "Unexpected extension size");
9212 if (OrigTy.getSizeInBits() >= 64)
9213 return N;
9214
9215 // Must extend size to at least 64 bits to be used as an operand for VMULL.
9216 EVT NewVT = getExtensionTo64Bits(OrigVT: OrigTy);
9217
9218 return DAG.getNode(Opcode: ExtOpcode, DL: SDLoc(N), VT: NewVT, Operand: N);
9219}
9220
9221/// SkipLoadExtensionForVMULL - return a load of the original vector size that
9222/// does not do any sign/zero extension. If the original vector is less
9223/// than 64 bits, an appropriate extension will be added after the load to
9224/// reach a total size of 64 bits. We have to add the extension separately
9225/// because ARM does not have a sign/zero extending load for vectors.
9226static SDValue SkipLoadExtensionForVMULL(LoadSDNode *LD, SelectionDAG& DAG) {
9227 EVT ExtendedTy = getExtensionTo64Bits(OrigVT: LD->getMemoryVT());
9228
9229 // The load already has the right type.
9230 if (ExtendedTy == LD->getMemoryVT())
9231 return DAG.getLoad(VT: LD->getMemoryVT(), dl: SDLoc(LD), Chain: LD->getChain(),
9232 Ptr: LD->getBasePtr(), PtrInfo: LD->getPointerInfo(), Alignment: LD->getAlign(),
9233 MMOFlags: LD->getMemOperand()->getFlags());
9234
9235 // We need to create a zextload/sextload. We cannot just create a load
9236 // followed by a zext/zext node because LowerMUL is also run during normal
9237 // operation legalization where we can't create illegal types.
9238 return DAG.getExtLoad(ExtType: LD->getExtensionType(), dl: SDLoc(LD), VT: ExtendedTy,
9239 Chain: LD->getChain(), Ptr: LD->getBasePtr(), PtrInfo: LD->getPointerInfo(),
9240 MemVT: LD->getMemoryVT(), Alignment: LD->getAlign(),
9241 MMOFlags: LD->getMemOperand()->getFlags());
9242}
9243
9244/// SkipExtensionForVMULL - For a node that is a SIGN_EXTEND, ZERO_EXTEND,
9245/// ANY_EXTEND, extending load, or BUILD_VECTOR with extended elements, return
9246/// the unextended value. The unextended vector should be 64 bits so that it can
9247/// be used as an operand to a VMULL instruction. If the original vector size
9248/// before extension is less than 64 bits we add a an extension to resize
9249/// the vector to 64 bits.
9250static SDValue SkipExtensionForVMULL(SDNode *N, SelectionDAG &DAG) {
9251 if (N->getOpcode() == ISD::SIGN_EXTEND ||
9252 N->getOpcode() == ISD::ZERO_EXTEND || N->getOpcode() == ISD::ANY_EXTEND)
9253 return AddRequiredExtensionForVMULL(N: N->getOperand(Num: 0), DAG,
9254 OrigTy: N->getOperand(Num: 0)->getValueType(ResNo: 0),
9255 ExtTy: N->getValueType(ResNo: 0),
9256 ExtOpcode: N->getOpcode());
9257
9258 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Val: N)) {
9259 assert((ISD::isSEXTLoad(LD) || ISD::isZEXTLoad(LD)) &&
9260 "Expected extending load");
9261
9262 SDValue newLoad = SkipLoadExtensionForVMULL(LD, DAG);
9263 DAG.ReplaceAllUsesOfValueWith(From: SDValue(LD, 1), To: newLoad.getValue(R: 1));
9264 unsigned Opcode = ISD::isSEXTLoad(N: LD) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
9265 SDValue extLoad =
9266 DAG.getNode(Opcode, DL: SDLoc(newLoad), VT: LD->getValueType(ResNo: 0), Operand: newLoad);
9267 DAG.ReplaceAllUsesOfValueWith(From: SDValue(LD, 0), To: extLoad);
9268
9269 return newLoad;
9270 }
9271
9272 // Otherwise, the value must be a BUILD_VECTOR. For v2i64, it will
9273 // have been legalized as a BITCAST from v4i32.
9274 if (N->getOpcode() == ISD::BITCAST) {
9275 SDNode *BVN = N->getOperand(Num: 0).getNode();
9276 assert(BVN->getOpcode() == ISD::BUILD_VECTOR &&
9277 BVN->getValueType(0) == MVT::v4i32 && "expected v4i32 BUILD_VECTOR");
9278 unsigned LowElt = DAG.getDataLayout().isBigEndian() ? 1 : 0;
9279 return DAG.getBuildVector(
9280 VT: MVT::v2i32, DL: SDLoc(N),
9281 Ops: {BVN->getOperand(Num: LowElt), BVN->getOperand(Num: LowElt + 2)});
9282 }
9283 // Construct a new BUILD_VECTOR with elements truncated to half the size.
9284 assert(N->getOpcode() == ISD::BUILD_VECTOR && "expected BUILD_VECTOR");
9285 EVT VT = N->getValueType(ResNo: 0);
9286 unsigned EltSize = VT.getScalarSizeInBits() / 2;
9287 unsigned NumElts = VT.getVectorNumElements();
9288 MVT TruncVT = MVT::getIntegerVT(BitWidth: EltSize);
9289 SmallVector<SDValue, 8> Ops;
9290 SDLoc dl(N);
9291 for (unsigned i = 0; i != NumElts; ++i) {
9292 const APInt &CInt = N->getConstantOperandAPInt(Num: i);
9293 // Element types smaller than 32 bits are not legal, so use i32 elements.
9294 // The values are implicitly truncated so sext vs. zext doesn't matter.
9295 Ops.push_back(Elt: DAG.getConstant(Val: CInt.zextOrTrunc(width: 32), DL: dl, VT: MVT::i32));
9296 }
9297 return DAG.getBuildVector(VT: MVT::getVectorVT(VT: TruncVT, NumElements: NumElts), DL: dl, Ops);
9298}
9299
9300static bool isAddSubSExt(SDNode *N, SelectionDAG &DAG) {
9301 unsigned Opcode = N->getOpcode();
9302 if (Opcode == ISD::ADD || Opcode == ISD::SUB) {
9303 SDNode *N0 = N->getOperand(Num: 0).getNode();
9304 SDNode *N1 = N->getOperand(Num: 1).getNode();
9305 return N0->hasOneUse() && N1->hasOneUse() &&
9306 isSignExtended(N: N0, DAG) && isSignExtended(N: N1, DAG);
9307 }
9308 return false;
9309}
9310
9311static bool isAddSubZExt(SDNode *N, SelectionDAG &DAG) {
9312 unsigned Opcode = N->getOpcode();
9313 if (Opcode == ISD::ADD || Opcode == ISD::SUB) {
9314 SDNode *N0 = N->getOperand(Num: 0).getNode();
9315 SDNode *N1 = N->getOperand(Num: 1).getNode();
9316 return N0->hasOneUse() && N1->hasOneUse() &&
9317 isZeroExtended(N: N0, DAG) && isZeroExtended(N: N1, DAG);
9318 }
9319 return false;
9320}
9321
9322static SDValue LowerMUL(SDValue Op, SelectionDAG &DAG) {
9323 // Multiplications are only custom-lowered for 128-bit vectors so that
9324 // VMULL can be detected. Otherwise v2i64 multiplications are not legal.
9325 EVT VT = Op.getValueType();
9326 assert(VT.is128BitVector() && VT.isInteger() &&
9327 "unexpected type for custom-lowering ISD::MUL");
9328 SDNode *N0 = Op.getOperand(i: 0).getNode();
9329 SDNode *N1 = Op.getOperand(i: 1).getNode();
9330 unsigned NewOpc = 0;
9331 bool isMLA = false;
9332 bool isN0SExt = isSignExtended(N: N0, DAG);
9333 bool isN1SExt = isSignExtended(N: N1, DAG);
9334 if (isN0SExt && isN1SExt)
9335 NewOpc = ARMISD::VMULLs;
9336 else {
9337 bool isN0ZExt = isZeroExtended(N: N0, DAG);
9338 bool isN1ZExt = isZeroExtended(N: N1, DAG);
9339 if (isN0ZExt && isN1ZExt)
9340 NewOpc = ARMISD::VMULLu;
9341 else if (isN1SExt || isN1ZExt) {
9342 // Look for (s/zext A + s/zext B) * (s/zext C). We want to turn these
9343 // into (s/zext A * s/zext C) + (s/zext B * s/zext C)
9344 if (isN1SExt && isAddSubSExt(N: N0, DAG)) {
9345 NewOpc = ARMISD::VMULLs;
9346 isMLA = true;
9347 } else if (isN1ZExt && isAddSubZExt(N: N0, DAG)) {
9348 NewOpc = ARMISD::VMULLu;
9349 isMLA = true;
9350 } else if (isN0ZExt && isAddSubZExt(N: N1, DAG)) {
9351 std::swap(a&: N0, b&: N1);
9352 NewOpc = ARMISD::VMULLu;
9353 isMLA = true;
9354 }
9355 }
9356
9357 if (!NewOpc) {
9358 if (VT == MVT::v2i64)
9359 // Fall through to expand this. It is not legal.
9360 return SDValue();
9361 else
9362 // Other vector multiplications are legal.
9363 return Op;
9364 }
9365 }
9366
9367 // Legalize to a VMULL instruction.
9368 SDLoc DL(Op);
9369 SDValue Op0;
9370 SDValue Op1 = SkipExtensionForVMULL(N: N1, DAG);
9371 if (!isMLA) {
9372 Op0 = SkipExtensionForVMULL(N: N0, DAG);
9373 assert(Op0.getValueType().is64BitVector() &&
9374 Op1.getValueType().is64BitVector() &&
9375 "unexpected types for extended operands to VMULL");
9376 return DAG.getNode(Opcode: NewOpc, DL, VT, N1: Op0, N2: Op1);
9377 }
9378
9379 // Optimizing (zext A + zext B) * C, to (VMULL A, C) + (VMULL B, C) during
9380 // isel lowering to take advantage of no-stall back to back vmul + vmla.
9381 // vmull q0, d4, d6
9382 // vmlal q0, d5, d6
9383 // is faster than
9384 // vaddl q0, d4, d5
9385 // vmovl q1, d6
9386 // vmul q0, q0, q1
9387 SDValue N00 = SkipExtensionForVMULL(N: N0->getOperand(Num: 0).getNode(), DAG);
9388 SDValue N01 = SkipExtensionForVMULL(N: N0->getOperand(Num: 1).getNode(), DAG);
9389 EVT Op1VT = Op1.getValueType();
9390 return DAG.getNode(Opcode: N0->getOpcode(), DL, VT,
9391 N1: DAG.getNode(Opcode: NewOpc, DL, VT,
9392 N1: DAG.getNode(Opcode: ISD::BITCAST, DL, VT: Op1VT, Operand: N00), N2: Op1),
9393 N2: DAG.getNode(Opcode: NewOpc, DL, VT,
9394 N1: DAG.getNode(Opcode: ISD::BITCAST, DL, VT: Op1VT, Operand: N01), N2: Op1));
9395}
9396
9397static SDValue LowerSDIV_v4i8(SDValue X, SDValue Y, const SDLoc &dl,
9398 SelectionDAG &DAG) {
9399 // TODO: Should this propagate fast-math-flags?
9400
9401 // Convert to float
9402 // float4 xf = vcvt_f32_s32(vmovl_s16(a.lo));
9403 // float4 yf = vcvt_f32_s32(vmovl_s16(b.lo));
9404 X = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: MVT::v4i32, Operand: X);
9405 Y = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: MVT::v4i32, Operand: Y);
9406 X = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: MVT::v4f32, Operand: X);
9407 Y = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: MVT::v4f32, Operand: Y);
9408 // Get reciprocal estimate.
9409 // float4 recip = vrecpeq_f32(yf);
9410 Y = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: MVT::v4f32,
9411 N1: DAG.getConstant(Val: Intrinsic::arm_neon_vrecpe, DL: dl, VT: MVT::i32),
9412 N2: Y);
9413 // Because char has a smaller range than uchar, we can actually get away
9414 // without any newton steps. This requires that we use a weird bias
9415 // of 0xb000, however (again, this has been exhaustively tested).
9416 // float4 result = as_float4(as_int4(xf*recip) + 0xb000);
9417 X = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::v4f32, N1: X, N2: Y);
9418 X = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v4i32, Operand: X);
9419 Y = DAG.getConstant(Val: 0xb000, DL: dl, VT: MVT::v4i32);
9420 X = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::v4i32, N1: X, N2: Y);
9421 X = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v4f32, Operand: X);
9422 // Convert back to short.
9423 X = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: dl, VT: MVT::v4i32, Operand: X);
9424 X = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::v4i16, Operand: X);
9425 return X;
9426}
9427
9428static SDValue LowerSDIV_v4i16(SDValue N0, SDValue N1, const SDLoc &dl,
9429 SelectionDAG &DAG) {
9430 // TODO: Should this propagate fast-math-flags?
9431
9432 SDValue N2;
9433 // Convert to float.
9434 // float4 yf = vcvt_f32_s32(vmovl_s16(y));
9435 // float4 xf = vcvt_f32_s32(vmovl_s16(x));
9436 N0 = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: MVT::v4i32, Operand: N0);
9437 N1 = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: MVT::v4i32, Operand: N1);
9438 N0 = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: MVT::v4f32, Operand: N0);
9439 N1 = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: MVT::v4f32, Operand: N1);
9440
9441 // Use reciprocal estimate and one refinement step.
9442 // float4 recip = vrecpeq_f32(yf);
9443 // recip *= vrecpsq_f32(yf, recip);
9444 N2 = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: MVT::v4f32,
9445 N1: DAG.getConstant(Val: Intrinsic::arm_neon_vrecpe, DL: dl, VT: MVT::i32),
9446 N2: N1);
9447 N1 = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: MVT::v4f32,
9448 N1: DAG.getConstant(Val: Intrinsic::arm_neon_vrecps, DL: dl, VT: MVT::i32),
9449 N2: N1, N3: N2);
9450 N2 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::v4f32, N1, N2);
9451 // Because short has a smaller range than ushort, we can actually get away
9452 // with only a single newton step. This requires that we use a weird bias
9453 // of 89, however (again, this has been exhaustively tested).
9454 // float4 result = as_float4(as_int4(xf*recip) + 0x89);
9455 N0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::v4f32, N1: N0, N2);
9456 N0 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v4i32, Operand: N0);
9457 N1 = DAG.getConstant(Val: 0x89, DL: dl, VT: MVT::v4i32);
9458 N0 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::v4i32, N1: N0, N2: N1);
9459 N0 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v4f32, Operand: N0);
9460 // Convert back to integer and return.
9461 // return vmovn_s32(vcvt_s32_f32(result));
9462 N0 = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: dl, VT: MVT::v4i32, Operand: N0);
9463 N0 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::v4i16, Operand: N0);
9464 return N0;
9465}
9466
9467static SDValue LowerSDIV(SDValue Op, SelectionDAG &DAG,
9468 const ARMSubtarget *ST) {
9469 EVT VT = Op.getValueType();
9470 assert((VT == MVT::v4i16 || VT == MVT::v8i8) &&
9471 "unexpected type for custom-lowering ISD::SDIV");
9472
9473 SDLoc dl(Op);
9474 SDValue N0 = Op.getOperand(i: 0);
9475 SDValue N1 = Op.getOperand(i: 1);
9476 SDValue N2, N3;
9477
9478 if (VT == MVT::v8i8) {
9479 N0 = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: MVT::v8i16, Operand: N0);
9480 N1 = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: MVT::v8i16, Operand: N1);
9481
9482 N2 = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: MVT::v4i16, N1: N0,
9483 N2: DAG.getIntPtrConstant(Val: 4, DL: dl));
9484 N3 = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: MVT::v4i16, N1,
9485 N2: DAG.getIntPtrConstant(Val: 4, DL: dl));
9486 N0 = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: MVT::v4i16, N1: N0,
9487 N2: DAG.getIntPtrConstant(Val: 0, DL: dl));
9488 N1 = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: MVT::v4i16, N1,
9489 N2: DAG.getIntPtrConstant(Val: 0, DL: dl));
9490
9491 N0 = LowerSDIV_v4i8(X: N0, Y: N1, dl, DAG); // v4i16
9492 N2 = LowerSDIV_v4i8(X: N2, Y: N3, dl, DAG); // v4i16
9493
9494 N0 = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: MVT::v8i16, N1: N0, N2);
9495 N0 = LowerCONCAT_VECTORS(Op: N0, DAG, ST);
9496
9497 N0 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::v8i8, Operand: N0);
9498 return N0;
9499 }
9500 return LowerSDIV_v4i16(N0, N1, dl, DAG);
9501}
9502
9503static SDValue LowerUDIV(SDValue Op, SelectionDAG &DAG,
9504 const ARMSubtarget *ST) {
9505 // TODO: Should this propagate fast-math-flags?
9506 EVT VT = Op.getValueType();
9507 assert((VT == MVT::v4i16 || VT == MVT::v8i8) &&
9508 "unexpected type for custom-lowering ISD::UDIV");
9509
9510 SDLoc dl(Op);
9511 SDValue N0 = Op.getOperand(i: 0);
9512 SDValue N1 = Op.getOperand(i: 1);
9513 SDValue N2, N3;
9514
9515 if (VT == MVT::v8i8) {
9516 N0 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: MVT::v8i16, Operand: N0);
9517 N1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: MVT::v8i16, Operand: N1);
9518
9519 N2 = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: MVT::v4i16, N1: N0,
9520 N2: DAG.getIntPtrConstant(Val: 4, DL: dl));
9521 N3 = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: MVT::v4i16, N1,
9522 N2: DAG.getIntPtrConstant(Val: 4, DL: dl));
9523 N0 = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: MVT::v4i16, N1: N0,
9524 N2: DAG.getIntPtrConstant(Val: 0, DL: dl));
9525 N1 = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: MVT::v4i16, N1,
9526 N2: DAG.getIntPtrConstant(Val: 0, DL: dl));
9527
9528 N0 = LowerSDIV_v4i16(N0, N1, dl, DAG); // v4i16
9529 N2 = LowerSDIV_v4i16(N0: N2, N1: N3, dl, DAG); // v4i16
9530
9531 N0 = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: MVT::v8i16, N1: N0, N2);
9532 N0 = LowerCONCAT_VECTORS(Op: N0, DAG, ST);
9533
9534 N0 = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: MVT::v8i8,
9535 N1: DAG.getConstant(Val: Intrinsic::arm_neon_vqmovnsu, DL: dl,
9536 VT: MVT::i32),
9537 N2: N0);
9538 return N0;
9539 }
9540
9541 // v4i16 sdiv ... Convert to float.
9542 // float4 yf = vcvt_f32_s32(vmovl_u16(y));
9543 // float4 xf = vcvt_f32_s32(vmovl_u16(x));
9544 N0 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: MVT::v4i32, Operand: N0);
9545 N1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: MVT::v4i32, Operand: N1);
9546 N0 = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: MVT::v4f32, Operand: N0);
9547 SDValue BN1 = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: MVT::v4f32, Operand: N1);
9548
9549 // Use reciprocal estimate and two refinement steps.
9550 // float4 recip = vrecpeq_f32(yf);
9551 // recip *= vrecpsq_f32(yf, recip);
9552 // recip *= vrecpsq_f32(yf, recip);
9553 N2 = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: MVT::v4f32,
9554 N1: DAG.getConstant(Val: Intrinsic::arm_neon_vrecpe, DL: dl, VT: MVT::i32),
9555 N2: BN1);
9556 N1 = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: MVT::v4f32,
9557 N1: DAG.getConstant(Val: Intrinsic::arm_neon_vrecps, DL: dl, VT: MVT::i32),
9558 N2: BN1, N3: N2);
9559 N2 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::v4f32, N1, N2);
9560 N1 = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: MVT::v4f32,
9561 N1: DAG.getConstant(Val: Intrinsic::arm_neon_vrecps, DL: dl, VT: MVT::i32),
9562 N2: BN1, N3: N2);
9563 N2 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::v4f32, N1, N2);
9564 // Simply multiplying by the reciprocal estimate can leave us a few ulps
9565 // too low, so we add 2 ulps (exhaustive testing shows that this is enough,
9566 // and that it will never cause us to return an answer too large).
9567 // float4 result = as_float4(as_int4(xf*recip) + 2);
9568 N0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::v4f32, N1: N0, N2);
9569 N0 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v4i32, Operand: N0);
9570 N1 = DAG.getConstant(Val: 2, DL: dl, VT: MVT::v4i32);
9571 N0 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::v4i32, N1: N0, N2: N1);
9572 N0 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v4f32, Operand: N0);
9573 // Convert back to integer and return.
9574 // return vmovn_u32(vcvt_s32_f32(result));
9575 N0 = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: dl, VT: MVT::v4i32, Operand: N0);
9576 N0 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::v4i16, Operand: N0);
9577 return N0;
9578}
9579
9580static SDValue LowerADDSUBO_CARRY(SDValue Op, SelectionDAG &DAG,
9581 unsigned Opcode, bool IsSigned) {
9582 EVT VT0 = Op.getValue(R: 0).getValueType();
9583 EVT VT1 = Op.getValue(R: 1).getValueType();
9584
9585 bool InvertCarry = Opcode == ARMISD::SUBE;
9586 SDValue OpLHS = Op.getOperand(i: 0);
9587 SDValue OpRHS = Op.getOperand(i: 1);
9588 SDValue OpCarryIn = valueToCarryFlag(Value: Op.getOperand(i: 2), DAG, Invert: InvertCarry);
9589
9590 SDLoc DL(Op);
9591
9592 SDValue Result = DAG.getNode(Opcode, DL, VTList: DAG.getVTList(VT1: VT0, VT2: MVT::i32), N1: OpLHS,
9593 N2: OpRHS, N3: OpCarryIn);
9594
9595 SDValue OutFlag =
9596 IsSigned ? overflowFlagToValue(Flags: Result.getValue(R: 1), VT: VT1, DAG)
9597 : carryFlagToValue(Flags: Result.getValue(R: 1), VT: VT1, DAG, Invert: InvertCarry);
9598
9599 return DAG.getMergeValues(Ops: {Result, OutFlag}, dl: DL);
9600}
9601
9602SDValue ARMTargetLowering::LowerWindowsDIVLibCall(SDValue Op, SelectionDAG &DAG,
9603 bool Signed,
9604 SDValue &Chain) const {
9605 EVT VT = Op.getValueType();
9606 assert((VT == MVT::i32 || VT == MVT::i64) &&
9607 "unexpected type for custom lowering DIV");
9608 SDLoc dl(Op);
9609
9610 const auto &DL = DAG.getDataLayout();
9611 RTLIB::Libcall LC;
9612 if (Signed)
9613 LC = VT == MVT::i32 ? RTLIB::SDIVREM_I32 : RTLIB::SDIVREM_I64;
9614 else
9615 LC = VT == MVT::i32 ? RTLIB::UDIVREM_I32 : RTLIB::UDIVREM_I64;
9616
9617 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
9618 SDValue ES = DAG.getExternalSymbol(LCImpl, VT: getPointerTy(DL));
9619
9620 ARMTargetLowering::ArgListTy Args;
9621
9622 for (auto AI : {1, 0}) {
9623 SDValue Operand = Op.getOperand(i: AI);
9624 Args.emplace_back(args&: Operand,
9625 args: Operand.getValueType().getTypeForEVT(Context&: *DAG.getContext()));
9626 }
9627
9628 CallLoweringInfo CLI(DAG);
9629 CLI.setDebugLoc(dl).setChain(Chain).setCallee(
9630 CC: DAG.getLibcalls().getLibcallImplCallingConv(Call: LCImpl),
9631 ResultType: VT.getTypeForEVT(Context&: *DAG.getContext()), Target: ES, ArgsList: std::move(Args));
9632
9633 return LowerCallTo(CLI).first;
9634}
9635
9636// This is a code size optimisation: return the original SDIV node to
9637// DAGCombiner when we don't want to expand SDIV into a sequence of
9638// instructions, and an empty node otherwise which will cause the
9639// SDIV to be expanded in DAGCombine.
9640SDValue
9641ARMTargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
9642 SelectionDAG &DAG,
9643 SmallVectorImpl<SDNode *> &Created) const {
9644 // TODO: Support SREM
9645 if (N->getOpcode() != ISD::SDIV)
9646 return SDValue();
9647
9648 const auto &ST = DAG.getSubtarget<ARMSubtarget>();
9649 const bool MinSize = ST.hasMinSize();
9650 const bool HasDivide = ST.isThumb() ? ST.hasDivideInThumbMode()
9651 : ST.hasDivideInARMMode();
9652
9653 // Don't touch vector types; rewriting this may lead to scalarizing
9654 // the int divs.
9655 if (N->getOperand(Num: 0).getValueType().isVector())
9656 return SDValue();
9657
9658 // Bail if MinSize is not set, and also for both ARM and Thumb mode we need
9659 // hwdiv support for this to be really profitable.
9660 if (!(MinSize && HasDivide))
9661 return SDValue();
9662
9663 // ARM mode is a bit simpler than Thumb: we can handle large power
9664 // of 2 immediates with 1 mov instruction; no further checks required,
9665 // just return the sdiv node.
9666 if (!ST.isThumb())
9667 return SDValue(N, 0);
9668
9669 // In Thumb mode, immediates larger than 128 need a wide 4-byte MOV,
9670 // and thus lose the code size benefits of a MOVS that requires only 2.
9671 // TargetTransformInfo and 'getIntImmCodeSizeCost' could be helpful here,
9672 // but as it's doing exactly this, it's not worth the trouble to get TTI.
9673 if (Divisor.sgt(RHS: 128))
9674 return SDValue();
9675
9676 return SDValue(N, 0);
9677}
9678
9679SDValue ARMTargetLowering::LowerDIV_Windows(SDValue Op, SelectionDAG &DAG,
9680 bool Signed) const {
9681 assert(Op.getValueType() == MVT::i32 &&
9682 "unexpected type for custom lowering DIV");
9683 SDLoc dl(Op);
9684
9685 SDValue DBZCHK = DAG.getNode(Opcode: ARMISD::WIN__DBZCHK, DL: dl, VT: MVT::Other,
9686 N1: DAG.getEntryNode(), N2: Op.getOperand(i: 1));
9687
9688 return LowerWindowsDIVLibCall(Op, DAG, Signed, Chain&: DBZCHK);
9689}
9690
9691static SDValue WinDBZCheckDenominator(SelectionDAG &DAG, SDNode *N, SDValue InChain) {
9692 SDLoc DL(N);
9693 SDValue Op = N->getOperand(Num: 1);
9694 if (N->getValueType(ResNo: 0) == MVT::i32)
9695 return DAG.getNode(Opcode: ARMISD::WIN__DBZCHK, DL, VT: MVT::Other, N1: InChain, N2: Op);
9696 SDValue Lo, Hi;
9697 std::tie(args&: Lo, args&: Hi) = DAG.SplitScalar(N: Op, DL, LoVT: MVT::i32, HiVT: MVT::i32);
9698 return DAG.getNode(Opcode: ARMISD::WIN__DBZCHK, DL, VT: MVT::Other, N1: InChain,
9699 N2: DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: Lo, N2: Hi));
9700}
9701
9702void ARMTargetLowering::ExpandDIV_Windows(
9703 SDValue Op, SelectionDAG &DAG, bool Signed,
9704 SmallVectorImpl<SDValue> &Results) const {
9705 const auto &DL = DAG.getDataLayout();
9706
9707 assert(Op.getValueType() == MVT::i64 &&
9708 "unexpected type for custom lowering DIV");
9709 SDLoc dl(Op);
9710
9711 SDValue DBZCHK = WinDBZCheckDenominator(DAG, N: Op.getNode(), InChain: DAG.getEntryNode());
9712
9713 SDValue Result = LowerWindowsDIVLibCall(Op, DAG, Signed, Chain&: DBZCHK);
9714
9715 SDValue Lower = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i32, Operand: Result);
9716 SDValue Upper = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i64, N1: Result,
9717 N2: DAG.getConstant(Val: 32, DL: dl, VT: getPointerTy(DL)));
9718 Upper = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i32, Operand: Upper);
9719
9720 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: MVT::i64, N1: Lower, N2: Upper));
9721}
9722
9723std::pair<SDValue, SDValue>
9724ARMTargetLowering::LowerAEABIUnalignedLoad(SDValue Op,
9725 SelectionDAG &DAG) const {
9726 // If we have an unaligned load from a i32 or i64 that would normally be
9727 // split into separate ldrb's, we can use the __aeabi_uread4/__aeabi_uread8
9728 // functions instead.
9729 LoadSDNode *LD = cast<LoadSDNode>(Val: Op.getNode());
9730 EVT MemVT = LD->getMemoryVT();
9731 if (MemVT != MVT::i32 && MemVT != MVT::i64)
9732 return std::make_pair(x: SDValue(), y: SDValue());
9733
9734 const auto &MF = DAG.getMachineFunction();
9735 unsigned AS = LD->getAddressSpace();
9736 Align Alignment = LD->getAlign();
9737 const DataLayout &DL = DAG.getDataLayout();
9738 bool AllowsUnaligned = Subtarget->allowsUnalignedMem();
9739 RTLIB::Libcall LC =
9740 (MemVT == MVT::i32) ? RTLIB::AEABI_UREAD4 : RTLIB::AEABI_UREAD8;
9741
9742 if (MF.getFunction().hasMinSize() && !AllowsUnaligned &&
9743 Alignment <= llvm::Align(2) && DAG.getLibcalls().getLibcallImpl(Call: LC)) {
9744 MakeLibCallOptions Opts;
9745 SDLoc dl(Op);
9746
9747 auto Pair = makeLibCall(DAG, LC, RetVT: MemVT.getSimpleVT(), Ops: LD->getBasePtr(),
9748 CallOptions: Opts, dl, Chain: LD->getChain());
9749
9750 // If necessary, extend the node to 64bit
9751 if (LD->getExtensionType() != ISD::NON_EXTLOAD) {
9752 unsigned ExtType = LD->getExtensionType() == ISD::SEXTLOAD
9753 ? ISD::SIGN_EXTEND
9754 : ISD::ZERO_EXTEND;
9755 SDValue EN = DAG.getNode(Opcode: ExtType, DL: dl, VT: LD->getValueType(ResNo: 0), Operand: Pair.first);
9756 Pair.first = EN;
9757 }
9758 return Pair;
9759 }
9760
9761 // Default expand to individual loads
9762 if (!allowsMemoryAccess(Context&: *DAG.getContext(), DL, VT: MemVT, AddrSpace: AS, Alignment))
9763 return expandUnalignedLoad(LD, DAG);
9764 return std::make_pair(x: SDValue(), y: SDValue());
9765}
9766
9767SDValue ARMTargetLowering::LowerAEABIUnalignedStore(SDValue Op,
9768 SelectionDAG &DAG) const {
9769 // If we have an unaligned store to a i32 or i64 that would normally be
9770 // split into separate ldrb's, we can use the __aeabi_uwrite4/__aeabi_uwrite8
9771 // functions instead.
9772 StoreSDNode *ST = cast<StoreSDNode>(Val: Op.getNode());
9773 EVT MemVT = ST->getMemoryVT();
9774 if (MemVT != MVT::i32 && MemVT != MVT::i64)
9775 return SDValue();
9776
9777 const auto &MF = DAG.getMachineFunction();
9778 unsigned AS = ST->getAddressSpace();
9779 Align Alignment = ST->getAlign();
9780 const DataLayout &DL = DAG.getDataLayout();
9781 bool AllowsUnaligned = Subtarget->allowsUnalignedMem();
9782 RTLIB::Libcall LC =
9783 (MemVT == MVT::i32) ? RTLIB::AEABI_UWRITE4 : RTLIB::AEABI_UWRITE8;
9784
9785 if (MF.getFunction().hasMinSize() && !AllowsUnaligned &&
9786 Alignment <= llvm::Align(2) && DAG.getLibcalls().getLibcallImpl(Call: LC)) {
9787
9788 SDLoc dl(Op);
9789
9790 // If necessary, trunc the value to 32bit
9791 SDValue StoreVal = ST->getOperand(Num: 1);
9792 if (ST->isTruncatingStore())
9793 StoreVal = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MemVT, Operand: ST->getOperand(Num: 1));
9794
9795 MakeLibCallOptions Opts;
9796 auto CallResult =
9797 makeLibCall(DAG, LC, RetVT: MVT::isVoid, Ops: {StoreVal, ST->getBasePtr()}, CallOptions: Opts,
9798 dl, Chain: ST->getChain());
9799
9800 return CallResult.second;
9801 }
9802
9803 // Default expand to individual stores
9804 if (!allowsMemoryAccess(Context&: *DAG.getContext(), DL, VT: MemVT, AddrSpace: AS, Alignment))
9805 return expandUnalignedStore(ST, DAG);
9806 return SDValue();
9807}
9808
9809static SDValue LowerPredicateLoad(SDValue Op, SelectionDAG &DAG) {
9810 LoadSDNode *LD = cast<LoadSDNode>(Val: Op.getNode());
9811 EVT MemVT = LD->getMemoryVT();
9812 assert((MemVT == MVT::v2i1 || MemVT == MVT::v4i1 || MemVT == MVT::v8i1 ||
9813 MemVT == MVT::v16i1) &&
9814 "Expected a predicate type!");
9815 assert(MemVT == Op.getValueType());
9816 assert(LD->getExtensionType() == ISD::NON_EXTLOAD &&
9817 "Expected a non-extending load");
9818 assert(LD->isUnindexed() && "Expected a unindexed load");
9819
9820 // The basic MVE VLDR on a v2i1/v4i1/v8i1 actually loads the entire 16bit
9821 // predicate, with the "v4i1" bits spread out over the 16 bits loaded. We
9822 // need to make sure that 8/4/2 bits are actually loaded into the correct
9823 // place, which means loading the value and then shuffling the values into
9824 // the bottom bits of the predicate.
9825 // Equally, VLDR for an v16i1 will actually load 32bits (so will be incorrect
9826 // for BE).
9827 // Speaking of BE, apparently the rest of llvm will assume a reverse order to
9828 // a natural VMSR(load), so needs to be reversed.
9829
9830 SDLoc dl(Op);
9831 SDValue Load = DAG.getExtLoad(
9832 ExtType: ISD::EXTLOAD, dl, VT: MVT::i32, Chain: LD->getChain(), Ptr: LD->getBasePtr(),
9833 MemVT: EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: MemVT.getSizeInBits()),
9834 MMO: LD->getMemOperand());
9835 SDValue Val = Load;
9836 if (DAG.getDataLayout().isBigEndian())
9837 Val = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i32,
9838 N1: DAG.getNode(Opcode: ISD::BITREVERSE, DL: dl, VT: MVT::i32, Operand: Load),
9839 N2: DAG.getConstant(Val: 32 - MemVT.getSizeInBits(), DL: dl, VT: MVT::i32));
9840 SDValue Pred = DAG.getNode(Opcode: ARMISD::PREDICATE_CAST, DL: dl, VT: MVT::v16i1, Operand: Val);
9841 if (MemVT != MVT::v16i1)
9842 Pred = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: MemVT, N1: Pred,
9843 N2: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
9844 return DAG.getMergeValues(Ops: {Pred, Load.getValue(R: 1)}, dl);
9845}
9846
9847void ARMTargetLowering::LowerLOAD(SDNode *N, SmallVectorImpl<SDValue> &Results,
9848 SelectionDAG &DAG) const {
9849 LoadSDNode *LD = cast<LoadSDNode>(Val: N);
9850 EVT MemVT = LD->getMemoryVT();
9851
9852 if (MemVT == MVT::i64 && Subtarget->hasV5TEOps() &&
9853 !Subtarget->isThumb1Only() && LD->isVolatile() &&
9854 LD->getAlign() >= Subtarget->getDualLoadStoreAlignment()) {
9855 assert(LD->isUnindexed() && "Loads should be unindexed at this point.");
9856 SDLoc dl(N);
9857 SDValue Result = DAG.getMemIntrinsicNode(
9858 Opcode: ARMISD::LDRD, dl, VTList: DAG.getVTList(VTs: {MVT::i32, MVT::i32, MVT::Other}),
9859 Ops: {LD->getChain(), LD->getBasePtr()}, MemVT, MMO: LD->getMemOperand());
9860 SDValue Lo = Result.getValue(R: DAG.getDataLayout().isLittleEndian() ? 0 : 1);
9861 SDValue Hi = Result.getValue(R: DAG.getDataLayout().isLittleEndian() ? 1 : 0);
9862 SDValue Pair = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: MVT::i64, N1: Lo, N2: Hi);
9863 Results.append(IL: {Pair, Result.getValue(R: 2)});
9864 } else if (MemVT == MVT::i32 || MemVT == MVT::i64) {
9865 auto Pair = LowerAEABIUnalignedLoad(Op: SDValue(N, 0), DAG);
9866 if (Pair.first) {
9867 Results.push_back(Elt: Pair.first);
9868 Results.push_back(Elt: Pair.second);
9869 }
9870 }
9871}
9872
9873static SDValue LowerPredicateStore(SDValue Op, SelectionDAG &DAG) {
9874 StoreSDNode *ST = cast<StoreSDNode>(Val: Op.getNode());
9875 EVT MemVT = ST->getMemoryVT();
9876 assert((MemVT == MVT::v2i1 || MemVT == MVT::v4i1 || MemVT == MVT::v8i1 ||
9877 MemVT == MVT::v16i1) &&
9878 "Expected a predicate type!");
9879 assert(MemVT == ST->getValue().getValueType());
9880 assert(!ST->isTruncatingStore() && "Expected a non-extending store");
9881 assert(ST->isUnindexed() && "Expected a unindexed store");
9882
9883 // Only store the v2i1 or v4i1 or v8i1 worth of bits, via a buildvector with
9884 // top bits unset and a scalar store.
9885 SDLoc dl(Op);
9886 SDValue Build = ST->getValue();
9887 if (MemVT != MVT::v16i1) {
9888 SmallVector<SDValue, 16> Ops;
9889 for (unsigned I = 0; I < MemVT.getVectorNumElements(); I++) {
9890 unsigned Elt = DAG.getDataLayout().isBigEndian()
9891 ? MemVT.getVectorNumElements() - I - 1
9892 : I;
9893 Ops.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: MVT::i32, N1: Build,
9894 N2: DAG.getConstant(Val: Elt, DL: dl, VT: MVT::i32)));
9895 }
9896 for (unsigned I = MemVT.getVectorNumElements(); I < 16; I++)
9897 Ops.push_back(Elt: DAG.getUNDEF(VT: MVT::i32));
9898 Build = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: dl, VT: MVT::v16i1, Ops);
9899 }
9900 SDValue GRP = DAG.getNode(Opcode: ARMISD::PREDICATE_CAST, DL: dl, VT: MVT::i32, Operand: Build);
9901 if (MemVT == MVT::v16i1 && DAG.getDataLayout().isBigEndian())
9902 GRP = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i32,
9903 N1: DAG.getNode(Opcode: ISD::BITREVERSE, DL: dl, VT: MVT::i32, Operand: GRP),
9904 N2: DAG.getConstant(Val: 16, DL: dl, VT: MVT::i32));
9905 return DAG.getTruncStore(
9906 Chain: ST->getChain(), dl, Val: GRP, Ptr: ST->getBasePtr(),
9907 SVT: EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: MemVT.getSizeInBits()),
9908 MMO: ST->getMemOperand());
9909}
9910
9911SDValue ARMTargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG,
9912 const ARMSubtarget *Subtarget) const {
9913 StoreSDNode *ST = cast<StoreSDNode>(Val: Op.getNode());
9914 EVT MemVT = ST->getMemoryVT();
9915
9916 if (MemVT == MVT::i64 && Subtarget->hasV5TEOps() &&
9917 !Subtarget->isThumb1Only() && ST->isVolatile() &&
9918 ST->getAlign() >= Subtarget->getDualLoadStoreAlignment()) {
9919 assert(ST->isUnindexed() && "Stores should be unindexed at this point.");
9920 SDNode *N = Op.getNode();
9921 SDLoc dl(N);
9922
9923 SDValue Lo = DAG.getNode(
9924 Opcode: ISD::EXTRACT_ELEMENT, DL: dl, VT: MVT::i32, N1: ST->getValue(),
9925 N2: DAG.getTargetConstant(Val: DAG.getDataLayout().isLittleEndian() ? 0 : 1, DL: dl,
9926 VT: MVT::i32));
9927 SDValue Hi = DAG.getNode(
9928 Opcode: ISD::EXTRACT_ELEMENT, DL: dl, VT: MVT::i32, N1: ST->getValue(),
9929 N2: DAG.getTargetConstant(Val: DAG.getDataLayout().isLittleEndian() ? 1 : 0, DL: dl,
9930 VT: MVT::i32));
9931
9932 return DAG.getMemIntrinsicNode(Opcode: ARMISD::STRD, dl, VTList: DAG.getVTList(VT: MVT::Other),
9933 Ops: {ST->getChain(), Lo, Hi, ST->getBasePtr()},
9934 MemVT, MMO: ST->getMemOperand());
9935 } else if (Subtarget->hasMVEIntegerOps() &&
9936 ((MemVT == MVT::v2i1 || MemVT == MVT::v4i1 || MemVT == MVT::v8i1 ||
9937 MemVT == MVT::v16i1))) {
9938 return LowerPredicateStore(Op, DAG);
9939 } else if (MemVT == MVT::i32 || MemVT == MVT::i64) {
9940 return LowerAEABIUnalignedStore(Op, DAG);
9941 }
9942 return SDValue();
9943}
9944
9945static bool isZeroVector(SDValue N) {
9946 return (ISD::isBuildVectorAllZeros(V: N) || (N->getOpcode() == ARMISD::VMOVIMM &&
9947 isNullConstant(V: N->getOperand(Num: 0))));
9948}
9949
9950static SDValue LowerMLOAD(SDValue Op, SelectionDAG &DAG) {
9951 MaskedLoadSDNode *N = cast<MaskedLoadSDNode>(Val: Op.getNode());
9952 MVT VT = Op.getSimpleValueType();
9953 SDValue Mask = N->getMask();
9954 SDValue PassThru = N->getPassThru();
9955 SDLoc dl(Op);
9956
9957 if (isZeroVector(N: PassThru))
9958 return Op;
9959
9960 // MVE Masked loads use zero as the passthru value. Here we convert undef to
9961 // zero too, and other values are lowered to a select.
9962 SDValue ZeroVec = DAG.getNode(Opcode: ARMISD::VMOVIMM, DL: dl, VT,
9963 Operand: DAG.getTargetConstant(Val: 0, DL: dl, VT: MVT::i32));
9964 SDValue NewLoad = DAG.getMaskedLoad(
9965 VT, dl, Chain: N->getChain(), Base: N->getBasePtr(), Offset: N->getOffset(), Mask, Src0: ZeroVec,
9966 MemVT: N->getMemoryVT(), MMO: N->getMemOperand(), AM: N->getAddressingMode(),
9967 N->getExtensionType(), IsExpanding: N->isExpandingLoad());
9968 SDValue Combo = NewLoad;
9969 bool PassThruIsCastZero = (PassThru.getOpcode() == ISD::BITCAST ||
9970 PassThru.getOpcode() == ARMISD::VECTOR_REG_CAST) &&
9971 isZeroVector(N: PassThru->getOperand(Num: 0));
9972 if (!PassThru.isUndef() && !PassThruIsCastZero)
9973 Combo = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT, N1: Mask, N2: NewLoad, N3: PassThru);
9974 return DAG.getMergeValues(Ops: {Combo, NewLoad.getValue(R: 1)}, dl);
9975}
9976
9977static SDValue LowerVecReduce(SDValue Op, SelectionDAG &DAG,
9978 const ARMSubtarget *ST) {
9979 if (!ST->hasMVEIntegerOps())
9980 return SDValue();
9981
9982 SDLoc dl(Op);
9983 unsigned BaseOpcode = 0;
9984 switch (Op->getOpcode()) {
9985 default: llvm_unreachable("Expected VECREDUCE opcode");
9986 case ISD::VECREDUCE_FADD: BaseOpcode = ISD::FADD; break;
9987 case ISD::VECREDUCE_FMUL: BaseOpcode = ISD::FMUL; break;
9988 case ISD::VECREDUCE_MUL: BaseOpcode = ISD::MUL; break;
9989 case ISD::VECREDUCE_AND: BaseOpcode = ISD::AND; break;
9990 case ISD::VECREDUCE_OR: BaseOpcode = ISD::OR; break;
9991 case ISD::VECREDUCE_XOR: BaseOpcode = ISD::XOR; break;
9992 case ISD::VECREDUCE_FMAX: BaseOpcode = ISD::FMAXNUM; break;
9993 case ISD::VECREDUCE_FMIN: BaseOpcode = ISD::FMINNUM; break;
9994 }
9995
9996 SDValue Op0 = Op->getOperand(Num: 0);
9997 EVT VT = Op0.getValueType();
9998 EVT EltVT = VT.getVectorElementType();
9999 unsigned NumElts = VT.getVectorNumElements();
10000 unsigned NumActiveLanes = NumElts;
10001
10002 assert((NumActiveLanes == 16 || NumActiveLanes == 8 || NumActiveLanes == 4 ||
10003 NumActiveLanes == 2) &&
10004 "Only expected a power 2 vector size");
10005
10006 // Use Mul(X, Rev(X)) until 4 items remain. Going down to 4 vector elements
10007 // allows us to easily extract vector elements from the lanes.
10008 while (NumActiveLanes > 4) {
10009 unsigned RevOpcode = NumActiveLanes == 16 ? ARMISD::VREV16 : ARMISD::VREV32;
10010 SDValue Rev = DAG.getNode(Opcode: RevOpcode, DL: dl, VT, Operand: Op0);
10011 Op0 = DAG.getNode(Opcode: BaseOpcode, DL: dl, VT, N1: Op0, N2: Rev);
10012 NumActiveLanes /= 2;
10013 }
10014
10015 SDValue Res;
10016 if (NumActiveLanes == 4) {
10017 // The remaining 4 elements are summed sequentially
10018 SDValue Ext0 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Op0,
10019 N2: DAG.getConstant(Val: 0 * NumElts / 4, DL: dl, VT: MVT::i32));
10020 SDValue Ext1 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Op0,
10021 N2: DAG.getConstant(Val: 1 * NumElts / 4, DL: dl, VT: MVT::i32));
10022 SDValue Ext2 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Op0,
10023 N2: DAG.getConstant(Val: 2 * NumElts / 4, DL: dl, VT: MVT::i32));
10024 SDValue Ext3 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Op0,
10025 N2: DAG.getConstant(Val: 3 * NumElts / 4, DL: dl, VT: MVT::i32));
10026 SDValue Res0 = DAG.getNode(Opcode: BaseOpcode, DL: dl, VT: EltVT, N1: Ext0, N2: Ext1, Flags: Op->getFlags());
10027 SDValue Res1 = DAG.getNode(Opcode: BaseOpcode, DL: dl, VT: EltVT, N1: Ext2, N2: Ext3, Flags: Op->getFlags());
10028 Res = DAG.getNode(Opcode: BaseOpcode, DL: dl, VT: EltVT, N1: Res0, N2: Res1, Flags: Op->getFlags());
10029 } else {
10030 SDValue Ext0 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Op0,
10031 N2: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
10032 SDValue Ext1 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Op0,
10033 N2: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
10034 Res = DAG.getNode(Opcode: BaseOpcode, DL: dl, VT: EltVT, N1: Ext0, N2: Ext1, Flags: Op->getFlags());
10035 }
10036
10037 // Result type may be wider than element type.
10038 if (EltVT != Op->getValueType(ResNo: 0))
10039 Res = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: Op->getValueType(ResNo: 0), Operand: Res);
10040 return Res;
10041}
10042
10043static SDValue LowerVecReduceF(SDValue Op, SelectionDAG &DAG,
10044 const ARMSubtarget *ST) {
10045 if (!ST->hasMVEFloatOps())
10046 return SDValue();
10047 return LowerVecReduce(Op, DAG, ST);
10048}
10049
10050static SDValue LowerVecReduceMinMax(SDValue Op, SelectionDAG &DAG,
10051 const ARMSubtarget *ST) {
10052 if (!ST->hasNEON())
10053 return SDValue();
10054
10055 SDLoc dl(Op);
10056 SDValue Op0 = Op->getOperand(Num: 0);
10057 EVT VT = Op0.getValueType();
10058 EVT EltVT = VT.getVectorElementType();
10059
10060 unsigned PairwiseIntrinsic = 0;
10061 switch (Op->getOpcode()) {
10062 default:
10063 llvm_unreachable("Expected VECREDUCE opcode");
10064 case ISD::VECREDUCE_UMIN:
10065 PairwiseIntrinsic = Intrinsic::arm_neon_vpminu;
10066 break;
10067 case ISD::VECREDUCE_UMAX:
10068 PairwiseIntrinsic = Intrinsic::arm_neon_vpmaxu;
10069 break;
10070 case ISD::VECREDUCE_SMIN:
10071 PairwiseIntrinsic = Intrinsic::arm_neon_vpmins;
10072 break;
10073 case ISD::VECREDUCE_SMAX:
10074 PairwiseIntrinsic = Intrinsic::arm_neon_vpmaxs;
10075 break;
10076 }
10077 SDValue PairwiseOp = DAG.getConstant(Val: PairwiseIntrinsic, DL: dl, VT: MVT::i32);
10078
10079 unsigned NumElts = VT.getVectorNumElements();
10080 unsigned NumActiveLanes = NumElts;
10081
10082 assert((NumActiveLanes == 16 || NumActiveLanes == 8 || NumActiveLanes == 4 ||
10083 NumActiveLanes == 2) &&
10084 "Only expected a power 2 vector size");
10085
10086 // Split 128-bit vectors, since vpmin/max takes 2 64-bit vectors.
10087 if (VT.is128BitVector()) {
10088 SDValue Lo, Hi;
10089 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Op0, DL: dl);
10090 VT = Lo.getValueType();
10091 Op0 = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT, Ops: {PairwiseOp, Lo, Hi});
10092 NumActiveLanes /= 2;
10093 }
10094
10095 // Use pairwise reductions until one lane remains
10096 while (NumActiveLanes > 1) {
10097 Op0 = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT, Ops: {PairwiseOp, Op0, Op0});
10098 NumActiveLanes /= 2;
10099 }
10100
10101 SDValue Res = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Op0,
10102 N2: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
10103
10104 // Result type may be wider than element type.
10105 if (EltVT != Op.getValueType()) {
10106 unsigned Extend = 0;
10107 switch (Op->getOpcode()) {
10108 default:
10109 llvm_unreachable("Expected VECREDUCE opcode");
10110 case ISD::VECREDUCE_UMIN:
10111 case ISD::VECREDUCE_UMAX:
10112 Extend = ISD::ZERO_EXTEND;
10113 break;
10114 case ISD::VECREDUCE_SMIN:
10115 case ISD::VECREDUCE_SMAX:
10116 Extend = ISD::SIGN_EXTEND;
10117 break;
10118 }
10119 Res = DAG.getNode(Opcode: Extend, DL: dl, VT: Op.getValueType(), Operand: Res);
10120 }
10121 return Res;
10122}
10123
10124static void ReplaceREADCYCLECOUNTER(SDNode *N,
10125 SmallVectorImpl<SDValue> &Results,
10126 SelectionDAG &DAG,
10127 const ARMSubtarget *Subtarget) {
10128 SDLoc DL(N);
10129 // Under Power Management extensions, the cycle-count is:
10130 // mrc p15, #0, <Rt>, c9, c13, #0
10131 SDValue Ops[] = { N->getOperand(Num: 0), // Chain
10132 DAG.getTargetConstant(Val: Intrinsic::arm_mrc, DL, VT: MVT::i32),
10133 DAG.getTargetConstant(Val: 15, DL, VT: MVT::i32),
10134 DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32),
10135 DAG.getTargetConstant(Val: 9, DL, VT: MVT::i32),
10136 DAG.getTargetConstant(Val: 13, DL, VT: MVT::i32),
10137 DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32)
10138 };
10139
10140 SDValue Cycles32 = DAG.getNode(Opcode: ISD::INTRINSIC_W_CHAIN, DL,
10141 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other), Ops);
10142 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: MVT::i64, N1: Cycles32,
10143 N2: DAG.getConstant(Val: 0, DL, VT: MVT::i32)));
10144 Results.push_back(Elt: Cycles32.getValue(R: 1));
10145}
10146
10147static SDValue createGPRPairNode2xi32(SelectionDAG &DAG, SDValue V0,
10148 SDValue V1) {
10149 SDLoc dl(V0.getNode());
10150 SDValue RegClass =
10151 DAG.getTargetConstant(Val: ARM::GPRPairRegClassID, DL: dl, VT: MVT::i32);
10152 SDValue SubReg0 = DAG.getTargetConstant(Val: ARM::gsub_0, DL: dl, VT: MVT::i32);
10153 SDValue SubReg1 = DAG.getTargetConstant(Val: ARM::gsub_1, DL: dl, VT: MVT::i32);
10154 const SDValue Ops[] = {RegClass, V0, SubReg0, V1, SubReg1};
10155 return SDValue(
10156 DAG.getMachineNode(Opcode: TargetOpcode::REG_SEQUENCE, dl, VT: MVT::Untyped, Ops), 0);
10157}
10158
10159static SDValue createGPRPairNodei64(SelectionDAG &DAG, SDValue V) {
10160 SDLoc dl(V.getNode());
10161 auto [VLo, VHi] = DAG.SplitScalar(N: V, DL: dl, LoVT: MVT::i32, HiVT: MVT::i32);
10162 bool isBigEndian = DAG.getDataLayout().isBigEndian();
10163 if (isBigEndian)
10164 std::swap(a&: VLo, b&: VHi);
10165 return createGPRPairNode2xi32(DAG, V0: VLo, V1: VHi);
10166}
10167
10168static void ReplaceCMP_SWAP_64Results(SDNode *N,
10169 SmallVectorImpl<SDValue> &Results,
10170 SelectionDAG &DAG) {
10171 assert(N->getValueType(0) == MVT::i64 &&
10172 "AtomicCmpSwap on types less than 64 should be legal");
10173 SDValue Ops[] = {
10174 createGPRPairNode2xi32(DAG, V0: N->getOperand(Num: 1),
10175 V1: DAG.getUNDEF(VT: MVT::i32)), // pointer, temp
10176 createGPRPairNodei64(DAG, V: N->getOperand(Num: 2)), // expected
10177 createGPRPairNodei64(DAG, V: N->getOperand(Num: 3)), // new
10178 N->getOperand(Num: 0), // chain in
10179 };
10180 SDNode *CmpSwap = DAG.getMachineNode(
10181 Opcode: ARM::CMP_SWAP_64, dl: SDLoc(N),
10182 VTs: DAG.getVTList(VT1: MVT::Untyped, VT2: MVT::Untyped, VT3: MVT::Other), Ops);
10183
10184 MachineMemOperand *MemOp = cast<MemSDNode>(Val: N)->getMemOperand();
10185 DAG.setNodeMemRefs(N: cast<MachineSDNode>(Val: CmpSwap), NewMemRefs: {MemOp});
10186
10187 bool isBigEndian = DAG.getDataLayout().isBigEndian();
10188
10189 SDValue Lo =
10190 DAG.getTargetExtractSubreg(SRIdx: isBigEndian ? ARM::gsub_1 : ARM::gsub_0,
10191 DL: SDLoc(N), VT: MVT::i32, Operand: SDValue(CmpSwap, 0));
10192 SDValue Hi =
10193 DAG.getTargetExtractSubreg(SRIdx: isBigEndian ? ARM::gsub_0 : ARM::gsub_1,
10194 DL: SDLoc(N), VT: MVT::i32, Operand: SDValue(CmpSwap, 0));
10195 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: SDLoc(N), VT: MVT::i64, N1: Lo, N2: Hi));
10196 Results.push_back(Elt: SDValue(CmpSwap, 2));
10197}
10198
10199SDValue ARMTargetLowering::LowerFSETCC(SDValue Op, SelectionDAG &DAG) const {
10200 SDLoc dl(Op);
10201 EVT VT = Op.getValueType();
10202 SDValue Chain = Op.getOperand(i: 0);
10203 SDValue LHS = Op.getOperand(i: 1);
10204 SDValue RHS = Op.getOperand(i: 2);
10205 ISD::CondCode CC = Op.getOperandCondCode(i: 3);
10206 bool IsSignaling = Op.getOpcode() == ISD::STRICT_FSETCCS;
10207
10208 // If we don't have instructions of this float type then soften to a libcall
10209 // and use SETCC instead.
10210 if (isUnsupportedFloatingType(VT: LHS.getValueType())) {
10211 softenSetCCOperands(DAG, VT: LHS.getValueType(), NewLHS&: LHS, NewRHS&: RHS, CCCode&: CC, DL: dl, OldLHS: LHS, OldRHS: RHS,
10212 Chain, IsSignaling);
10213 if (!RHS.getNode()) {
10214 RHS = DAG.getConstant(Val: 0, DL: dl, VT: LHS.getValueType());
10215 CC = ISD::SETNE;
10216 }
10217 SDValue Result = DAG.getNode(Opcode: ISD::SETCC, DL: dl, VT, N1: LHS, N2: RHS,
10218 N3: DAG.getCondCode(Cond: CC));
10219 return DAG.getMergeValues(Ops: {Result, Chain}, dl);
10220 }
10221
10222 ARMCC::CondCodes CondCode, CondCode2;
10223 FPCCToARMCC(CC, CondCode, CondCode2);
10224
10225 SDValue True = DAG.getConstant(Val: 1, DL: dl, VT);
10226 SDValue False = DAG.getConstant(Val: 0, DL: dl, VT);
10227 SDValue ARMcc = DAG.getConstant(Val: CondCode, DL: dl, VT: MVT::i32);
10228 SDValue Cmp = getVFPCmp(LHS, RHS, DAG, dl, Signaling: IsSignaling);
10229 SDValue Result = getCMOV(dl, VT, FalseVal: False, TrueVal: True, ARMcc, Flags: Cmp, DAG);
10230 if (CondCode2 != ARMCC::AL) {
10231 ARMcc = DAG.getConstant(Val: CondCode2, DL: dl, VT: MVT::i32);
10232 Result = getCMOV(dl, VT, FalseVal: Result, TrueVal: True, ARMcc, Flags: Cmp, DAG);
10233 }
10234 return DAG.getMergeValues(Ops: {Result, Chain}, dl);
10235}
10236
10237SDValue ARMTargetLowering::LowerSPONENTRY(SDValue Op, SelectionDAG &DAG) const {
10238 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
10239
10240 EVT VT = getPointerTy(DL: DAG.getDataLayout());
10241 int FI = MFI.CreateFixedObject(Size: 4, SPOffset: 0, IsImmutable: false);
10242 return DAG.getFrameIndex(FI, VT);
10243}
10244
10245SDValue ARMTargetLowering::LowerFP_TO_BF16(SDValue Op,
10246 SelectionDAG &DAG) const {
10247 SDLoc DL(Op);
10248 MakeLibCallOptions CallOptions;
10249 MVT SVT = Op.getOperand(i: 0).getSimpleValueType();
10250 RTLIB::Libcall LC = RTLIB::getFPROUND(OpVT: SVT, RetVT: MVT::bf16);
10251 SDValue Res =
10252 makeLibCall(DAG, LC, RetVT: MVT::f32, Ops: Op.getOperand(i: 0), CallOptions, dl: DL).first;
10253 return DAG.getBitcast(VT: MVT::i32, V: Res);
10254}
10255
10256SDValue ARMTargetLowering::LowerCMP(SDValue Op, SelectionDAG &DAG) const {
10257 SDLoc dl(Op);
10258 SDValue LHS = Op.getOperand(i: 0);
10259 SDValue RHS = Op.getOperand(i: 1);
10260
10261 // Determine if this is signed or unsigned comparison
10262 bool IsSigned = (Op.getOpcode() == ISD::SCMP);
10263
10264 // Special case for Thumb1 UCMP only
10265 if (!IsSigned && Subtarget->isThumb1Only()) {
10266 // For Thumb unsigned comparison, use this sequence:
10267 // subs r2, r0, r1 ; r2 = LHS - RHS, sets flags
10268 // sbc r2, r2 ; r2 = r2 - r2 - !carry
10269 // cmp r1, r0 ; compare RHS with LHS
10270 // sbc r1, r1 ; r1 = r1 - r1 - !carry
10271 // subs r0, r2, r1 ; r0 = r2 - r1 (final result)
10272
10273 // First subtraction: LHS - RHS
10274 SDValue Sub1WithFlags = DAG.getNode(
10275 Opcode: ARMISD::SUBC, DL: dl, VTList: DAG.getVTList(VT1: MVT::i32, VT2: FlagsVT), N1: LHS, N2: RHS);
10276 SDValue Sub1Result = Sub1WithFlags.getValue(R: 0);
10277 SDValue Flags1 = Sub1WithFlags.getValue(R: 1);
10278
10279 // SUBE: Sub1Result - Sub1Result - !carry
10280 // This gives 0 if LHS >= RHS (unsigned), -1 if LHS < RHS (unsigned)
10281 SDValue Sbc1 =
10282 DAG.getNode(Opcode: ARMISD::SUBE, DL: dl, VTList: DAG.getVTList(VT1: MVT::i32, VT2: FlagsVT),
10283 N1: Sub1Result, N2: Sub1Result, N3: Flags1);
10284 SDValue Sbc1Result = Sbc1.getValue(R: 0);
10285
10286 // Second comparison: RHS vs LHS (reverse comparison)
10287 SDValue CmpFlags = DAG.getNode(Opcode: ARMISD::CMP, DL: dl, VT: FlagsVT, N1: RHS, N2: LHS);
10288
10289 // SUBE: RHS - RHS - !carry
10290 // This gives 0 if RHS <= LHS (unsigned), -1 if RHS > LHS (unsigned)
10291 SDValue Sbc2 = DAG.getNode(
10292 Opcode: ARMISD::SUBE, DL: dl, VTList: DAG.getVTList(VT1: MVT::i32, VT2: FlagsVT), N1: RHS, N2: RHS, N3: CmpFlags);
10293 SDValue Sbc2Result = Sbc2.getValue(R: 0);
10294
10295 // Final subtraction: Sbc1Result - Sbc2Result (no flags needed)
10296 SDValue Result =
10297 DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32, N1: Sbc1Result, N2: Sbc2Result);
10298 if (Op.getValueType() != MVT::i32)
10299 Result = DAG.getSExtOrTrunc(Op: Result, DL: dl, VT: Op.getValueType());
10300
10301 return Result;
10302 }
10303
10304 // For the ARM assembly pattern:
10305 // subs r0, r0, r1 ; subtract RHS from LHS and set flags
10306 // movgt r0, #1 ; if LHS > RHS, set result to 1 (GT for signed, HI for
10307 // unsigned) mvnlt r0, #0 ; if LHS < RHS, set result to -1 (LT for
10308 // signed, LO for unsigned)
10309 // ; if LHS == RHS, result remains 0 from the subs
10310
10311 // Optimization: if RHS is a subtraction against 0, use ADDC instead of SUBC
10312 unsigned Opcode = ARMISD::SUBC;
10313
10314 // Check if RHS is a subtraction against 0: (0 - X)
10315 if (RHS.getOpcode() == ISD::SUB) {
10316 SDValue SubLHS = RHS.getOperand(i: 0);
10317 SDValue SubRHS = RHS.getOperand(i: 1);
10318
10319 // Check if it's 0 - X
10320 if (isNullConstant(V: SubLHS)) {
10321 bool CanUseAdd = false;
10322 if (IsSigned) {
10323 // For SCMP: only if X is known to never be INT_MIN (to avoid overflow)
10324 if (RHS->getFlags().hasNoSignedWrap() || !DAG.computeKnownBits(Op: SubRHS)
10325 .getSignedMinValue()
10326 .isMinSignedValue()) {
10327 CanUseAdd = true;
10328 }
10329 } else {
10330 // For UCMP: only if X is known to never be zero
10331 if (DAG.isKnownNeverZero(Op: SubRHS)) {
10332 CanUseAdd = true;
10333 }
10334 }
10335
10336 if (CanUseAdd) {
10337 Opcode = ARMISD::ADDC;
10338 RHS = SubRHS; // Replace RHS with X, so we do LHS + X instead of
10339 // LHS - (0 - X)
10340 }
10341 }
10342 }
10343
10344 // Generate the operation with flags
10345 SDValue OpWithFlags =
10346 DAG.getNode(Opcode, DL: dl, VTList: DAG.getVTList(VT1: MVT::i32, VT2: FlagsVT), N1: LHS, N2: RHS);
10347
10348 SDValue OpResult = OpWithFlags.getValue(R: 0);
10349 SDValue Flags = OpWithFlags.getValue(R: 1);
10350
10351 // Constants for conditional moves
10352 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32);
10353 SDValue MinusOne = DAG.getAllOnesConstant(DL: dl, VT: MVT::i32);
10354
10355 // Select condition codes based on signed vs unsigned
10356 ARMCC::CondCodes GTCond = IsSigned ? ARMCC::GT : ARMCC::HI;
10357 ARMCC::CondCodes LTCond = IsSigned ? ARMCC::LT : ARMCC::LO;
10358
10359 // First conditional move: if greater than, set to 1
10360 SDValue GTCondValue = DAG.getConstant(Val: GTCond, DL: dl, VT: MVT::i32);
10361 SDValue Result1 = DAG.getNode(Opcode: ARMISD::CMOV, DL: dl, VT: MVT::i32, N1: OpResult, N2: One,
10362 N3: GTCondValue, N4: Flags);
10363
10364 // Second conditional move: if less than, set to -1
10365 SDValue LTCondValue = DAG.getConstant(Val: LTCond, DL: dl, VT: MVT::i32);
10366 SDValue Result2 = DAG.getNode(Opcode: ARMISD::CMOV, DL: dl, VT: MVT::i32, N1: Result1, N2: MinusOne,
10367 N3: LTCondValue, N4: Flags);
10368
10369 if (Op.getValueType() != MVT::i32)
10370 Result2 = DAG.getSExtOrTrunc(Op: Result2, DL: dl, VT: Op.getValueType());
10371
10372 return Result2;
10373}
10374
10375SDValue ARMTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
10376 LLVM_DEBUG(dbgs() << "Lowering node: "; Op.dump());
10377 switch (Op.getOpcode()) {
10378 default: llvm_unreachable("Don't know how to custom lower this!");
10379 case ISD::WRITE_REGISTER: return LowerWRITE_REGISTER(Op, DAG);
10380 case ISD::ConstantPool: return LowerConstantPool(Op, DAG);
10381 case ISD::BlockAddress: return LowerBlockAddress(Op, DAG);
10382 case ISD::GlobalAddress: return LowerGlobalAddress(Op, DAG);
10383 case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG);
10384 case ISD::SELECT: return LowerSELECT(Op, DAG);
10385 case ISD::SELECT_CC: return LowerSELECT_CC(Op, DAG);
10386 case ISD::BRCOND: return LowerBRCOND(Op, DAG);
10387 case ISD::BR_CC: return LowerBR_CC(Op, DAG);
10388 case ISD::BR_JT: return LowerBR_JT(Op, DAG);
10389 case ISD::VASTART: return LowerVASTART(Op, DAG);
10390 case ISD::ATOMIC_FENCE: return LowerATOMIC_FENCE(Op, DAG, Subtarget);
10391 case ISD::PREFETCH: return LowerPREFETCH(Op, DAG, Subtarget);
10392 case ISD::STRICT_UINT_TO_FP:
10393 case ISD::STRICT_SINT_TO_FP:
10394 case ISD::SINT_TO_FP:
10395 case ISD::UINT_TO_FP: return LowerINT_TO_FP(Op, DAG);
10396 case ISD::STRICT_FP_TO_SINT:
10397 case ISD::STRICT_FP_TO_UINT:
10398 case ISD::FP_TO_SINT:
10399 case ISD::FP_TO_UINT: return LowerFP_TO_INT(Op, DAG);
10400 case ISD::FP_TO_SINT_SAT:
10401 case ISD::FP_TO_UINT_SAT: return LowerFP_TO_INT_SAT(Op, DAG, Subtarget);
10402 case ISD::FCOPYSIGN: return LowerFCOPYSIGN(Op, DAG);
10403 case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG);
10404 case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG);
10405 case ISD::EH_SJLJ_SETJMP: return LowerEH_SJLJ_SETJMP(Op, DAG);
10406 case ISD::EH_SJLJ_LONGJMP: return LowerEH_SJLJ_LONGJMP(Op, DAG);
10407 case ISD::EH_SJLJ_SETUP_DISPATCH: return LowerEH_SJLJ_SETUP_DISPATCH(Op, DAG);
10408 case ISD::INTRINSIC_VOID: return LowerINTRINSIC_VOID(Op, DAG, Subtarget);
10409 case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG,
10410 Subtarget);
10411 case ISD::BITCAST: return ExpandBITCAST(N: Op.getNode(), DAG, Subtarget);
10412 case ISD::SHL:
10413 case ISD::SRL:
10414 case ISD::SRA: return LowerShift(N: Op.getNode(), DAG, ST: Subtarget);
10415 case ISD::SREM: return LowerREM(N: Op.getNode(), DAG);
10416 case ISD::UREM: return LowerREM(N: Op.getNode(), DAG);
10417 case ISD::SHL_PARTS: return LowerShiftLeftParts(Op, DAG);
10418 case ISD::SRL_PARTS:
10419 case ISD::SRA_PARTS: return LowerShiftRightParts(Op, DAG);
10420 case ISD::CTTZ:
10421 case ISD::CTTZ_ZERO_POISON: return LowerCTTZ(N: Op.getNode(), DAG, ST: Subtarget);
10422 case ISD::CTPOP: return LowerCTPOP(N: Op.getNode(), DAG, ST: Subtarget);
10423 case ISD::SETCC: return LowerVSETCC(Op, DAG, ST: Subtarget);
10424 case ISD::SETCCCARRY: return LowerSETCCCARRY(Op, DAG);
10425 case ISD::ConstantFP: return LowerConstantFP(Op, DAG, ST: Subtarget);
10426 case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG, ST: Subtarget);
10427 case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG, ST: Subtarget);
10428 case ISD::EXTRACT_SUBVECTOR: return LowerEXTRACT_SUBVECTOR(Op, DAG, ST: Subtarget);
10429 case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG);
10430 case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG, ST: Subtarget);
10431 case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG, ST: Subtarget);
10432 case ISD::TRUNCATE: return LowerTruncate(N: Op.getNode(), DAG, Subtarget);
10433 case ISD::SIGN_EXTEND:
10434 case ISD::ZERO_EXTEND: return LowerVectorExtend(N: Op.getNode(), DAG, Subtarget);
10435 case ISD::GET_ROUNDING: return LowerGET_ROUNDING(Op, DAG);
10436 case ISD::SET_ROUNDING: return LowerSET_ROUNDING(Op, DAG);
10437 case ISD::SET_FPMODE:
10438 return LowerSET_FPMODE(Op, DAG);
10439 case ISD::RESET_FPMODE:
10440 return LowerRESET_FPMODE(Op, DAG);
10441 case ISD::MUL: return LowerMUL(Op, DAG);
10442 case ISD::SDIV:
10443 if (getTargetMachine().getTargetTriple().isOSWindows() &&
10444 !Op.getValueType().isVector())
10445 return LowerDIV_Windows(Op, DAG, /* Signed */ true);
10446 return LowerSDIV(Op, DAG, ST: Subtarget);
10447 case ISD::UDIV:
10448 if (getTargetMachine().getTargetTriple().isOSWindows() &&
10449 !Op.getValueType().isVector())
10450 return LowerDIV_Windows(Op, DAG, /* Signed */ false);
10451 return LowerUDIV(Op, DAG, ST: Subtarget);
10452 case ISD::UADDO_CARRY:
10453 return LowerADDSUBO_CARRY(Op, DAG, Opcode: ARMISD::ADDE, IsSigned: false /*unsigned*/);
10454 case ISD::USUBO_CARRY:
10455 return LowerADDSUBO_CARRY(Op, DAG, Opcode: ARMISD::SUBE, IsSigned: false /*unsigned*/);
10456 case ISD::SADDO_CARRY:
10457 return LowerADDSUBO_CARRY(Op, DAG, Opcode: ARMISD::ADDE, IsSigned: true /*signed*/);
10458 case ISD::SSUBO_CARRY:
10459 return LowerADDSUBO_CARRY(Op, DAG, Opcode: ARMISD::SUBE, IsSigned: true /*signed*/);
10460 case ISD::UADDO:
10461 case ISD::USUBO:
10462 case ISD::UMULO:
10463 case ISD::SADDO:
10464 case ISD::SSUBO:
10465 case ISD::SMULO:
10466 return LowerALUO(Op, DAG);
10467 case ISD::SADDSAT:
10468 case ISD::SSUBSAT:
10469 case ISD::UADDSAT:
10470 case ISD::USUBSAT:
10471 return LowerADDSUBSAT(Op, DAG, Subtarget);
10472 case ISD::LOAD: {
10473 auto *LD = cast<LoadSDNode>(Val&: Op);
10474 EVT MemVT = LD->getMemoryVT();
10475 if (Subtarget->hasMVEIntegerOps() &&
10476 (MemVT == MVT::v2i1 || MemVT == MVT::v4i1 || MemVT == MVT::v8i1 ||
10477 MemVT == MVT::v16i1))
10478 return LowerPredicateLoad(Op, DAG);
10479
10480 auto Pair = LowerAEABIUnalignedLoad(Op, DAG);
10481 if (Pair.first)
10482 return DAG.getMergeValues(Ops: {Pair.first, Pair.second}, dl: SDLoc(Pair.first));
10483 return SDValue();
10484 }
10485 case ISD::STORE:
10486 return LowerSTORE(Op, DAG, Subtarget);
10487 case ISD::MLOAD:
10488 return LowerMLOAD(Op, DAG);
10489 case ISD::VECREDUCE_MUL:
10490 case ISD::VECREDUCE_AND:
10491 case ISD::VECREDUCE_OR:
10492 case ISD::VECREDUCE_XOR:
10493 return LowerVecReduce(Op, DAG, ST: Subtarget);
10494 case ISD::VECREDUCE_FADD:
10495 case ISD::VECREDUCE_FMUL:
10496 case ISD::VECREDUCE_FMIN:
10497 case ISD::VECREDUCE_FMAX:
10498 return LowerVecReduceF(Op, DAG, ST: Subtarget);
10499 case ISD::VECREDUCE_UMIN:
10500 case ISD::VECREDUCE_UMAX:
10501 case ISD::VECREDUCE_SMIN:
10502 case ISD::VECREDUCE_SMAX:
10503 return LowerVecReduceMinMax(Op, DAG, ST: Subtarget);
10504 case ISD::SDIVREM:
10505 case ISD::UDIVREM: return LowerDivRem(Op, DAG);
10506 case ISD::DYNAMIC_STACKALLOC:
10507 if (getTargetMachine().getTargetTriple().isOSWindows())
10508 return LowerDYNAMIC_STACKALLOC(Op, DAG);
10509 llvm_unreachable("Don't know how to custom lower this!");
10510 case ISD::STRICT_FP_ROUND:
10511 case ISD::FP_ROUND: return LowerFP_ROUND(Op, DAG);
10512 case ISD::STRICT_FP_EXTEND:
10513 case ISD::FP_EXTEND: return LowerFP_EXTEND(Op, DAG);
10514 case ISD::STRICT_FSETCC:
10515 case ISD::STRICT_FSETCCS: return LowerFSETCC(Op, DAG);
10516 case ISD::SPONENTRY:
10517 return LowerSPONENTRY(Op, DAG);
10518 case ISD::FP_TO_BF16:
10519 return LowerFP_TO_BF16(Op, DAG);
10520 case ARMISD::WIN__DBZCHK: return SDValue();
10521 case ISD::UCMP:
10522 case ISD::SCMP:
10523 return LowerCMP(Op, DAG);
10524 case ISD::ABS:
10525 return LowerABS(Op, DAG);
10526 case ISD::STRICT_LROUND:
10527 case ISD::STRICT_LLROUND:
10528 case ISD::STRICT_LRINT:
10529 case ISD::STRICT_LLRINT: {
10530 assert((Op.getOperand(1).getValueType() == MVT::f16 ||
10531 Op.getOperand(1).getValueType() == MVT::bf16) &&
10532 "Expected custom lowering of rounding operations only for f16");
10533 SDLoc DL(Op);
10534 SDValue Ext = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL, ResultTys: {MVT::f32, MVT::Other},
10535 Ops: {Op.getOperand(i: 0), Op.getOperand(i: 1)});
10536 return DAG.getNode(Opcode: Op.getOpcode(), DL, ResultTys: {Op.getValueType(), MVT::Other},
10537 Ops: {Ext.getValue(R: 1), Ext.getValue(R: 0)});
10538 }
10539 }
10540}
10541
10542static void ReplaceLongIntrinsic(SDNode *N, SmallVectorImpl<SDValue> &Results,
10543 SelectionDAG &DAG) {
10544 unsigned IntNo = N->getConstantOperandVal(Num: 0);
10545 unsigned Opc = 0;
10546 if (IntNo == Intrinsic::arm_smlald)
10547 Opc = ARMISD::SMLALD;
10548 else if (IntNo == Intrinsic::arm_smlaldx)
10549 Opc = ARMISD::SMLALDX;
10550 else if (IntNo == Intrinsic::arm_smlsld)
10551 Opc = ARMISD::SMLSLD;
10552 else if (IntNo == Intrinsic::arm_smlsldx)
10553 Opc = ARMISD::SMLSLDX;
10554 else
10555 return;
10556
10557 SDLoc dl(N);
10558 SDValue Lo, Hi;
10559 std::tie(args&: Lo, args&: Hi) = DAG.SplitScalar(N: N->getOperand(Num: 3), DL: dl, LoVT: MVT::i32, HiVT: MVT::i32);
10560
10561 SDValue LongMul = DAG.getNode(Opcode: Opc, DL: dl,
10562 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
10563 N1: N->getOperand(Num: 1), N2: N->getOperand(Num: 2),
10564 N3: Lo, N4: Hi);
10565 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: MVT::i64,
10566 N1: LongMul.getValue(R: 0), N2: LongMul.getValue(R: 1)));
10567}
10568
10569/// ReplaceNodeResults - Replace the results of node with an illegal result
10570/// type with new values built out of custom code.
10571void ARMTargetLowering::ReplaceNodeResults(SDNode *N,
10572 SmallVectorImpl<SDValue> &Results,
10573 SelectionDAG &DAG) const {
10574 SDValue Res;
10575 switch (N->getOpcode()) {
10576 default:
10577 llvm_unreachable("Don't know how to custom expand this!");
10578 case ISD::READ_REGISTER:
10579 ExpandREAD_REGISTER(N, Results, DAG);
10580 break;
10581 case ISD::BITCAST:
10582 Res = ExpandBITCAST(N, DAG, Subtarget);
10583 break;
10584 case ISD::SRL:
10585 case ISD::SRA:
10586 case ISD::SHL:
10587 Res = Expand64BitShift(N, DAG, ST: Subtarget);
10588 break;
10589 case ISD::SREM:
10590 case ISD::UREM:
10591 Res = LowerREM(N, DAG);
10592 break;
10593 case ISD::SDIVREM:
10594 case ISD::UDIVREM:
10595 Res = LowerDivRem(Op: SDValue(N, 0), DAG);
10596 assert(Res.getNumOperands() == 2 && "DivRem needs two values");
10597 Results.push_back(Elt: Res.getValue(R: 0));
10598 Results.push_back(Elt: Res.getValue(R: 1));
10599 return;
10600 case ISD::SADDSAT:
10601 case ISD::SSUBSAT:
10602 case ISD::UADDSAT:
10603 case ISD::USUBSAT:
10604 Res = LowerADDSUBSAT(Op: SDValue(N, 0), DAG, Subtarget);
10605 break;
10606 case ISD::READCYCLECOUNTER:
10607 ReplaceREADCYCLECOUNTER(N, Results, DAG, Subtarget);
10608 return;
10609 case ISD::UDIV:
10610 case ISD::SDIV:
10611 assert(getTargetMachine().getTargetTriple().isOSWindows() &&
10612 "can only expand DIV on Windows");
10613 return ExpandDIV_Windows(Op: SDValue(N, 0), DAG, Signed: N->getOpcode() == ISD::SDIV,
10614 Results);
10615 case ISD::ATOMIC_CMP_SWAP:
10616 ReplaceCMP_SWAP_64Results(N, Results, DAG);
10617 return;
10618 case ISD::INTRINSIC_WO_CHAIN:
10619 return ReplaceLongIntrinsic(N, Results, DAG);
10620 case ISD::LOAD:
10621 LowerLOAD(N, Results, DAG);
10622 break;
10623 case ISD::STORE:
10624 Res = LowerAEABIUnalignedStore(Op: SDValue(N, 0), DAG);
10625 break;
10626 case ISD::TRUNCATE:
10627 Res = LowerTruncate(N, DAG, Subtarget);
10628 break;
10629 case ISD::SIGN_EXTEND:
10630 case ISD::ZERO_EXTEND:
10631 Res = LowerVectorExtend(N, DAG, Subtarget);
10632 break;
10633 case ISD::FP_TO_SINT_SAT:
10634 case ISD::FP_TO_UINT_SAT:
10635 Res = LowerFP_TO_INT_SAT(Op: SDValue(N, 0), DAG, Subtarget);
10636 break;
10637 }
10638 if (Res.getNode())
10639 Results.push_back(Elt: Res);
10640}
10641
10642//===----------------------------------------------------------------------===//
10643// ARM Scheduler Hooks
10644//===----------------------------------------------------------------------===//
10645
10646/// SetupEntryBlockForSjLj - Insert code into the entry block that creates and
10647/// registers the function context.
10648void ARMTargetLowering::SetupEntryBlockForSjLj(MachineInstr &MI,
10649 MachineBasicBlock *MBB,
10650 MachineBasicBlock *DispatchBB,
10651 int FI) const {
10652 assert(!Subtarget->isROPI() && !Subtarget->isRWPI() &&
10653 "ROPI/RWPI not currently supported with SjLj");
10654 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
10655 DebugLoc dl = MI.getDebugLoc();
10656 MachineFunction *MF = MBB->getParent();
10657 MachineRegisterInfo *MRI = &MF->getRegInfo();
10658 MachineConstantPool *MCP = MF->getConstantPool();
10659 ARMFunctionInfo *AFI = MF->getInfo<ARMFunctionInfo>();
10660 const Function &F = MF->getFunction();
10661
10662 bool isThumb = Subtarget->isThumb();
10663 bool isThumb2 = Subtarget->isThumb2();
10664
10665 unsigned PCLabelId = AFI->createPICLabelUId();
10666 unsigned PCAdj = (isThumb || isThumb2) ? 4 : 8;
10667 ARMConstantPoolValue *CPV =
10668 ARMConstantPoolMBB::Create(C&: F.getContext(), mbb: DispatchBB, ID: PCLabelId, PCAdj);
10669 unsigned CPI = MCP->getConstantPoolIndex(V: CPV, Alignment: Align(4));
10670
10671 const TargetRegisterClass *TRC = isThumb ? &ARM::tGPRRegClass
10672 : &ARM::GPRRegClass;
10673
10674 // Grab constant pool and fixed stack memory operands.
10675 MachineMemOperand *CPMMO =
10676 MF->getMachineMemOperand(PtrInfo: MachinePointerInfo::getConstantPool(MF&: *MF),
10677 F: MachineMemOperand::MOLoad, Size: 4, BaseAlignment: Align(4));
10678
10679 MachineMemOperand *FIMMOSt =
10680 MF->getMachineMemOperand(PtrInfo: MachinePointerInfo::getFixedStack(MF&: *MF, FI),
10681 F: MachineMemOperand::MOStore, Size: 4, BaseAlignment: Align(4));
10682
10683 // Load the address of the dispatch MBB into the jump buffer.
10684 if (isThumb2) {
10685 // Incoming value: jbuf
10686 // ldr.n r5, LCPI1_1
10687 // orr r5, r5, #1
10688 // add r5, pc
10689 // str r5, [$jbuf, #+4] ; &jbuf[1]
10690 Register NewVReg1 = MRI->createVirtualRegister(RegClass: TRC);
10691 BuildMI(BB&: *MBB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: ARM::t2LDRpci), DestReg: NewVReg1)
10692 .addConstantPoolIndex(Idx: CPI)
10693 .addMemOperand(MMO: CPMMO)
10694 .add(MOs: predOps(Pred: ARMCC::AL));
10695 // Set the low bit because of thumb mode.
10696 Register NewVReg2 = MRI->createVirtualRegister(RegClass: TRC);
10697 BuildMI(BB&: *MBB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: ARM::t2ORRri), DestReg: NewVReg2)
10698 .addReg(RegNo: NewVReg1)
10699 .addImm(Val: 0x01)
10700 .add(MOs: predOps(Pred: ARMCC::AL))
10701 .add(MO: condCodeOp());
10702 Register NewVReg3 = MRI->createVirtualRegister(RegClass: TRC);
10703 BuildMI(BB&: *MBB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: ARM::tPICADD), DestReg: NewVReg3)
10704 .addReg(RegNo: NewVReg2)
10705 .addImm(Val: PCLabelId);
10706 BuildMI(BB&: *MBB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: ARM::t2STRi12))
10707 .addReg(RegNo: NewVReg3)
10708 .addFrameIndex(Idx: FI)
10709 .addImm(Val: 36) // &jbuf[1] :: pc
10710 .addMemOperand(MMO: FIMMOSt)
10711 .add(MOs: predOps(Pred: ARMCC::AL));
10712 } else if (isThumb) {
10713 // Incoming value: jbuf
10714 // ldr.n r1, LCPI1_4
10715 // add r1, pc
10716 // mov r2, #1
10717 // orrs r1, r2
10718 // add r2, $jbuf, #+4 ; &jbuf[1]
10719 // str r1, [r2]
10720 Register NewVReg1 = MRI->createVirtualRegister(RegClass: TRC);
10721 BuildMI(BB&: *MBB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: ARM::tLDRpci), DestReg: NewVReg1)
10722 .addConstantPoolIndex(Idx: CPI)
10723 .addMemOperand(MMO: CPMMO)
10724 .add(MOs: predOps(Pred: ARMCC::AL));
10725 Register NewVReg2 = MRI->createVirtualRegister(RegClass: TRC);
10726 BuildMI(BB&: *MBB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: ARM::tPICADD), DestReg: NewVReg2)
10727 .addReg(RegNo: NewVReg1)
10728 .addImm(Val: PCLabelId);
10729 // Set the low bit because of thumb mode.
10730 Register NewVReg3 = MRI->createVirtualRegister(RegClass: TRC);
10731 BuildMI(BB&: *MBB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: ARM::tMOVi8), DestReg: NewVReg3)
10732 .addReg(RegNo: ARM::CPSR, Flags: RegState::Define)
10733 .addImm(Val: 1)
10734 .add(MOs: predOps(Pred: ARMCC::AL));
10735 Register NewVReg4 = MRI->createVirtualRegister(RegClass: TRC);
10736 BuildMI(BB&: *MBB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: ARM::tORR), DestReg: NewVReg4)
10737 .addReg(RegNo: ARM::CPSR, Flags: RegState::Define)
10738 .addReg(RegNo: NewVReg2)
10739 .addReg(RegNo: NewVReg3)
10740 .add(MOs: predOps(Pred: ARMCC::AL));
10741 Register NewVReg5 = MRI->createVirtualRegister(RegClass: TRC);
10742 BuildMI(BB&: *MBB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: ARM::tADDframe), DestReg: NewVReg5)
10743 .addFrameIndex(Idx: FI)
10744 .addImm(Val: 36); // &jbuf[1] :: pc
10745 BuildMI(BB&: *MBB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: ARM::tSTRi))
10746 .addReg(RegNo: NewVReg4)
10747 .addReg(RegNo: NewVReg5)
10748 .addImm(Val: 0)
10749 .addMemOperand(MMO: FIMMOSt)
10750 .add(MOs: predOps(Pred: ARMCC::AL));
10751 } else {
10752 // Incoming value: jbuf
10753 // ldr r1, LCPI1_1
10754 // add r1, pc, r1
10755 // str r1, [$jbuf, #+4] ; &jbuf[1]
10756 Register NewVReg1 = MRI->createVirtualRegister(RegClass: TRC);
10757 BuildMI(BB&: *MBB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: ARM::LDRi12), DestReg: NewVReg1)
10758 .addConstantPoolIndex(Idx: CPI)
10759 .addImm(Val: 0)
10760 .addMemOperand(MMO: CPMMO)
10761 .add(MOs: predOps(Pred: ARMCC::AL));
10762 Register NewVReg2 = MRI->createVirtualRegister(RegClass: TRC);
10763 BuildMI(BB&: *MBB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: ARM::PICADD), DestReg: NewVReg2)
10764 .addReg(RegNo: NewVReg1)
10765 .addImm(Val: PCLabelId)
10766 .add(MOs: predOps(Pred: ARMCC::AL));
10767 BuildMI(BB&: *MBB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: ARM::STRi12))
10768 .addReg(RegNo: NewVReg2)
10769 .addFrameIndex(Idx: FI)
10770 .addImm(Val: 36) // &jbuf[1] :: pc
10771 .addMemOperand(MMO: FIMMOSt)
10772 .add(MOs: predOps(Pred: ARMCC::AL));
10773 }
10774}
10775
10776void ARMTargetLowering::EmitSjLjDispatchBlock(MachineInstr &MI,
10777 MachineBasicBlock *MBB) const {
10778 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
10779 DebugLoc dl = MI.getDebugLoc();
10780 MachineFunction *MF = MBB->getParent();
10781 MachineRegisterInfo *MRI = &MF->getRegInfo();
10782 MachineFrameInfo &MFI = MF->getFrameInfo();
10783 int FI = MFI.getFunctionContextIndex();
10784
10785 const TargetRegisterClass *TRC = Subtarget->isThumb() ? &ARM::tGPRRegClass
10786 : &ARM::GPRnopcRegClass;
10787
10788 // Get a mapping of the call site numbers to all of the landing pads they're
10789 // associated with.
10790 DenseMap<unsigned, SmallVector<MachineBasicBlock*, 2>> CallSiteNumToLPad;
10791 unsigned MaxCSNum = 0;
10792 for (MachineBasicBlock &BB : *MF) {
10793 if (!BB.isEHPad())
10794 continue;
10795
10796 // FIXME: We should assert that the EH_LABEL is the first MI in the landing
10797 // pad.
10798 for (MachineInstr &II : BB) {
10799 if (!II.isEHLabel())
10800 continue;
10801
10802 MCSymbol *Sym = II.getOperand(i: 0).getMCSymbol();
10803 if (!MF->hasCallSiteLandingPad(Sym)) continue;
10804
10805 SmallVectorImpl<unsigned> &CallSiteIdxs = MF->getCallSiteLandingPad(Sym);
10806 for (unsigned Idx : CallSiteIdxs) {
10807 CallSiteNumToLPad[Idx].push_back(Elt: &BB);
10808 MaxCSNum = std::max(a: MaxCSNum, b: Idx);
10809 }
10810 break;
10811 }
10812 }
10813
10814 // Get an ordered list of the machine basic blocks for the jump table.
10815 std::vector<MachineBasicBlock*> LPadList;
10816 SmallPtrSet<MachineBasicBlock*, 32> InvokeBBs;
10817 LPadList.reserve(n: CallSiteNumToLPad.size());
10818 for (unsigned I = 1; I <= MaxCSNum; ++I) {
10819 SmallVectorImpl<MachineBasicBlock*> &MBBList = CallSiteNumToLPad[I];
10820 for (MachineBasicBlock *MBB : MBBList) {
10821 LPadList.push_back(x: MBB);
10822 InvokeBBs.insert_range(R: MBB->predecessors());
10823 }
10824 }
10825
10826 assert(!LPadList.empty() &&
10827 "No landing pad destinations for the dispatch jump table!");
10828
10829 // Create the jump table and associated information.
10830 MachineJumpTableInfo *JTI =
10831 MF->getOrCreateJumpTableInfo(JTEntryKind: MachineJumpTableInfo::EK_Inline);
10832 unsigned MJTI = JTI->createJumpTableIndex(DestBBs: LPadList);
10833
10834 // Create the MBBs for the dispatch code.
10835
10836 // Shove the dispatch's address into the return slot in the function context.
10837 MachineBasicBlock *DispatchBB = MF->CreateMachineBasicBlock();
10838 DispatchBB->setIsEHPad();
10839
10840 MachineBasicBlock *TrapBB = MF->CreateMachineBasicBlock();
10841
10842 BuildMI(BB: TrapBB, MIMD: dl, MCID: TII->get(Opcode: Subtarget->isThumb() ? ARM::tTRAP : ARM::TRAP));
10843 DispatchBB->addSuccessor(Succ: TrapBB);
10844
10845 MachineBasicBlock *DispContBB = MF->CreateMachineBasicBlock();
10846 DispatchBB->addSuccessor(Succ: DispContBB);
10847
10848 // Insert and MBBs.
10849 MF->insert(MBBI: MF->end(), MBB: DispatchBB);
10850 MF->insert(MBBI: MF->end(), MBB: DispContBB);
10851 MF->insert(MBBI: MF->end(), MBB: TrapBB);
10852
10853 // Insert code into the entry block that creates and registers the function
10854 // context.
10855 SetupEntryBlockForSjLj(MI, MBB, DispatchBB, FI);
10856
10857 MachineMemOperand *FIMMOLd = MF->getMachineMemOperand(
10858 PtrInfo: MachinePointerInfo::getFixedStack(MF&: *MF, FI),
10859 F: MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile, Size: 4, BaseAlignment: Align(4));
10860
10861 MachineInstrBuilder MIB;
10862 MIB = BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::Int_eh_sjlj_dispatchsetup));
10863
10864 const ARMBaseInstrInfo *AII = static_cast<const ARMBaseInstrInfo*>(TII);
10865 const ARMBaseRegisterInfo &RI = AII->getRegisterInfo();
10866
10867 // Add a register mask with no preserved registers. This results in all
10868 // registers being marked as clobbered. This can't work if the dispatch block
10869 // is in a Thumb1 function and is linked with ARM code which uses the FP
10870 // registers, as there is no way to preserve the FP registers in Thumb1 mode.
10871 MIB.addRegMask(Mask: RI.getSjLjDispatchPreservedMask(MF: *MF));
10872
10873 bool IsPositionIndependent = isPositionIndependent();
10874 unsigned NumLPads = LPadList.size();
10875 if (Subtarget->isThumb2()) {
10876 Register NewVReg1 = MRI->createVirtualRegister(RegClass: TRC);
10877 BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::t2LDRi12), DestReg: NewVReg1)
10878 .addFrameIndex(Idx: FI)
10879 .addImm(Val: 4)
10880 .addMemOperand(MMO: FIMMOLd)
10881 .add(MOs: predOps(Pred: ARMCC::AL));
10882
10883 if (NumLPads < 256) {
10884 BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::t2CMPri))
10885 .addReg(RegNo: NewVReg1)
10886 .addImm(Val: LPadList.size())
10887 .add(MOs: predOps(Pred: ARMCC::AL));
10888 } else {
10889 Register VReg1 = MRI->createVirtualRegister(RegClass: TRC);
10890 BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::t2MOVi16), DestReg: VReg1)
10891 .addImm(Val: NumLPads & 0xFFFF)
10892 .add(MOs: predOps(Pred: ARMCC::AL));
10893
10894 unsigned VReg2 = VReg1;
10895 if ((NumLPads & 0xFFFF0000) != 0) {
10896 VReg2 = MRI->createVirtualRegister(RegClass: TRC);
10897 BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::t2MOVTi16), DestReg: VReg2)
10898 .addReg(RegNo: VReg1)
10899 .addImm(Val: NumLPads >> 16)
10900 .add(MOs: predOps(Pred: ARMCC::AL));
10901 }
10902
10903 BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::t2CMPrr))
10904 .addReg(RegNo: NewVReg1)
10905 .addReg(RegNo: VReg2)
10906 .add(MOs: predOps(Pred: ARMCC::AL));
10907 }
10908
10909 BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::t2Bcc))
10910 .addMBB(MBB: TrapBB)
10911 .addImm(Val: ARMCC::HI)
10912 .addReg(RegNo: ARM::CPSR);
10913
10914 Register NewVReg3 = MRI->createVirtualRegister(RegClass: TRC);
10915 BuildMI(BB: DispContBB, MIMD: dl, MCID: TII->get(Opcode: ARM::t2LEApcrelJT), DestReg: NewVReg3)
10916 .addJumpTableIndex(Idx: MJTI)
10917 .add(MOs: predOps(Pred: ARMCC::AL));
10918
10919 Register NewVReg4 = MRI->createVirtualRegister(RegClass: TRC);
10920 BuildMI(BB: DispContBB, MIMD: dl, MCID: TII->get(Opcode: ARM::t2ADDrs), DestReg: NewVReg4)
10921 .addReg(RegNo: NewVReg3)
10922 .addReg(RegNo: NewVReg1)
10923 .addImm(Val: ARM_AM::getSORegOpc(ShOp: ARM_AM::lsl, Imm: 2))
10924 .add(MOs: predOps(Pred: ARMCC::AL))
10925 .add(MO: condCodeOp());
10926
10927 BuildMI(BB: DispContBB, MIMD: dl, MCID: TII->get(Opcode: ARM::t2BR_JT))
10928 .addReg(RegNo: NewVReg4)
10929 .addReg(RegNo: NewVReg1)
10930 .addJumpTableIndex(Idx: MJTI);
10931 } else if (Subtarget->isThumb()) {
10932 Register NewVReg1 = MRI->createVirtualRegister(RegClass: TRC);
10933 BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::tLDRspi), DestReg: NewVReg1)
10934 .addFrameIndex(Idx: FI)
10935 .addImm(Val: 1)
10936 .addMemOperand(MMO: FIMMOLd)
10937 .add(MOs: predOps(Pred: ARMCC::AL));
10938
10939 if (NumLPads < 256) {
10940 BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::tCMPi8))
10941 .addReg(RegNo: NewVReg1)
10942 .addImm(Val: NumLPads)
10943 .add(MOs: predOps(Pred: ARMCC::AL));
10944 } else {
10945 MachineConstantPool *ConstantPool = MF->getConstantPool();
10946 Type *Int32Ty = Type::getInt32Ty(C&: MF->getFunction().getContext());
10947 const Constant *C = ConstantInt::get(Ty: Int32Ty, V: NumLPads);
10948
10949 // MachineConstantPool wants an explicit alignment.
10950 Align Alignment = MF->getDataLayout().getPrefTypeAlign(Ty: Int32Ty);
10951 unsigned Idx = ConstantPool->getConstantPoolIndex(C, Alignment);
10952
10953 Register VReg1 = MRI->createVirtualRegister(RegClass: TRC);
10954 BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::tLDRpci))
10955 .addReg(RegNo: VReg1, Flags: RegState::Define)
10956 .addConstantPoolIndex(Idx)
10957 .add(MOs: predOps(Pred: ARMCC::AL));
10958 BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::tCMPr))
10959 .addReg(RegNo: NewVReg1)
10960 .addReg(RegNo: VReg1)
10961 .add(MOs: predOps(Pred: ARMCC::AL));
10962 }
10963
10964 BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::tBcc))
10965 .addMBB(MBB: TrapBB)
10966 .addImm(Val: ARMCC::HI)
10967 .addReg(RegNo: ARM::CPSR);
10968
10969 Register NewVReg2 = MRI->createVirtualRegister(RegClass: TRC);
10970 BuildMI(BB: DispContBB, MIMD: dl, MCID: TII->get(Opcode: ARM::tLSLri), DestReg: NewVReg2)
10971 .addReg(RegNo: ARM::CPSR, Flags: RegState::Define)
10972 .addReg(RegNo: NewVReg1)
10973 .addImm(Val: 2)
10974 .add(MOs: predOps(Pred: ARMCC::AL));
10975
10976 Register NewVReg3 = MRI->createVirtualRegister(RegClass: TRC);
10977 BuildMI(BB: DispContBB, MIMD: dl, MCID: TII->get(Opcode: ARM::tLEApcrelJT), DestReg: NewVReg3)
10978 .addJumpTableIndex(Idx: MJTI)
10979 .add(MOs: predOps(Pred: ARMCC::AL));
10980
10981 Register NewVReg4 = MRI->createVirtualRegister(RegClass: TRC);
10982 BuildMI(BB: DispContBB, MIMD: dl, MCID: TII->get(Opcode: ARM::tADDrr), DestReg: NewVReg4)
10983 .addReg(RegNo: ARM::CPSR, Flags: RegState::Define)
10984 .addReg(RegNo: NewVReg2)
10985 .addReg(RegNo: NewVReg3)
10986 .add(MOs: predOps(Pred: ARMCC::AL));
10987
10988 MachineMemOperand *JTMMOLd =
10989 MF->getMachineMemOperand(PtrInfo: MachinePointerInfo::getJumpTable(MF&: *MF),
10990 F: MachineMemOperand::MOLoad, Size: 4, BaseAlignment: Align(4));
10991
10992 Register NewVReg5 = MRI->createVirtualRegister(RegClass: TRC);
10993 BuildMI(BB: DispContBB, MIMD: dl, MCID: TII->get(Opcode: ARM::tLDRi), DestReg: NewVReg5)
10994 .addReg(RegNo: NewVReg4)
10995 .addImm(Val: 0)
10996 .addMemOperand(MMO: JTMMOLd)
10997 .add(MOs: predOps(Pred: ARMCC::AL));
10998
10999 unsigned NewVReg6 = NewVReg5;
11000 if (IsPositionIndependent) {
11001 NewVReg6 = MRI->createVirtualRegister(RegClass: TRC);
11002 BuildMI(BB: DispContBB, MIMD: dl, MCID: TII->get(Opcode: ARM::tADDrr), DestReg: NewVReg6)
11003 .addReg(RegNo: ARM::CPSR, Flags: RegState::Define)
11004 .addReg(RegNo: NewVReg5)
11005 .addReg(RegNo: NewVReg3)
11006 .add(MOs: predOps(Pred: ARMCC::AL));
11007 }
11008
11009 BuildMI(BB: DispContBB, MIMD: dl, MCID: TII->get(Opcode: ARM::tBR_JTr))
11010 .addReg(RegNo: NewVReg6)
11011 .addJumpTableIndex(Idx: MJTI);
11012 } else {
11013 Register NewVReg1 = MRI->createVirtualRegister(RegClass: TRC);
11014 BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::LDRi12), DestReg: NewVReg1)
11015 .addFrameIndex(Idx: FI)
11016 .addImm(Val: 4)
11017 .addMemOperand(MMO: FIMMOLd)
11018 .add(MOs: predOps(Pred: ARMCC::AL));
11019
11020 if (NumLPads < 256) {
11021 BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::CMPri))
11022 .addReg(RegNo: NewVReg1)
11023 .addImm(Val: NumLPads)
11024 .add(MOs: predOps(Pred: ARMCC::AL));
11025 } else if (Subtarget->hasV6T2Ops() && isUInt<16>(x: NumLPads)) {
11026 Register VReg1 = MRI->createVirtualRegister(RegClass: TRC);
11027 BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::MOVi16), DestReg: VReg1)
11028 .addImm(Val: NumLPads & 0xFFFF)
11029 .add(MOs: predOps(Pred: ARMCC::AL));
11030
11031 unsigned VReg2 = VReg1;
11032 if ((NumLPads & 0xFFFF0000) != 0) {
11033 VReg2 = MRI->createVirtualRegister(RegClass: TRC);
11034 BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::MOVTi16), DestReg: VReg2)
11035 .addReg(RegNo: VReg1)
11036 .addImm(Val: NumLPads >> 16)
11037 .add(MOs: predOps(Pred: ARMCC::AL));
11038 }
11039
11040 BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::CMPrr))
11041 .addReg(RegNo: NewVReg1)
11042 .addReg(RegNo: VReg2)
11043 .add(MOs: predOps(Pred: ARMCC::AL));
11044 } else {
11045 MachineConstantPool *ConstantPool = MF->getConstantPool();
11046 Type *Int32Ty = Type::getInt32Ty(C&: MF->getFunction().getContext());
11047 const Constant *C = ConstantInt::get(Ty: Int32Ty, V: NumLPads);
11048
11049 // MachineConstantPool wants an explicit alignment.
11050 Align Alignment = MF->getDataLayout().getPrefTypeAlign(Ty: Int32Ty);
11051 unsigned Idx = ConstantPool->getConstantPoolIndex(C, Alignment);
11052
11053 Register VReg1 = MRI->createVirtualRegister(RegClass: TRC);
11054 BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::LDRcp))
11055 .addReg(RegNo: VReg1, Flags: RegState::Define)
11056 .addConstantPoolIndex(Idx)
11057 .addImm(Val: 0)
11058 .add(MOs: predOps(Pred: ARMCC::AL));
11059 BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::CMPrr))
11060 .addReg(RegNo: NewVReg1)
11061 .addReg(RegNo: VReg1)
11062 .add(MOs: predOps(Pred: ARMCC::AL));
11063 }
11064
11065 BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::Bcc))
11066 .addMBB(MBB: TrapBB)
11067 .addImm(Val: ARMCC::HI)
11068 .addReg(RegNo: ARM::CPSR);
11069
11070 Register NewVReg3 = MRI->createVirtualRegister(RegClass: TRC);
11071 BuildMI(BB: DispContBB, MIMD: dl, MCID: TII->get(Opcode: ARM::MOVsi), DestReg: NewVReg3)
11072 .addReg(RegNo: NewVReg1)
11073 .addImm(Val: ARM_AM::getSORegOpc(ShOp: ARM_AM::lsl, Imm: 2))
11074 .add(MOs: predOps(Pred: ARMCC::AL))
11075 .add(MO: condCodeOp());
11076 Register NewVReg4 = MRI->createVirtualRegister(RegClass: TRC);
11077 BuildMI(BB: DispContBB, MIMD: dl, MCID: TII->get(Opcode: ARM::LEApcrelJT), DestReg: NewVReg4)
11078 .addJumpTableIndex(Idx: MJTI)
11079 .add(MOs: predOps(Pred: ARMCC::AL));
11080
11081 MachineMemOperand *JTMMOLd =
11082 MF->getMachineMemOperand(PtrInfo: MachinePointerInfo::getJumpTable(MF&: *MF),
11083 F: MachineMemOperand::MOLoad, Size: 4, BaseAlignment: Align(4));
11084 Register NewVReg5 = MRI->createVirtualRegister(RegClass: TRC);
11085 BuildMI(BB: DispContBB, MIMD: dl, MCID: TII->get(Opcode: ARM::LDRrs), DestReg: NewVReg5)
11086 .addReg(RegNo: NewVReg3)
11087 .addReg(RegNo: NewVReg4)
11088 .addImm(Val: 0)
11089 .addMemOperand(MMO: JTMMOLd)
11090 .add(MOs: predOps(Pred: ARMCC::AL));
11091
11092 if (IsPositionIndependent) {
11093 BuildMI(BB: DispContBB, MIMD: dl, MCID: TII->get(Opcode: ARM::BR_JTadd))
11094 .addReg(RegNo: NewVReg5)
11095 .addReg(RegNo: NewVReg4)
11096 .addJumpTableIndex(Idx: MJTI);
11097 } else {
11098 BuildMI(BB: DispContBB, MIMD: dl, MCID: TII->get(Opcode: ARM::BR_JTr))
11099 .addReg(RegNo: NewVReg5)
11100 .addJumpTableIndex(Idx: MJTI);
11101 }
11102 }
11103
11104 // Add the jump table entries as successors to the MBB.
11105 SmallPtrSet<MachineBasicBlock*, 8> SeenMBBs;
11106 for (MachineBasicBlock *CurMBB : LPadList) {
11107 if (SeenMBBs.insert(Ptr: CurMBB).second)
11108 DispContBB->addSuccessor(Succ: CurMBB);
11109 }
11110
11111 // N.B. the order the invoke BBs are processed in doesn't matter here.
11112 const MCPhysReg *SavedRegs = RI.getCalleeSavedRegs(MF);
11113 SmallVector<MachineBasicBlock*, 64> MBBLPads;
11114 for (MachineBasicBlock *BB : InvokeBBs) {
11115
11116 // Remove the landing pad successor from the invoke block and replace it
11117 // with the new dispatch block.
11118 SmallVector<MachineBasicBlock*, 4> Successors(BB->successors());
11119 while (!Successors.empty()) {
11120 MachineBasicBlock *SMBB = Successors.pop_back_val();
11121 if (SMBB->isEHPad()) {
11122 BB->removeSuccessor(Succ: SMBB);
11123 MBBLPads.push_back(Elt: SMBB);
11124 }
11125 }
11126
11127 BB->addSuccessor(Succ: DispatchBB, Prob: BranchProbability::getZero());
11128 BB->normalizeSuccProbs();
11129
11130 // Find the invoke call and mark all of the callee-saved registers as
11131 // 'implicit defined' so that they're spilled. This prevents code from
11132 // moving instructions to before the EH block, where they will never be
11133 // executed.
11134 for (MachineBasicBlock::reverse_iterator
11135 II = BB->rbegin(), IE = BB->rend(); II != IE; ++II) {
11136 if (!II->isCall()) continue;
11137
11138 DenseSet<unsigned> DefRegs;
11139 for (MachineInstr::mop_iterator
11140 OI = II->operands_begin(), OE = II->operands_end();
11141 OI != OE; ++OI) {
11142 if (!OI->isReg()) continue;
11143 DefRegs.insert(V: OI->getReg());
11144 }
11145
11146 MachineInstrBuilder MIB(*MF, &*II);
11147
11148 for (unsigned i = 0; SavedRegs[i] != 0; ++i) {
11149 unsigned Reg = SavedRegs[i];
11150 if (Subtarget->isThumb2() &&
11151 !ARM::tGPRRegClass.contains(Reg) &&
11152 !ARM::hGPRRegClass.contains(Reg))
11153 continue;
11154 if (Subtarget->isThumb1Only() && !ARM::tGPRRegClass.contains(Reg))
11155 continue;
11156 if (!Subtarget->isThumb() && !ARM::GPRRegClass.contains(Reg))
11157 continue;
11158 if (!DefRegs.contains(V: Reg))
11159 MIB.addReg(RegNo: Reg, Flags: RegState::ImplicitDefine | RegState::Dead);
11160 }
11161
11162 break;
11163 }
11164 }
11165
11166 // Mark all former landing pads as non-landing pads. The dispatch is the only
11167 // landing pad now.
11168 for (MachineBasicBlock *MBBLPad : MBBLPads)
11169 MBBLPad->setIsEHPad(false);
11170
11171 // The instruction is gone now.
11172 MI.eraseFromParent();
11173}
11174
11175static
11176MachineBasicBlock *OtherSucc(MachineBasicBlock *MBB, MachineBasicBlock *Succ) {
11177 for (MachineBasicBlock *S : MBB->successors())
11178 if (S != Succ)
11179 return S;
11180 llvm_unreachable("Expecting a BB with two successors!");
11181}
11182
11183/// Return the load opcode for a given load size. If load size >= 8,
11184/// neon opcode will be returned.
11185static unsigned getLdOpcode(unsigned LdSize, bool IsThumb1, bool IsThumb2) {
11186 if (LdSize >= 8)
11187 return LdSize == 16 ? ARM::VLD1q32wb_fixed
11188 : LdSize == 8 ? ARM::VLD1d32wb_fixed : 0;
11189 if (IsThumb1)
11190 return LdSize == 4 ? ARM::tLDRi
11191 : LdSize == 2 ? ARM::tLDRHi
11192 : LdSize == 1 ? ARM::tLDRBi : 0;
11193 if (IsThumb2)
11194 return LdSize == 4 ? ARM::t2LDR_POST
11195 : LdSize == 2 ? ARM::t2LDRH_POST
11196 : LdSize == 1 ? ARM::t2LDRB_POST : 0;
11197 return LdSize == 4 ? ARM::LDR_POST_IMM
11198 : LdSize == 2 ? ARM::LDRH_POST
11199 : LdSize == 1 ? ARM::LDRB_POST_IMM : 0;
11200}
11201
11202/// Return the store opcode for a given store size. If store size >= 8,
11203/// neon opcode will be returned.
11204static unsigned getStOpcode(unsigned StSize, bool IsThumb1, bool IsThumb2) {
11205 if (StSize >= 8)
11206 return StSize == 16 ? ARM::VST1q32wb_fixed
11207 : StSize == 8 ? ARM::VST1d32wb_fixed : 0;
11208 if (IsThumb1)
11209 return StSize == 4 ? ARM::tSTRi
11210 : StSize == 2 ? ARM::tSTRHi
11211 : StSize == 1 ? ARM::tSTRBi : 0;
11212 if (IsThumb2)
11213 return StSize == 4 ? ARM::t2STR_POST
11214 : StSize == 2 ? ARM::t2STRH_POST
11215 : StSize == 1 ? ARM::t2STRB_POST : 0;
11216 return StSize == 4 ? ARM::STR_POST_IMM
11217 : StSize == 2 ? ARM::STRH_POST
11218 : StSize == 1 ? ARM::STRB_POST_IMM : 0;
11219}
11220
11221/// Emit a post-increment load operation with given size. The instructions
11222/// will be added to BB at Pos.
11223static void emitPostLd(MachineBasicBlock *BB, MachineBasicBlock::iterator Pos,
11224 const TargetInstrInfo *TII, const DebugLoc &dl,
11225 unsigned LdSize, unsigned Data, unsigned AddrIn,
11226 unsigned AddrOut, bool IsThumb1, bool IsThumb2) {
11227 unsigned LdOpc = getLdOpcode(LdSize, IsThumb1, IsThumb2);
11228 assert(LdOpc != 0 && "Should have a load opcode");
11229 if (LdSize >= 8) {
11230 BuildMI(BB&: *BB, I: Pos, MIMD: dl, MCID: TII->get(Opcode: LdOpc), DestReg: Data)
11231 .addReg(RegNo: AddrOut, Flags: RegState::Define)
11232 .addReg(RegNo: AddrIn)
11233 .addImm(Val: 0)
11234 .add(MOs: predOps(Pred: ARMCC::AL));
11235 } else if (IsThumb1) {
11236 // load + update AddrIn
11237 BuildMI(BB&: *BB, I: Pos, MIMD: dl, MCID: TII->get(Opcode: LdOpc), DestReg: Data)
11238 .addReg(RegNo: AddrIn)
11239 .addImm(Val: 0)
11240 .add(MOs: predOps(Pred: ARMCC::AL));
11241 BuildMI(BB&: *BB, I: Pos, MIMD: dl, MCID: TII->get(Opcode: ARM::tADDi8), DestReg: AddrOut)
11242 .add(MO: t1CondCodeOp())
11243 .addReg(RegNo: AddrIn)
11244 .addImm(Val: LdSize)
11245 .add(MOs: predOps(Pred: ARMCC::AL));
11246 } else if (IsThumb2) {
11247 BuildMI(BB&: *BB, I: Pos, MIMD: dl, MCID: TII->get(Opcode: LdOpc), DestReg: Data)
11248 .addReg(RegNo: AddrOut, Flags: RegState::Define)
11249 .addReg(RegNo: AddrIn)
11250 .addImm(Val: LdSize)
11251 .add(MOs: predOps(Pred: ARMCC::AL));
11252 } else { // arm
11253 BuildMI(BB&: *BB, I: Pos, MIMD: dl, MCID: TII->get(Opcode: LdOpc), DestReg: Data)
11254 .addReg(RegNo: AddrOut, Flags: RegState::Define)
11255 .addReg(RegNo: AddrIn)
11256 .addReg(RegNo: 0)
11257 .addImm(Val: LdSize)
11258 .add(MOs: predOps(Pred: ARMCC::AL));
11259 }
11260}
11261
11262/// Emit a post-increment store operation with given size. The instructions
11263/// will be added to BB at Pos.
11264static void emitPostSt(MachineBasicBlock *BB, MachineBasicBlock::iterator Pos,
11265 const TargetInstrInfo *TII, const DebugLoc &dl,
11266 unsigned StSize, unsigned Data, unsigned AddrIn,
11267 unsigned AddrOut, bool IsThumb1, bool IsThumb2) {
11268 unsigned StOpc = getStOpcode(StSize, IsThumb1, IsThumb2);
11269 assert(StOpc != 0 && "Should have a store opcode");
11270 if (StSize >= 8) {
11271 BuildMI(BB&: *BB, I: Pos, MIMD: dl, MCID: TII->get(Opcode: StOpc), DestReg: AddrOut)
11272 .addReg(RegNo: AddrIn)
11273 .addImm(Val: 0)
11274 .addReg(RegNo: Data)
11275 .add(MOs: predOps(Pred: ARMCC::AL));
11276 } else if (IsThumb1) {
11277 // store + update AddrIn
11278 BuildMI(BB&: *BB, I: Pos, MIMD: dl, MCID: TII->get(Opcode: StOpc))
11279 .addReg(RegNo: Data)
11280 .addReg(RegNo: AddrIn)
11281 .addImm(Val: 0)
11282 .add(MOs: predOps(Pred: ARMCC::AL));
11283 BuildMI(BB&: *BB, I: Pos, MIMD: dl, MCID: TII->get(Opcode: ARM::tADDi8), DestReg: AddrOut)
11284 .add(MO: t1CondCodeOp())
11285 .addReg(RegNo: AddrIn)
11286 .addImm(Val: StSize)
11287 .add(MOs: predOps(Pred: ARMCC::AL));
11288 } else if (IsThumb2) {
11289 BuildMI(BB&: *BB, I: Pos, MIMD: dl, MCID: TII->get(Opcode: StOpc), DestReg: AddrOut)
11290 .addReg(RegNo: Data)
11291 .addReg(RegNo: AddrIn)
11292 .addImm(Val: StSize)
11293 .add(MOs: predOps(Pred: ARMCC::AL));
11294 } else { // arm
11295 BuildMI(BB&: *BB, I: Pos, MIMD: dl, MCID: TII->get(Opcode: StOpc), DestReg: AddrOut)
11296 .addReg(RegNo: Data)
11297 .addReg(RegNo: AddrIn)
11298 .addReg(RegNo: 0)
11299 .addImm(Val: StSize)
11300 .add(MOs: predOps(Pred: ARMCC::AL));
11301 }
11302}
11303
11304MachineBasicBlock *
11305ARMTargetLowering::EmitStructByval(MachineInstr &MI,
11306 MachineBasicBlock *BB) const {
11307 // This pseudo instruction has 3 operands: dst, src, size
11308 // We expand it to a loop if size > Subtarget->getMaxInlineSizeThreshold().
11309 // Otherwise, we will generate unrolled scalar copies.
11310 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
11311 const BasicBlock *LLVM_BB = BB->getBasicBlock();
11312 MachineFunction::iterator It = ++BB->getIterator();
11313
11314 Register dest = MI.getOperand(i: 0).getReg();
11315 Register src = MI.getOperand(i: 1).getReg();
11316 unsigned SizeVal = MI.getOperand(i: 2).getImm();
11317 unsigned Alignment = MI.getOperand(i: 3).getImm();
11318 DebugLoc dl = MI.getDebugLoc();
11319
11320 MachineFunction *MF = BB->getParent();
11321 MachineRegisterInfo &MRI = MF->getRegInfo();
11322 unsigned UnitSize = 0;
11323 const TargetRegisterClass *TRC = nullptr;
11324 const TargetRegisterClass *VecTRC = nullptr;
11325
11326 bool IsThumb1 = Subtarget->isThumb1Only();
11327 bool IsThumb2 = Subtarget->isThumb2();
11328 bool IsThumb = Subtarget->isThumb();
11329
11330 if (Alignment & 1) {
11331 UnitSize = 1;
11332 } else if (Alignment & 2) {
11333 UnitSize = 2;
11334 } else {
11335 // Check whether we can use NEON instructions.
11336 if (!MF->getFunction().hasFnAttribute(Kind: Attribute::NoImplicitFloat) &&
11337 Subtarget->hasNEON()) {
11338 if ((Alignment % 16 == 0) && SizeVal >= 16)
11339 UnitSize = 16;
11340 else if ((Alignment % 8 == 0) && SizeVal >= 8)
11341 UnitSize = 8;
11342 }
11343 // Can't use NEON instructions.
11344 if (UnitSize == 0)
11345 UnitSize = 4;
11346 }
11347
11348 // Select the correct opcode and register class for unit size load/store
11349 bool IsNeon = UnitSize >= 8;
11350 TRC = IsThumb ? &ARM::tGPRRegClass : &ARM::GPRRegClass;
11351 if (IsNeon)
11352 VecTRC = UnitSize == 16 ? &ARM::DPairRegClass
11353 : UnitSize == 8 ? &ARM::DPRRegClass
11354 : nullptr;
11355
11356 unsigned BytesLeft = SizeVal % UnitSize;
11357 unsigned LoopSize = SizeVal - BytesLeft;
11358
11359 if (SizeVal <= Subtarget->getMaxInlineSizeThreshold()) {
11360 // Use LDR and STR to copy.
11361 // [scratch, srcOut] = LDR_POST(srcIn, UnitSize)
11362 // [destOut] = STR_POST(scratch, destIn, UnitSize)
11363 unsigned srcIn = src;
11364 unsigned destIn = dest;
11365 for (unsigned i = 0; i < LoopSize; i+=UnitSize) {
11366 Register srcOut = MRI.createVirtualRegister(RegClass: TRC);
11367 Register destOut = MRI.createVirtualRegister(RegClass: TRC);
11368 Register scratch = MRI.createVirtualRegister(RegClass: IsNeon ? VecTRC : TRC);
11369 emitPostLd(BB, Pos: MI, TII, dl, LdSize: UnitSize, Data: scratch, AddrIn: srcIn, AddrOut: srcOut,
11370 IsThumb1, IsThumb2);
11371 emitPostSt(BB, Pos: MI, TII, dl, StSize: UnitSize, Data: scratch, AddrIn: destIn, AddrOut: destOut,
11372 IsThumb1, IsThumb2);
11373 srcIn = srcOut;
11374 destIn = destOut;
11375 }
11376
11377 // Handle the leftover bytes with LDRB and STRB.
11378 // [scratch, srcOut] = LDRB_POST(srcIn, 1)
11379 // [destOut] = STRB_POST(scratch, destIn, 1)
11380 for (unsigned i = 0; i < BytesLeft; i++) {
11381 Register srcOut = MRI.createVirtualRegister(RegClass: TRC);
11382 Register destOut = MRI.createVirtualRegister(RegClass: TRC);
11383 Register scratch = MRI.createVirtualRegister(RegClass: TRC);
11384 emitPostLd(BB, Pos: MI, TII, dl, LdSize: 1, Data: scratch, AddrIn: srcIn, AddrOut: srcOut,
11385 IsThumb1, IsThumb2);
11386 emitPostSt(BB, Pos: MI, TII, dl, StSize: 1, Data: scratch, AddrIn: destIn, AddrOut: destOut,
11387 IsThumb1, IsThumb2);
11388 srcIn = srcOut;
11389 destIn = destOut;
11390 }
11391 MI.eraseFromParent(); // The instruction is gone now.
11392 return BB;
11393 }
11394
11395 // Expand the pseudo op to a loop.
11396 // thisMBB:
11397 // ...
11398 // movw varEnd, # --> with thumb2
11399 // movt varEnd, #
11400 // ldrcp varEnd, idx --> without thumb2
11401 // fallthrough --> loopMBB
11402 // loopMBB:
11403 // PHI varPhi, varEnd, varLoop
11404 // PHI srcPhi, src, srcLoop
11405 // PHI destPhi, dst, destLoop
11406 // [scratch, srcLoop] = LDR_POST(srcPhi, UnitSize)
11407 // [destLoop] = STR_POST(scratch, destPhi, UnitSize)
11408 // subs varLoop, varPhi, #UnitSize
11409 // bne loopMBB
11410 // fallthrough --> exitMBB
11411 // exitMBB:
11412 // epilogue to handle left-over bytes
11413 // [scratch, srcOut] = LDRB_POST(srcLoop, 1)
11414 // [destOut] = STRB_POST(scratch, destLoop, 1)
11415 MachineBasicBlock *loopMBB = MF->CreateMachineBasicBlock(BB: LLVM_BB);
11416 MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(BB: LLVM_BB);
11417 MF->insert(MBBI: It, MBB: loopMBB);
11418 MF->insert(MBBI: It, MBB: exitMBB);
11419
11420 // Set the call frame size on entry to the new basic blocks.
11421 unsigned CallFrameSize = TII->getCallFrameSizeAt(MI);
11422 loopMBB->setCallFrameSize(CallFrameSize);
11423 exitMBB->setCallFrameSize(CallFrameSize);
11424
11425 // Transfer the remainder of BB and its successor edges to exitMBB.
11426 exitMBB->splice(Where: exitMBB->begin(), Other: BB,
11427 From: std::next(x: MachineBasicBlock::iterator(MI)), To: BB->end());
11428 exitMBB->transferSuccessorsAndUpdatePHIs(FromMBB: BB);
11429
11430 // Load an immediate to varEnd.
11431 Register varEnd = MRI.createVirtualRegister(RegClass: TRC);
11432 if (Subtarget->useMovt()) {
11433 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: IsThumb ? ARM::t2MOVi32imm : ARM::MOVi32imm),
11434 DestReg: varEnd)
11435 .addImm(Val: LoopSize);
11436 } else if (Subtarget->genExecuteOnly()) {
11437 assert(IsThumb && "Non-thumb expected to have used movt");
11438 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: ARM::tMOVi32imm), DestReg: varEnd).addImm(Val: LoopSize);
11439 } else {
11440 MachineConstantPool *ConstantPool = MF->getConstantPool();
11441 Type *Int32Ty = Type::getInt32Ty(C&: MF->getFunction().getContext());
11442 const Constant *C = ConstantInt::get(Ty: Int32Ty, V: LoopSize);
11443
11444 // MachineConstantPool wants an explicit alignment.
11445 Align Alignment = MF->getDataLayout().getPrefTypeAlign(Ty: Int32Ty);
11446 unsigned Idx = ConstantPool->getConstantPoolIndex(C, Alignment);
11447 MachineMemOperand *CPMMO =
11448 MF->getMachineMemOperand(PtrInfo: MachinePointerInfo::getConstantPool(MF&: *MF),
11449 F: MachineMemOperand::MOLoad, Size: 4, BaseAlignment: Align(4));
11450
11451 if (IsThumb)
11452 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: ARM::tLDRpci))
11453 .addReg(RegNo: varEnd, Flags: RegState::Define)
11454 .addConstantPoolIndex(Idx)
11455 .add(MOs: predOps(Pred: ARMCC::AL))
11456 .addMemOperand(MMO: CPMMO);
11457 else
11458 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: ARM::LDRcp))
11459 .addReg(RegNo: varEnd, Flags: RegState::Define)
11460 .addConstantPoolIndex(Idx)
11461 .addImm(Val: 0)
11462 .add(MOs: predOps(Pred: ARMCC::AL))
11463 .addMemOperand(MMO: CPMMO);
11464 }
11465 BB->addSuccessor(Succ: loopMBB);
11466
11467 // Generate the loop body:
11468 // varPhi = PHI(varLoop, varEnd)
11469 // srcPhi = PHI(srcLoop, src)
11470 // destPhi = PHI(destLoop, dst)
11471 MachineBasicBlock *entryBB = BB;
11472 BB = loopMBB;
11473 Register varLoop = MRI.createVirtualRegister(RegClass: TRC);
11474 Register varPhi = MRI.createVirtualRegister(RegClass: TRC);
11475 Register srcLoop = MRI.createVirtualRegister(RegClass: TRC);
11476 Register srcPhi = MRI.createVirtualRegister(RegClass: TRC);
11477 Register destLoop = MRI.createVirtualRegister(RegClass: TRC);
11478 Register destPhi = MRI.createVirtualRegister(RegClass: TRC);
11479
11480 BuildMI(BB&: *BB, I: BB->begin(), MIMD: dl, MCID: TII->get(Opcode: ARM::PHI), DestReg: varPhi)
11481 .addReg(RegNo: varLoop).addMBB(MBB: loopMBB)
11482 .addReg(RegNo: varEnd).addMBB(MBB: entryBB);
11483 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: ARM::PHI), DestReg: srcPhi)
11484 .addReg(RegNo: srcLoop).addMBB(MBB: loopMBB)
11485 .addReg(RegNo: src).addMBB(MBB: entryBB);
11486 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: ARM::PHI), DestReg: destPhi)
11487 .addReg(RegNo: destLoop).addMBB(MBB: loopMBB)
11488 .addReg(RegNo: dest).addMBB(MBB: entryBB);
11489
11490 // [scratch, srcLoop] = LDR_POST(srcPhi, UnitSize)
11491 // [destLoop] = STR_POST(scratch, destPhi, UnitSiz)
11492 Register scratch = MRI.createVirtualRegister(RegClass: IsNeon ? VecTRC : TRC);
11493 emitPostLd(BB, Pos: BB->end(), TII, dl, LdSize: UnitSize, Data: scratch, AddrIn: srcPhi, AddrOut: srcLoop,
11494 IsThumb1, IsThumb2);
11495 emitPostSt(BB, Pos: BB->end(), TII, dl, StSize: UnitSize, Data: scratch, AddrIn: destPhi, AddrOut: destLoop,
11496 IsThumb1, IsThumb2);
11497
11498 // Decrement loop variable by UnitSize.
11499 if (IsThumb1) {
11500 BuildMI(BB&: *BB, I: BB->end(), MIMD: dl, MCID: TII->get(Opcode: ARM::tSUBi8), DestReg: varLoop)
11501 .add(MO: t1CondCodeOp())
11502 .addReg(RegNo: varPhi)
11503 .addImm(Val: UnitSize)
11504 .add(MOs: predOps(Pred: ARMCC::AL));
11505 } else {
11506 MachineInstrBuilder MIB =
11507 BuildMI(BB&: *BB, I: BB->end(), MIMD: dl,
11508 MCID: TII->get(Opcode: IsThumb2 ? ARM::t2SUBri : ARM::SUBri), DestReg: varLoop);
11509 MIB.addReg(RegNo: varPhi)
11510 .addImm(Val: UnitSize)
11511 .add(MOs: predOps(Pred: ARMCC::AL))
11512 .add(MO: condCodeOp());
11513 MIB->getOperand(i: 5).setReg(ARM::CPSR);
11514 MIB->getOperand(i: 5).setIsDef(true);
11515 }
11516 BuildMI(BB&: *BB, I: BB->end(), MIMD: dl,
11517 MCID: TII->get(Opcode: IsThumb1 ? ARM::tBcc : IsThumb2 ? ARM::t2Bcc : ARM::Bcc))
11518 .addMBB(MBB: loopMBB).addImm(Val: ARMCC::NE).addReg(RegNo: ARM::CPSR);
11519
11520 // loopMBB can loop back to loopMBB or fall through to exitMBB.
11521 BB->addSuccessor(Succ: loopMBB);
11522 BB->addSuccessor(Succ: exitMBB);
11523
11524 // Add epilogue to handle BytesLeft.
11525 BB = exitMBB;
11526 auto StartOfExit = exitMBB->begin();
11527
11528 // [scratch, srcOut] = LDRB_POST(srcLoop, 1)
11529 // [destOut] = STRB_POST(scratch, destLoop, 1)
11530 unsigned srcIn = srcLoop;
11531 unsigned destIn = destLoop;
11532 for (unsigned i = 0; i < BytesLeft; i++) {
11533 Register srcOut = MRI.createVirtualRegister(RegClass: TRC);
11534 Register destOut = MRI.createVirtualRegister(RegClass: TRC);
11535 Register scratch = MRI.createVirtualRegister(RegClass: TRC);
11536 emitPostLd(BB, Pos: StartOfExit, TII, dl, LdSize: 1, Data: scratch, AddrIn: srcIn, AddrOut: srcOut,
11537 IsThumb1, IsThumb2);
11538 emitPostSt(BB, Pos: StartOfExit, TII, dl, StSize: 1, Data: scratch, AddrIn: destIn, AddrOut: destOut,
11539 IsThumb1, IsThumb2);
11540 srcIn = srcOut;
11541 destIn = destOut;
11542 }
11543
11544 MI.eraseFromParent(); // The instruction is gone now.
11545 return BB;
11546}
11547
11548MachineBasicBlock *
11549ARMTargetLowering::EmitLowered__chkstk(MachineInstr &MI,
11550 MachineBasicBlock *MBB) const {
11551 const TargetMachine &TM = getTargetMachine();
11552 const TargetInstrInfo &TII = *Subtarget->getInstrInfo();
11553 DebugLoc DL = MI.getDebugLoc();
11554
11555 assert(TM.getTargetTriple().isOSWindows() &&
11556 "__chkstk is only supported on Windows");
11557 assert(Subtarget->isThumb2() && "Windows on ARM requires Thumb-2 mode");
11558
11559 // __chkstk takes the number of words to allocate on the stack in R4, and
11560 // returns the stack adjustment in number of bytes in R4. This will not
11561 // clober any other registers (other than the obvious lr).
11562 //
11563 // Although, technically, IP should be considered a register which may be
11564 // clobbered, the call itself will not touch it. Windows on ARM is a pure
11565 // thumb-2 environment, so there is no interworking required. As a result, we
11566 // do not expect a veneer to be emitted by the linker, clobbering IP.
11567 //
11568 // Each module receives its own copy of __chkstk, so no import thunk is
11569 // required, again, ensuring that IP is not clobbered.
11570 //
11571 // Finally, although some linkers may theoretically provide a trampoline for
11572 // out of range calls (which is quite common due to a 32M range limitation of
11573 // branches for Thumb), we can generate the long-call version via
11574 // -mcmodel=large, alleviating the need for the trampoline which may clobber
11575 // IP.
11576
11577 RTLIB::LibcallImpl ChkStkLibcall = getLibcallImpl(Call: RTLIB::STACK_PROBE);
11578 if (ChkStkLibcall == RTLIB::Unsupported)
11579 reportFatalUsageError(reason: "no available implementation of __chkstk");
11580
11581 const char *ChkStk = getLibcallImplName(Call: ChkStkLibcall).data();
11582 switch (TM.getCodeModel()) {
11583 case CodeModel::Tiny:
11584 llvm_unreachable("Tiny code model not available on ARM.");
11585 case CodeModel::Small:
11586 case CodeModel::Medium:
11587 case CodeModel::Kernel:
11588 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: ARM::tBL))
11589 .add(MOs: predOps(Pred: ARMCC::AL))
11590 .addExternalSymbol(FnName: ChkStk)
11591 .setOperandDead(3) // implicit-def $lr
11592 .addReg(RegNo: ARM::R4, Flags: RegState::Implicit | RegState::Kill)
11593 .addReg(RegNo: ARM::R4, Flags: RegState::Implicit | RegState::Define)
11594 .addReg(RegNo: ARM::R12,
11595 Flags: RegState::Implicit | RegState::Define | RegState::Dead)
11596 .addReg(RegNo: ARM::CPSR,
11597 Flags: RegState::Implicit | RegState::Define | RegState::Dead);
11598 break;
11599 case CodeModel::Large: {
11600 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
11601 Register Reg = MRI.createVirtualRegister(RegClass: &ARM::rGPRRegClass);
11602
11603 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: ARM::t2MOVi32imm), DestReg: Reg)
11604 .addExternalSymbol(FnName: ChkStk);
11605 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: gettBLXrOpcode(MF: *MBB->getParent())))
11606 .add(MOs: predOps(Pred: ARMCC::AL))
11607 .addReg(RegNo: Reg)
11608 .setOperandDead(3) // implicit-def $lr
11609 .addReg(RegNo: ARM::R4, Flags: RegState::Implicit | RegState::Kill)
11610 .addReg(RegNo: ARM::R4, Flags: RegState::Implicit | RegState::Define)
11611 .addReg(RegNo: ARM::R12,
11612 Flags: RegState::Implicit | RegState::Define | RegState::Dead)
11613 .addReg(RegNo: ARM::CPSR,
11614 Flags: RegState::Implicit | RegState::Define | RegState::Dead);
11615 break;
11616 }
11617 }
11618
11619 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: ARM::t2SUBrr), DestReg: ARM::SP)
11620 .addReg(RegNo: ARM::SP, Flags: RegState::Kill)
11621 .addReg(RegNo: ARM::R4, Flags: RegState::Kill)
11622 .setMIFlags(MachineInstr::FrameSetup)
11623 .add(MOs: predOps(Pred: ARMCC::AL))
11624 .add(MO: condCodeOp());
11625
11626 MI.eraseFromParent();
11627 return MBB;
11628}
11629
11630MachineBasicBlock *
11631ARMTargetLowering::EmitLowered__dbzchk(MachineInstr &MI,
11632 MachineBasicBlock *MBB) const {
11633 DebugLoc DL = MI.getDebugLoc();
11634 MachineFunction *MF = MBB->getParent();
11635 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
11636
11637 MachineBasicBlock *ContBB = MF->CreateMachineBasicBlock();
11638 MF->insert(MBBI: ++MBB->getIterator(), MBB: ContBB);
11639 ContBB->splice(Where: ContBB->begin(), Other: MBB,
11640 From: std::next(x: MachineBasicBlock::iterator(MI)), To: MBB->end());
11641 ContBB->transferSuccessorsAndUpdatePHIs(FromMBB: MBB);
11642 MBB->addSuccessor(Succ: ContBB);
11643
11644 MachineBasicBlock *TrapBB = MF->CreateMachineBasicBlock();
11645 BuildMI(BB: TrapBB, MIMD: DL, MCID: TII->get(Opcode: ARM::t__brkdiv0));
11646 MF->push_back(MBB: TrapBB);
11647 MBB->addSuccessor(Succ: TrapBB);
11648
11649 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: ARM::tCMPi8))
11650 .addReg(RegNo: MI.getOperand(i: 0).getReg())
11651 .addImm(Val: 0)
11652 .add(MOs: predOps(Pred: ARMCC::AL));
11653 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: ARM::t2Bcc))
11654 .addMBB(MBB: TrapBB)
11655 .addImm(Val: ARMCC::EQ)
11656 .addReg(RegNo: ARM::CPSR);
11657
11658 MI.eraseFromParent();
11659 return ContBB;
11660}
11661
11662// The CPSR operand of SelectItr might be missing a kill marker
11663// because there were multiple uses of CPSR, and ISel didn't know
11664// which to mark. Figure out whether SelectItr should have had a
11665// kill marker, and set it if it should. Returns the correct kill
11666// marker value.
11667static bool checkAndUpdateCPSRKill(MachineBasicBlock::iterator SelectItr,
11668 MachineBasicBlock* BB,
11669 const TargetRegisterInfo* TRI) {
11670 // Scan forward through BB for a use/def of CPSR.
11671 MachineBasicBlock::iterator miI(std::next(x: SelectItr));
11672 for (MachineBasicBlock::iterator miE = BB->end(); miI != miE; ++miI) {
11673 const MachineInstr& mi = *miI;
11674 if (mi.readsRegister(Reg: ARM::CPSR, /*TRI=*/nullptr))
11675 return false;
11676 if (mi.definesRegister(Reg: ARM::CPSR, /*TRI=*/nullptr))
11677 break; // Should have kill-flag - update below.
11678 }
11679
11680 // If we hit the end of the block, check whether CPSR is live into a
11681 // successor.
11682 if (miI == BB->end()) {
11683 for (MachineBasicBlock *Succ : BB->successors())
11684 if (Succ->isLiveIn(Reg: ARM::CPSR))
11685 return false;
11686 }
11687
11688 // We found a def, or hit the end of the basic block and CPSR wasn't live
11689 // out. SelectMI should have a kill flag on CPSR.
11690 SelectItr->addRegisterKilled(IncomingReg: ARM::CPSR, RegInfo: TRI);
11691 return true;
11692}
11693
11694/// Adds logic in loop entry MBB to calculate loop iteration count and adds
11695/// t2WhileLoopSetup and t2WhileLoopStart to generate WLS loop
11696static Register genTPEntry(MachineBasicBlock *TpEntry,
11697 MachineBasicBlock *TpLoopBody,
11698 MachineBasicBlock *TpExit, Register OpSizeReg,
11699 const TargetInstrInfo *TII, DebugLoc Dl,
11700 MachineRegisterInfo &MRI) {
11701 // Calculates loop iteration count = ceil(n/16) = (n + 15) >> 4.
11702 Register AddDestReg = MRI.createVirtualRegister(RegClass: &ARM::rGPRRegClass);
11703 BuildMI(BB: TpEntry, MIMD: Dl, MCID: TII->get(Opcode: ARM::t2ADDri), DestReg: AddDestReg)
11704 .addUse(RegNo: OpSizeReg)
11705 .addImm(Val: 15)
11706 .add(MOs: predOps(Pred: ARMCC::AL))
11707 .addReg(RegNo: 0);
11708
11709 Register LsrDestReg = MRI.createVirtualRegister(RegClass: &ARM::rGPRRegClass);
11710 BuildMI(BB: TpEntry, MIMD: Dl, MCID: TII->get(Opcode: ARM::t2LSRri), DestReg: LsrDestReg)
11711 .addUse(RegNo: AddDestReg)
11712 .addImm(Val: 4)
11713 .add(MOs: predOps(Pred: ARMCC::AL))
11714 .addReg(RegNo: 0);
11715
11716 Register TotalIterationsReg = MRI.createVirtualRegister(RegClass: &ARM::GPRlrRegClass);
11717 BuildMI(BB: TpEntry, MIMD: Dl, MCID: TII->get(Opcode: ARM::t2WhileLoopSetup), DestReg: TotalIterationsReg)
11718 .addUse(RegNo: LsrDestReg);
11719
11720 BuildMI(BB: TpEntry, MIMD: Dl, MCID: TII->get(Opcode: ARM::t2WhileLoopStart))
11721 .addUse(RegNo: TotalIterationsReg)
11722 .addMBB(MBB: TpExit);
11723
11724 BuildMI(BB: TpEntry, MIMD: Dl, MCID: TII->get(Opcode: ARM::t2B))
11725 .addMBB(MBB: TpLoopBody)
11726 .add(MOs: predOps(Pred: ARMCC::AL));
11727
11728 return TotalIterationsReg;
11729}
11730
11731/// Adds logic in the loopBody MBB to generate MVE_VCTP, t2DoLoopDec and
11732/// t2DoLoopEnd. These are used by later passes to generate tail predicated
11733/// loops.
11734static void genTPLoopBody(MachineBasicBlock *TpLoopBody,
11735 MachineBasicBlock *TpEntry, MachineBasicBlock *TpExit,
11736 const TargetInstrInfo *TII, DebugLoc Dl,
11737 MachineRegisterInfo &MRI, Register OpSrcReg,
11738 Register OpDestReg, Register ElementCountReg,
11739 Register TotalIterationsReg, bool IsMemcpy) {
11740 // First insert 4 PHI nodes for: Current pointer to Src (if memcpy), Dest
11741 // array, loop iteration counter, predication counter.
11742
11743 Register SrcPhiReg, CurrSrcReg;
11744 if (IsMemcpy) {
11745 // Current position in the src array
11746 SrcPhiReg = MRI.createVirtualRegister(RegClass: &ARM::rGPRRegClass);
11747 CurrSrcReg = MRI.createVirtualRegister(RegClass: &ARM::rGPRRegClass);
11748 BuildMI(BB: TpLoopBody, MIMD: Dl, MCID: TII->get(Opcode: ARM::PHI), DestReg: SrcPhiReg)
11749 .addUse(RegNo: OpSrcReg)
11750 .addMBB(MBB: TpEntry)
11751 .addUse(RegNo: CurrSrcReg)
11752 .addMBB(MBB: TpLoopBody);
11753 }
11754
11755 // Current position in the dest array
11756 Register DestPhiReg = MRI.createVirtualRegister(RegClass: &ARM::rGPRRegClass);
11757 Register CurrDestReg = MRI.createVirtualRegister(RegClass: &ARM::rGPRRegClass);
11758 BuildMI(BB: TpLoopBody, MIMD: Dl, MCID: TII->get(Opcode: ARM::PHI), DestReg: DestPhiReg)
11759 .addUse(RegNo: OpDestReg)
11760 .addMBB(MBB: TpEntry)
11761 .addUse(RegNo: CurrDestReg)
11762 .addMBB(MBB: TpLoopBody);
11763
11764 // Current loop counter
11765 Register LoopCounterPhiReg = MRI.createVirtualRegister(RegClass: &ARM::GPRlrRegClass);
11766 Register RemainingLoopIterationsReg =
11767 MRI.createVirtualRegister(RegClass: &ARM::GPRlrRegClass);
11768 BuildMI(BB: TpLoopBody, MIMD: Dl, MCID: TII->get(Opcode: ARM::PHI), DestReg: LoopCounterPhiReg)
11769 .addUse(RegNo: TotalIterationsReg)
11770 .addMBB(MBB: TpEntry)
11771 .addUse(RegNo: RemainingLoopIterationsReg)
11772 .addMBB(MBB: TpLoopBody);
11773
11774 // Predication counter
11775 Register PredCounterPhiReg = MRI.createVirtualRegister(RegClass: &ARM::rGPRRegClass);
11776 Register RemainingElementsReg = MRI.createVirtualRegister(RegClass: &ARM::rGPRRegClass);
11777 BuildMI(BB: TpLoopBody, MIMD: Dl, MCID: TII->get(Opcode: ARM::PHI), DestReg: PredCounterPhiReg)
11778 .addUse(RegNo: ElementCountReg)
11779 .addMBB(MBB: TpEntry)
11780 .addUse(RegNo: RemainingElementsReg)
11781 .addMBB(MBB: TpLoopBody);
11782
11783 // Pass predication counter to VCTP
11784 Register VccrReg = MRI.createVirtualRegister(RegClass: &ARM::VCCRRegClass);
11785 BuildMI(BB: TpLoopBody, MIMD: Dl, MCID: TII->get(Opcode: ARM::MVE_VCTP8), DestReg: VccrReg)
11786 .addUse(RegNo: PredCounterPhiReg)
11787 .addImm(Val: ARMVCC::None)
11788 .addReg(RegNo: 0)
11789 .addReg(RegNo: 0);
11790
11791 BuildMI(BB: TpLoopBody, MIMD: Dl, MCID: TII->get(Opcode: ARM::t2SUBri), DestReg: RemainingElementsReg)
11792 .addUse(RegNo: PredCounterPhiReg)
11793 .addImm(Val: 16)
11794 .add(MOs: predOps(Pred: ARMCC::AL))
11795 .addReg(RegNo: 0);
11796
11797 // VLDRB (only if memcpy) and VSTRB instructions, predicated using VPR
11798 Register SrcValueReg;
11799 if (IsMemcpy) {
11800 SrcValueReg = MRI.createVirtualRegister(RegClass: &ARM::MQPRRegClass);
11801 BuildMI(BB: TpLoopBody, MIMD: Dl, MCID: TII->get(Opcode: ARM::MVE_VLDRBU8_post))
11802 .addDef(RegNo: CurrSrcReg)
11803 .addDef(RegNo: SrcValueReg)
11804 .addReg(RegNo: SrcPhiReg)
11805 .addImm(Val: 16)
11806 .addImm(Val: ARMVCC::Then)
11807 .addUse(RegNo: VccrReg)
11808 .addReg(RegNo: 0);
11809 } else
11810 SrcValueReg = OpSrcReg;
11811
11812 BuildMI(BB: TpLoopBody, MIMD: Dl, MCID: TII->get(Opcode: ARM::MVE_VSTRBU8_post))
11813 .addDef(RegNo: CurrDestReg)
11814 .addUse(RegNo: SrcValueReg)
11815 .addReg(RegNo: DestPhiReg)
11816 .addImm(Val: 16)
11817 .addImm(Val: ARMVCC::Then)
11818 .addUse(RegNo: VccrReg)
11819 .addReg(RegNo: 0);
11820
11821 // Add the pseudoInstrs for decrementing the loop counter and marking the
11822 // end:t2DoLoopDec and t2DoLoopEnd
11823 BuildMI(BB: TpLoopBody, MIMD: Dl, MCID: TII->get(Opcode: ARM::t2LoopDec), DestReg: RemainingLoopIterationsReg)
11824 .addUse(RegNo: LoopCounterPhiReg)
11825 .addImm(Val: 1);
11826
11827 BuildMI(BB: TpLoopBody, MIMD: Dl, MCID: TII->get(Opcode: ARM::t2LoopEnd))
11828 .addUse(RegNo: RemainingLoopIterationsReg)
11829 .addMBB(MBB: TpLoopBody);
11830
11831 BuildMI(BB: TpLoopBody, MIMD: Dl, MCID: TII->get(Opcode: ARM::t2B))
11832 .addMBB(MBB: TpExit)
11833 .add(MOs: predOps(Pred: ARMCC::AL));
11834}
11835
11836bool ARMTargetLowering::supportKCFIBundles() const {
11837 // KCFI is supported in all ARM/Thumb modes
11838 return true;
11839}
11840
11841MachineInstr *
11842ARMTargetLowering::EmitKCFICheck(MachineBasicBlock &MBB,
11843 MachineBasicBlock::instr_iterator &MBBI,
11844 const TargetInstrInfo *TII) const {
11845 assert(MBBI->isCall() && MBBI->getCFIType() &&
11846 "Invalid call instruction for a KCFI check");
11847
11848 MachineOperand *TargetOp = nullptr;
11849 switch (MBBI->getOpcode()) {
11850 // ARM mode opcodes
11851 case ARM::BLX:
11852 case ARM::BLX_pred:
11853 case ARM::BLX_noip:
11854 case ARM::BLX_pred_noip:
11855 case ARM::BX_CALL:
11856 TargetOp = &MBBI->getOperand(i: 0);
11857 break;
11858 case ARM::TCRETURNri:
11859 case ARM::TCRETURNrinotr12:
11860 case ARM::TAILJMPr:
11861 case ARM::TAILJMPr4:
11862 TargetOp = &MBBI->getOperand(i: 0);
11863 break;
11864 // Thumb mode opcodes (Thumb1 and Thumb2)
11865 // Note: Most Thumb call instructions have predicate operands before the
11866 // target register Format: tBLXr pred, predreg, target_register, ...
11867 case ARM::tBLXr: // Thumb1/Thumb2: BLX register (requires V5T)
11868 case ARM::tBLXr_noip: // Thumb1/Thumb2: BLX register, no IP clobber
11869 case ARM::tBX_CALL: // Thumb1 only: BX call (push LR, BX)
11870 TargetOp = &MBBI->getOperand(i: 2);
11871 break;
11872 // Tail call instructions don't have predicates, target is operand 0
11873 case ARM::tTAILJMPr: // Thumb1/Thumb2: Tail call via register
11874 TargetOp = &MBBI->getOperand(i: 0);
11875 break;
11876 default:
11877 llvm_unreachable("Unexpected CFI call opcode");
11878 }
11879
11880 assert(TargetOp && TargetOp->isReg() && "Invalid target operand");
11881 TargetOp->setIsRenamable(false);
11882
11883 // Select the appropriate KCFI_CHECK variant based on the instruction set
11884 unsigned KCFICheckOpcode;
11885 if (Subtarget->isThumb()) {
11886 if (Subtarget->isThumb2()) {
11887 KCFICheckOpcode = ARM::KCFI_CHECK_Thumb2;
11888 } else {
11889 KCFICheckOpcode = ARM::KCFI_CHECK_Thumb1;
11890 }
11891 } else {
11892 KCFICheckOpcode = ARM::KCFI_CHECK_ARM;
11893 }
11894
11895 return BuildMI(BB&: MBB, I: MBBI, MIMD: MBBI->getDebugLoc(), MCID: TII->get(Opcode: KCFICheckOpcode))
11896 .addReg(RegNo: TargetOp->getReg())
11897 .addImm(Val: MBBI->getCFIType())
11898 .getInstr();
11899}
11900
11901MachineBasicBlock *
11902ARMTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
11903 MachineBasicBlock *BB) const {
11904 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
11905 DebugLoc dl = MI.getDebugLoc();
11906 bool isThumb2 = Subtarget->isThumb2();
11907 switch (MI.getOpcode()) {
11908 default: {
11909 MI.print(OS&: errs());
11910 llvm_unreachable("Unexpected instr type to insert");
11911 }
11912
11913 // Thumb1 post-indexed loads are really just single-register LDMs.
11914 case ARM::tLDR_postidx: {
11915 MachineOperand Def(MI.getOperand(i: 1));
11916 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: ARM::tLDMIA_UPD))
11917 .add(MO: Def) // Rn_wb
11918 .add(MO: MI.getOperand(i: 2)) // Rn
11919 .add(MO: MI.getOperand(i: 3)) // PredImm
11920 .add(MO: MI.getOperand(i: 4)) // PredReg
11921 .add(MO: MI.getOperand(i: 0)) // Rt
11922 .cloneMemRefs(OtherMI: MI);
11923 MI.eraseFromParent();
11924 return BB;
11925 }
11926
11927 case ARM::MVE_MEMCPYLOOPINST:
11928 case ARM::MVE_MEMSETLOOPINST: {
11929
11930 // Transformation below expands MVE_MEMCPYLOOPINST/MVE_MEMSETLOOPINST Pseudo
11931 // into a Tail Predicated (TP) Loop. It adds the instructions to calculate
11932 // the iteration count =ceil(size_in_bytes/16)) in the TP entry block and
11933 // adds the relevant instructions in the TP loop Body for generation of a
11934 // WLSTP loop.
11935
11936 // Below is relevant portion of the CFG after the transformation.
11937 // The Machine Basic Blocks are shown along with branch conditions (in
11938 // brackets). Note that TP entry/exit MBBs depict the entry/exit of this
11939 // portion of the CFG and may not necessarily be the entry/exit of the
11940 // function.
11941
11942 // (Relevant) CFG after transformation:
11943 // TP entry MBB
11944 // |
11945 // |-----------------|
11946 // (n <= 0) (n > 0)
11947 // | |
11948 // | TP loop Body MBB<--|
11949 // | | |
11950 // \ |___________|
11951 // \ /
11952 // TP exit MBB
11953
11954 MachineFunction *MF = BB->getParent();
11955 MachineFunctionProperties &Properties = MF->getProperties();
11956 MachineRegisterInfo &MRI = MF->getRegInfo();
11957
11958 Register OpDestReg = MI.getOperand(i: 0).getReg();
11959 Register OpSrcReg = MI.getOperand(i: 1).getReg();
11960 Register OpSizeReg = MI.getOperand(i: 2).getReg();
11961
11962 // Allocate the required MBBs and add to parent function.
11963 MachineBasicBlock *TpEntry = BB;
11964 MachineBasicBlock *TpLoopBody = MF->CreateMachineBasicBlock();
11965 MachineBasicBlock *TpExit;
11966
11967 MF->push_back(MBB: TpLoopBody);
11968
11969 // If any instructions are present in the current block after
11970 // MVE_MEMCPYLOOPINST or MVE_MEMSETLOOPINST, split the current block and
11971 // move the instructions into the newly created exit block. If there are no
11972 // instructions add an explicit branch to the FallThrough block and then
11973 // split.
11974 //
11975 // The split is required for two reasons:
11976 // 1) A terminator(t2WhileLoopStart) will be placed at that site.
11977 // 2) Since a TPLoopBody will be added later, any phis in successive blocks
11978 // need to be updated. splitAt() already handles this.
11979 TpExit = BB->splitAt(SplitInst&: MI, UpdateLiveIns: false);
11980 if (TpExit == BB) {
11981 assert(BB->canFallThrough() && "Exit Block must be Fallthrough of the "
11982 "block containing memcpy/memset Pseudo");
11983 TpExit = BB->getFallThrough();
11984 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: ARM::t2B))
11985 .addMBB(MBB: TpExit)
11986 .add(MOs: predOps(Pred: ARMCC::AL));
11987 TpExit = BB->splitAt(SplitInst&: MI, UpdateLiveIns: false);
11988 }
11989
11990 // Add logic for iteration count
11991 Register TotalIterationsReg =
11992 genTPEntry(TpEntry, TpLoopBody, TpExit, OpSizeReg, TII, Dl: dl, MRI);
11993
11994 // Add the vectorized (and predicated) loads/store instructions
11995 bool IsMemcpy = MI.getOpcode() == ARM::MVE_MEMCPYLOOPINST;
11996 genTPLoopBody(TpLoopBody, TpEntry, TpExit, TII, Dl: dl, MRI, OpSrcReg,
11997 OpDestReg, ElementCountReg: OpSizeReg, TotalIterationsReg, IsMemcpy);
11998
11999 // Required to avoid conflict with the MachineVerifier during testing.
12000 Properties.resetNoPHIs();
12001
12002 // Connect the blocks
12003 TpEntry->addSuccessor(Succ: TpLoopBody);
12004 TpLoopBody->addSuccessor(Succ: TpLoopBody);
12005 TpLoopBody->addSuccessor(Succ: TpExit);
12006
12007 // Reorder for a more natural layout
12008 TpLoopBody->moveAfter(NewBefore: TpEntry);
12009 TpExit->moveAfter(NewBefore: TpLoopBody);
12010
12011 // Finally, remove the memcpy Pseudo Instruction
12012 MI.eraseFromParent();
12013
12014 // Return the exit block as it may contain other instructions requiring a
12015 // custom inserter
12016 return TpExit;
12017 }
12018
12019 // The Thumb2 pre-indexed stores have the same MI operands, they just
12020 // define them differently in the .td files from the isel patterns, so
12021 // they need pseudos.
12022 case ARM::t2STR_preidx:
12023 MI.setDesc(TII->get(Opcode: ARM::t2STR_PRE));
12024 return BB;
12025 case ARM::t2STRB_preidx:
12026 MI.setDesc(TII->get(Opcode: ARM::t2STRB_PRE));
12027 return BB;
12028 case ARM::t2STRH_preidx:
12029 MI.setDesc(TII->get(Opcode: ARM::t2STRH_PRE));
12030 return BB;
12031
12032 case ARM::STRi_preidx:
12033 case ARM::STRBi_preidx: {
12034 unsigned NewOpc = MI.getOpcode() == ARM::STRi_preidx ? ARM::STR_PRE_IMM
12035 : ARM::STRB_PRE_IMM;
12036 // Decode the offset.
12037 unsigned Offset = MI.getOperand(i: 4).getImm();
12038 bool isSub = ARM_AM::getAM2Op(AM2Opc: Offset) == ARM_AM::sub;
12039 Offset = ARM_AM::getAM2Offset(AM2Opc: Offset);
12040 if (isSub)
12041 Offset = -Offset;
12042
12043 MachineMemOperand *MMO = *MI.memoperands_begin();
12044 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: NewOpc))
12045 .add(MO: MI.getOperand(i: 0)) // Rn_wb
12046 .add(MO: MI.getOperand(i: 1)) // Rt
12047 .add(MO: MI.getOperand(i: 2)) // Rn
12048 .addImm(Val: Offset) // offset (skip GPR==zero_reg)
12049 .add(MO: MI.getOperand(i: 5)) // pred
12050 .add(MO: MI.getOperand(i: 6))
12051 .addMemOperand(MMO);
12052 MI.eraseFromParent();
12053 return BB;
12054 }
12055 case ARM::STRr_preidx:
12056 case ARM::STRBr_preidx:
12057 case ARM::STRH_preidx: {
12058 unsigned NewOpc;
12059 switch (MI.getOpcode()) {
12060 default: llvm_unreachable("unexpected opcode!");
12061 case ARM::STRr_preidx: NewOpc = ARM::STR_PRE_REG; break;
12062 case ARM::STRBr_preidx: NewOpc = ARM::STRB_PRE_REG; break;
12063 case ARM::STRH_preidx: NewOpc = ARM::STRH_PRE; break;
12064 }
12065 MachineInstrBuilder MIB = BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: NewOpc));
12066 for (const MachineOperand &MO : MI.operands())
12067 MIB.add(MO);
12068 MI.eraseFromParent();
12069 return BB;
12070 }
12071
12072 case ARM::tMOVCCr_pseudo: {
12073 // To "insert" a SELECT_CC instruction, we actually have to insert the
12074 // diamond control-flow pattern. The incoming instruction knows the
12075 // destination vreg to set, the condition code register to branch on, the
12076 // true/false values to select between, and a branch opcode to use.
12077 const BasicBlock *LLVM_BB = BB->getBasicBlock();
12078 MachineFunction::iterator It = ++BB->getIterator();
12079
12080 // thisMBB:
12081 // ...
12082 // TrueVal = ...
12083 // cmpTY ccX, r1, r2
12084 // bCC copy1MBB
12085 // fallthrough --> copy0MBB
12086 MachineBasicBlock *thisMBB = BB;
12087 MachineFunction *F = BB->getParent();
12088 MachineBasicBlock *copy0MBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
12089 MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
12090 F->insert(MBBI: It, MBB: copy0MBB);
12091 F->insert(MBBI: It, MBB: sinkMBB);
12092
12093 // Set the call frame size on entry to the new basic blocks.
12094 unsigned CallFrameSize = TII->getCallFrameSizeAt(MI);
12095 copy0MBB->setCallFrameSize(CallFrameSize);
12096 sinkMBB->setCallFrameSize(CallFrameSize);
12097
12098 // Check whether CPSR is live past the tMOVCCr_pseudo.
12099 const TargetRegisterInfo *TRI = Subtarget->getRegisterInfo();
12100 if (!MI.killsRegister(Reg: ARM::CPSR, /*TRI=*/nullptr) &&
12101 !checkAndUpdateCPSRKill(SelectItr: MI, BB: thisMBB, TRI)) {
12102 copy0MBB->addLiveIn(PhysReg: ARM::CPSR);
12103 sinkMBB->addLiveIn(PhysReg: ARM::CPSR);
12104 }
12105
12106 // Transfer the remainder of BB and its successor edges to sinkMBB.
12107 sinkMBB->splice(Where: sinkMBB->begin(), Other: BB,
12108 From: std::next(x: MachineBasicBlock::iterator(MI)), To: BB->end());
12109 sinkMBB->transferSuccessorsAndUpdatePHIs(FromMBB: BB);
12110
12111 BB->addSuccessor(Succ: copy0MBB);
12112 BB->addSuccessor(Succ: sinkMBB);
12113
12114 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: ARM::tBcc))
12115 .addMBB(MBB: sinkMBB)
12116 .addImm(Val: MI.getOperand(i: 3).getImm())
12117 .addReg(RegNo: MI.getOperand(i: 4).getReg());
12118
12119 // copy0MBB:
12120 // %FalseValue = ...
12121 // # fallthrough to sinkMBB
12122 BB = copy0MBB;
12123
12124 // Update machine-CFG edges
12125 BB->addSuccessor(Succ: sinkMBB);
12126
12127 // sinkMBB:
12128 // %Result = phi [ %FalseValue, copy0MBB ], [ %TrueValue, thisMBB ]
12129 // ...
12130 BB = sinkMBB;
12131 BuildMI(BB&: *BB, I: BB->begin(), MIMD: dl, MCID: TII->get(Opcode: ARM::PHI), DestReg: MI.getOperand(i: 0).getReg())
12132 .addReg(RegNo: MI.getOperand(i: 1).getReg())
12133 .addMBB(MBB: copy0MBB)
12134 .addReg(RegNo: MI.getOperand(i: 2).getReg())
12135 .addMBB(MBB: thisMBB);
12136
12137 MI.eraseFromParent(); // The pseudo instruction is gone now.
12138 return BB;
12139 }
12140
12141 case ARM::BCCi64:
12142 case ARM::BCCZi64: {
12143 // If there is an unconditional branch to the other successor, remove it.
12144 BB->erase(I: std::next(x: MachineBasicBlock::iterator(MI)), E: BB->end());
12145
12146 // Compare both parts that make up the double comparison separately for
12147 // equality.
12148 bool RHSisZero = MI.getOpcode() == ARM::BCCZi64;
12149
12150 Register LHS1 = MI.getOperand(i: 1).getReg();
12151 Register LHS2 = MI.getOperand(i: 2).getReg();
12152 if (RHSisZero) {
12153 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: isThumb2 ? ARM::t2CMPri : ARM::CMPri))
12154 .addReg(RegNo: LHS1)
12155 .addImm(Val: 0)
12156 .add(MOs: predOps(Pred: ARMCC::AL));
12157 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: isThumb2 ? ARM::t2CMPri : ARM::CMPri))
12158 .addReg(RegNo: LHS2).addImm(Val: 0)
12159 .addImm(Val: ARMCC::EQ).addReg(RegNo: ARM::CPSR);
12160 } else {
12161 Register RHS1 = MI.getOperand(i: 3).getReg();
12162 Register RHS2 = MI.getOperand(i: 4).getReg();
12163 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: isThumb2 ? ARM::t2CMPrr : ARM::CMPrr))
12164 .addReg(RegNo: LHS1)
12165 .addReg(RegNo: RHS1)
12166 .add(MOs: predOps(Pred: ARMCC::AL));
12167 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: isThumb2 ? ARM::t2CMPrr : ARM::CMPrr))
12168 .addReg(RegNo: LHS2).addReg(RegNo: RHS2)
12169 .addImm(Val: ARMCC::EQ).addReg(RegNo: ARM::CPSR);
12170 }
12171
12172 MachineBasicBlock *destMBB = MI.getOperand(i: RHSisZero ? 3 : 5).getMBB();
12173 MachineBasicBlock *exitMBB = OtherSucc(MBB: BB, Succ: destMBB);
12174 if (MI.getOperand(i: 0).getImm() == ARMCC::NE)
12175 std::swap(a&: destMBB, b&: exitMBB);
12176
12177 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: isThumb2 ? ARM::t2Bcc : ARM::Bcc))
12178 .addMBB(MBB: destMBB).addImm(Val: ARMCC::EQ).addReg(RegNo: ARM::CPSR);
12179 if (isThumb2)
12180 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: ARM::t2B))
12181 .addMBB(MBB: exitMBB)
12182 .add(MOs: predOps(Pred: ARMCC::AL));
12183 else
12184 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: ARM::B)) .addMBB(MBB: exitMBB);
12185
12186 MI.eraseFromParent(); // The pseudo instruction is gone now.
12187 return BB;
12188 }
12189
12190 case ARM::Int_eh_sjlj_setjmp:
12191 case ARM::Int_eh_sjlj_setjmp_nofp:
12192 case ARM::tInt_eh_sjlj_setjmp:
12193 case ARM::t2Int_eh_sjlj_setjmp:
12194 case ARM::t2Int_eh_sjlj_setjmp_nofp:
12195 return BB;
12196
12197 case ARM::Int_eh_sjlj_setup_dispatch:
12198 EmitSjLjDispatchBlock(MI, MBB: BB);
12199 return BB;
12200 case ARM::COPY_STRUCT_BYVAL_I32:
12201 ++NumLoopByVals;
12202 return EmitStructByval(MI, BB);
12203 case ARM::WIN__CHKSTK:
12204 return EmitLowered__chkstk(MI, MBB: BB);
12205 case ARM::WIN__DBZCHK:
12206 return EmitLowered__dbzchk(MI, MBB: BB);
12207 }
12208}
12209
12210/// Attaches vregs to MEMCPY that it will use as scratch registers
12211/// when it is expanded into LDM/STM. This is done as a post-isel lowering
12212/// instead of as a custom inserter because we need the use list from the SDNode.
12213static void attachMEMCPYScratchRegs(const ARMSubtarget *Subtarget,
12214 MachineInstr &MI, const SDNode *Node) {
12215 bool isThumb1 = Subtarget->isThumb1Only();
12216
12217 MachineFunction *MF = MI.getParent()->getParent();
12218 MachineRegisterInfo &MRI = MF->getRegInfo();
12219 MachineInstrBuilder MIB(*MF, MI);
12220
12221 // If the new dst/src is unused mark it as dead.
12222 if (!Node->hasAnyUseOfValue(Value: 0)) {
12223 MI.getOperand(i: 0).setIsDead(true);
12224 }
12225 if (!Node->hasAnyUseOfValue(Value: 1)) {
12226 MI.getOperand(i: 1).setIsDead(true);
12227 }
12228
12229 // The MEMCPY both defines and kills the scratch registers.
12230 for (unsigned I = 0; I != MI.getOperand(i: 4).getImm(); ++I) {
12231 Register TmpReg = MRI.createVirtualRegister(RegClass: isThumb1 ? &ARM::tGPRRegClass
12232 : &ARM::GPRRegClass);
12233 MIB.addReg(RegNo: TmpReg, Flags: RegState::Define|RegState::Dead);
12234 }
12235}
12236
12237void ARMTargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI,
12238 SDNode *Node) const {
12239 if (MI.getOpcode() == ARM::MEMCPY) {
12240 attachMEMCPYScratchRegs(Subtarget, MI, Node);
12241 return;
12242 }
12243
12244 const MCInstrDesc *MCID = &MI.getDesc();
12245 // Adjust potentially 's' setting instructions after isel, i.e. ADC, SBC, RSB,
12246 // RSC. Coming out of isel, they have an implicit CPSR def, but the optional
12247 // operand is still set to noreg. If needed, set the optional operand's
12248 // register to CPSR, and remove the redundant implicit def.
12249 //
12250 // e.g. ADCS (..., implicit-def CPSR) -> ADC (... opt:def CPSR).
12251
12252 // Rename pseudo opcodes.
12253 unsigned NewOpc = convertAddSubFlagsOpcode(OldOpc: MI.getOpcode());
12254 unsigned ccOutIdx;
12255 if (NewOpc) {
12256 const ARMBaseInstrInfo *TII = Subtarget->getInstrInfo();
12257 MCID = &TII->get(Opcode: NewOpc);
12258
12259 assert(MCID->getNumOperands() ==
12260 MI.getDesc().getNumOperands() + 5 - MI.getDesc().getSize()
12261 && "converted opcode should be the same except for cc_out"
12262 " (and, on Thumb1, pred)");
12263
12264 MI.setDesc(*MCID);
12265
12266 // Add the optional cc_out operand
12267 MI.addOperand(Op: MachineOperand::CreateReg(Reg: 0, /*isDef=*/true));
12268
12269 // On Thumb1, move all input operands to the end, then add the predicate
12270 if (Subtarget->isThumb1Only()) {
12271 for (unsigned c = MCID->getNumOperands() - 4; c--;) {
12272 MI.addOperand(Op: MI.getOperand(i: 1));
12273 MI.removeOperand(OpNo: 1);
12274 }
12275
12276 // Restore the ties
12277 for (unsigned i = MI.getNumOperands(); i--;) {
12278 const MachineOperand& op = MI.getOperand(i);
12279 if (op.isReg() && op.isUse()) {
12280 int DefIdx = MCID->getOperandConstraint(OpNum: i, Constraint: MCOI::TIED_TO);
12281 if (DefIdx != -1)
12282 MI.tieOperands(DefIdx, UseIdx: i);
12283 }
12284 }
12285
12286 MI.addOperand(Op: MachineOperand::CreateImm(Val: ARMCC::AL));
12287 MI.addOperand(Op: MachineOperand::CreateReg(Reg: 0, /*isDef=*/false));
12288 ccOutIdx = 1;
12289 } else
12290 ccOutIdx = MCID->getNumOperands() - 1;
12291 } else
12292 ccOutIdx = MCID->getNumOperands() - 1;
12293
12294 // Any ARM instruction that sets the 's' bit should specify an optional
12295 // "cc_out" operand in the last operand position.
12296 if (!MI.hasOptionalDef() || !MCID->operands()[ccOutIdx].isOptionalDef()) {
12297 assert(!NewOpc && "Optional cc_out operand required");
12298 return;
12299 }
12300 // Look for an implicit def of CPSR added by MachineInstr ctor. Remove it
12301 // since we already have an optional CPSR def.
12302 bool definesCPSR = false;
12303 bool deadCPSR = false;
12304 for (unsigned i = MCID->getNumOperands(), e = MI.getNumOperands(); i != e;
12305 ++i) {
12306 const MachineOperand &MO = MI.getOperand(i);
12307 if (MO.isReg() && MO.isDef() && MO.getReg() == ARM::CPSR) {
12308 definesCPSR = true;
12309 if (MO.isDead())
12310 deadCPSR = true;
12311 MI.removeOperand(OpNo: i);
12312 break;
12313 }
12314 }
12315 if (!definesCPSR) {
12316 assert(!NewOpc && "Optional cc_out operand required");
12317 return;
12318 }
12319 assert(deadCPSR == !Node->hasAnyUseOfValue(1) && "inconsistent dead flag");
12320 if (deadCPSR) {
12321 assert(!MI.getOperand(ccOutIdx).getReg() &&
12322 "expect uninitialized optional cc_out operand");
12323 // Thumb1 instructions must have the S bit even if the CPSR is dead.
12324 if (!Subtarget->isThumb1Only())
12325 return;
12326 }
12327
12328 // If this instruction was defined with an optional CPSR def and its dag node
12329 // had a live implicit CPSR def, then activate the optional CPSR def.
12330 MachineOperand &MO = MI.getOperand(i: ccOutIdx);
12331 MO.setReg(ARM::CPSR);
12332 MO.setIsDef(true);
12333 MO.setIsDead(deadCPSR);
12334}
12335
12336//===----------------------------------------------------------------------===//
12337// ARM Optimization Hooks
12338//===----------------------------------------------------------------------===//
12339
12340// Helper function that checks if N is a null or all ones constant.
12341static inline bool isZeroOrAllOnes(SDValue N, bool AllOnes) {
12342 return AllOnes ? isAllOnesConstant(V: N) : isNullConstant(V: N);
12343}
12344
12345// Return true if N is conditionally 0 or all ones.
12346// Detects these expressions where cc is an i1 value:
12347//
12348// (select cc 0, y) [AllOnes=0]
12349// (select cc y, 0) [AllOnes=0]
12350// (zext cc) [AllOnes=0]
12351// (sext cc) [AllOnes=0/1]
12352// (select cc -1, y) [AllOnes=1]
12353// (select cc y, -1) [AllOnes=1]
12354//
12355// Invert is set when N is the null/all ones constant when CC is false.
12356// OtherOp is set to the alternative value of N.
12357static bool isConditionalZeroOrAllOnes(SDNode *N, bool AllOnes,
12358 SDValue &CC, bool &Invert,
12359 SDValue &OtherOp,
12360 SelectionDAG &DAG) {
12361 switch (N->getOpcode()) {
12362 default: return false;
12363 case ISD::SELECT: {
12364 CC = N->getOperand(Num: 0);
12365 SDValue N1 = N->getOperand(Num: 1);
12366 SDValue N2 = N->getOperand(Num: 2);
12367 if (isZeroOrAllOnes(N: N1, AllOnes)) {
12368 Invert = false;
12369 OtherOp = N2;
12370 return true;
12371 }
12372 if (isZeroOrAllOnes(N: N2, AllOnes)) {
12373 Invert = true;
12374 OtherOp = N1;
12375 return true;
12376 }
12377 return false;
12378 }
12379 case ISD::ZERO_EXTEND:
12380 // (zext cc) can never be the all ones value.
12381 if (AllOnes)
12382 return false;
12383 [[fallthrough]];
12384 case ISD::SIGN_EXTEND: {
12385 SDLoc dl(N);
12386 EVT VT = N->getValueType(ResNo: 0);
12387 CC = N->getOperand(Num: 0);
12388 if (CC.getValueType() != MVT::i1 || CC.getOpcode() != ISD::SETCC)
12389 return false;
12390 Invert = !AllOnes;
12391 if (AllOnes)
12392 // When looking for an AllOnes constant, N is an sext, and the 'other'
12393 // value is 0.
12394 OtherOp = DAG.getConstant(Val: 0, DL: dl, VT);
12395 else if (N->getOpcode() == ISD::ZERO_EXTEND)
12396 // When looking for a 0 constant, N can be zext or sext.
12397 OtherOp = DAG.getConstant(Val: 1, DL: dl, VT);
12398 else
12399 OtherOp = DAG.getAllOnesConstant(DL: dl, VT);
12400 return true;
12401 }
12402 }
12403}
12404
12405// Combine a constant select operand into its use:
12406//
12407// (add (select cc, 0, c), x) -> (select cc, x, (add, x, c))
12408// (sub x, (select cc, 0, c)) -> (select cc, x, (sub, x, c))
12409// (and (select cc, -1, c), x) -> (select cc, x, (and, x, c)) [AllOnes=1]
12410// (or (select cc, 0, c), x) -> (select cc, x, (or, x, c))
12411// (xor (select cc, 0, c), x) -> (select cc, x, (xor, x, c))
12412//
12413// The transform is rejected if the select doesn't have a constant operand that
12414// is null, or all ones when AllOnes is set.
12415//
12416// Also recognize sext/zext from i1:
12417//
12418// (add (zext cc), x) -> (select cc (add x, 1), x)
12419// (add (sext cc), x) -> (select cc (add x, -1), x)
12420//
12421// These transformations eventually create predicated instructions.
12422//
12423// @param N The node to transform.
12424// @param Slct The N operand that is a select.
12425// @param OtherOp The other N operand (x above).
12426// @param DCI Context.
12427// @param AllOnes Require the select constant to be all ones instead of null.
12428// @returns The new node, or SDValue() on failure.
12429static
12430SDValue combineSelectAndUse(SDNode *N, SDValue Slct, SDValue OtherOp,
12431 TargetLowering::DAGCombinerInfo &DCI,
12432 bool AllOnes = false) {
12433 SelectionDAG &DAG = DCI.DAG;
12434 EVT VT = N->getValueType(ResNo: 0);
12435 SDValue NonConstantVal;
12436 SDValue CCOp;
12437 bool SwapSelectOps;
12438 if (!isConditionalZeroOrAllOnes(N: Slct.getNode(), AllOnes, CC&: CCOp, Invert&: SwapSelectOps,
12439 OtherOp&: NonConstantVal, DAG))
12440 return SDValue();
12441
12442 // Slct is now know to be the desired identity constant when CC is true.
12443 SDValue TrueVal = OtherOp;
12444 SDValue FalseVal = DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT,
12445 N1: OtherOp, N2: NonConstantVal);
12446 // Unless SwapSelectOps says CC should be false.
12447 if (SwapSelectOps)
12448 std::swap(a&: TrueVal, b&: FalseVal);
12449
12450 return DAG.getNode(Opcode: ISD::SELECT, DL: SDLoc(N), VT,
12451 N1: CCOp, N2: TrueVal, N3: FalseVal);
12452}
12453
12454// Attempt combineSelectAndUse on each operand of a commutative operator N.
12455static
12456SDValue combineSelectAndUseCommutative(SDNode *N, bool AllOnes,
12457 TargetLowering::DAGCombinerInfo &DCI) {
12458 SDValue N0 = N->getOperand(Num: 0);
12459 SDValue N1 = N->getOperand(Num: 1);
12460 if (N0.getNode()->hasOneUse())
12461 if (SDValue Result = combineSelectAndUse(N, Slct: N0, OtherOp: N1, DCI, AllOnes))
12462 return Result;
12463 if (N1.getNode()->hasOneUse())
12464 if (SDValue Result = combineSelectAndUse(N, Slct: N1, OtherOp: N0, DCI, AllOnes))
12465 return Result;
12466 return SDValue();
12467}
12468
12469static bool IsVUZPShuffleNode(SDNode *N) {
12470 // VUZP shuffle node.
12471 if (N->getOpcode() == ARMISD::VUZP)
12472 return true;
12473
12474 // "VUZP" on i32 is an alias for VTRN.
12475 if (N->getOpcode() == ARMISD::VTRN && N->getValueType(ResNo: 0) == MVT::v2i32)
12476 return true;
12477
12478 return false;
12479}
12480
12481static SDValue AddCombineToVPADD(SDNode *N, SDValue N0, SDValue N1,
12482 TargetLowering::DAGCombinerInfo &DCI,
12483 const ARMSubtarget *Subtarget) {
12484 // Look for ADD(VUZP.0, VUZP.1).
12485 if (!IsVUZPShuffleNode(N: N0.getNode()) || N0.getNode() != N1.getNode() ||
12486 N0 == N1)
12487 return SDValue();
12488
12489 // Make sure the ADD is a 64-bit add; there is no 128-bit VPADD.
12490 if (!N->getValueType(ResNo: 0).is64BitVector())
12491 return SDValue();
12492
12493 // Generate vpadd.
12494 SelectionDAG &DAG = DCI.DAG;
12495 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
12496 SDLoc dl(N);
12497 SDNode *Unzip = N0.getNode();
12498 EVT VT = N->getValueType(ResNo: 0);
12499
12500 SmallVector<SDValue, 8> Ops;
12501 Ops.push_back(Elt: DAG.getConstant(Val: Intrinsic::arm_neon_vpadd, DL: dl,
12502 VT: TLI.getPointerTy(DL: DAG.getDataLayout())));
12503 Ops.push_back(Elt: Unzip->getOperand(Num: 0));
12504 Ops.push_back(Elt: Unzip->getOperand(Num: 1));
12505
12506 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT, Ops);
12507}
12508
12509static SDValue AddCombineVUZPToVPADDL(SDNode *N, SDValue N0, SDValue N1,
12510 TargetLowering::DAGCombinerInfo &DCI,
12511 const ARMSubtarget *Subtarget) {
12512 // Check for two extended operands.
12513 if (!(N0.getOpcode() == ISD::SIGN_EXTEND &&
12514 N1.getOpcode() == ISD::SIGN_EXTEND) &&
12515 !(N0.getOpcode() == ISD::ZERO_EXTEND &&
12516 N1.getOpcode() == ISD::ZERO_EXTEND))
12517 return SDValue();
12518
12519 SDValue N00 = N0.getOperand(i: 0);
12520 SDValue N10 = N1.getOperand(i: 0);
12521
12522 // Look for ADD(SEXT(VUZP.0), SEXT(VUZP.1))
12523 if (!IsVUZPShuffleNode(N: N00.getNode()) || N00.getNode() != N10.getNode() ||
12524 N00 == N10)
12525 return SDValue();
12526
12527 // We only recognize Q register paddl here; this can't be reached until
12528 // after type legalization.
12529 if (!N00.getValueType().is64BitVector() ||
12530 !N0.getValueType().is128BitVector())
12531 return SDValue();
12532
12533 // Generate vpaddl.
12534 SelectionDAG &DAG = DCI.DAG;
12535 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
12536 SDLoc dl(N);
12537 EVT VT = N->getValueType(ResNo: 0);
12538
12539 SmallVector<SDValue, 8> Ops;
12540 // Form vpaddl.sN or vpaddl.uN depending on the kind of extension.
12541 unsigned Opcode;
12542 if (N0.getOpcode() == ISD::SIGN_EXTEND)
12543 Opcode = Intrinsic::arm_neon_vpaddls;
12544 else
12545 Opcode = Intrinsic::arm_neon_vpaddlu;
12546 Ops.push_back(Elt: DAG.getConstant(Val: Opcode, DL: dl,
12547 VT: TLI.getPointerTy(DL: DAG.getDataLayout())));
12548 EVT ElemTy = N00.getValueType().getVectorElementType();
12549 unsigned NumElts = VT.getVectorNumElements();
12550 EVT ConcatVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ElemTy, NumElements: NumElts * 2);
12551 SDValue Concat = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(N), VT: ConcatVT,
12552 N1: N00.getOperand(i: 0), N2: N00.getOperand(i: 1));
12553 Ops.push_back(Elt: Concat);
12554
12555 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT, Ops);
12556}
12557
12558// FIXME: This function shouldn't be necessary; if we lower BUILD_VECTOR in
12559// an appropriate manner, we end up with ADD(VUZP(ZEXT(N))), which is
12560// much easier to match.
12561static SDValue
12562AddCombineBUILD_VECTORToVPADDL(SDNode *N, SDValue N0, SDValue N1,
12563 TargetLowering::DAGCombinerInfo &DCI,
12564 const ARMSubtarget *Subtarget) {
12565 // Only perform optimization if after legalize, and if NEON is available. We
12566 // also expected both operands to be BUILD_VECTORs.
12567 if (DCI.isBeforeLegalize() || !Subtarget->hasNEON()
12568 || N0.getOpcode() != ISD::BUILD_VECTOR
12569 || N1.getOpcode() != ISD::BUILD_VECTOR)
12570 return SDValue();
12571
12572 // Check output type since VPADDL operand elements can only be 8, 16, or 32.
12573 EVT VT = N->getValueType(ResNo: 0);
12574 if (!VT.isInteger() || VT.getVectorElementType() == MVT::i64)
12575 return SDValue();
12576
12577 // Check that the vector operands are of the right form.
12578 // N0 and N1 are BUILD_VECTOR nodes with N number of EXTRACT_VECTOR
12579 // operands, where N is the size of the formed vector.
12580 // Each EXTRACT_VECTOR should have the same input vector and odd or even
12581 // index such that we have a pair wise add pattern.
12582
12583 // Grab the vector that all EXTRACT_VECTOR nodes should be referencing.
12584 if (N0->getOperand(Num: 0)->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
12585 return SDValue();
12586 SDValue Vec = N0->getOperand(Num: 0)->getOperand(Num: 0);
12587 SDNode *V = Vec.getNode();
12588 unsigned nextIndex = 0;
12589
12590 // For each operands to the ADD which are BUILD_VECTORs,
12591 // check to see if each of their operands are an EXTRACT_VECTOR with
12592 // the same vector and appropriate index.
12593 for (unsigned i = 0, e = N0->getNumOperands(); i != e; ++i) {
12594 if (N0->getOperand(Num: i)->getOpcode() == ISD::EXTRACT_VECTOR_ELT
12595 && N1->getOperand(Num: i)->getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
12596
12597 SDValue ExtVec0 = N0->getOperand(Num: i);
12598 SDValue ExtVec1 = N1->getOperand(Num: i);
12599
12600 // First operand is the vector, verify its the same.
12601 if (V != ExtVec0->getOperand(Num: 0).getNode() ||
12602 V != ExtVec1->getOperand(Num: 0).getNode())
12603 return SDValue();
12604
12605 // Second is the constant, verify its correct.
12606 ConstantSDNode *C0 = dyn_cast<ConstantSDNode>(Val: ExtVec0->getOperand(Num: 1));
12607 ConstantSDNode *C1 = dyn_cast<ConstantSDNode>(Val: ExtVec1->getOperand(Num: 1));
12608
12609 // For the constant, we want to see all the even or all the odd.
12610 if (!C0 || !C1 || C0->getZExtValue() != nextIndex
12611 || C1->getZExtValue() != nextIndex+1)
12612 return SDValue();
12613
12614 // Increment index.
12615 nextIndex+=2;
12616 } else
12617 return SDValue();
12618 }
12619
12620 // Don't generate vpaddl+vmovn; we'll match it to vpadd later. Also make sure
12621 // we're using the entire input vector, otherwise there's a size/legality
12622 // mismatch somewhere.
12623 if (nextIndex != Vec.getValueType().getVectorNumElements() ||
12624 Vec.getValueType().getVectorElementType() == VT.getVectorElementType())
12625 return SDValue();
12626
12627 // Create VPADDL node.
12628 SelectionDAG &DAG = DCI.DAG;
12629 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
12630
12631 SDLoc dl(N);
12632
12633 // Build operand list.
12634 SmallVector<SDValue, 8> Ops;
12635 Ops.push_back(Elt: DAG.getConstant(Val: Intrinsic::arm_neon_vpaddls, DL: dl,
12636 VT: TLI.getPointerTy(DL: DAG.getDataLayout())));
12637
12638 // Input is the vector.
12639 Ops.push_back(Elt: Vec);
12640
12641 // Get widened type and narrowed type.
12642 MVT widenType;
12643 unsigned numElem = VT.getVectorNumElements();
12644
12645 EVT inputLaneType = Vec.getValueType().getVectorElementType();
12646 switch (inputLaneType.getSimpleVT().SimpleTy) {
12647 case MVT::i8: widenType = MVT::getVectorVT(VT: MVT::i16, NumElements: numElem); break;
12648 case MVT::i16: widenType = MVT::getVectorVT(VT: MVT::i32, NumElements: numElem); break;
12649 case MVT::i32: widenType = MVT::getVectorVT(VT: MVT::i64, NumElements: numElem); break;
12650 default:
12651 llvm_unreachable("Invalid vector element type for padd optimization.");
12652 }
12653
12654 SDValue tmp = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: widenType, Ops);
12655 unsigned ExtOp = VT.bitsGT(VT: tmp.getValueType()) ? ISD::ANY_EXTEND : ISD::TRUNCATE;
12656 return DAG.getNode(Opcode: ExtOp, DL: dl, VT, Operand: tmp);
12657}
12658
12659static SDValue findMUL_LOHI(SDValue V) {
12660 if (V->getOpcode() == ISD::UMUL_LOHI ||
12661 V->getOpcode() == ISD::SMUL_LOHI)
12662 return V;
12663 return SDValue();
12664}
12665
12666static SDValue AddCombineTo64BitSMLAL16(SDNode *AddcNode, SDNode *AddeNode,
12667 TargetLowering::DAGCombinerInfo &DCI,
12668 const ARMSubtarget *Subtarget) {
12669 if (!Subtarget->hasBaseDSP())
12670 return SDValue();
12671
12672 // SMLALBB, SMLALBT, SMLALTB, SMLALTT multiply two 16-bit values and
12673 // accumulates the product into a 64-bit value. The 16-bit values will
12674 // be sign extended somehow or SRA'd into 32-bit values
12675 // (addc (adde (mul 16bit, 16bit), lo), hi)
12676 SDValue Mul = AddcNode->getOperand(Num: 0);
12677 SDValue Lo = AddcNode->getOperand(Num: 1);
12678 if (Mul.getOpcode() != ISD::MUL) {
12679 Lo = AddcNode->getOperand(Num: 0);
12680 Mul = AddcNode->getOperand(Num: 1);
12681 if (Mul.getOpcode() != ISD::MUL)
12682 return SDValue();
12683 }
12684
12685 SDValue SRA = AddeNode->getOperand(Num: 0);
12686 SDValue Hi = AddeNode->getOperand(Num: 1);
12687 if (SRA.getOpcode() != ISD::SRA) {
12688 SRA = AddeNode->getOperand(Num: 1);
12689 Hi = AddeNode->getOperand(Num: 0);
12690 if (SRA.getOpcode() != ISD::SRA)
12691 return SDValue();
12692 }
12693 if (auto Const = dyn_cast<ConstantSDNode>(Val: SRA.getOperand(i: 1))) {
12694 if (Const->getZExtValue() != 31)
12695 return SDValue();
12696 } else
12697 return SDValue();
12698
12699 if (SRA.getOperand(i: 0) != Mul)
12700 return SDValue();
12701
12702 SelectionDAG &DAG = DCI.DAG;
12703 SDLoc dl(AddcNode);
12704 unsigned Opcode = 0;
12705 SDValue Op0;
12706 SDValue Op1;
12707
12708 if (isS16(Op: Mul.getOperand(i: 0), DAG) && isS16(Op: Mul.getOperand(i: 1), DAG)) {
12709 Opcode = ARMISD::SMLALBB;
12710 Op0 = Mul.getOperand(i: 0);
12711 Op1 = Mul.getOperand(i: 1);
12712 } else if (isS16(Op: Mul.getOperand(i: 0), DAG) && isSRA16(Op: Mul.getOperand(i: 1))) {
12713 Opcode = ARMISD::SMLALBT;
12714 Op0 = Mul.getOperand(i: 0);
12715 Op1 = Mul.getOperand(i: 1).getOperand(i: 0);
12716 } else if (isSRA16(Op: Mul.getOperand(i: 0)) && isS16(Op: Mul.getOperand(i: 1), DAG)) {
12717 Opcode = ARMISD::SMLALTB;
12718 Op0 = Mul.getOperand(i: 0).getOperand(i: 0);
12719 Op1 = Mul.getOperand(i: 1);
12720 } else if (isSRA16(Op: Mul.getOperand(i: 0)) && isSRA16(Op: Mul.getOperand(i: 1))) {
12721 Opcode = ARMISD::SMLALTT;
12722 Op0 = Mul->getOperand(Num: 0).getOperand(i: 0);
12723 Op1 = Mul->getOperand(Num: 1).getOperand(i: 0);
12724 }
12725
12726 if (!Op0 || !Op1)
12727 return SDValue();
12728
12729 SDValue SMLAL = DAG.getNode(Opcode, DL: dl, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
12730 N1: Op0, N2: Op1, N3: Lo, N4: Hi);
12731 // Replace the ADDs' nodes uses by the MLA node's values.
12732 SDValue HiMLALResult(SMLAL.getNode(), 1);
12733 SDValue LoMLALResult(SMLAL.getNode(), 0);
12734
12735 DAG.ReplaceAllUsesOfValueWith(From: SDValue(AddcNode, 0), To: LoMLALResult);
12736 DAG.ReplaceAllUsesOfValueWith(From: SDValue(AddeNode, 0), To: HiMLALResult);
12737 DCI.recursivelyDeleteUnusedNodes(N: AddeNode);
12738
12739 // Return original node to notify the driver to stop replacing.
12740 return SDValue(AddeNode, 0);
12741}
12742
12743static SDValue AddCombineTo64bitMLAL(SDNode *AddeSubeNode,
12744 TargetLowering::DAGCombinerInfo &DCI,
12745 const ARMSubtarget *Subtarget) {
12746 // Look for multiply add opportunities.
12747 // The pattern is a ISD::UMUL_LOHI followed by two add nodes, where
12748 // each add nodes consumes a value from ISD::UMUL_LOHI and there is
12749 // a glue link from the first add to the second add.
12750 // If we find this pattern, we can replace the U/SMUL_LOHI, ADDC, and ADDE by
12751 // a S/UMLAL instruction.
12752 // UMUL_LOHI
12753 // / :lo \ :hi
12754 // V \ [no multiline comment]
12755 // loAdd -> ADDC |
12756 // \ :carry /
12757 // V V
12758 // ADDE <- hiAdd
12759 //
12760 // In the special case where only the higher part of a signed result is used
12761 // and the add to the low part of the result of ISD::UMUL_LOHI adds or subtracts
12762 // a constant with the exact value of 0x80000000, we recognize we are dealing
12763 // with a "rounded multiply and add" (or subtract) and transform it into
12764 // either a ARMISD::SMMLAR or ARMISD::SMMLSR respectively.
12765
12766 assert((AddeSubeNode->getOpcode() == ARMISD::ADDE ||
12767 AddeSubeNode->getOpcode() == ARMISD::SUBE) &&
12768 "Expect an ADDE or SUBE");
12769
12770 assert(AddeSubeNode->getNumOperands() == 3 &&
12771 AddeSubeNode->getOperand(2).getValueType() == MVT::i32 &&
12772 "ADDE node has the wrong inputs");
12773
12774 if (AddeSubeNode->hasAnyUseOfValue(Value: 1))
12775 return SDValue();
12776
12777 // Check that we are chained to the right ADDC or SUBC node.
12778 SDNode *AddcSubcNode = AddeSubeNode->getOperand(Num: 2).getNode();
12779 if ((AddeSubeNode->getOpcode() == ARMISD::ADDE &&
12780 AddcSubcNode->getOpcode() != ARMISD::ADDC) ||
12781 (AddeSubeNode->getOpcode() == ARMISD::SUBE &&
12782 AddcSubcNode->getOpcode() != ARMISD::SUBC))
12783 return SDValue();
12784
12785 SDValue AddcSubcOp0 = AddcSubcNode->getOperand(Num: 0);
12786 SDValue AddcSubcOp1 = AddcSubcNode->getOperand(Num: 1);
12787
12788 // Check if the two operands are from the same mul_lohi node.
12789 if (AddcSubcOp0.getNode() == AddcSubcOp1.getNode())
12790 return SDValue();
12791
12792 assert(AddcSubcNode->getNumValues() == 2 &&
12793 AddcSubcNode->getValueType(0) == MVT::i32 &&
12794 "Expect ADDC with two result values. First: i32");
12795
12796 // Check that the ADDC adds the low result of the S/UMUL_LOHI. If not, it
12797 // maybe a SMLAL which multiplies two 16-bit values.
12798 if (AddeSubeNode->getOpcode() == ARMISD::ADDE &&
12799 AddcSubcOp0->getOpcode() != ISD::UMUL_LOHI &&
12800 AddcSubcOp0->getOpcode() != ISD::SMUL_LOHI &&
12801 AddcSubcOp1->getOpcode() != ISD::UMUL_LOHI &&
12802 AddcSubcOp1->getOpcode() != ISD::SMUL_LOHI)
12803 return AddCombineTo64BitSMLAL16(AddcNode: AddcSubcNode, AddeNode: AddeSubeNode, DCI, Subtarget);
12804
12805 // Check for the triangle shape.
12806 SDValue AddeSubeOp0 = AddeSubeNode->getOperand(Num: 0);
12807 SDValue AddeSubeOp1 = AddeSubeNode->getOperand(Num: 1);
12808
12809 // Make sure that the ADDE/SUBE operands are not coming from the same node.
12810 if (AddeSubeOp0.getNode() == AddeSubeOp1.getNode())
12811 return SDValue();
12812
12813 // Find the MUL_LOHI node walking up ADDE/SUBE's operands.
12814 bool IsLeftOperandMUL = false;
12815 SDValue MULOp = findMUL_LOHI(V: AddeSubeOp0);
12816 if (MULOp == SDValue())
12817 MULOp = findMUL_LOHI(V: AddeSubeOp1);
12818 else
12819 IsLeftOperandMUL = true;
12820 if (MULOp == SDValue())
12821 return SDValue();
12822
12823 // Figure out the right opcode.
12824 unsigned Opc = MULOp->getOpcode();
12825 unsigned FinalOpc = (Opc == ISD::SMUL_LOHI) ? ARMISD::SMLAL : ARMISD::UMLAL;
12826
12827 // Figure out the high and low input values to the MLAL node.
12828 SDValue *HiAddSub = nullptr;
12829 SDValue *LoMul = nullptr;
12830 SDValue *LowAddSub = nullptr;
12831
12832 // Ensure that ADDE/SUBE is from high result of ISD::xMUL_LOHI.
12833 if ((AddeSubeOp0 != MULOp.getValue(R: 1)) && (AddeSubeOp1 != MULOp.getValue(R: 1)))
12834 return SDValue();
12835
12836 if (IsLeftOperandMUL)
12837 HiAddSub = &AddeSubeOp1;
12838 else
12839 HiAddSub = &AddeSubeOp0;
12840
12841 // Ensure that LoMul and LowAddSub are taken from correct ISD::SMUL_LOHI node
12842 // whose low result is fed to the ADDC/SUBC we are checking.
12843
12844 if (AddcSubcOp0 == MULOp.getValue(R: 0)) {
12845 LoMul = &AddcSubcOp0;
12846 LowAddSub = &AddcSubcOp1;
12847 }
12848 if (AddcSubcOp1 == MULOp.getValue(R: 0)) {
12849 LoMul = &AddcSubcOp1;
12850 LowAddSub = &AddcSubcOp0;
12851 }
12852
12853 if (!LoMul)
12854 return SDValue();
12855
12856 // If HiAddSub is the same node as ADDC/SUBC or is a predecessor of ADDC/SUBC
12857 // the replacement below will create a cycle.
12858 if (AddcSubcNode == HiAddSub->getNode() ||
12859 AddcSubcNode->isPredecessorOf(N: HiAddSub->getNode()))
12860 return SDValue();
12861
12862 // Create the merged node.
12863 SelectionDAG &DAG = DCI.DAG;
12864
12865 // Start building operand list.
12866 SmallVector<SDValue, 8> Ops;
12867 Ops.push_back(Elt: LoMul->getOperand(i: 0));
12868 Ops.push_back(Elt: LoMul->getOperand(i: 1));
12869
12870 // Check whether we can use SMMLAR, SMMLSR or SMMULR instead. For this to be
12871 // the case, we must be doing signed multiplication and only use the higher
12872 // part of the result of the MLAL, furthermore the LowAddSub must be a constant
12873 // addition or subtraction with the value of 0x800000.
12874 if (Subtarget->hasV6Ops() && Subtarget->hasDSP() && Subtarget->useMulOps() &&
12875 FinalOpc == ARMISD::SMLAL &&
12876 LowAddSub->getNode()->getOpcode() == ISD::Constant &&
12877 static_cast<ConstantSDNode *>(LowAddSub->getNode())->getZExtValue() ==
12878 0x80000000) {
12879 Ops.push_back(Elt: *HiAddSub);
12880 if (AddcSubcNode->getOpcode() == ARMISD::SUBC) {
12881 FinalOpc = ARMISD::SMMLSR;
12882 } else {
12883 FinalOpc = ARMISD::SMMLAR;
12884 }
12885 SDValue NewNode = DAG.getNode(Opcode: FinalOpc, DL: SDLoc(AddcSubcNode), VT: MVT::i32, Ops);
12886 DAG.ReplaceAllUsesOfValueWith(From: SDValue(AddeSubeNode, 0), To: NewNode);
12887
12888 return SDValue(AddeSubeNode, 0);
12889 } else if (AddcSubcNode->getOpcode() == ARMISD::SUBC)
12890 // SMMLS is generated during instruction selection and the rest of this
12891 // function can not handle the case where AddcSubcNode is a SUBC.
12892 return SDValue();
12893
12894 // Finish building the operand list for {U/S}MLAL
12895 Ops.push_back(Elt: *LowAddSub);
12896 Ops.push_back(Elt: *HiAddSub);
12897
12898 SDValue MLALNode = DAG.getNode(Opcode: FinalOpc, DL: SDLoc(AddcSubcNode),
12899 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), Ops);
12900
12901 // Replace the ADDs' nodes uses by the MLA node's values.
12902 SDValue HiMLALResult(MLALNode.getNode(), 1);
12903 DAG.ReplaceAllUsesOfValueWith(From: SDValue(AddeSubeNode, 0), To: HiMLALResult);
12904
12905 SDValue LoMLALResult(MLALNode.getNode(), 0);
12906 DAG.ReplaceAllUsesOfValueWith(From: SDValue(AddcSubcNode, 0), To: LoMLALResult);
12907
12908 // Return original node to notify the driver to stop replacing.
12909 return SDValue(AddeSubeNode, 0);
12910}
12911
12912static SDValue AddCombineTo64bitUMAAL(SDNode *AddeNode,
12913 TargetLowering::DAGCombinerInfo &DCI,
12914 const ARMSubtarget *Subtarget) {
12915 // UMAAL is similar to UMLAL except that it adds two unsigned values.
12916 // While trying to combine for the other MLAL nodes, first search for the
12917 // chance to use UMAAL. Check if Addc uses a node which has already
12918 // been combined into a UMLAL. The other pattern is UMLAL using Addc/Adde
12919 // as the addend, and it's handled in PerformUMLALCombine.
12920
12921 if (AddeNode->hasAnyUseOfValue(Value: 1))
12922 return SDValue();
12923
12924 if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP())
12925 return AddCombineTo64bitMLAL(AddeSubeNode: AddeNode, DCI, Subtarget);
12926
12927 // Check that we have a glued ADDC node.
12928 SDNode* AddcNode = AddeNode->getOperand(Num: 2).getNode();
12929 if (AddcNode->getOpcode() != ARMISD::ADDC)
12930 return SDValue();
12931
12932 // Find the converted UMAAL or quit if it doesn't exist.
12933 SDNode *UmlalNode = nullptr;
12934 SDValue AddHi;
12935 if (AddcNode->getOperand(Num: 0).getOpcode() == ARMISD::UMLAL) {
12936 UmlalNode = AddcNode->getOperand(Num: 0).getNode();
12937 AddHi = AddcNode->getOperand(Num: 1);
12938 } else if (AddcNode->getOperand(Num: 1).getOpcode() == ARMISD::UMLAL) {
12939 UmlalNode = AddcNode->getOperand(Num: 1).getNode();
12940 AddHi = AddcNode->getOperand(Num: 0);
12941 } else {
12942 return AddCombineTo64bitMLAL(AddeSubeNode: AddeNode, DCI, Subtarget);
12943 }
12944
12945 // The ADDC should be glued to an ADDE node, which uses the same UMLAL as
12946 // the ADDC as well as Zero.
12947 if (!isNullConstant(V: UmlalNode->getOperand(Num: 3)))
12948 return SDValue();
12949
12950 if ((isNullConstant(V: AddeNode->getOperand(Num: 0)) &&
12951 AddeNode->getOperand(Num: 1).getNode() == UmlalNode) ||
12952 (AddeNode->getOperand(Num: 0).getNode() == UmlalNode &&
12953 isNullConstant(V: AddeNode->getOperand(Num: 1)))) {
12954 SelectionDAG &DAG = DCI.DAG;
12955 SDValue Ops[] = { UmlalNode->getOperand(Num: 0), UmlalNode->getOperand(Num: 1),
12956 UmlalNode->getOperand(Num: 2), AddHi };
12957 SDValue UMAAL = DAG.getNode(Opcode: ARMISD::UMAAL, DL: SDLoc(AddcNode),
12958 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), Ops);
12959
12960 // Replace the ADDs' nodes uses by the UMAAL node's values.
12961 DAG.ReplaceAllUsesOfValueWith(From: SDValue(AddeNode, 0), To: SDValue(UMAAL.getNode(), 1));
12962 DAG.ReplaceAllUsesOfValueWith(From: SDValue(AddcNode, 0), To: SDValue(UMAAL.getNode(), 0));
12963
12964 // Return original node to notify the driver to stop replacing.
12965 return SDValue(AddeNode, 0);
12966 }
12967 return SDValue();
12968}
12969
12970static SDValue PerformUMLALCombine(SDNode *N, SelectionDAG &DAG,
12971 const ARMSubtarget *Subtarget) {
12972 if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP())
12973 return SDValue();
12974
12975 // Check that we have a pair of ADDC and ADDE as operands.
12976 // Both addends of the ADDE must be zero.
12977 SDNode* AddcNode = N->getOperand(Num: 2).getNode();
12978 SDNode* AddeNode = N->getOperand(Num: 3).getNode();
12979 if ((AddcNode->getOpcode() == ARMISD::ADDC) &&
12980 (AddeNode->getOpcode() == ARMISD::ADDE) &&
12981 isNullConstant(V: AddeNode->getOperand(Num: 0)) &&
12982 isNullConstant(V: AddeNode->getOperand(Num: 1)) &&
12983 (AddeNode->getOperand(Num: 2).getNode() == AddcNode))
12984 return DAG.getNode(Opcode: ARMISD::UMAAL, DL: SDLoc(N),
12985 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
12986 Ops: {N->getOperand(Num: 0), N->getOperand(Num: 1),
12987 AddcNode->getOperand(Num: 0), AddcNode->getOperand(Num: 1)});
12988 else
12989 return SDValue();
12990}
12991
12992static SDValue PerformAddcSubcCombine(SDNode *N,
12993 TargetLowering::DAGCombinerInfo &DCI,
12994 const ARMSubtarget *Subtarget) {
12995 SelectionDAG &DAG(DCI.DAG);
12996
12997 if (N->getOpcode() == ARMISD::SUBC && N->hasAnyUseOfValue(Value: 1)) {
12998 // (SUBC (ADDE 0, 0, C), 1) -> C
12999 SDValue LHS = N->getOperand(Num: 0);
13000 SDValue RHS = N->getOperand(Num: 1);
13001 if (LHS->getOpcode() == ARMISD::ADDE &&
13002 isNullConstant(V: LHS->getOperand(Num: 0)) &&
13003 isNullConstant(V: LHS->getOperand(Num: 1)) && isOneConstant(V: RHS)) {
13004 return DCI.CombineTo(N, Res0: SDValue(N, 0), Res1: LHS->getOperand(Num: 2));
13005 }
13006 }
13007
13008 if (Subtarget->isThumb1Only()) {
13009 SDValue RHS = N->getOperand(Num: 1);
13010 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: RHS)) {
13011 int32_t imm = C->getSExtValue();
13012 if (imm < 0 && imm > std::numeric_limits<int>::min()) {
13013 SDLoc DL(N);
13014 RHS = DAG.getConstant(Val: -imm, DL, VT: MVT::i32);
13015 unsigned Opcode = (N->getOpcode() == ARMISD::ADDC) ? ARMISD::SUBC
13016 : ARMISD::ADDC;
13017 return DAG.getNode(Opcode, DL, VTList: N->getVTList(), N1: N->getOperand(Num: 0), N2: RHS);
13018 }
13019 }
13020 }
13021
13022 return SDValue();
13023}
13024
13025static SDValue PerformAddeSubeCombine(SDNode *N,
13026 TargetLowering::DAGCombinerInfo &DCI,
13027 const ARMSubtarget *Subtarget) {
13028 if (Subtarget->isThumb1Only()) {
13029 SelectionDAG &DAG = DCI.DAG;
13030 SDValue RHS = N->getOperand(Num: 1);
13031 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: RHS)) {
13032 int64_t imm = C->getSExtValue();
13033 if (imm < 0) {
13034 SDLoc DL(N);
13035
13036 // The with-carry-in form matches bitwise not instead of the negation.
13037 // Effectively, the inverse interpretation of the carry flag already
13038 // accounts for part of the negation.
13039 RHS = DAG.getConstant(Val: ~imm, DL, VT: MVT::i32);
13040
13041 unsigned Opcode = (N->getOpcode() == ARMISD::ADDE) ? ARMISD::SUBE
13042 : ARMISD::ADDE;
13043 return DAG.getNode(Opcode, DL, VTList: N->getVTList(),
13044 N1: N->getOperand(Num: 0), N2: RHS, N3: N->getOperand(Num: 2));
13045 }
13046 }
13047 } else if (N->getOperand(Num: 1)->getOpcode() == ISD::SMUL_LOHI) {
13048 return AddCombineTo64bitMLAL(AddeSubeNode: N, DCI, Subtarget);
13049 }
13050 return SDValue();
13051}
13052
13053static SDValue PerformSELECTCombine(SDNode *N,
13054 TargetLowering::DAGCombinerInfo &DCI,
13055 const ARMSubtarget *Subtarget) {
13056 if (!Subtarget->hasMVEIntegerOps())
13057 return SDValue();
13058
13059 SDLoc dl(N);
13060 SDValue SetCC;
13061 SDValue LHS;
13062 SDValue RHS;
13063 ISD::CondCode CC;
13064 SDValue TrueVal;
13065 SDValue FalseVal;
13066
13067 if (N->getOpcode() == ISD::SELECT &&
13068 N->getOperand(Num: 0)->getOpcode() == ISD::SETCC) {
13069 SetCC = N->getOperand(Num: 0);
13070 LHS = SetCC->getOperand(Num: 0);
13071 RHS = SetCC->getOperand(Num: 1);
13072 CC = SetCC->getOperandCondCode(Num: 2);
13073 TrueVal = N->getOperand(Num: 1);
13074 FalseVal = N->getOperand(Num: 2);
13075 } else if (N->getOpcode() == ISD::SELECT_CC) {
13076 LHS = N->getOperand(Num: 0);
13077 RHS = N->getOperand(Num: 1);
13078 CC = N->getOperandCondCode(Num: 4);
13079 TrueVal = N->getOperand(Num: 2);
13080 FalseVal = N->getOperand(Num: 3);
13081 } else {
13082 return SDValue();
13083 }
13084
13085 unsigned int Opcode = 0;
13086 if ((TrueVal->getOpcode() == ISD::VECREDUCE_UMIN ||
13087 FalseVal->getOpcode() == ISD::VECREDUCE_UMIN) &&
13088 (CC == ISD::SETULT || CC == ISD::SETUGT)) {
13089 Opcode = ARMISD::VMINVu;
13090 if (CC == ISD::SETUGT)
13091 std::swap(a&: TrueVal, b&: FalseVal);
13092 } else if ((TrueVal->getOpcode() == ISD::VECREDUCE_SMIN ||
13093 FalseVal->getOpcode() == ISD::VECREDUCE_SMIN) &&
13094 (CC == ISD::SETLT || CC == ISD::SETGT)) {
13095 Opcode = ARMISD::VMINVs;
13096 if (CC == ISD::SETGT)
13097 std::swap(a&: TrueVal, b&: FalseVal);
13098 } else if ((TrueVal->getOpcode() == ISD::VECREDUCE_UMAX ||
13099 FalseVal->getOpcode() == ISD::VECREDUCE_UMAX) &&
13100 (CC == ISD::SETUGT || CC == ISD::SETULT)) {
13101 Opcode = ARMISD::VMAXVu;
13102 if (CC == ISD::SETULT)
13103 std::swap(a&: TrueVal, b&: FalseVal);
13104 } else if ((TrueVal->getOpcode() == ISD::VECREDUCE_SMAX ||
13105 FalseVal->getOpcode() == ISD::VECREDUCE_SMAX) &&
13106 (CC == ISD::SETGT || CC == ISD::SETLT)) {
13107 Opcode = ARMISD::VMAXVs;
13108 if (CC == ISD::SETLT)
13109 std::swap(a&: TrueVal, b&: FalseVal);
13110 } else
13111 return SDValue();
13112
13113 // Normalise to the right hand side being the vector reduction
13114 switch (TrueVal->getOpcode()) {
13115 case ISD::VECREDUCE_UMIN:
13116 case ISD::VECREDUCE_SMIN:
13117 case ISD::VECREDUCE_UMAX:
13118 case ISD::VECREDUCE_SMAX:
13119 std::swap(a&: LHS, b&: RHS);
13120 std::swap(a&: TrueVal, b&: FalseVal);
13121 break;
13122 }
13123
13124 EVT VectorType = FalseVal->getOperand(Num: 0).getValueType();
13125
13126 if (VectorType != MVT::v16i8 && VectorType != MVT::v8i16 &&
13127 VectorType != MVT::v4i32)
13128 return SDValue();
13129
13130 EVT VectorScalarType = VectorType.getVectorElementType();
13131
13132 // The values being selected must also be the ones being compared
13133 if (TrueVal != LHS || FalseVal != RHS)
13134 return SDValue();
13135
13136 EVT LeftType = LHS->getValueType(ResNo: 0);
13137 EVT RightType = RHS->getValueType(ResNo: 0);
13138
13139 // The types must match the reduced type too
13140 if (LeftType != VectorScalarType || RightType != VectorScalarType)
13141 return SDValue();
13142
13143 // Legalise the scalar to an i32
13144 if (VectorScalarType != MVT::i32)
13145 LHS = DCI.DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: MVT::i32, Operand: LHS);
13146
13147 // Generate the reduction as an i32 for legalisation purposes
13148 auto Reduction =
13149 DCI.DAG.getNode(Opcode, DL: dl, VT: MVT::i32, N1: LHS, N2: RHS->getOperand(Num: 0));
13150
13151 // The result isn't actually an i32 so truncate it back to its original type
13152 if (VectorScalarType != MVT::i32)
13153 Reduction = DCI.DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: VectorScalarType, Operand: Reduction);
13154
13155 return Reduction;
13156}
13157
13158// A special combine for the vqdmulh family of instructions. This is one of the
13159// potential set of patterns that could patch this instruction. The base pattern
13160// you would expect to be min(max(ashr(mul(mul(sext(x), 2), sext(y)), 16))).
13161// This matches the different min(max(ashr(mul(mul(sext(x), sext(y)), 2), 16))),
13162// which llvm will have optimized to min(ashr(mul(sext(x), sext(y)), 15))) as
13163// the max is unnecessary.
13164static SDValue PerformVQDMULHCombine(SDNode *N, SelectionDAG &DAG) {
13165 EVT VT = N->getValueType(ResNo: 0);
13166 SDValue Shft;
13167 ConstantSDNode *Clamp;
13168
13169 if (!VT.isVector() || VT.getScalarSizeInBits() > 64)
13170 return SDValue();
13171
13172 if (N->getOpcode() == ISD::SMIN) {
13173 Shft = N->getOperand(Num: 0);
13174 Clamp = isConstOrConstSplat(N: N->getOperand(Num: 1));
13175 } else if (N->getOpcode() == ISD::VSELECT) {
13176 // Detect a SMIN, which for an i64 node will be a vselect/setcc, not a smin.
13177 SDValue Cmp = N->getOperand(Num: 0);
13178 if (Cmp.getOpcode() != ISD::SETCC ||
13179 Cmp.getOperandCondCode(i: 2) != ISD::SETLT ||
13180 Cmp.getOperand(i: 0) != N->getOperand(Num: 1) ||
13181 Cmp.getOperand(i: 1) != N->getOperand(Num: 2))
13182 return SDValue();
13183 Shft = N->getOperand(Num: 1);
13184 Clamp = isConstOrConstSplat(N: N->getOperand(Num: 2));
13185 } else
13186 return SDValue();
13187
13188 if (!Clamp)
13189 return SDValue();
13190
13191 MVT ScalarType;
13192 int ShftAmt = 0;
13193 switch (Clamp->getSExtValue()) {
13194 case (1 << 7) - 1:
13195 ScalarType = MVT::i8;
13196 ShftAmt = 7;
13197 break;
13198 case (1 << 15) - 1:
13199 ScalarType = MVT::i16;
13200 ShftAmt = 15;
13201 break;
13202 case (1ULL << 31) - 1:
13203 ScalarType = MVT::i32;
13204 ShftAmt = 31;
13205 break;
13206 default:
13207 return SDValue();
13208 }
13209
13210 if (Shft.getOpcode() != ISD::SRA)
13211 return SDValue();
13212 ConstantSDNode *N1 = isConstOrConstSplat(N: Shft.getOperand(i: 1));
13213 if (!N1 || N1->getSExtValue() != ShftAmt)
13214 return SDValue();
13215
13216 SDValue Mul = Shft.getOperand(i: 0);
13217 if (Mul.getOpcode() != ISD::MUL)
13218 return SDValue();
13219
13220 SDValue Ext0 = Mul.getOperand(i: 0);
13221 SDValue Ext1 = Mul.getOperand(i: 1);
13222 if (Ext0.getOpcode() != ISD::SIGN_EXTEND ||
13223 Ext1.getOpcode() != ISD::SIGN_EXTEND)
13224 return SDValue();
13225 EVT VecVT = Ext0.getOperand(i: 0).getValueType();
13226 if (!VecVT.isPow2VectorType() || VecVT.getVectorNumElements() == 1)
13227 return SDValue();
13228 if (Ext1.getOperand(i: 0).getValueType() != VecVT ||
13229 VecVT.getScalarType() != ScalarType ||
13230 VT.getScalarSizeInBits() < ScalarType.getScalarSizeInBits() * 2)
13231 return SDValue();
13232
13233 SDLoc DL(Mul);
13234 unsigned LegalLanes = 128 / (ShftAmt + 1);
13235 EVT LegalVecVT = MVT::getVectorVT(VT: ScalarType, NumElements: LegalLanes);
13236 // For types smaller than legal vectors extend to be legal and only use needed
13237 // lanes.
13238 if (VecVT.getSizeInBits() < 128) {
13239 EVT ExtVecVT =
13240 MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: 128 / VecVT.getVectorNumElements()),
13241 NumElements: VecVT.getVectorNumElements());
13242 SDValue Inp0 =
13243 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: ExtVecVT, Operand: Ext0.getOperand(i: 0));
13244 SDValue Inp1 =
13245 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: ExtVecVT, Operand: Ext1.getOperand(i: 0));
13246 Inp0 = DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL, VT: LegalVecVT, Operand: Inp0);
13247 Inp1 = DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL, VT: LegalVecVT, Operand: Inp1);
13248 SDValue VQDMULH = DAG.getNode(Opcode: ARMISD::VQDMULH, DL, VT: LegalVecVT, N1: Inp0, N2: Inp1);
13249 SDValue Trunc = DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL, VT: ExtVecVT, Operand: VQDMULH);
13250 Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: VecVT, Operand: Trunc);
13251 return DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT, Operand: Trunc);
13252 }
13253
13254 // For larger types, split into legal sized chunks.
13255 assert(VecVT.getSizeInBits() % 128 == 0 && "Expected a power2 type");
13256 unsigned NumParts = VecVT.getSizeInBits() / 128;
13257 SmallVector<SDValue> Parts;
13258 for (unsigned I = 0; I < NumParts; ++I) {
13259 SDValue Inp0 =
13260 DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: LegalVecVT, N1: Ext0.getOperand(i: 0),
13261 N2: DAG.getVectorIdxConstant(Val: I * LegalLanes, DL));
13262 SDValue Inp1 =
13263 DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: LegalVecVT, N1: Ext1.getOperand(i: 0),
13264 N2: DAG.getVectorIdxConstant(Val: I * LegalLanes, DL));
13265 SDValue VQDMULH = DAG.getNode(Opcode: ARMISD::VQDMULH, DL, VT: LegalVecVT, N1: Inp0, N2: Inp1);
13266 Parts.push_back(Elt: VQDMULH);
13267 }
13268 return DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT,
13269 Operand: DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: VecVT, Ops: Parts));
13270}
13271
13272static SDValue PerformVSELECTCombine(SDNode *N,
13273 TargetLowering::DAGCombinerInfo &DCI,
13274 const ARMSubtarget *Subtarget) {
13275 if (!Subtarget->hasMVEIntegerOps())
13276 return SDValue();
13277
13278 // Constant fold vselect 0, A, B -> B
13279 // and vselect 0xffff, A, B -> A
13280 if (N->getOperand(Num: 0).getOpcode() == ARMISD::PREDICATE_CAST &&
13281 isa<ConstantSDNode>(Val: N->getOperand(Num: 0).getOperand(i: 0))) {
13282 unsigned C = N->getOperand(Num: 0).getConstantOperandVal(i: 0);
13283 if (C == 0)
13284 return N->getOperand(Num: 2);
13285 if (C == 0xffff)
13286 return N->getOperand(Num: 1);
13287 }
13288
13289 if (SDValue V = PerformVQDMULHCombine(N, DAG&: DCI.DAG))
13290 return V;
13291
13292 // Transforms vselect(not(cond), lhs, rhs) into vselect(cond, rhs, lhs).
13293 //
13294 // We need to re-implement this optimization here as the implementation in the
13295 // Target-Independent DAGCombiner does not handle the kind of constant we make
13296 // (it calls isConstOrConstSplat with AllowTruncation set to false - and for
13297 // good reason, allowing truncation there would break other targets).
13298 //
13299 // Currently, this is only done for MVE, as it's the only target that benefits
13300 // from this transformation (e.g. VPNOT+VPSEL becomes a single VPSEL).
13301 if (N->getOperand(Num: 0).getOpcode() != ISD::XOR)
13302 return SDValue();
13303 SDValue XOR = N->getOperand(Num: 0);
13304
13305 // Check if the XOR's RHS is either a 1, or a BUILD_VECTOR of 1s.
13306 // It is important to check with truncation allowed as the BUILD_VECTORs we
13307 // generate in those situations will truncate their operands.
13308 ConstantSDNode *Const =
13309 isConstOrConstSplat(N: XOR->getOperand(Num: 1), /*AllowUndefs*/ false,
13310 /*AllowTruncation*/ true);
13311 if (!Const || !Const->isOne())
13312 return SDValue();
13313
13314 // Rewrite into vselect(cond, rhs, lhs).
13315 SDValue Cond = XOR->getOperand(Num: 0);
13316 SDValue LHS = N->getOperand(Num: 1);
13317 SDValue RHS = N->getOperand(Num: 2);
13318 EVT Type = N->getValueType(ResNo: 0);
13319 return DCI.DAG.getNode(Opcode: ISD::VSELECT, DL: SDLoc(N), VT: Type, N1: Cond, N2: RHS, N3: LHS);
13320}
13321
13322// Convert vsetcc([0,1,2,..], splat(n), ult) -> vctp n
13323static SDValue PerformVSetCCToVCTPCombine(SDNode *N,
13324 TargetLowering::DAGCombinerInfo &DCI,
13325 const ARMSubtarget *Subtarget) {
13326 SDValue Op0 = N->getOperand(Num: 0);
13327 SDValue Op1 = N->getOperand(Num: 1);
13328 ISD::CondCode CC = N->getOperandCondCode(Num: 2);
13329 EVT VT = N->getValueType(ResNo: 0);
13330
13331 if (!Subtarget->hasMVEIntegerOps() ||
13332 !DCI.DAG.getTargetLoweringInfo().isTypeLegal(VT))
13333 return SDValue();
13334
13335 if (CC == ISD::SETUGT) {
13336 std::swap(a&: Op0, b&: Op1);
13337 CC = ISD::SETULT;
13338 }
13339
13340 if (CC != ISD::SETULT || VT.getScalarSizeInBits() != 1 ||
13341 Op0.getOpcode() != ISD::BUILD_VECTOR)
13342 return SDValue();
13343
13344 // Check first operand is BuildVector of 0,1,2,...
13345 for (unsigned I = 0; I < VT.getVectorNumElements(); I++) {
13346 if (!Op0.getOperand(i: I).isUndef() &&
13347 !(isa<ConstantSDNode>(Val: Op0.getOperand(i: I)) &&
13348 Op0.getConstantOperandVal(i: I) == I))
13349 return SDValue();
13350 }
13351
13352 // The second is a Splat of Op1S
13353 SDValue Op1S = DCI.DAG.getSplatValue(V: Op1);
13354 if (!Op1S)
13355 return SDValue();
13356
13357 unsigned Opc;
13358 switch (VT.getVectorNumElements()) {
13359 case 4:
13360 Opc = Intrinsic::arm_mve_vctp32;
13361 break;
13362 case 8:
13363 Opc = Intrinsic::arm_mve_vctp16;
13364 break;
13365 case 16:
13366 Opc = Intrinsic::arm_mve_vctp8;
13367 break;
13368 default:
13369 return SDValue();
13370 }
13371
13372 SDLoc DL(N);
13373 return DCI.DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT,
13374 N1: DCI.DAG.getConstant(Val: Opc, DL, VT: MVT::i32),
13375 N2: DCI.DAG.getZExtOrTrunc(Op: Op1S, DL, VT: MVT::i32));
13376}
13377
13378/// PerformADDECombine - Target-specific dag combine transform from
13379/// ARMISD::ADDC, ARMISD::ADDE, and ISD::MUL_LOHI to MLAL or
13380/// ARMISD::ADDC, ARMISD::ADDE and ARMISD::UMLAL to ARMISD::UMAAL
13381static SDValue PerformADDECombine(SDNode *N,
13382 TargetLowering::DAGCombinerInfo &DCI,
13383 const ARMSubtarget *Subtarget) {
13384 // Only ARM and Thumb2 support UMLAL/SMLAL.
13385 if (Subtarget->isThumb1Only())
13386 return PerformAddeSubeCombine(N, DCI, Subtarget);
13387
13388 // Only perform the checks after legalize when the pattern is available.
13389 if (DCI.isBeforeLegalize()) return SDValue();
13390
13391 return AddCombineTo64bitUMAAL(AddeNode: N, DCI, Subtarget);
13392}
13393
13394/// PerformADDCombineWithOperands - Try DAG combinations for an ADD with
13395/// operands N0 and N1. This is a helper for PerformADDCombine that is
13396/// called with the default operands, and if that fails, with commuted
13397/// operands.
13398static SDValue PerformADDCombineWithOperands(SDNode *N, SDValue N0, SDValue N1,
13399 TargetLowering::DAGCombinerInfo &DCI,
13400 const ARMSubtarget *Subtarget){
13401 // Attempt to create vpadd for this add.
13402 if (SDValue Result = AddCombineToVPADD(N, N0, N1, DCI, Subtarget))
13403 return Result;
13404
13405 // Attempt to create vpaddl for this add.
13406 if (SDValue Result = AddCombineVUZPToVPADDL(N, N0, N1, DCI, Subtarget))
13407 return Result;
13408 if (SDValue Result = AddCombineBUILD_VECTORToVPADDL(N, N0, N1, DCI,
13409 Subtarget))
13410 return Result;
13411
13412 // fold (add (select cc, 0, c), x) -> (select cc, x, (add, x, c))
13413 if (N0.getNode()->hasOneUse())
13414 if (SDValue Result = combineSelectAndUse(N, Slct: N0, OtherOp: N1, DCI))
13415 return Result;
13416 return SDValue();
13417}
13418
13419static SDValue TryDistrubutionADDVecReduce(SDNode *N, SelectionDAG &DAG) {
13420 EVT VT = N->getValueType(ResNo: 0);
13421 SDValue N0 = N->getOperand(Num: 0);
13422 SDValue N1 = N->getOperand(Num: 1);
13423 SDLoc dl(N);
13424
13425 auto IsVecReduce = [](SDValue Op) {
13426 switch (Op.getOpcode()) {
13427 case ISD::VECREDUCE_ADD:
13428 case ARMISD::VADDVs:
13429 case ARMISD::VADDVu:
13430 case ARMISD::VMLAVs:
13431 case ARMISD::VMLAVu:
13432 return true;
13433 }
13434 return false;
13435 };
13436
13437 auto DistrubuteAddAddVecReduce = [&](SDValue N0, SDValue N1) {
13438 // Distribute add(X, add(vecreduce(Y), vecreduce(Z))) ->
13439 // add(add(X, vecreduce(Y)), vecreduce(Z))
13440 // to make better use of vaddva style instructions.
13441 if (VT == MVT::i32 && N1.getOpcode() == ISD::ADD && !IsVecReduce(N0) &&
13442 IsVecReduce(N1.getOperand(i: 0)) && IsVecReduce(N1.getOperand(i: 1)) &&
13443 !isa<ConstantSDNode>(Val: N0) && N1->hasOneUse()) {
13444 SDValue Add0 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: N0, N2: N1.getOperand(i: 0));
13445 return DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Add0, N2: N1.getOperand(i: 1));
13446 }
13447 // And turn add(add(A, reduce(B)), add(C, reduce(D))) ->
13448 // add(add(add(A, C), reduce(B)), reduce(D))
13449 if (VT == MVT::i32 && N0.getOpcode() == ISD::ADD &&
13450 N1.getOpcode() == ISD::ADD && N0->hasOneUse() && N1->hasOneUse()) {
13451 unsigned N0RedOp = 0;
13452 if (!IsVecReduce(N0.getOperand(i: N0RedOp))) {
13453 N0RedOp = 1;
13454 if (!IsVecReduce(N0.getOperand(i: N0RedOp)))
13455 return SDValue();
13456 }
13457
13458 unsigned N1RedOp = 0;
13459 if (!IsVecReduce(N1.getOperand(i: N1RedOp)))
13460 N1RedOp = 1;
13461 if (!IsVecReduce(N1.getOperand(i: N1RedOp)))
13462 return SDValue();
13463
13464 SDValue Add0 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: N0.getOperand(i: 1 - N0RedOp),
13465 N2: N1.getOperand(i: 1 - N1RedOp));
13466 SDValue Add1 =
13467 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Add0, N2: N0.getOperand(i: N0RedOp));
13468 return DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Add1, N2: N1.getOperand(i: N1RedOp));
13469 }
13470 return SDValue();
13471 };
13472 if (SDValue R = DistrubuteAddAddVecReduce(N0, N1))
13473 return R;
13474 if (SDValue R = DistrubuteAddAddVecReduce(N1, N0))
13475 return R;
13476
13477 // Distribute add(vecreduce(load(Y)), vecreduce(load(Z)))
13478 // Or add(add(X, vecreduce(load(Y))), vecreduce(load(Z)))
13479 // by ascending load offsets. This can help cores prefetch if the order of
13480 // loads is more predictable.
13481 auto DistrubuteVecReduceLoad = [&](SDValue N0, SDValue N1, bool IsForward) {
13482 // Check if two reductions are known to load data where one is before/after
13483 // another. Return negative if N0 loads data before N1, positive if N1 is
13484 // before N0 and 0 otherwise if nothing is known.
13485 auto IsKnownOrderedLoad = [&](SDValue N0, SDValue N1) {
13486 // Look through to the first operand of a MUL, for the VMLA case.
13487 // Currently only looks at the first operand, in the hope they are equal.
13488 if (N0.getOpcode() == ISD::MUL)
13489 N0 = N0.getOperand(i: 0);
13490 if (N1.getOpcode() == ISD::MUL)
13491 N1 = N1.getOperand(i: 0);
13492
13493 // Return true if the two operands are loads to the same object and the
13494 // offset of the first is known to be less than the offset of the second.
13495 LoadSDNode *Load0 = dyn_cast<LoadSDNode>(Val&: N0);
13496 LoadSDNode *Load1 = dyn_cast<LoadSDNode>(Val&: N1);
13497 if (!Load0 || !Load1 || Load0->getChain() != Load1->getChain() ||
13498 !Load0->isSimple() || !Load1->isSimple() || Load0->isIndexed() ||
13499 Load1->isIndexed())
13500 return 0;
13501
13502 auto BaseLocDecomp0 = BaseIndexOffset::match(N: Load0, DAG);
13503 auto BaseLocDecomp1 = BaseIndexOffset::match(N: Load1, DAG);
13504
13505 if (!BaseLocDecomp0.getBase() ||
13506 BaseLocDecomp0.getBase() != BaseLocDecomp1.getBase() ||
13507 !BaseLocDecomp0.hasValidOffset() || !BaseLocDecomp1.hasValidOffset())
13508 return 0;
13509 if (BaseLocDecomp0.getOffset() < BaseLocDecomp1.getOffset())
13510 return -1;
13511 if (BaseLocDecomp0.getOffset() > BaseLocDecomp1.getOffset())
13512 return 1;
13513 return 0;
13514 };
13515
13516 SDValue X;
13517 if (N0.getOpcode() == ISD::ADD && N0->hasOneUse()) {
13518 if (IsVecReduce(N0.getOperand(i: 0)) && IsVecReduce(N0.getOperand(i: 1))) {
13519 int IsBefore = IsKnownOrderedLoad(N0.getOperand(i: 0).getOperand(i: 0),
13520 N0.getOperand(i: 1).getOperand(i: 0));
13521 if (IsBefore < 0) {
13522 X = N0.getOperand(i: 0);
13523 N0 = N0.getOperand(i: 1);
13524 } else if (IsBefore > 0) {
13525 X = N0.getOperand(i: 1);
13526 N0 = N0.getOperand(i: 0);
13527 } else
13528 return SDValue();
13529 } else if (IsVecReduce(N0.getOperand(i: 0))) {
13530 X = N0.getOperand(i: 1);
13531 N0 = N0.getOperand(i: 0);
13532 } else if (IsVecReduce(N0.getOperand(i: 1))) {
13533 X = N0.getOperand(i: 0);
13534 N0 = N0.getOperand(i: 1);
13535 } else
13536 return SDValue();
13537 } else if (IsForward && IsVecReduce(N0) && IsVecReduce(N1) &&
13538 IsKnownOrderedLoad(N0.getOperand(i: 0), N1.getOperand(i: 0)) < 0) {
13539 // Note this is backward to how you would expect. We create
13540 // add(reduce(load + 16), reduce(load + 0)) so that the
13541 // add(reduce(load+16), X) is combined into VADDVA(X, load+16)), leaving
13542 // the X as VADDV(load + 0)
13543 return DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1, N2: N0);
13544 } else
13545 return SDValue();
13546
13547 if (!IsVecReduce(N0) || !IsVecReduce(N1))
13548 return SDValue();
13549
13550 if (IsKnownOrderedLoad(N1.getOperand(i: 0), N0.getOperand(i: 0)) >= 0)
13551 return SDValue();
13552
13553 // Switch from add(add(X, N0), N1) to add(add(X, N1), N0)
13554 SDValue Add0 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: X, N2: N1);
13555 return DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Add0, N2: N0);
13556 };
13557 if (SDValue R = DistrubuteVecReduceLoad(N0, N1, true))
13558 return R;
13559 if (SDValue R = DistrubuteVecReduceLoad(N1, N0, false))
13560 return R;
13561 return SDValue();
13562}
13563
13564static SDValue PerformADDVecReduce(SDNode *N, SelectionDAG &DAG,
13565 const ARMSubtarget *Subtarget) {
13566 if (!Subtarget->hasMVEIntegerOps())
13567 return SDValue();
13568
13569 if (SDValue R = TryDistrubutionADDVecReduce(N, DAG))
13570 return R;
13571
13572 EVT VT = N->getValueType(ResNo: 0);
13573 SDValue N0 = N->getOperand(Num: 0);
13574 SDValue N1 = N->getOperand(Num: 1);
13575 SDLoc dl(N);
13576
13577 if (VT != MVT::i64)
13578 return SDValue();
13579
13580 // We are looking for a i64 add of a VADDLVx. Due to these being i64's, this
13581 // will look like:
13582 // t1: i32,i32 = ARMISD::VADDLVs x
13583 // t2: i64 = build_pair t1, t1:1
13584 // t3: i64 = add t2, y
13585 // Otherwise we try to push the add up above VADDLVAx, to potentially allow
13586 // the add to be simplified separately.
13587 // We also need to check for sext / zext and commutitive adds.
13588 auto MakeVecReduce = [&](unsigned Opcode, unsigned OpcodeA, SDValue NA,
13589 SDValue NB) {
13590 if (NB->getOpcode() != ISD::BUILD_PAIR)
13591 return SDValue();
13592 SDValue VecRed = NB->getOperand(Num: 0);
13593 if ((VecRed->getOpcode() != Opcode && VecRed->getOpcode() != OpcodeA) ||
13594 VecRed.getResNo() != 0 ||
13595 NB->getOperand(Num: 1) != SDValue(VecRed.getNode(), 1))
13596 return SDValue();
13597
13598 if (VecRed->getOpcode() == OpcodeA) {
13599 // add(NA, VADDLVA(Inp), Y) -> VADDLVA(add(NA, Inp), Y)
13600 SDValue Inp = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: MVT::i64,
13601 N1: VecRed.getOperand(i: 0), N2: VecRed.getOperand(i: 1));
13602 NA = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::i64, N1: Inp, N2: NA);
13603 }
13604
13605 SmallVector<SDValue, 4> Ops(2);
13606 std::tie(args&: Ops[0], args&: Ops[1]) = DAG.SplitScalar(N: NA, DL: dl, LoVT: MVT::i32, HiVT: MVT::i32);
13607
13608 unsigned S = VecRed->getOpcode() == OpcodeA ? 2 : 0;
13609 for (unsigned I = S, E = VecRed.getNumOperands(); I < E; I++)
13610 Ops.push_back(Elt: VecRed->getOperand(Num: I));
13611 SDValue Red =
13612 DAG.getNode(Opcode: OpcodeA, DL: dl, VTList: DAG.getVTList(VTs: {MVT::i32, MVT::i32}), Ops);
13613 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: MVT::i64, N1: Red,
13614 N2: SDValue(Red.getNode(), 1));
13615 };
13616
13617 if (SDValue M = MakeVecReduce(ARMISD::VADDLVs, ARMISD::VADDLVAs, N0, N1))
13618 return M;
13619 if (SDValue M = MakeVecReduce(ARMISD::VADDLVu, ARMISD::VADDLVAu, N0, N1))
13620 return M;
13621 if (SDValue M = MakeVecReduce(ARMISD::VADDLVs, ARMISD::VADDLVAs, N1, N0))
13622 return M;
13623 if (SDValue M = MakeVecReduce(ARMISD::VADDLVu, ARMISD::VADDLVAu, N1, N0))
13624 return M;
13625 if (SDValue M = MakeVecReduce(ARMISD::VADDLVps, ARMISD::VADDLVAps, N0, N1))
13626 return M;
13627 if (SDValue M = MakeVecReduce(ARMISD::VADDLVpu, ARMISD::VADDLVApu, N0, N1))
13628 return M;
13629 if (SDValue M = MakeVecReduce(ARMISD::VADDLVps, ARMISD::VADDLVAps, N1, N0))
13630 return M;
13631 if (SDValue M = MakeVecReduce(ARMISD::VADDLVpu, ARMISD::VADDLVApu, N1, N0))
13632 return M;
13633 if (SDValue M = MakeVecReduce(ARMISD::VMLALVs, ARMISD::VMLALVAs, N0, N1))
13634 return M;
13635 if (SDValue M = MakeVecReduce(ARMISD::VMLALVu, ARMISD::VMLALVAu, N0, N1))
13636 return M;
13637 if (SDValue M = MakeVecReduce(ARMISD::VMLALVs, ARMISD::VMLALVAs, N1, N0))
13638 return M;
13639 if (SDValue M = MakeVecReduce(ARMISD::VMLALVu, ARMISD::VMLALVAu, N1, N0))
13640 return M;
13641 if (SDValue M = MakeVecReduce(ARMISD::VMLALVps, ARMISD::VMLALVAps, N0, N1))
13642 return M;
13643 if (SDValue M = MakeVecReduce(ARMISD::VMLALVpu, ARMISD::VMLALVApu, N0, N1))
13644 return M;
13645 if (SDValue M = MakeVecReduce(ARMISD::VMLALVps, ARMISD::VMLALVAps, N1, N0))
13646 return M;
13647 if (SDValue M = MakeVecReduce(ARMISD::VMLALVpu, ARMISD::VMLALVApu, N1, N0))
13648 return M;
13649 return SDValue();
13650}
13651
13652bool
13653ARMTargetLowering::isDesirableToCommuteWithShift(const SDNode *N,
13654 CombineLevel Level) const {
13655 assert((N->getOpcode() == ISD::SHL || N->getOpcode() == ISD::SRA ||
13656 N->getOpcode() == ISD::SRL) &&
13657 "Expected shift op");
13658
13659 SDValue ShiftLHS = N->getOperand(Num: 0);
13660 if (!ShiftLHS->hasOneUse())
13661 return false;
13662
13663 if (ShiftLHS.getOpcode() == ISD::SIGN_EXTEND &&
13664 !ShiftLHS.getOperand(i: 0)->hasOneUse())
13665 return false;
13666
13667 if (Level == BeforeLegalizeTypes)
13668 return true;
13669
13670 if (N->getOpcode() != ISD::SHL)
13671 return true;
13672
13673 if (Subtarget->isThumb1Only()) {
13674 // Avoid making expensive immediates by commuting shifts. (This logic
13675 // only applies to Thumb1 because ARM and Thumb2 immediates can be shifted
13676 // for free.)
13677 if (N->getOpcode() != ISD::SHL)
13678 return true;
13679 SDValue N1 = N->getOperand(Num: 0);
13680 if (N1->getOpcode() != ISD::ADD && N1->getOpcode() != ISD::AND &&
13681 N1->getOpcode() != ISD::OR && N1->getOpcode() != ISD::XOR)
13682 return true;
13683 if (auto *Const = dyn_cast<ConstantSDNode>(Val: N1->getOperand(Num: 1))) {
13684 if (Const->getAPIntValue().ult(RHS: 256))
13685 return false;
13686 if (N1->getOpcode() == ISD::ADD && Const->getAPIntValue().slt(RHS: 0) &&
13687 Const->getAPIntValue().sgt(RHS: -256))
13688 return false;
13689 }
13690 return true;
13691 }
13692
13693 // Turn off commute-with-shift transform after legalization, so it doesn't
13694 // conflict with PerformSHLSimplify. (We could try to detect when
13695 // PerformSHLSimplify would trigger more precisely, but it isn't
13696 // really necessary.)
13697 return false;
13698}
13699
13700bool ARMTargetLowering::isDesirableToCommuteXorWithShift(
13701 const SDNode *N) const {
13702 assert(N->getOpcode() == ISD::XOR &&
13703 (N->getOperand(0).getOpcode() == ISD::SHL ||
13704 N->getOperand(0).getOpcode() == ISD::SRL) &&
13705 "Expected XOR(SHIFT) pattern");
13706
13707 // Only commute if the entire NOT mask is a hidden shifted mask.
13708 auto *XorC = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
13709 auto *ShiftC = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 0).getOperand(i: 1));
13710 if (XorC && ShiftC) {
13711 unsigned MaskIdx, MaskLen;
13712 if (XorC->getAPIntValue().isShiftedMask(MaskIdx, MaskLen)) {
13713 unsigned ShiftAmt = ShiftC->getZExtValue();
13714 unsigned BitWidth = N->getValueType(ResNo: 0).getScalarSizeInBits();
13715 if (N->getOperand(Num: 0).getOpcode() == ISD::SHL)
13716 return MaskIdx == ShiftAmt && MaskLen == (BitWidth - ShiftAmt);
13717 return MaskIdx == 0 && MaskLen == (BitWidth - ShiftAmt);
13718 }
13719 }
13720
13721 return false;
13722}
13723
13724bool ARMTargetLowering::shouldFoldConstantShiftPairToMask(
13725 const SDNode *N) const {
13726 assert(((N->getOpcode() == ISD::SHL &&
13727 N->getOperand(0).getOpcode() == ISD::SRL) ||
13728 (N->getOpcode() == ISD::SRL &&
13729 N->getOperand(0).getOpcode() == ISD::SHL)) &&
13730 "Expected shift-shift mask");
13731
13732 if (!Subtarget->isThumb1Only())
13733 return true;
13734
13735 EVT VT = N->getValueType(ResNo: 0);
13736 if (VT.getScalarSizeInBits() > 32)
13737 return true;
13738
13739 return false;
13740}
13741
13742bool ARMTargetLowering::shouldFoldSelectWithIdentityConstant(
13743 unsigned BinOpcode, EVT VT, unsigned SelectOpcode, SDValue X,
13744 SDValue Y) const {
13745 return Subtarget->hasMVEIntegerOps() && isTypeLegal(VT) &&
13746 SelectOpcode == ISD::VSELECT;
13747}
13748
13749bool ARMTargetLowering::preferIncOfAddToSubOfNot(EVT VT) const {
13750 if (!Subtarget->hasNEON() && !Subtarget->hasMVEIntegerOps()) {
13751 if (Subtarget->isThumb1Only())
13752 return VT.getScalarSizeInBits() <= 32;
13753 return true;
13754 }
13755 return VT.isScalarInteger();
13756}
13757
13758bool ARMTargetLowering::shouldConvertFpToSat(unsigned Op, EVT FPVT,
13759 EVT VT) const {
13760 if (!isOperationLegalOrCustom(Op, VT) || !FPVT.isSimple())
13761 return false;
13762
13763 switch (FPVT.getSimpleVT().SimpleTy) {
13764 case MVT::f16:
13765 return Subtarget->hasVFP2Base();
13766 case MVT::f32:
13767 return Subtarget->hasVFP2Base();
13768 case MVT::f64:
13769 return Subtarget->hasFP64();
13770 case MVT::v4f32:
13771 case MVT::v8f16:
13772 return Subtarget->hasMVEFloatOps();
13773 default:
13774 return false;
13775 }
13776}
13777
13778static SDValue PerformSHLSimplify(SDNode *N,
13779 TargetLowering::DAGCombinerInfo &DCI,
13780 const ARMSubtarget *ST) {
13781 // Allow the generic combiner to identify potential bswaps.
13782 if (DCI.isBeforeLegalize())
13783 return SDValue();
13784
13785 // DAG combiner will fold:
13786 // (shl (add x, c1), c2) -> (add (shl x, c2), c1 << c2)
13787 // (shl (or x, c1), c2) -> (or (shl x, c2), c1 << c2
13788 // Other code patterns that can be also be modified have the following form:
13789 // b + ((a << 1) | 510)
13790 // b + ((a << 1) & 510)
13791 // b + ((a << 1) ^ 510)
13792 // b + ((a << 1) + 510)
13793
13794 // Many instructions can perform the shift for free, but it requires both
13795 // the operands to be registers. If c1 << c2 is too large, a mov immediate
13796 // instruction will needed. So, unfold back to the original pattern if:
13797 // - if c1 and c2 are small enough that they don't require mov imms.
13798 // - the user(s) of the node can perform an shl
13799
13800 // No shifted operands for 16-bit instructions.
13801 if (ST->isThumb1Only())
13802 return SDValue();
13803
13804 // Check that all the users could perform the shl themselves.
13805 for (auto *U : N->users()) {
13806 switch(U->getOpcode()) {
13807 default:
13808 return SDValue();
13809 case ISD::SUB:
13810 case ISD::ADD:
13811 case ISD::AND:
13812 case ISD::OR:
13813 case ISD::XOR:
13814 case ISD::SETCC:
13815 case ARMISD::CMP:
13816 // Check that the user isn't already using a constant because there
13817 // aren't any instructions that support an immediate operand and a
13818 // shifted operand.
13819 if (isa<ConstantSDNode>(Val: U->getOperand(Num: 0)) ||
13820 isa<ConstantSDNode>(Val: U->getOperand(Num: 1)))
13821 return SDValue();
13822
13823 // Check that it's not already using a shift.
13824 if (U->getOperand(Num: 0).getOpcode() == ISD::SHL ||
13825 U->getOperand(Num: 1).getOpcode() == ISD::SHL)
13826 return SDValue();
13827 break;
13828 }
13829 }
13830
13831 if (N->getOpcode() != ISD::ADD && N->getOpcode() != ISD::OR &&
13832 N->getOpcode() != ISD::XOR && N->getOpcode() != ISD::AND)
13833 return SDValue();
13834
13835 if (N->getOperand(Num: 0).getOpcode() != ISD::SHL)
13836 return SDValue();
13837
13838 SDValue SHL = N->getOperand(Num: 0);
13839
13840 auto *C1ShlC2 = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
13841 auto *C2 = dyn_cast<ConstantSDNode>(Val: SHL.getOperand(i: 1));
13842 if (!C1ShlC2 || !C2)
13843 return SDValue();
13844
13845 APInt C2Int = C2->getAPIntValue();
13846 APInt C1Int = C1ShlC2->getAPIntValue();
13847 unsigned C2Width = C2Int.getBitWidth();
13848 if (C2Int.uge(RHS: C2Width))
13849 return SDValue();
13850 uint64_t C2Value = C2Int.getZExtValue();
13851
13852 // Check that performing a lshr will not lose any information.
13853 APInt Mask = APInt::getHighBitsSet(numBits: C2Width, hiBitsSet: C2Width - C2Value);
13854 if ((C1Int & Mask) != C1Int)
13855 return SDValue();
13856
13857 // Shift the first constant.
13858 C1Int.lshrInPlace(ShiftAmt: C2Int);
13859
13860 // The immediates are encoded as an 8-bit value that can be rotated.
13861 auto LargeImm = [](const APInt &Imm) {
13862 unsigned Zeros = Imm.countl_zero() + Imm.countr_zero();
13863 return Imm.getBitWidth() - Zeros > 8;
13864 };
13865
13866 if (LargeImm(C1Int) || LargeImm(C2Int))
13867 return SDValue();
13868
13869 SelectionDAG &DAG = DCI.DAG;
13870 SDLoc dl(N);
13871 SDValue X = SHL.getOperand(i: 0);
13872 SDValue BinOp = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: MVT::i32, N1: X,
13873 N2: DAG.getConstant(Val: C1Int, DL: dl, VT: MVT::i32));
13874 // Shift left to compensate for the lshr of C1Int.
13875 SDValue Res = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: MVT::i32, N1: BinOp, N2: SHL.getOperand(i: 1));
13876
13877 LLVM_DEBUG(dbgs() << "Simplify shl use:\n"; SHL.getOperand(0).dump();
13878 SHL.dump(); N->dump());
13879 LLVM_DEBUG(dbgs() << "Into:\n"; X.dump(); BinOp.dump(); Res.dump());
13880 return Res;
13881}
13882
13883
13884/// PerformADDCombine - Target-specific dag combine xforms for ISD::ADD.
13885///
13886static SDValue PerformADDCombine(SDNode *N,
13887 TargetLowering::DAGCombinerInfo &DCI,
13888 const ARMSubtarget *Subtarget) {
13889 SDValue N0 = N->getOperand(Num: 0);
13890 SDValue N1 = N->getOperand(Num: 1);
13891
13892 // Only works one way, because it needs an immediate operand.
13893 if (SDValue Result = PerformSHLSimplify(N, DCI, ST: Subtarget))
13894 return Result;
13895
13896 if (SDValue Result = PerformADDVecReduce(N, DAG&: DCI.DAG, Subtarget))
13897 return Result;
13898
13899 // First try with the default operand order.
13900 if (SDValue Result = PerformADDCombineWithOperands(N, N0, N1, DCI, Subtarget))
13901 return Result;
13902
13903 // If that didn't work, try again with the operands commuted.
13904 return PerformADDCombineWithOperands(N, N0: N1, N1: N0, DCI, Subtarget);
13905}
13906
13907// Combine (sub 0, (csinc X, Y, CC)) -> (csinv -X, Y, CC)
13908// providing -X is as cheap as X (currently, just a constant).
13909static SDValue PerformSubCSINCCombine(SDNode *N, SelectionDAG &DAG) {
13910 if (N->getValueType(ResNo: 0) != MVT::i32 || !isNullConstant(V: N->getOperand(Num: 0)))
13911 return SDValue();
13912 SDValue CSINC = N->getOperand(Num: 1);
13913 if (CSINC.getOpcode() != ARMISD::CSINC || !CSINC.hasOneUse())
13914 return SDValue();
13915
13916 ConstantSDNode *X = dyn_cast<ConstantSDNode>(Val: CSINC.getOperand(i: 0));
13917 if (!X)
13918 return SDValue();
13919
13920 return DAG.getNode(Opcode: ARMISD::CSINV, DL: SDLoc(N), VT: MVT::i32,
13921 N1: DAG.getNode(Opcode: ISD::SUB, DL: SDLoc(N), VT: MVT::i32, N1: N->getOperand(Num: 0),
13922 N2: CSINC.getOperand(i: 0)),
13923 N2: CSINC.getOperand(i: 1), N3: CSINC.getOperand(i: 2),
13924 N4: CSINC.getOperand(i: 3));
13925}
13926
13927static int getNegationCost(SDValue Op) {
13928 // Free to negate.
13929 if (isa<ConstantSDNode>(Val: Op))
13930 return 0;
13931
13932 // Will save one instruction.
13933 if (Op.getOpcode() == ISD::SUB && isNullConstant(V: Op.getOperand(i: 0)))
13934 return -1;
13935
13936 // Can freely negate by converting sra <-> srl.
13937 if (Op.getOpcode() == ISD::SRA || Op.getOpcode() == ISD::SRL) {
13938 ConstantSDNode *ShiftAmt = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1));
13939 if (Op.hasOneUse() && ShiftAmt &&
13940 ShiftAmt->getZExtValue() == Op.getValueType().getScalarSizeInBits() - 1)
13941 return 0;
13942 }
13943
13944 // Will have to create sub.
13945 return 1;
13946}
13947
13948// Try to fold
13949//
13950// (neg (cmov X, Y)) -> (cmov (neg X), (neg Y))
13951//
13952// The folding helps cmov to be matched with csneg without generating
13953// redundant neg instruction.
13954static SDValue performNegCMovCombine(SDNode *N, SelectionDAG &DAG) {
13955 assert(N->getOpcode() == ISD::SUB);
13956 if (!isNullConstant(V: N->getOperand(Num: 0)))
13957 return SDValue();
13958
13959 SDValue CMov = N->getOperand(Num: 1);
13960 if (CMov.getOpcode() != ARMISD::CMOV || !CMov->hasOneUse())
13961 return SDValue();
13962
13963 SDValue N0 = CMov.getOperand(i: 0);
13964 SDValue N1 = CMov.getOperand(i: 1);
13965
13966 // Only perform the fold if we actually save something.
13967 if (getNegationCost(Op: N0) + getNegationCost(Op: N1) > 0)
13968 return SDValue();
13969
13970 SDLoc DL(N);
13971 EVT VT = CMov.getValueType();
13972
13973 SDValue N0N = DAG.getNegative(Val: N0, DL, VT);
13974 SDValue N1N = DAG.getNegative(Val: N1, DL, VT);
13975 return DAG.getNode(Opcode: ARMISD::CMOV, DL, VT, N1: N0N, N2: N1N, N3: CMov.getOperand(i: 2),
13976 N4: CMov.getOperand(i: 3));
13977}
13978
13979/// PerformSUBCombine - Target-specific dag combine xforms for ISD::SUB.
13980///
13981static SDValue PerformSUBCombine(SDNode *N,
13982 TargetLowering::DAGCombinerInfo &DCI,
13983 const ARMSubtarget *Subtarget) {
13984 SDValue N0 = N->getOperand(Num: 0);
13985 SDValue N1 = N->getOperand(Num: 1);
13986
13987 // fold (sub x, (select cc, 0, c)) -> (select cc, x, (sub, x, c))
13988 if (N1.getNode()->hasOneUse())
13989 if (SDValue Result = combineSelectAndUse(N, Slct: N1, OtherOp: N0, DCI))
13990 return Result;
13991
13992 if (SDValue R = PerformSubCSINCCombine(N, DAG&: DCI.DAG))
13993 return R;
13994
13995 if (SDValue Val = performNegCMovCombine(N, DAG&: DCI.DAG))
13996 return Val;
13997
13998 if (!Subtarget->hasMVEIntegerOps() || !N->getValueType(ResNo: 0).isVector())
13999 return SDValue();
14000
14001 // Fold (sub (ARMvmovImm 0), (ARMvdup x)) -> (ARMvdup (sub 0, x))
14002 // so that we can readily pattern match more mve instructions which can use
14003 // a scalar operand.
14004 SDValue VDup = N->getOperand(Num: 1);
14005 if (VDup->getOpcode() != ARMISD::VDUP)
14006 return SDValue();
14007
14008 SDValue VMov = N->getOperand(Num: 0);
14009 if (VMov->getOpcode() == ISD::BITCAST)
14010 VMov = VMov->getOperand(Num: 0);
14011
14012 if (VMov->getOpcode() != ARMISD::VMOVIMM || !isZeroVector(N: VMov))
14013 return SDValue();
14014
14015 SDLoc dl(N);
14016 SDValue Negate = DCI.DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32,
14017 N1: DCI.DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32),
14018 N2: VDup->getOperand(Num: 0));
14019 return DCI.DAG.getNode(Opcode: ARMISD::VDUP, DL: dl, VT: N->getValueType(ResNo: 0), Operand: Negate);
14020}
14021
14022/// PerformVMULCombine
14023/// Distribute (A + B) * C to (A * C) + (B * C) to take advantage of the
14024/// special multiplier accumulator forwarding.
14025/// vmul d3, d0, d2
14026/// vmla d3, d1, d2
14027/// is faster than
14028/// vadd d3, d0, d1
14029/// vmul d3, d3, d2
14030// However, for (A + B) * (A + B),
14031// vadd d2, d0, d1
14032// vmul d3, d0, d2
14033// vmla d3, d1, d2
14034// is slower than
14035// vadd d2, d0, d1
14036// vmul d3, d2, d2
14037static SDValue PerformVMULCombine(SDNode *N,
14038 TargetLowering::DAGCombinerInfo &DCI,
14039 const ARMSubtarget *Subtarget) {
14040 if (!Subtarget->hasVMLxForwarding())
14041 return SDValue();
14042
14043 SelectionDAG &DAG = DCI.DAG;
14044 SDValue N0 = N->getOperand(Num: 0);
14045 SDValue N1 = N->getOperand(Num: 1);
14046 unsigned Opcode = N0.getOpcode();
14047 if (Opcode != ISD::ADD && Opcode != ISD::SUB &&
14048 Opcode != ISD::FADD && Opcode != ISD::FSUB) {
14049 Opcode = N1.getOpcode();
14050 if (Opcode != ISD::ADD && Opcode != ISD::SUB &&
14051 Opcode != ISD::FADD && Opcode != ISD::FSUB)
14052 return SDValue();
14053 std::swap(a&: N0, b&: N1);
14054 }
14055
14056 if (N0 == N1)
14057 return SDValue();
14058
14059 EVT VT = N->getValueType(ResNo: 0);
14060 SDLoc DL(N);
14061 SDValue N00 = N0->getOperand(Num: 0);
14062 SDValue N01 = N0->getOperand(Num: 1);
14063 return DAG.getNode(Opcode, DL, VT,
14064 N1: DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: N00, N2: N1),
14065 N2: DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: N01, N2: N1));
14066}
14067
14068static SDValue PerformMVEVMULLCombine(SDNode *N, SelectionDAG &DAG,
14069 const ARMSubtarget *Subtarget) {
14070 EVT VT = N->getValueType(ResNo: 0);
14071 if (VT != MVT::v2i64)
14072 return SDValue();
14073
14074 SDValue N0 = N->getOperand(Num: 0);
14075 SDValue N1 = N->getOperand(Num: 1);
14076
14077 auto IsSignExt = [&](SDValue Op) {
14078 if (Op->getOpcode() != ISD::SIGN_EXTEND_INREG)
14079 return SDValue();
14080 EVT VT = cast<VTSDNode>(Val: Op->getOperand(Num: 1))->getVT();
14081 if (VT.getScalarSizeInBits() == 32)
14082 return Op->getOperand(Num: 0);
14083 return SDValue();
14084 };
14085 auto IsZeroExt = [&](SDValue Op) {
14086 // Zero extends are a little more awkward. At the point we are matching
14087 // this, we are looking for an AND with a (-1, 0, -1, 0) buildvector mask.
14088 // That might be before of after a bitcast depending on how the and is
14089 // placed. Because this has to look through bitcasts, it is currently only
14090 // supported on LE.
14091 if (!Subtarget->isLittle())
14092 return SDValue();
14093
14094 SDValue And = Op;
14095 if (And->getOpcode() == ISD::BITCAST)
14096 And = And->getOperand(Num: 0);
14097 if (And->getOpcode() != ISD::AND)
14098 return SDValue();
14099 SDValue Mask = And->getOperand(Num: 1);
14100 if (Mask->getOpcode() == ISD::BITCAST)
14101 Mask = Mask->getOperand(Num: 0);
14102
14103 if (Mask->getOpcode() != ISD::BUILD_VECTOR ||
14104 Mask.getValueType() != MVT::v4i32)
14105 return SDValue();
14106 if (isAllOnesConstant(V: Mask->getOperand(Num: 0)) &&
14107 isNullConstant(V: Mask->getOperand(Num: 1)) &&
14108 isAllOnesConstant(V: Mask->getOperand(Num: 2)) &&
14109 isNullConstant(V: Mask->getOperand(Num: 3)))
14110 return And->getOperand(Num: 0);
14111 return SDValue();
14112 };
14113
14114 SDLoc dl(N);
14115 if (SDValue Op0 = IsSignExt(N0)) {
14116 if (SDValue Op1 = IsSignExt(N1)) {
14117 SDValue New0a = DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT: MVT::v4i32, Operand: Op0);
14118 SDValue New1a = DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT: MVT::v4i32, Operand: Op1);
14119 return DAG.getNode(Opcode: ARMISD::VMULLs, DL: dl, VT, N1: New0a, N2: New1a);
14120 }
14121 }
14122 if (SDValue Op0 = IsZeroExt(N0)) {
14123 if (SDValue Op1 = IsZeroExt(N1)) {
14124 SDValue New0a = DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT: MVT::v4i32, Operand: Op0);
14125 SDValue New1a = DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT: MVT::v4i32, Operand: Op1);
14126 return DAG.getNode(Opcode: ARMISD::VMULLu, DL: dl, VT, N1: New0a, N2: New1a);
14127 }
14128 }
14129
14130 return SDValue();
14131}
14132
14133static SDValue PerformMULCombine(SDNode *N,
14134 TargetLowering::DAGCombinerInfo &DCI,
14135 const ARMSubtarget *Subtarget) {
14136 SelectionDAG &DAG = DCI.DAG;
14137
14138 EVT VT = N->getValueType(ResNo: 0);
14139 if (Subtarget->hasMVEIntegerOps() && VT == MVT::v2i64)
14140 return PerformMVEVMULLCombine(N, DAG, Subtarget);
14141
14142 if (Subtarget->isThumb1Only())
14143 return SDValue();
14144
14145 if (DCI.isBeforeLegalize() || DCI.isCalledByLegalizer())
14146 return SDValue();
14147
14148 if (VT.is64BitVector() || VT.is128BitVector())
14149 return PerformVMULCombine(N, DCI, Subtarget);
14150 if (VT != MVT::i32)
14151 return SDValue();
14152
14153 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
14154 if (!C)
14155 return SDValue();
14156
14157 int64_t MulAmt = C->getSExtValue();
14158 unsigned ShiftAmt = llvm::countr_zero<uint64_t>(Val: MulAmt);
14159
14160 ShiftAmt = ShiftAmt & (32 - 1);
14161 SDValue V = N->getOperand(Num: 0);
14162 SDLoc DL(N);
14163
14164 SDValue Res;
14165 MulAmt >>= ShiftAmt;
14166
14167 if (MulAmt >= 0) {
14168 if (llvm::has_single_bit<uint32_t>(Value: MulAmt - 1)) {
14169 // (mul x, 2^N + 1) => (add (shl x, N), x)
14170 Res = DAG.getNode(Opcode: ISD::ADD, DL, VT,
14171 N1: V,
14172 N2: DAG.getNode(Opcode: ISD::SHL, DL, VT,
14173 N1: V,
14174 N2: DAG.getConstant(Val: Log2_32(Value: MulAmt - 1), DL,
14175 VT: MVT::i32)));
14176 } else if (llvm::has_single_bit<uint32_t>(Value: MulAmt + 1)) {
14177 // (mul x, 2^N - 1) => (sub (shl x, N), x)
14178 Res = DAG.getNode(Opcode: ISD::SUB, DL, VT,
14179 N1: DAG.getNode(Opcode: ISD::SHL, DL, VT,
14180 N1: V,
14181 N2: DAG.getConstant(Val: Log2_32(Value: MulAmt + 1), DL,
14182 VT: MVT::i32)),
14183 N2: V);
14184 } else
14185 return SDValue();
14186 } else {
14187 uint64_t MulAmtAbs = -MulAmt;
14188 if (llvm::has_single_bit<uint32_t>(Value: MulAmtAbs + 1)) {
14189 // (mul x, -(2^N - 1)) => (sub x, (shl x, N))
14190 Res = DAG.getNode(Opcode: ISD::SUB, DL, VT,
14191 N1: V,
14192 N2: DAG.getNode(Opcode: ISD::SHL, DL, VT,
14193 N1: V,
14194 N2: DAG.getConstant(Val: Log2_32(Value: MulAmtAbs + 1), DL,
14195 VT: MVT::i32)));
14196 } else if (llvm::has_single_bit<uint32_t>(Value: MulAmtAbs - 1)) {
14197 // (mul x, -(2^N + 1)) => - (add (shl x, N), x)
14198 Res = DAG.getNode(Opcode: ISD::ADD, DL, VT,
14199 N1: V,
14200 N2: DAG.getNode(Opcode: ISD::SHL, DL, VT,
14201 N1: V,
14202 N2: DAG.getConstant(Val: Log2_32(Value: MulAmtAbs - 1), DL,
14203 VT: MVT::i32)));
14204 Res = DAG.getNode(Opcode: ISD::SUB, DL, VT,
14205 N1: DAG.getConstant(Val: 0, DL, VT: MVT::i32), N2: Res);
14206 } else
14207 return SDValue();
14208 }
14209
14210 if (ShiftAmt != 0)
14211 Res = DAG.getNode(Opcode: ISD::SHL, DL, VT,
14212 N1: Res, N2: DAG.getConstant(Val: ShiftAmt, DL, VT: MVT::i32));
14213
14214 // Do not add new nodes to DAG combiner worklist.
14215 DCI.CombineTo(N, Res, AddTo: false);
14216 return SDValue();
14217}
14218
14219static SDValue CombineANDShift(SDNode *N,
14220 TargetLowering::DAGCombinerInfo &DCI,
14221 const ARMSubtarget *Subtarget) {
14222 // Allow DAGCombine to pattern-match before we touch the canonical form.
14223 if (DCI.isBeforeLegalize() || DCI.isCalledByLegalizer())
14224 return SDValue();
14225
14226 if (N->getValueType(ResNo: 0) != MVT::i32)
14227 return SDValue();
14228
14229 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
14230 if (!N1C)
14231 return SDValue();
14232
14233 uint32_t C1 = (uint32_t)N1C->getZExtValue();
14234 // Don't transform uxtb/uxth.
14235 if (C1 == 255 || C1 == 65535)
14236 return SDValue();
14237
14238 SDNode *N0 = N->getOperand(Num: 0).getNode();
14239 if (!N0->hasOneUse())
14240 return SDValue();
14241
14242 if (N0->getOpcode() != ISD::SHL && N0->getOpcode() != ISD::SRL)
14243 return SDValue();
14244
14245 bool LeftShift = N0->getOpcode() == ISD::SHL;
14246
14247 ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(Val: N0->getOperand(Num: 1));
14248 if (!N01C)
14249 return SDValue();
14250
14251 uint32_t C2 = (uint32_t)N01C->getZExtValue();
14252 if (!C2 || C2 >= 32)
14253 return SDValue();
14254
14255 // Clear irrelevant bits in the mask.
14256 if (LeftShift)
14257 C1 &= (-1U << C2);
14258 else
14259 C1 &= (-1U >> C2);
14260
14261 SelectionDAG &DAG = DCI.DAG;
14262 SDLoc DL(N);
14263
14264 // We have a pattern of the form "(and (shl x, c2) c1)" or
14265 // "(and (srl x, c2) c1)", where c1 is a shifted mask. Try to
14266 // transform to a pair of shifts, to save materializing c1.
14267
14268 // First pattern: right shift, then mask off leading bits.
14269 // FIXME: Use demanded bits?
14270 if (!LeftShift && isMask_32(Value: C1)) {
14271 uint32_t C3 = llvm::countl_zero(Val: C1);
14272 if (C2 < C3) {
14273 SDValue SHL = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i32, N1: N0->getOperand(Num: 0),
14274 N2: DAG.getConstant(Val: C3 - C2, DL, VT: MVT::i32));
14275 return DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: SHL,
14276 N2: DAG.getConstant(Val: C3, DL, VT: MVT::i32));
14277 }
14278 }
14279
14280 // First pattern, reversed: left shift, then mask off trailing bits.
14281 if (LeftShift && isMask_32(Value: ~C1)) {
14282 uint32_t C3 = llvm::countr_zero(Val: C1);
14283 if (C2 < C3) {
14284 SDValue SHL = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: N0->getOperand(Num: 0),
14285 N2: DAG.getConstant(Val: C3 - C2, DL, VT: MVT::i32));
14286 return DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i32, N1: SHL,
14287 N2: DAG.getConstant(Val: C3, DL, VT: MVT::i32));
14288 }
14289 }
14290
14291 // Second pattern: left shift, then mask off leading bits.
14292 // FIXME: Use demanded bits?
14293 if (LeftShift && isShiftedMask_32(Value: C1)) {
14294 uint32_t Trailing = llvm::countr_zero(Val: C1);
14295 uint32_t C3 = llvm::countl_zero(Val: C1);
14296 if (Trailing == C2 && C2 + C3 < 32) {
14297 SDValue SHL = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i32, N1: N0->getOperand(Num: 0),
14298 N2: DAG.getConstant(Val: C2 + C3, DL, VT: MVT::i32));
14299 return DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: SHL,
14300 N2: DAG.getConstant(Val: C3, DL, VT: MVT::i32));
14301 }
14302 }
14303
14304 // Second pattern, reversed: right shift, then mask off trailing bits.
14305 // FIXME: Handle other patterns of known/demanded bits.
14306 if (!LeftShift && isShiftedMask_32(Value: C1)) {
14307 uint32_t Leading = llvm::countl_zero(Val: C1);
14308 uint32_t C3 = llvm::countr_zero(Val: C1);
14309 if (Leading == C2 && C2 + C3 < 32) {
14310 SDValue SHL = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: N0->getOperand(Num: 0),
14311 N2: DAG.getConstant(Val: C2 + C3, DL, VT: MVT::i32));
14312 return DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i32, N1: SHL,
14313 N2: DAG.getConstant(Val: C3, DL, VT: MVT::i32));
14314 }
14315 }
14316
14317 // Transform "(and (shl x, c2) c1)" into "(shl (and x, c1>>c2), c2)"
14318 // if "c1 >> c2" is a cheaper immediate than "c1"
14319 if (LeftShift &&
14320 HasLowerConstantMaterializationCost(Val1: C1 >> C2, Val2: C1, Subtarget)) {
14321
14322 SDValue And = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: N0->getOperand(Num: 0),
14323 N2: DAG.getConstant(Val: C1 >> C2, DL, VT: MVT::i32));
14324 return DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i32, N1: And,
14325 N2: DAG.getConstant(Val: C2, DL, VT: MVT::i32));
14326 }
14327
14328 return SDValue();
14329}
14330
14331static SDValue PerformANDCombine(SDNode *N,
14332 TargetLowering::DAGCombinerInfo &DCI,
14333 const ARMSubtarget *Subtarget) {
14334 // Attempt to use immediate-form VBIC
14335 BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(Val: N->getOperand(Num: 1));
14336 SDLoc dl(N);
14337 EVT VT = N->getValueType(ResNo: 0);
14338 SelectionDAG &DAG = DCI.DAG;
14339
14340 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT) || VT == MVT::v2i1 ||
14341 VT == MVT::v4i1 || VT == MVT::v8i1 || VT == MVT::v16i1)
14342 return SDValue();
14343
14344 APInt SplatBits, SplatUndef;
14345 unsigned SplatBitSize;
14346 bool HasAnyUndefs;
14347 if (BVN && (Subtarget->hasNEON() || Subtarget->hasMVEIntegerOps()) &&
14348 BVN->isConstantSplat(SplatValue&: SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
14349 if (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32 ||
14350 SplatBitSize == 64) {
14351 EVT VbicVT;
14352 SDValue Val = isVMOVModifiedImm(SplatBits: (~SplatBits).getZExtValue(),
14353 SplatUndef: SplatUndef.getZExtValue(), SplatBitSize,
14354 DAG, dl, VT&: VbicVT, VectorVT: VT, type: OtherModImm);
14355 if (Val.getNode()) {
14356 SDValue Input =
14357 DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT: VbicVT, Operand: N->getOperand(Num: 0));
14358 SDValue Vbic = DAG.getNode(Opcode: ARMISD::VBICIMM, DL: dl, VT: VbicVT, N1: Input, N2: Val);
14359 return DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT, Operand: Vbic);
14360 }
14361 }
14362 }
14363
14364 if (!Subtarget->isThumb1Only()) {
14365 // fold (and (select cc, -1, c), x) -> (select cc, x, (and, x, c))
14366 if (SDValue Result = combineSelectAndUseCommutative(N, AllOnes: true, DCI))
14367 return Result;
14368
14369 if (SDValue Result = PerformSHLSimplify(N, DCI, ST: Subtarget))
14370 return Result;
14371 }
14372
14373 if (Subtarget->isThumb1Only())
14374 if (SDValue Result = CombineANDShift(N, DCI, Subtarget))
14375 return Result;
14376
14377 return SDValue();
14378}
14379
14380// Try combining OR nodes to SMULWB, SMULWT.
14381static SDValue PerformORCombineToSMULWBT(SDNode *OR,
14382 TargetLowering::DAGCombinerInfo &DCI,
14383 const ARMSubtarget *Subtarget) {
14384 if (!Subtarget->hasV6Ops() ||
14385 (Subtarget->isThumb() &&
14386 (!Subtarget->hasThumb2() || !Subtarget->hasDSP())))
14387 return SDValue();
14388
14389 SDValue SRL = OR->getOperand(Num: 0);
14390 SDValue SHL = OR->getOperand(Num: 1);
14391
14392 if (SRL.getOpcode() != ISD::SRL || SHL.getOpcode() != ISD::SHL) {
14393 SRL = OR->getOperand(Num: 1);
14394 SHL = OR->getOperand(Num: 0);
14395 }
14396 if (!isSRL16(Op: SRL) || !isSHL16(Op: SHL))
14397 return SDValue();
14398
14399 // The first operands to the shifts need to be the two results from the
14400 // same smul_lohi node.
14401 if ((SRL.getOperand(i: 0).getNode() != SHL.getOperand(i: 0).getNode()) ||
14402 SRL.getOperand(i: 0).getOpcode() != ISD::SMUL_LOHI)
14403 return SDValue();
14404
14405 SDNode *SMULLOHI = SRL.getOperand(i: 0).getNode();
14406 if (SRL.getOperand(i: 0) != SDValue(SMULLOHI, 0) ||
14407 SHL.getOperand(i: 0) != SDValue(SMULLOHI, 1))
14408 return SDValue();
14409
14410 // Now we have:
14411 // (or (srl (smul_lohi ?, ?), 16), (shl (smul_lohi ?, ?), 16)))
14412 // For SMUL[B|T] smul_lohi will take a 32-bit and a 16-bit arguments.
14413 // For SMUWB the 16-bit value will signed extended somehow.
14414 // For SMULWT only the SRA is required.
14415 // Check both sides of SMUL_LOHI
14416 SDValue OpS16 = SMULLOHI->getOperand(Num: 0);
14417 SDValue OpS32 = SMULLOHI->getOperand(Num: 1);
14418
14419 SelectionDAG &DAG = DCI.DAG;
14420 if (!isS16(Op: OpS16, DAG) && !isSRA16(Op: OpS16)) {
14421 OpS16 = OpS32;
14422 OpS32 = SMULLOHI->getOperand(Num: 0);
14423 }
14424
14425 SDLoc dl(OR);
14426 unsigned Opcode = 0;
14427 if (isS16(Op: OpS16, DAG))
14428 Opcode = ARMISD::SMULWB;
14429 else if (isSRA16(Op: OpS16)) {
14430 Opcode = ARMISD::SMULWT;
14431 OpS16 = OpS16->getOperand(Num: 0);
14432 }
14433 else
14434 return SDValue();
14435
14436 SDValue Res = DAG.getNode(Opcode, DL: dl, VT: MVT::i32, N1: OpS32, N2: OpS16);
14437 DAG.ReplaceAllUsesOfValueWith(From: SDValue(OR, 0), To: Res);
14438 return SDValue(OR, 0);
14439}
14440
14441static SDValue PerformORCombineToBFI(SDNode *N,
14442 TargetLowering::DAGCombinerInfo &DCI,
14443 const ARMSubtarget *Subtarget) {
14444 // BFI is only available on V6T2+
14445 if (Subtarget->isThumb1Only() || !Subtarget->hasV6T2Ops())
14446 return SDValue();
14447
14448 EVT VT = N->getValueType(ResNo: 0);
14449 SDValue N0 = N->getOperand(Num: 0);
14450 SDValue N1 = N->getOperand(Num: 1);
14451 SelectionDAG &DAG = DCI.DAG;
14452 SDLoc DL(N);
14453 // 1) or (and A, mask), val => ARMbfi A, val, mask
14454 // iff (val & mask) == val
14455 //
14456 // 2) or (and A, mask), (and B, mask2) => ARMbfi A, (lsr B, amt), mask
14457 // 2a) iff isBitFieldInvertedMask(mask) && isBitFieldInvertedMask(~mask2)
14458 // && mask == ~mask2
14459 // 2b) iff isBitFieldInvertedMask(~mask) && isBitFieldInvertedMask(mask2)
14460 // && ~mask == mask2
14461 // (i.e., copy a bitfield value into another bitfield of the same width)
14462
14463 if (VT != MVT::i32)
14464 return SDValue();
14465
14466 SDValue N00 = N0.getOperand(i: 0);
14467
14468 // The value and the mask need to be constants so we can verify this is
14469 // actually a bitfield set. If the mask is 0xffff, we can do better
14470 // via a movt instruction, so don't use BFI in that case.
14471 SDValue MaskOp = N0.getOperand(i: 1);
14472 ConstantSDNode *MaskC = dyn_cast<ConstantSDNode>(Val&: MaskOp);
14473 if (!MaskC)
14474 return SDValue();
14475 unsigned Mask = MaskC->getZExtValue();
14476 if (Mask == 0xffff)
14477 return SDValue();
14478 SDValue Res;
14479 // Case (1): or (and A, mask), val => ARMbfi A, val, mask
14480 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(Val&: N1);
14481 if (N1C) {
14482 unsigned Val = N1C->getZExtValue();
14483 if ((Val & ~Mask) != Val)
14484 return SDValue();
14485
14486 if (ARM::isBitFieldInvertedMask(v: Mask)) {
14487 Val >>= llvm::countr_zero(Val: ~Mask);
14488
14489 Res = DAG.getNode(Opcode: ARMISD::BFI, DL, VT, N1: N00,
14490 N2: DAG.getConstant(Val, DL, VT: MVT::i32),
14491 N3: DAG.getConstant(Val: Mask, DL, VT: MVT::i32));
14492
14493 DCI.CombineTo(N, Res, AddTo: false);
14494 // Return value from the original node to inform the combiner than N is
14495 // now dead.
14496 return SDValue(N, 0);
14497 }
14498 } else if (N1.getOpcode() == ISD::AND) {
14499 // case (2) or (and A, mask), (and B, mask2) => ARMbfi A, (lsr B, amt), mask
14500 ConstantSDNode *N11C = dyn_cast<ConstantSDNode>(Val: N1.getOperand(i: 1));
14501 if (!N11C)
14502 return SDValue();
14503 unsigned Mask2 = N11C->getZExtValue();
14504
14505 // Mask and ~Mask2 (or reverse) must be equivalent for the BFI pattern
14506 // as is to match.
14507 if (ARM::isBitFieldInvertedMask(v: Mask) &&
14508 (Mask == ~Mask2)) {
14509 // The pack halfword instruction works better for masks that fit it,
14510 // so use that when it's available.
14511 if (Subtarget->hasDSP() &&
14512 (Mask == 0xffff || Mask == 0xffff0000))
14513 return SDValue();
14514 // 2a
14515 unsigned amt = llvm::countr_zero(Val: Mask2);
14516 Res = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: N1.getOperand(i: 0),
14517 N2: DAG.getConstant(Val: amt, DL, VT: MVT::i32));
14518 Res = DAG.getNode(Opcode: ARMISD::BFI, DL, VT, N1: N00, N2: Res,
14519 N3: DAG.getConstant(Val: Mask, DL, VT: MVT::i32));
14520 DCI.CombineTo(N, Res, AddTo: false);
14521 // Return value from the original node to inform the combiner than N is
14522 // now dead.
14523 return SDValue(N, 0);
14524 } else if (ARM::isBitFieldInvertedMask(v: ~Mask) &&
14525 (~Mask == Mask2)) {
14526 // The pack halfword instruction works better for masks that fit it,
14527 // so use that when it's available.
14528 if (Subtarget->hasDSP() &&
14529 (Mask2 == 0xffff || Mask2 == 0xffff0000))
14530 return SDValue();
14531 // 2b
14532 unsigned lsb = llvm::countr_zero(Val: Mask);
14533 Res = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: N00,
14534 N2: DAG.getConstant(Val: lsb, DL, VT: MVT::i32));
14535 Res = DAG.getNode(Opcode: ARMISD::BFI, DL, VT, N1: N1.getOperand(i: 0), N2: Res,
14536 N3: DAG.getConstant(Val: Mask2, DL, VT: MVT::i32));
14537 DCI.CombineTo(N, Res, AddTo: false);
14538 // Return value from the original node to inform the combiner than N is
14539 // now dead.
14540 return SDValue(N, 0);
14541 }
14542 }
14543
14544 if (DAG.MaskedValueIsZero(Op: N1, Mask: MaskC->getAPIntValue()) &&
14545 N00.getOpcode() == ISD::SHL && isa<ConstantSDNode>(Val: N00.getOperand(i: 1)) &&
14546 ARM::isBitFieldInvertedMask(v: ~Mask)) {
14547 // Case (3): or (and (shl A, #shamt), mask), B => ARMbfi B, A, ~mask
14548 // where lsb(mask) == #shamt and masked bits of B are known zero.
14549 SDValue ShAmt = N00.getOperand(i: 1);
14550 unsigned ShAmtC = ShAmt->getAsZExtVal();
14551 unsigned LSB = llvm::countr_zero(Val: Mask);
14552 if (ShAmtC != LSB)
14553 return SDValue();
14554
14555 Res = DAG.getNode(Opcode: ARMISD::BFI, DL, VT, N1, N2: N00.getOperand(i: 0),
14556 N3: DAG.getConstant(Val: ~Mask, DL, VT: MVT::i32));
14557
14558 DCI.CombineTo(N, Res, AddTo: false);
14559 // Return value from the original node to inform the combiner than N is
14560 // now dead.
14561 return SDValue(N, 0);
14562 }
14563
14564 return SDValue();
14565}
14566
14567static bool isValidMVECond(unsigned CC, bool IsFloat) {
14568 switch (CC) {
14569 case ARMCC::EQ:
14570 case ARMCC::NE:
14571 case ARMCC::LE:
14572 case ARMCC::GT:
14573 case ARMCC::GE:
14574 case ARMCC::LT:
14575 return true;
14576 case ARMCC::HS:
14577 case ARMCC::HI:
14578 return !IsFloat;
14579 default:
14580 return false;
14581 };
14582}
14583
14584static ARMCC::CondCodes getVCMPCondCode(SDValue N) {
14585 if (N->getOpcode() == ARMISD::VCMP)
14586 return (ARMCC::CondCodes)N->getConstantOperandVal(Num: 2);
14587 else if (N->getOpcode() == ARMISD::VCMPZ)
14588 return (ARMCC::CondCodes)N->getConstantOperandVal(Num: 1);
14589 else
14590 llvm_unreachable("Not a VCMP/VCMPZ!");
14591}
14592
14593static bool CanInvertMVEVCMP(SDValue N) {
14594 ARMCC::CondCodes CC = ARMCC::getOppositeCondition(CC: getVCMPCondCode(N));
14595 return isValidMVECond(CC, IsFloat: N->getOperand(Num: 0).getValueType().isFloatingPoint());
14596}
14597
14598static SDValue PerformORCombine_i1(SDNode *N, SelectionDAG &DAG,
14599 const ARMSubtarget *Subtarget) {
14600 // Try to invert "or A, B" -> "and ~A, ~B", as the "and" is easier to chain
14601 // together with predicates
14602 EVT VT = N->getValueType(ResNo: 0);
14603 SDLoc DL(N);
14604 SDValue N0 = N->getOperand(Num: 0);
14605 SDValue N1 = N->getOperand(Num: 1);
14606
14607 auto IsFreelyInvertable = [&](SDValue V) {
14608 if (V->getOpcode() == ARMISD::VCMP || V->getOpcode() == ARMISD::VCMPZ)
14609 return CanInvertMVEVCMP(N: V);
14610 return false;
14611 };
14612
14613 // At least one operand must be freely invertable.
14614 if (!(IsFreelyInvertable(N0) || IsFreelyInvertable(N1)))
14615 return SDValue();
14616
14617 SDValue NewN0 = DAG.getLogicalNOT(DL, Val: N0, VT);
14618 SDValue NewN1 = DAG.getLogicalNOT(DL, Val: N1, VT);
14619 SDValue And = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: NewN0, N2: NewN1);
14620 return DAG.getLogicalNOT(DL, Val: And, VT);
14621}
14622
14623// Try to form a NEON shift-{right, left}-and-insert (VSRI/VSLI) from:
14624// (or (and X, splat (i32 C1)), (srl Y, splat (i32 C2))) -> VSRI X, Y, #C2
14625// (or (and X, splat (i32 C1)), (shl Y, splat (i32 C2))) -> VSLI X, Y, #C2
14626// where C1 is a mask that preserves the bits not written by the shift/insert,
14627// i.e. `C1 == (1 << C2) - 1`.
14628static SDValue PerformORCombineToShiftInsert(SelectionDAG &DAG, SDValue AndOp,
14629 SDValue ShiftOp, EVT VT,
14630 SDLoc dl) {
14631 // Match (and X, Mask)
14632 if (AndOp.getOpcode() != ISD::AND)
14633 return SDValue();
14634
14635 SDValue X = AndOp.getOperand(i: 0);
14636 SDValue Mask = AndOp.getOperand(i: 1);
14637
14638 ConstantSDNode *MaskC = isConstOrConstSplat(N: Mask, AllowUndefs: false, AllowTruncation: true);
14639 if (!MaskC)
14640 return SDValue();
14641 APInt MaskBits =
14642 MaskC->getAPIntValue().trunc(width: Mask.getScalarValueSizeInBits());
14643
14644 // Match shift (srl/shl Y, CntVec)
14645 int64_t Cnt = 0;
14646 bool IsShiftRight = false;
14647 SDValue Y;
14648
14649 if (ShiftOp.getOpcode() == ARMISD::VSHRuIMM) {
14650 IsShiftRight = true;
14651 Y = ShiftOp.getOperand(i: 0);
14652 Cnt = ShiftOp.getConstantOperandVal(i: 1);
14653 } else if (ShiftOp.getOpcode() == ARMISD::VSHLIMM) {
14654 Y = ShiftOp.getOperand(i: 0);
14655 Cnt = ShiftOp.getConstantOperandVal(i: 1);
14656 } else {
14657 return SDValue();
14658 }
14659
14660 unsigned ElemBits = VT.getScalarSizeInBits();
14661 APInt RequiredMask = IsShiftRight
14662 ? APInt::getHighBitsSet(numBits: ElemBits, hiBitsSet: (unsigned)Cnt)
14663 : APInt::getLowBitsSet(numBits: ElemBits, loBitsSet: (unsigned)Cnt);
14664 if (MaskBits != RequiredMask)
14665 return SDValue();
14666
14667 unsigned Opc = IsShiftRight ? ARMISD::VSRIIMM : ARMISD::VSLIIMM;
14668 return DAG.getNode(Opcode: Opc, DL: dl, VT, N1: X, N2: Y, N3: DAG.getConstant(Val: Cnt, DL: dl, VT: MVT::i32));
14669}
14670
14671/// PerformORCombine - Target-specific dag combine xforms for ISD::OR
14672static SDValue PerformORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
14673 const ARMSubtarget *Subtarget) {
14674 // Attempt to use immediate-form VORR
14675 BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(Val: N->getOperand(Num: 1));
14676 SDLoc dl(N);
14677 EVT VT = N->getValueType(ResNo: 0);
14678 SelectionDAG &DAG = DCI.DAG;
14679
14680 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
14681 return SDValue();
14682
14683 if (Subtarget->hasMVEIntegerOps() && (VT == MVT::v2i1 || VT == MVT::v4i1 ||
14684 VT == MVT::v8i1 || VT == MVT::v16i1))
14685 return PerformORCombine_i1(N, DAG, Subtarget);
14686
14687 APInt SplatBits, SplatUndef;
14688 unsigned SplatBitSize;
14689 bool HasAnyUndefs;
14690 if (BVN && (Subtarget->hasNEON() || Subtarget->hasMVEIntegerOps()) &&
14691 BVN->isConstantSplat(SplatValue&: SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
14692 if (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32 ||
14693 SplatBitSize == 64) {
14694 EVT VorrVT;
14695 SDValue Val =
14696 isVMOVModifiedImm(SplatBits: SplatBits.getZExtValue(), SplatUndef: SplatUndef.getZExtValue(),
14697 SplatBitSize, DAG, dl, VT&: VorrVT, VectorVT: VT, type: OtherModImm);
14698 if (Val.getNode()) {
14699 SDValue Input =
14700 DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT: VorrVT, Operand: N->getOperand(Num: 0));
14701 SDValue Vorr = DAG.getNode(Opcode: ARMISD::VORRIMM, DL: dl, VT: VorrVT, N1: Input, N2: Val);
14702 return DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT, Operand: Vorr);
14703 }
14704 }
14705 }
14706
14707 if (!Subtarget->isThumb1Only()) {
14708 // fold (or (select cc, 0, c), x) -> (select cc, x, (or, x, c))
14709 if (SDValue Result = combineSelectAndUseCommutative(N, AllOnes: false, DCI))
14710 return Result;
14711 if (SDValue Result = PerformORCombineToSMULWBT(OR: N, DCI, Subtarget))
14712 return Result;
14713 }
14714
14715 SDValue N0 = N->getOperand(Num: 0);
14716 SDValue N1 = N->getOperand(Num: 1);
14717
14718 // (or (and X, C1), (srl Y, C2)) -> VSRI X, Y, #C2
14719 // (or (and X, C1), (shl Y, C2)) -> VSLI X, Y, #C2
14720 if (VT.isVector() &&
14721 ((Subtarget->hasNEON() && DAG.getTargetLoweringInfo().isTypeLegal(VT)) ||
14722 (Subtarget->hasMVEIntegerOps() &&
14723 (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32)))) {
14724 if (SDValue ShiftInsert =
14725 PerformORCombineToShiftInsert(DAG, AndOp: N0, ShiftOp: N1, VT, dl))
14726 return ShiftInsert;
14727
14728 if (SDValue ShiftInsert =
14729 PerformORCombineToShiftInsert(DAG, AndOp: N1, ShiftOp: N0, VT, dl))
14730 return ShiftInsert;
14731 }
14732
14733 // (or (and B, A), (and C, ~A)) => (VBSL A, B, C) when A is a constant.
14734 if (Subtarget->hasNEON() && N1.getOpcode() == ISD::AND && VT.isVector() &&
14735 DAG.getTargetLoweringInfo().isTypeLegal(VT)) {
14736
14737 // The code below optimizes (or (and X, Y), Z).
14738 // The AND operand needs to have a single user to make these optimizations
14739 // profitable.
14740 if (N0.getOpcode() != ISD::AND || !N0.hasOneUse())
14741 return SDValue();
14742
14743 APInt SplatUndef;
14744 unsigned SplatBitSize;
14745 bool HasAnyUndefs;
14746
14747 APInt SplatBits0, SplatBits1;
14748 BuildVectorSDNode *BVN0 = dyn_cast<BuildVectorSDNode>(Val: N0->getOperand(Num: 1));
14749 BuildVectorSDNode *BVN1 = dyn_cast<BuildVectorSDNode>(Val: N1->getOperand(Num: 1));
14750 // Ensure that the second operand of both ands are constants
14751 if (BVN0 && BVN0->isConstantSplat(SplatValue&: SplatBits0, SplatUndef, SplatBitSize,
14752 HasAnyUndefs) && !HasAnyUndefs) {
14753 if (BVN1 && BVN1->isConstantSplat(SplatValue&: SplatBits1, SplatUndef, SplatBitSize,
14754 HasAnyUndefs) && !HasAnyUndefs) {
14755 // Ensure that the bit width of the constants are the same and that
14756 // the splat arguments are logical inverses as per the pattern we
14757 // are trying to simplify.
14758 if (SplatBits0.getBitWidth() == SplatBits1.getBitWidth() &&
14759 SplatBits0 == ~SplatBits1) {
14760 // Canonicalize the vector type to make instruction selection
14761 // simpler.
14762 EVT CanonicalVT = VT.is128BitVector() ? MVT::v4i32 : MVT::v2i32;
14763 SDValue Mask = DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl,
14764 VT: CanonicalVT, Operand: N0->getOperand(Num: 1));
14765 SDValue LHS = DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl,
14766 VT: CanonicalVT, Operand: N0->getOperand(Num: 0));
14767 SDValue RHS = DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl,
14768 VT: CanonicalVT, Operand: N1->getOperand(Num: 0));
14769 SDValue Result =
14770 DAG.getNode(Opcode: ARMISD::VBSP, DL: dl, VT: CanonicalVT, N1: Mask, N2: LHS, N3: RHS);
14771 return DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT, Operand: Result);
14772 }
14773 }
14774 }
14775 }
14776
14777 // Try to use the ARM/Thumb2 BFI (bitfield insert) instruction when
14778 // reasonable.
14779 if (N0.getOpcode() == ISD::AND && N0.hasOneUse()) {
14780 if (SDValue Res = PerformORCombineToBFI(N, DCI, Subtarget))
14781 return Res;
14782 }
14783
14784 if (SDValue Result = PerformSHLSimplify(N, DCI, ST: Subtarget))
14785 return Result;
14786
14787 // (or x, (csinc 0, 0, cc)) -> (csinc x, 0, cc)
14788 // providing that the x is 0 or 1.
14789 SDValue CSINC = N1;
14790 SDValue Other = N0;
14791 if (CSINC.getOpcode() != ARMISD::CSINC)
14792 std::swap(a&: CSINC, b&: Other);
14793 if (CSINC.getOpcode() == ARMISD::CSINC &&
14794 isNullConstant(V: CSINC.getOperand(i: 0)) &&
14795 isNullConstant(V: CSINC.getOperand(i: 1)) &&
14796 DAG.MaskedValueIsZero(Op: Other, Mask: APInt::getHighBitsSet(numBits: 32, hiBitsSet: 31)))
14797 return DAG.getNode(Opcode: ARMISD::CSINC, DL: dl, VT, N1: Other, N2: CSINC.getOperand(i: 1),
14798 N3: CSINC.getOperand(i: 2), N4: CSINC.getOperand(i: 3));
14799
14800 return SDValue();
14801}
14802
14803static SDValue PerformXORCombine(SDNode *N,
14804 TargetLowering::DAGCombinerInfo &DCI,
14805 const ARMSubtarget *Subtarget) {
14806 EVT VT = N->getValueType(ResNo: 0);
14807 SelectionDAG &DAG = DCI.DAG;
14808
14809 if(!DAG.getTargetLoweringInfo().isTypeLegal(VT))
14810 return SDValue();
14811
14812 if (!Subtarget->isThumb1Only()) {
14813 // fold (xor (select cc, 0, c), x) -> (select cc, x, (xor, x, c))
14814 if (SDValue Result = combineSelectAndUseCommutative(N, AllOnes: false, DCI))
14815 return Result;
14816
14817 if (SDValue Result = PerformSHLSimplify(N, DCI, ST: Subtarget))
14818 return Result;
14819 }
14820
14821 if (Subtarget->hasMVEIntegerOps()) {
14822 // fold (xor(vcmp/z, 1)) into a vcmp with the opposite condition.
14823 SDValue N0 = N->getOperand(Num: 0);
14824 SDValue N1 = N->getOperand(Num: 1);
14825 const TargetLowering *TLI = Subtarget->getTargetLowering();
14826 if (TLI->isConstTrueVal(N: N1) &&
14827 (N0->getOpcode() == ARMISD::VCMP || N0->getOpcode() == ARMISD::VCMPZ)) {
14828 if (CanInvertMVEVCMP(N: N0)) {
14829 SDLoc DL(N0);
14830 ARMCC::CondCodes CC = ARMCC::getOppositeCondition(CC: getVCMPCondCode(N: N0));
14831
14832 SmallVector<SDValue, 4> Ops;
14833 Ops.push_back(Elt: N0->getOperand(Num: 0));
14834 if (N0->getOpcode() == ARMISD::VCMP)
14835 Ops.push_back(Elt: N0->getOperand(Num: 1));
14836 Ops.push_back(Elt: DAG.getConstant(Val: CC, DL, VT: MVT::i32));
14837 return DAG.getNode(Opcode: N0->getOpcode(), DL, VT: N0->getValueType(ResNo: 0), Ops);
14838 }
14839 }
14840 }
14841
14842 return SDValue();
14843}
14844
14845// ParseBFI - given a BFI instruction in N, extract the "from" value (Rn) and return it,
14846// and fill in FromMask and ToMask with (consecutive) bits in "from" to be extracted and
14847// their position in "to" (Rd).
14848static SDValue ParseBFI(SDNode *N, APInt &ToMask, APInt &FromMask) {
14849 assert(N->getOpcode() == ARMISD::BFI);
14850
14851 SDValue From = N->getOperand(Num: 1);
14852 ToMask = ~N->getConstantOperandAPInt(Num: 2);
14853 FromMask = APInt::getLowBitsSet(numBits: ToMask.getBitWidth(), loBitsSet: ToMask.popcount());
14854
14855 // If the Base came from a SHR #C, we can deduce that it is really testing bit
14856 // #C in the base of the SHR.
14857 if (From->getOpcode() == ISD::SRL &&
14858 isa<ConstantSDNode>(Val: From->getOperand(Num: 1))) {
14859 APInt Shift = From->getConstantOperandAPInt(Num: 1);
14860 assert(Shift.getLimitedValue() < 32 && "Shift too large!");
14861 FromMask <<= Shift.getLimitedValue(Limit: 31);
14862 From = From->getOperand(Num: 0);
14863 }
14864
14865 return From;
14866}
14867
14868// If A and B contain one contiguous set of bits, does A | B == A . B?
14869//
14870// Neither A nor B must be zero.
14871static bool BitsProperlyConcatenate(const APInt &A, const APInt &B) {
14872 unsigned LastActiveBitInA = A.countr_zero();
14873 unsigned FirstActiveBitInB = B.getBitWidth() - B.countl_zero() - 1;
14874 return LastActiveBitInA - 1 == FirstActiveBitInB;
14875}
14876
14877static SDValue FindBFIToCombineWith(SDNode *N) {
14878 // We have a BFI in N. Find a BFI it can combine with, if one exists.
14879 APInt ToMask, FromMask;
14880 SDValue From = ParseBFI(N, ToMask, FromMask);
14881 SDValue To = N->getOperand(Num: 0);
14882
14883 SDValue V = To;
14884 if (V.getOpcode() != ARMISD::BFI)
14885 return SDValue();
14886
14887 APInt NewToMask, NewFromMask;
14888 SDValue NewFrom = ParseBFI(N: V.getNode(), ToMask&: NewToMask, FromMask&: NewFromMask);
14889 if (NewFrom != From)
14890 return SDValue();
14891
14892 // Do the written bits conflict with any we've seen so far?
14893 if ((NewToMask & ToMask).getBoolValue())
14894 // Conflicting bits.
14895 return SDValue();
14896
14897 // Are the new bits contiguous when combined with the old bits?
14898 if (BitsProperlyConcatenate(A: ToMask, B: NewToMask) &&
14899 BitsProperlyConcatenate(A: FromMask, B: NewFromMask))
14900 return V;
14901 if (BitsProperlyConcatenate(A: NewToMask, B: ToMask) &&
14902 BitsProperlyConcatenate(A: NewFromMask, B: FromMask))
14903 return V;
14904
14905 return SDValue();
14906}
14907
14908static SDValue PerformBFICombine(SDNode *N, SelectionDAG &DAG) {
14909 SDValue N0 = N->getOperand(Num: 0);
14910 SDValue N1 = N->getOperand(Num: 1);
14911
14912 if (N1.getOpcode() == ISD::AND) {
14913 // (bfi A, (and B, Mask1), Mask2) -> (bfi A, B, Mask2) iff
14914 // the bits being cleared by the AND are not demanded by the BFI.
14915 ConstantSDNode *N11C = dyn_cast<ConstantSDNode>(Val: N1.getOperand(i: 1));
14916 if (!N11C)
14917 return SDValue();
14918 unsigned InvMask = N->getConstantOperandVal(Num: 2);
14919 unsigned LSB = llvm::countr_zero(Val: ~InvMask);
14920 unsigned Width = llvm::bit_width<unsigned>(Value: ~InvMask) - LSB;
14921 assert(Width <
14922 static_cast<unsigned>(std::numeric_limits<unsigned>::digits) &&
14923 "undefined behavior");
14924 unsigned Mask = (1u << Width) - 1;
14925 unsigned Mask2 = N11C->getZExtValue();
14926 if ((Mask & (~Mask2)) == 0)
14927 return DAG.getNode(Opcode: ARMISD::BFI, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
14928 N1: N->getOperand(Num: 0), N2: N1.getOperand(i: 0), N3: N->getOperand(Num: 2));
14929 return SDValue();
14930 }
14931
14932 // Look for another BFI to combine with.
14933 if (SDValue CombineBFI = FindBFIToCombineWith(N)) {
14934 // We've found a BFI.
14935 APInt ToMask1, FromMask1;
14936 SDValue From1 = ParseBFI(N, ToMask&: ToMask1, FromMask&: FromMask1);
14937
14938 APInt ToMask2, FromMask2;
14939 SDValue From2 = ParseBFI(N: CombineBFI.getNode(), ToMask&: ToMask2, FromMask&: FromMask2);
14940 assert(From1 == From2);
14941 (void)From2;
14942
14943 // Create a new BFI, combining the two together.
14944 APInt NewFromMask = FromMask1 | FromMask2;
14945 APInt NewToMask = ToMask1 | ToMask2;
14946
14947 EVT VT = N->getValueType(ResNo: 0);
14948 SDLoc dl(N);
14949
14950 if (NewFromMask[0] == 0)
14951 From1 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: From1,
14952 N2: DAG.getConstant(Val: NewFromMask.countr_zero(), DL: dl, VT));
14953 return DAG.getNode(Opcode: ARMISD::BFI, DL: dl, VT, N1: CombineBFI.getOperand(i: 0), N2: From1,
14954 N3: DAG.getConstant(Val: ~NewToMask, DL: dl, VT));
14955 }
14956
14957 // Reassociate BFI(BFI (A, B, M1), C, M2) to BFI(BFI (A, C, M2), B, M1) so
14958 // that lower bit insertions are performed first, providing that M1 and M2
14959 // do no overlap. This can allow multiple BFI instructions to be combined
14960 // together by the other folds above.
14961 if (N->getOperand(Num: 0).getOpcode() == ARMISD::BFI) {
14962 APInt ToMask1 = ~N->getConstantOperandAPInt(Num: 2);
14963 APInt ToMask2 = ~N0.getConstantOperandAPInt(i: 2);
14964
14965 if (!N0.hasOneUse() || (ToMask1 & ToMask2) != 0 ||
14966 ToMask1.countl_zero() < ToMask2.countl_zero())
14967 return SDValue();
14968
14969 EVT VT = N->getValueType(ResNo: 0);
14970 SDLoc dl(N);
14971 SDValue BFI1 = DAG.getNode(Opcode: ARMISD::BFI, DL: dl, VT, N1: N0.getOperand(i: 0),
14972 N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2));
14973 return DAG.getNode(Opcode: ARMISD::BFI, DL: dl, VT, N1: BFI1, N2: N0.getOperand(i: 1),
14974 N3: N0.getOperand(i: 2));
14975 }
14976
14977 return SDValue();
14978}
14979
14980// Check that N is CMPZ(CSINC(0, 0, CC, X)),
14981// or CMPZ(CMOV(1, 0, CC, X))
14982// return X if valid.
14983static SDValue IsCMPZCSINC(SDNode *Cmp, ARMCC::CondCodes &CC) {
14984 if (Cmp->getOpcode() != ARMISD::CMPZ || !isNullConstant(V: Cmp->getOperand(Num: 1)))
14985 return SDValue();
14986 SDValue CSInc = Cmp->getOperand(Num: 0);
14987
14988 // Ignore any `And 1` nodes that may not yet have been removed. We are
14989 // looking for a value that produces 1/0, so these have no effect on the
14990 // code.
14991 while (CSInc.getOpcode() == ISD::AND &&
14992 isa<ConstantSDNode>(Val: CSInc.getOperand(i: 1)) &&
14993 CSInc.getConstantOperandVal(i: 1) == 1 && CSInc->hasOneUse())
14994 CSInc = CSInc.getOperand(i: 0);
14995
14996 if (CSInc.getOpcode() == ARMISD::CSINC &&
14997 isNullConstant(V: CSInc.getOperand(i: 0)) &&
14998 isNullConstant(V: CSInc.getOperand(i: 1)) && CSInc->hasOneUse()) {
14999 CC = (ARMCC::CondCodes)CSInc.getConstantOperandVal(i: 2);
15000 return CSInc.getOperand(i: 3);
15001 }
15002 if (CSInc.getOpcode() == ARMISD::CMOV && isOneConstant(V: CSInc.getOperand(i: 0)) &&
15003 isNullConstant(V: CSInc.getOperand(i: 1)) && CSInc->hasOneUse()) {
15004 CC = (ARMCC::CondCodes)CSInc.getConstantOperandVal(i: 2);
15005 return CSInc.getOperand(i: 3);
15006 }
15007 if (CSInc.getOpcode() == ARMISD::CMOV && isOneConstant(V: CSInc.getOperand(i: 1)) &&
15008 isNullConstant(V: CSInc.getOperand(i: 0)) && CSInc->hasOneUse()) {
15009 CC = ARMCC::getOppositeCondition(
15010 CC: (ARMCC::CondCodes)CSInc.getConstantOperandVal(i: 2));
15011 return CSInc.getOperand(i: 3);
15012 }
15013 return SDValue();
15014}
15015
15016static SDValue PerformCMPZCombine(SDNode *N, SelectionDAG &DAG) {
15017 // Given CMPZ(CSINC(C, 0, 0, EQ), 0), we can just use C directly. As in
15018 // t92: flags = ARMISD::CMPZ t74, 0
15019 // t93: i32 = ARMISD::CSINC 0, 0, 1, t92
15020 // t96: flags = ARMISD::CMPZ t93, 0
15021 // t114: i32 = ARMISD::CSINV 0, 0, 0, t96
15022 ARMCC::CondCodes Cond;
15023 if (SDValue C = IsCMPZCSINC(Cmp: N, CC&: Cond))
15024 if (Cond == ARMCC::EQ)
15025 return C;
15026 return SDValue();
15027}
15028
15029static SDValue PerformCSETCombine(SDNode *N, SelectionDAG &DAG) {
15030 // Fold away an unnecessary CMPZ/CSINC
15031 // CSXYZ A, B, C1 (CMPZ (CSINC 0, 0, C2, D), 0) ->
15032 // if C1==EQ -> CSXYZ A, B, C2, D
15033 // if C1==NE -> CSXYZ A, B, NOT(C2), D
15034 ARMCC::CondCodes Cond;
15035 if (SDValue C = IsCMPZCSINC(Cmp: N->getOperand(Num: 3).getNode(), CC&: Cond)) {
15036 if (N->getConstantOperandVal(Num: 2) == ARMCC::EQ)
15037 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: MVT::i32, N1: N->getOperand(Num: 0),
15038 N2: N->getOperand(Num: 1),
15039 N3: DAG.getConstant(Val: Cond, DL: SDLoc(N), VT: MVT::i32), N4: C);
15040 if (N->getConstantOperandVal(Num: 2) == ARMCC::NE)
15041 return DAG.getNode(
15042 Opcode: N->getOpcode(), DL: SDLoc(N), VT: MVT::i32, N1: N->getOperand(Num: 0),
15043 N2: N->getOperand(Num: 1),
15044 N3: DAG.getConstant(Val: ARMCC::getOppositeCondition(CC: Cond), DL: SDLoc(N), VT: MVT::i32), N4: C);
15045 }
15046 return SDValue();
15047}
15048
15049/// PerformVMOVRRDCombine - Target-specific dag combine xforms for
15050/// ARMISD::VMOVRRD.
15051static SDValue PerformVMOVRRDCombine(SDNode *N,
15052 TargetLowering::DAGCombinerInfo &DCI,
15053 const ARMSubtarget *Subtarget) {
15054 // vmovrrd(vmovdrr x, y) -> x,y
15055 SDValue InDouble = N->getOperand(Num: 0);
15056 if (InDouble.getOpcode() == ARMISD::VMOVDRR && Subtarget->hasFP64())
15057 return DCI.CombineTo(N, Res0: InDouble.getOperand(i: 0), Res1: InDouble.getOperand(i: 1));
15058
15059 // vmovrrd(load f64) -> (load i32), (load i32)
15060 SDNode *InNode = InDouble.getNode();
15061 if (ISD::isNormalLoad(N: InNode) && InNode->hasOneUse() &&
15062 InNode->getValueType(ResNo: 0) == MVT::f64 &&
15063 InNode->getOperand(Num: 1).getOpcode() == ISD::FrameIndex &&
15064 !cast<LoadSDNode>(Val: InNode)->isVolatile()) {
15065 // TODO: Should this be done for non-FrameIndex operands?
15066 LoadSDNode *LD = cast<LoadSDNode>(Val: InNode);
15067
15068 SelectionDAG &DAG = DCI.DAG;
15069 SDLoc DL(LD);
15070 SDValue BasePtr = LD->getBasePtr();
15071 SDValue NewLD1 =
15072 DAG.getLoad(VT: MVT::i32, dl: DL, Chain: LD->getChain(), Ptr: BasePtr, PtrInfo: LD->getPointerInfo(),
15073 Alignment: LD->getAlign(), MMOFlags: LD->getMemOperand()->getFlags());
15074
15075 SDValue OffsetPtr = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i32, N1: BasePtr,
15076 N2: DAG.getConstant(Val: 4, DL, VT: MVT::i32));
15077
15078 SDValue NewLD2 = DAG.getLoad(VT: MVT::i32, dl: DL, Chain: LD->getChain(), Ptr: OffsetPtr,
15079 PtrInfo: LD->getPointerInfo().getWithOffset(O: 4),
15080 Alignment: commonAlignment(A: LD->getAlign(), Offset: 4),
15081 MMOFlags: LD->getMemOperand()->getFlags());
15082
15083 DAG.ReplaceAllUsesOfValueWith(From: SDValue(LD, 1), To: NewLD2.getValue(R: 1));
15084 if (DCI.DAG.getDataLayout().isBigEndian())
15085 std::swap (a&: NewLD1, b&: NewLD2);
15086 SDValue Result = DCI.CombineTo(N, Res0: NewLD1, Res1: NewLD2);
15087 return Result;
15088 }
15089
15090 // VMOVRRD(extract(..(build_vector(a, b, c, d)))) -> a,b or c,d
15091 // VMOVRRD(extract(insert_vector(insert_vector(.., a, l1), b, l2))) -> a,b
15092 if (InDouble.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
15093 isa<ConstantSDNode>(Val: InDouble.getOperand(i: 1))) {
15094 SDValue BV = InDouble.getOperand(i: 0);
15095 // Look up through any nop bitcasts and vector_reg_casts. bitcasts may
15096 // change lane order under big endian.
15097 bool BVSwap = BV.getOpcode() == ISD::BITCAST;
15098 while (
15099 (BV.getOpcode() == ISD::BITCAST ||
15100 BV.getOpcode() == ARMISD::VECTOR_REG_CAST) &&
15101 (BV.getValueType() == MVT::v2f64 || BV.getValueType() == MVT::v2i64)) {
15102 BVSwap = BV.getOpcode() == ISD::BITCAST;
15103 BV = BV.getOperand(i: 0);
15104 }
15105 if (BV.getValueType() != MVT::v4i32)
15106 return SDValue();
15107
15108 // Handle buildvectors, pulling out the correct lane depending on
15109 // endianness.
15110 unsigned Offset = InDouble.getConstantOperandVal(i: 1) == 1 ? 2 : 0;
15111 if (BV.getOpcode() == ISD::BUILD_VECTOR) {
15112 SDValue Op0 = BV.getOperand(i: Offset);
15113 SDValue Op1 = BV.getOperand(i: Offset + 1);
15114 if (!Subtarget->isLittle() && BVSwap)
15115 std::swap(a&: Op0, b&: Op1);
15116
15117 return DCI.DAG.getMergeValues(Ops: {Op0, Op1}, dl: SDLoc(N));
15118 }
15119
15120 // A chain of insert_vectors, grabbing the correct value of the chain of
15121 // inserts.
15122 SDValue Op0, Op1;
15123 while (BV.getOpcode() == ISD::INSERT_VECTOR_ELT) {
15124 if (isa<ConstantSDNode>(Val: BV.getOperand(i: 2))) {
15125 if (BV.getConstantOperandVal(i: 2) == Offset && !Op0)
15126 Op0 = BV.getOperand(i: 1);
15127 if (BV.getConstantOperandVal(i: 2) == Offset + 1 && !Op1)
15128 Op1 = BV.getOperand(i: 1);
15129 }
15130 BV = BV.getOperand(i: 0);
15131 }
15132 if (!Subtarget->isLittle() && BVSwap)
15133 std::swap(a&: Op0, b&: Op1);
15134 if (Op0 && Op1)
15135 return DCI.DAG.getMergeValues(Ops: {Op0, Op1}, dl: SDLoc(N));
15136 }
15137
15138 return SDValue();
15139}
15140
15141/// PerformVMOVDRRCombine - Target-specific dag combine xforms for
15142/// ARMISD::VMOVDRR. This is also used for BUILD_VECTORs with 2 operands.
15143static SDValue PerformVMOVDRRCombine(SDNode *N, SelectionDAG &DAG) {
15144 // N=vmovrrd(X); vmovdrr(N:0, N:1) -> bit_convert(X)
15145 SDValue Op0 = N->getOperand(Num: 0);
15146 SDValue Op1 = N->getOperand(Num: 1);
15147 if (Op0.getOpcode() == ISD::BITCAST)
15148 Op0 = Op0.getOperand(i: 0);
15149 if (Op1.getOpcode() == ISD::BITCAST)
15150 Op1 = Op1.getOperand(i: 0);
15151 if (Op0.getOpcode() == ARMISD::VMOVRRD &&
15152 Op0.getNode() == Op1.getNode() &&
15153 Op0.getResNo() == 0 && Op1.getResNo() == 1)
15154 return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N),
15155 VT: N->getValueType(ResNo: 0), Operand: Op0.getOperand(i: 0));
15156 return SDValue();
15157}
15158
15159static SDValue PerformVMOVhrCombine(SDNode *N,
15160 TargetLowering::DAGCombinerInfo &DCI) {
15161 SDValue Op0 = N->getOperand(Num: 0);
15162
15163 // VMOVhr (VMOVrh (X)) -> X
15164 if (Op0->getOpcode() == ARMISD::VMOVrh)
15165 return Op0->getOperand(Num: 0);
15166
15167 // FullFP16: half values are passed in S-registers, and we don't
15168 // need any of the bitcast and moves:
15169 //
15170 // t2: f32,ch1,gl1? = CopyFromReg ch, Register:f32 %0, gl?
15171 // t5: i32 = bitcast t2
15172 // t18: f16 = ARMISD::VMOVhr t5
15173 // =>
15174 // tN: f16,ch2,gl2? = CopyFromReg ch, Register::f32 %0, gl?
15175 if (Op0->getOpcode() == ISD::BITCAST) {
15176 SDValue Copy = Op0->getOperand(Num: 0);
15177 if (Copy.getValueType() == MVT::f32 &&
15178 Copy->getOpcode() == ISD::CopyFromReg) {
15179 bool HasGlue = Copy->getNumOperands() == 3;
15180 SDValue Ops[] = {Copy->getOperand(Num: 0), Copy->getOperand(Num: 1),
15181 HasGlue ? Copy->getOperand(Num: 2) : SDValue()};
15182 EVT OutTys[] = {N->getValueType(ResNo: 0), MVT::Other, MVT::Glue};
15183 SDValue NewCopy =
15184 DCI.DAG.getNode(Opcode: ISD::CopyFromReg, DL: SDLoc(N),
15185 VTList: DCI.DAG.getVTList(VTs: ArrayRef(OutTys, HasGlue ? 3 : 2)),
15186 Ops: ArrayRef(Ops, HasGlue ? 3 : 2));
15187
15188 // Update Users, Chains, and Potential Glue.
15189 DCI.DAG.ReplaceAllUsesOfValueWith(From: SDValue(N, 0), To: NewCopy.getValue(R: 0));
15190 DCI.DAG.ReplaceAllUsesOfValueWith(From: Copy.getValue(R: 1), To: NewCopy.getValue(R: 1));
15191 if (HasGlue)
15192 DCI.DAG.ReplaceAllUsesOfValueWith(From: Copy.getValue(R: 2),
15193 To: NewCopy.getValue(R: 2));
15194
15195 return NewCopy;
15196 }
15197 }
15198
15199 // fold (VMOVhr (load x)) -> (load (f16*)x)
15200 if (LoadSDNode *LN0 = dyn_cast<LoadSDNode>(Val&: Op0)) {
15201 if (LN0->hasOneUse() && LN0->isUnindexed() &&
15202 LN0->getMemoryVT() == MVT::i16) {
15203 SDValue Load =
15204 DCI.DAG.getLoad(VT: N->getValueType(ResNo: 0), dl: SDLoc(N), Chain: LN0->getChain(),
15205 Ptr: LN0->getBasePtr(), MMO: LN0->getMemOperand());
15206 DCI.DAG.ReplaceAllUsesOfValueWith(From: SDValue(N, 0), To: Load.getValue(R: 0));
15207 DCI.DAG.ReplaceAllUsesOfValueWith(From: Op0.getValue(R: 1), To: Load.getValue(R: 1));
15208 return Load;
15209 }
15210 }
15211
15212 // Only the bottom 16 bits of the source register are used.
15213 APInt DemandedMask = APInt::getLowBitsSet(numBits: 32, loBitsSet: 16);
15214 const TargetLowering &TLI = DCI.DAG.getTargetLoweringInfo();
15215 if (TLI.SimplifyDemandedBits(Op: Op0, DemandedBits: DemandedMask, DCI))
15216 return SDValue(N, 0);
15217
15218 return SDValue();
15219}
15220
15221static SDValue PerformVMOVrhCombine(SDNode *N, SelectionDAG &DAG) {
15222 SDValue N0 = N->getOperand(Num: 0);
15223 EVT VT = N->getValueType(ResNo: 0);
15224
15225 // fold (VMOVrh (fpconst x)) -> const x
15226 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val&: N0)) {
15227 APFloat V = C->getValueAPF();
15228 return DAG.getConstant(Val: V.bitcastToAPInt().getZExtValue(), DL: SDLoc(N), VT);
15229 }
15230
15231 // fold (VMOVrh (load x)) -> (zextload (i16*)x)
15232 if (ISD::isNormalLoad(N: N0.getNode()) && N0.hasOneUse()) {
15233 LoadSDNode *LN0 = cast<LoadSDNode>(Val&: N0);
15234
15235 SDValue Load =
15236 DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl: SDLoc(N), VT, Chain: LN0->getChain(),
15237 Ptr: LN0->getBasePtr(), MemVT: MVT::i16, MMO: LN0->getMemOperand());
15238 DAG.ReplaceAllUsesOfValueWith(From: SDValue(N, 0), To: Load.getValue(R: 0));
15239 DAG.ReplaceAllUsesOfValueWith(From: N0.getValue(R: 1), To: Load.getValue(R: 1));
15240 return Load;
15241 }
15242
15243 // Fold VMOVrh(extract(x, n)) -> vgetlaneu(x, n)
15244 if (N0->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
15245 isa<ConstantSDNode>(Val: N0->getOperand(Num: 1)))
15246 return DAG.getNode(Opcode: ARMISD::VGETLANEu, DL: SDLoc(N), VT, N1: N0->getOperand(Num: 0),
15247 N2: N0->getOperand(Num: 1));
15248
15249 return SDValue();
15250}
15251
15252static SDValue PerformVMOVrsCombine(SDNode *N, SelectionDAG &DAG) {
15253 SDValue Op0 = N->getOperand(Num: 0);
15254 if (Op0->getOpcode() == ARMISD::VMOVsr)
15255 return Op0->getOperand(Num: 0);
15256 return SDValue();
15257}
15258
15259/// hasNormalLoadOperand - Check if any of the operands of a BUILD_VECTOR node
15260/// are normal, non-volatile loads. If so, it is profitable to bitcast an
15261/// i64 vector to have f64 elements, since the value can then be loaded
15262/// directly into a VFP register.
15263static bool hasNormalLoadOperand(SDNode *N) {
15264 unsigned NumElts = N->getValueType(ResNo: 0).getVectorNumElements();
15265 for (unsigned i = 0; i < NumElts; ++i) {
15266 SDNode *Elt = N->getOperand(Num: i).getNode();
15267 if (ISD::isNormalLoad(N: Elt) && !cast<LoadSDNode>(Val: Elt)->isVolatile())
15268 return true;
15269 }
15270 return false;
15271}
15272
15273/// PerformBUILD_VECTORCombine - Target-specific dag combine xforms for
15274/// ISD::BUILD_VECTOR.
15275static SDValue PerformBUILD_VECTORCombine(SDNode *N,
15276 TargetLowering::DAGCombinerInfo &DCI,
15277 const ARMSubtarget *Subtarget) {
15278 // build_vector(N=ARMISD::VMOVRRD(X), N:1) -> bit_convert(X):
15279 // VMOVRRD is introduced when legalizing i64 types. It forces the i64 value
15280 // into a pair of GPRs, which is fine when the value is used as a scalar,
15281 // but if the i64 value is converted to a vector, we need to undo the VMOVRRD.
15282 SelectionDAG &DAG = DCI.DAG;
15283 if (N->getNumOperands() == 2)
15284 if (SDValue RV = PerformVMOVDRRCombine(N, DAG))
15285 return RV;
15286
15287 // Load i64 elements as f64 values so that type legalization does not split
15288 // them up into i32 values.
15289 EVT VT = N->getValueType(ResNo: 0);
15290 if (VT.getVectorElementType() != MVT::i64 || !hasNormalLoadOperand(N))
15291 return SDValue();
15292 SDLoc dl(N);
15293 SmallVector<SDValue, 8> Ops;
15294 unsigned NumElts = VT.getVectorNumElements();
15295 for (unsigned i = 0; i < NumElts; ++i) {
15296 SDValue V = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::f64, Operand: N->getOperand(Num: i));
15297 Ops.push_back(Elt: V);
15298 // Make the DAGCombiner fold the bitcast.
15299 DCI.AddToWorklist(N: V.getNode());
15300 }
15301 EVT FloatVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::f64, NumElements: NumElts);
15302 SDValue BV = DAG.getBuildVector(VT: FloatVT, DL: dl, Ops);
15303 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: BV);
15304}
15305
15306static SDValue PerformLegalizedVECTOR_DEINTERLEAVECombine(
15307 SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG,
15308 const ARMSubtarget *Subtarget) {
15309 // Type legalization splits a wide load into consecutive legal loads. Combine
15310 // each group used by a legal VECTOR_DEINTERLEAVE into a structured load.
15311 if (DCI.getDAGCombineLevel() < AfterLegalizeTypes)
15312 return SDValue();
15313
15314 unsigned NumParts = N->getNumOperands();
15315 if (NumParts < 2 || NumParts > 4)
15316 return SDValue();
15317
15318 if (NumParts == 3 && Subtarget->hasMVEIntegerOps())
15319 return SDValue();
15320
15321 EVT SubVecTy = N->getValueType(ResNo: 0);
15322 if (Subtarget->hasNEON() && !SubVecTy.is64BitVector() &&
15323 !SubVecTy.is128BitVector())
15324 return SDValue();
15325 if (Subtarget->hasMVEIntegerOps() && !SubVecTy.is128BitVector())
15326 return SDValue();
15327 unsigned EltBits = SubVecTy.getScalarSizeInBits();
15328 if (EltBits != 8 && EltBits != 16 && EltBits != 32)
15329 return SDValue();
15330
15331 SmallVector<LoadSDNode *, 4> Loads;
15332 for (const SDValue &Operand : N->op_values()) {
15333 auto *Load = dyn_cast<LoadSDNode>(Val: Operand);
15334 if (!Load || !Operand.hasOneUse())
15335 return SDValue();
15336 Loads.push_back(Elt: Load);
15337 }
15338
15339 LoadSDNode *BaseLoad = Loads[0];
15340 if (Subtarget->hasMVEIntegerOps() && BaseLoad->getAlign() < EltBits / 8)
15341 return SDValue();
15342 unsigned Bytes = SubVecTy.getStoreSize().getFixedValue();
15343 for (auto [Idx, Load] : enumerate(First&: Loads)) {
15344 if (!DAG.areNonVolatileConsecutiveLoads(LD: Load, Base: BaseLoad, Bytes, Dist: Idx))
15345 return SDValue();
15346 }
15347
15348 SDLoc DL(N);
15349 EVT MemVT =
15350 EVT::getVectorVT(Context&: *DAG.getContext(), VT: SubVecTy.getVectorElementType(),
15351 EC: SubVecTy.getVectorElementCount() * NumParts);
15352 MachineFunction &MF = DAG.getMachineFunction();
15353 MachineMemOperand *MMO =
15354 MF.getMachineMemOperand(MMO: BaseLoad->getMemOperand(), Offset: 0, Size: NumParts * Bytes);
15355
15356 SmallVector<EVT, 5> ResVTs(NumParts, SubVecTy);
15357 ResVTs.push_back(Elt: MVT::Other);
15358
15359 SmallVector<SDValue> NewLdOps;
15360 NewLdOps.push_back(Elt: BaseLoad->getChain());
15361 Intrinsic::ID IID = 0;
15362 static constexpr Intrinsic::ID NEONLoads[] = {Intrinsic::arm_neon_vld2,
15363 Intrinsic::arm_neon_vld3,
15364 Intrinsic::arm_neon_vld4};
15365 if (Subtarget->hasNEON())
15366 IID = NEONLoads[NumParts - 2];
15367 else
15368 IID = NumParts == 2 ? Intrinsic::arm_mve_vld2q : Intrinsic::arm_mve_vld4q;
15369
15370 NewLdOps.push_back(Elt: DAG.getTargetConstant(Val: IID, DL, VT: MVT::i64));
15371 NewLdOps.push_back(Elt: BaseLoad->getBasePtr());
15372 // We can now generate a structured load!
15373 SDValue NewLoad = DAG.getMemIntrinsicNode(
15374 Opcode: ISD::INTRINSIC_W_CHAIN, dl: DL, VTList: DAG.getVTList(VTs: ResVTs), Ops: NewLdOps, MemVT, MMO);
15375
15376 SmallVector<SDValue, 4> ResOps;
15377 for (unsigned I = 0; I != NumParts; ++I)
15378 ResOps.push_back(Elt: NewLoad.getValue(R: I));
15379
15380 // Preserve the memory ordering of each original load with the new load.
15381 for (LoadSDNode *Load : Loads)
15382 DAG.makeEquivalentMemoryOrdering(OldChain: SDValue(Load, 1),
15383 NewMemOpChain: NewLoad.getValue(R: NumParts));
15384 return DCI.CombineTo(N, To: ResOps, AddTo: false);
15385}
15386
15387// Combine a deinterleave of adjacent subvectors from one load into a
15388// structured NEON load.
15389static SDValue
15390PerformVECTOR_DEINTERLEAVECombine(SDNode *N,
15391 TargetLowering::DAGCombinerInfo &DCI,
15392 const ARMSubtarget *Subtarget) {
15393 if (!Subtarget->hasNEON() && !Subtarget->hasMVEIntegerOps())
15394 return SDValue();
15395
15396 if (SDValue Res = PerformLegalizedVECTOR_DEINTERLEAVECombine(N, DCI, DAG&: DCI.DAG,
15397 Subtarget))
15398 return Res;
15399
15400 if (!DCI.isBeforeLegalize())
15401 return SDValue();
15402
15403 SelectionDAG &DAG = DCI.DAG;
15404 unsigned NumParts = N->getNumOperands();
15405 if (NumParts != 2 && NumParts != 3 && NumParts != 4)
15406 return SDValue();
15407
15408 if (NumParts == 3 && Subtarget->hasMVEIntegerOps())
15409 return SDValue();
15410
15411 EVT SubVecTy = N->getValueType(ResNo: 0);
15412
15413 if (Subtarget->hasNEON() && !SubVecTy.is64BitVector() &&
15414 !SubVecTy.is128BitVector())
15415 return SDValue();
15416 if (Subtarget->hasMVEIntegerOps() && !SubVecTy.is128BitVector())
15417 return SDValue();
15418 unsigned EltBits = SubVecTy.getScalarSizeInBits();
15419 if (EltBits != 8 && EltBits != 16 && EltBits != 32)
15420 return SDValue();
15421
15422 // Make sure each input operand is the correct extract_subvector of the same
15423 // wider vector.
15424 unsigned MinNumElements = SubVecTy.getVectorMinNumElements();
15425 SDValue Op0 = N->getOperand(Num: 0);
15426 for (unsigned I = 0; I < NumParts; I++) {
15427 SDValue OpI = N->getOperand(Num: I);
15428 if (!OpI.hasOneUse() || OpI.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
15429 OpI.getOperand(i: 0) != Op0.getOperand(i: 0))
15430 return SDValue();
15431 if (OpI.getConstantOperandVal(i: 1) != (I * MinNumElements))
15432 return SDValue();
15433 }
15434
15435 SDValue WideVec = Op0.getOperand(i: 0);
15436 SDLoc DL(N);
15437
15438 SmallVector<EVT, 5> ResVTs(NumParts, SubVecTy);
15439 ResVTs.push_back(Elt: MVT::Other);
15440 SDVTList ResVTList = DAG.getVTList(VTs: ResVTs);
15441 auto *Load = dyn_cast<LoadSDNode>(Val&: WideVec);
15442 if (!Load || !Load->hasNUsesOfValue(NUses: NumParts, Value: 0) || !Load->isSimple() ||
15443 !ISD::isNormalLoad(N: Load) || !Load->getOffset().isUndef())
15444 return SDValue();
15445 if (Subtarget->hasMVEIntegerOps() && Load->getAlign() < EltBits / 8)
15446 return SDValue();
15447
15448 static constexpr Intrinsic::ID NEONLoads[] = {Intrinsic::arm_neon_vld2,
15449 Intrinsic::arm_neon_vld3,
15450 Intrinsic::arm_neon_vld4};
15451 SmallVector<SDValue> NewLdOps;
15452 Intrinsic::ID IID = 0;
15453 NewLdOps.push_back(Elt: Load->getChain());
15454 if (Subtarget->hasNEON())
15455 IID = NEONLoads[NumParts - 2];
15456 else
15457 IID = NumParts == 2 ? Intrinsic::arm_mve_vld2q : Intrinsic::arm_mve_vld4q;
15458 NewLdOps.push_back(Elt: DAG.getTargetConstant(Val: IID, DL, VT: MVT::i64));
15459
15460 NewLdOps.push_back(Elt: Load->getBasePtr());
15461
15462 // We can now generate a structured load!
15463 SDValue Res =
15464 DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_W_CHAIN, dl: DL, VTList: ResVTList, Ops: NewLdOps,
15465 MemVT: Load->getMemoryVT(), MMO: Load->getMemOperand());
15466
15467 SmallVector<SDValue, 4> ResOps(NumParts);
15468 for (unsigned Idx = 0; Idx < NumParts; Idx++)
15469 ResOps[Idx] = Res.getValue(R: Idx);
15470
15471 // Preserve the original load's memory ordering with the new load.
15472 DAG.makeEquivalentMemoryOrdering(OldChain: WideVec.getValue(R: 1), NewMemOpChain: Res.getValue(R: NumParts));
15473 return DCI.CombineTo(N, To: ResOps, AddTo: false);
15474}
15475
15476/// Target-specific dag combine xforms for ARMISD::BUILD_VECTOR.
15477static SDValue
15478PerformARMBUILD_VECTORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
15479 SelectionDAG &DAG = DCI.DAG;
15480 EVT VT = N->getValueType(ResNo: 0);
15481
15482 // Fold buildvector(extractelt(bitcast(build_vector(a,b,_,_))),
15483 // extractelt(bitcast(build_vector(_,_,c,d))))
15484 // -> bitcast(build_vector(a,b,c,d))
15485 if (VT.getScalarSizeInBits() == 64 && N->getNumOperands() == 2) {
15486 auto IsExtractFromBuildVec = [](SDValue V, SDValue &BV, int64_t &Lane) {
15487 if (!sd_match(N: V, P: m_ExtractElt(Vec: m_Value(N&: BV), Idx: m_ConstInt(V&: Lane))))
15488 return false;
15489 while (BV.getOpcode() == ISD::BITCAST)
15490 BV = BV.getOperand(i: 0);
15491 return BV.getOpcode() == ARMISD::BUILD_VECTOR &&
15492 BV.getNumOperands() == 4 &&
15493 BV.getValueType().getScalarSizeInBits() == 32;
15494 };
15495 SDValue BV1, BV2;
15496 int64_t Lane1, Lane2;
15497 if (IsExtractFromBuildVec(N->getOperand(Num: 0), BV1, Lane1) &&
15498 IsExtractFromBuildVec(N->getOperand(Num: 1), BV2, Lane2))
15499 return DAG.getBitcast(
15500 VT, V: DAG.getNode(
15501 Opcode: ARMISD::BUILD_VECTOR, DL: N, VT: MVT::v4f32,
15502 N1: DAG.getBitcast(VT: MVT::f32, V: BV1->getOperand(Num: Lane1 * 2 + 0)),
15503 N2: DAG.getBitcast(VT: MVT::f32, V: BV1->getOperand(Num: Lane1 * 2 + 1)),
15504 N3: DAG.getBitcast(VT: MVT::f32, V: BV2->getOperand(Num: Lane2 * 2 + 0)),
15505 N4: DAG.getBitcast(VT: MVT::f32, V: BV2->getOperand(Num: Lane2 * 2 + 1))));
15506 }
15507
15508 // ARMISD::BUILD_VECTOR is introduced when legalizing ISD::BUILD_VECTOR.
15509 // At that time, we may have inserted bitcasts from integer to float.
15510 // If these bitcasts have survived DAGCombine, change the lowering of this
15511 // BUILD_VECTOR in something more vector friendly, i.e., that does not
15512 // force to use floating point types.
15513
15514 // Make sure we can change the type of the vector.
15515 // This is possible iff:
15516 // 1. The vector is only used in a bitcast to a integer type. I.e.,
15517 // 1.1. Vector is used only once.
15518 // 1.2. Use is a bit convert to an integer type.
15519 // 2. The size of its operands are 32-bits (64-bits are not legal).
15520 EVT EltVT = VT.getVectorElementType();
15521
15522 // Check 1.1. and 2.
15523 if (EltVT.getSizeInBits() != 32 || !N->hasOneUse())
15524 return SDValue();
15525
15526 // By construction, the input type must be float.
15527 assert(EltVT == MVT::f32 && "Unexpected type!");
15528
15529 // Check 1.2.
15530 SDNode *Use = *N->user_begin();
15531 if (Use->getOpcode() != ISD::BITCAST ||
15532 Use->getValueType(ResNo: 0).isFloatingPoint())
15533 return SDValue();
15534
15535 // Check profitability.
15536 // Model is, if more than half of the relevant operands are bitcast from
15537 // i32, turn the build_vector into a sequence of insert_vector_elt.
15538 // Relevant operands are everything that is not statically
15539 // (i.e., at compile time) bitcasted.
15540 unsigned NumOfBitCastedElts = 0;
15541 unsigned NumElts = VT.getVectorNumElements();
15542 unsigned NumOfRelevantElts = NumElts;
15543 for (unsigned Idx = 0; Idx < NumElts; ++Idx) {
15544 SDValue Elt = N->getOperand(Num: Idx);
15545 if (Elt->getOpcode() == ISD::BITCAST) {
15546 // Assume only bit cast to i32 will go away.
15547 if (Elt->getOperand(Num: 0).getValueType() == MVT::i32)
15548 ++NumOfBitCastedElts;
15549 } else if (Elt.isUndef() || isa<ConstantSDNode>(Val: Elt))
15550 // Constants are statically casted, thus do not count them as
15551 // relevant operands.
15552 --NumOfRelevantElts;
15553 }
15554
15555 // Check if more than half of the elements require a non-free bitcast.
15556 if (NumOfBitCastedElts <= NumOfRelevantElts / 2)
15557 return SDValue();
15558
15559 // Create the new vector type.
15560 EVT VecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32, NumElements: NumElts);
15561 // Check if the type is legal.
15562 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
15563 if (!TLI.isTypeLegal(VT: VecVT))
15564 return SDValue();
15565
15566 // Combine:
15567 // ARMISD::BUILD_VECTOR E1, E2, ..., EN.
15568 // => BITCAST INSERT_VECTOR_ELT
15569 // (INSERT_VECTOR_ELT (...), (BITCAST EN-1), N-1),
15570 // (BITCAST EN), N.
15571 SDValue Vec = DAG.getUNDEF(VT: VecVT);
15572 SDLoc dl(N);
15573 for (unsigned Idx = 0 ; Idx < NumElts; ++Idx) {
15574 SDValue V = N->getOperand(Num: Idx);
15575 if (V.isUndef())
15576 continue;
15577 if (V.getOpcode() == ISD::BITCAST &&
15578 V->getOperand(Num: 0).getValueType() == MVT::i32)
15579 // Fold obvious case.
15580 V = V.getOperand(i: 0);
15581 else {
15582 V = DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(V), VT: MVT::i32, Operand: V);
15583 // Make the DAGCombiner fold the bitcasts.
15584 DCI.AddToWorklist(N: V.getNode());
15585 }
15586 SDValue LaneIdx = DAG.getConstant(Val: Idx, DL: dl, VT: MVT::i32);
15587 Vec = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: VecVT, N1: Vec, N2: V, N3: LaneIdx);
15588 }
15589 Vec = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: Vec);
15590 // Make the DAGCombiner fold the bitcasts.
15591 DCI.AddToWorklist(N: Vec.getNode());
15592 return Vec;
15593}
15594
15595static SDValue
15596PerformPREDICATE_CASTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
15597 EVT VT = N->getValueType(ResNo: 0);
15598 SDValue Op = N->getOperand(Num: 0);
15599 SDLoc dl(N);
15600
15601 // PREDICATE_CAST(PREDICATE_CAST(x)) == PREDICATE_CAST(x)
15602 if (Op->getOpcode() == ARMISD::PREDICATE_CAST) {
15603 // If the valuetypes are the same, we can remove the cast entirely.
15604 if (Op->getOperand(Num: 0).getValueType() == VT)
15605 return Op->getOperand(Num: 0);
15606 return DCI.DAG.getNode(Opcode: ARMISD::PREDICATE_CAST, DL: dl, VT, Operand: Op->getOperand(Num: 0));
15607 }
15608
15609 // Turn pred_cast(xor x, -1) into xor(pred_cast x, -1), in order to produce
15610 // more VPNOT which might get folded as else predicates.
15611 if (Op.getValueType() == MVT::i32 && isBitwiseNot(V: Op)) {
15612 SDValue X =
15613 DCI.DAG.getNode(Opcode: ARMISD::PREDICATE_CAST, DL: dl, VT, Operand: Op->getOperand(Num: 0));
15614 SDValue C = DCI.DAG.getNode(Opcode: ARMISD::PREDICATE_CAST, DL: dl, VT,
15615 Operand: DCI.DAG.getConstant(Val: 65535, DL: dl, VT: MVT::i32));
15616 return DCI.DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: X, N2: C);
15617 }
15618
15619 // Only the bottom 16 bits of the source register are used.
15620 if (Op.getValueType() == MVT::i32) {
15621 APInt DemandedMask = APInt::getLowBitsSet(numBits: 32, loBitsSet: 16);
15622 const TargetLowering &TLI = DCI.DAG.getTargetLoweringInfo();
15623 if (TLI.SimplifyDemandedBits(Op, DemandedBits: DemandedMask, DCI))
15624 return SDValue(N, 0);
15625 }
15626 return SDValue();
15627}
15628
15629static SDValue PerformVECTOR_REG_CASTCombine(SDNode *N, SelectionDAG &DAG,
15630 const ARMSubtarget *ST) {
15631 EVT VT = N->getValueType(ResNo: 0);
15632 SDValue Op = N->getOperand(Num: 0);
15633 SDLoc dl(N);
15634
15635 // Under Little endian, a VECTOR_REG_CAST is equivalent to a BITCAST
15636 if (ST->isLittle())
15637 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: Op);
15638
15639 // VT VECTOR_REG_CAST (VT Op) -> Op
15640 if (Op.getValueType() == VT)
15641 return Op;
15642 // VECTOR_REG_CAST undef -> undef
15643 if (Op.isUndef())
15644 return DAG.getUNDEF(VT);
15645
15646 // VECTOR_REG_CAST(VECTOR_REG_CAST(x)) == VECTOR_REG_CAST(x)
15647 if (Op->getOpcode() == ARMISD::VECTOR_REG_CAST) {
15648 // If the valuetypes are the same, we can remove the cast entirely.
15649 if (Op->getOperand(Num: 0).getValueType() == VT)
15650 return Op->getOperand(Num: 0);
15651 return DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT, Operand: Op->getOperand(Num: 0));
15652 }
15653
15654 return SDValue();
15655}
15656
15657static SDValue PerformVCMPCombine(SDNode *N, SelectionDAG &DAG,
15658 const ARMSubtarget *Subtarget) {
15659 if (!Subtarget->hasMVEIntegerOps())
15660 return SDValue();
15661
15662 EVT VT = N->getValueType(ResNo: 0);
15663 SDValue Op0 = N->getOperand(Num: 0);
15664 SDValue Op1 = N->getOperand(Num: 1);
15665 ARMCC::CondCodes Cond = (ARMCC::CondCodes)N->getConstantOperandVal(Num: 2);
15666 SDLoc dl(N);
15667
15668 // vcmp X, 0, cc -> vcmpz X, cc
15669 if (isZeroVector(N: Op1))
15670 return DAG.getNode(Opcode: ARMISD::VCMPZ, DL: dl, VT, N1: Op0, N2: N->getOperand(Num: 2));
15671
15672 unsigned SwappedCond = getSwappedCondition(CC: Cond);
15673 if (isValidMVECond(CC: SwappedCond, IsFloat: VT.isFloatingPoint())) {
15674 // vcmp 0, X, cc -> vcmpz X, reversed(cc)
15675 if (isZeroVector(N: Op0))
15676 return DAG.getNode(Opcode: ARMISD::VCMPZ, DL: dl, VT, N1: Op1,
15677 N2: DAG.getConstant(Val: SwappedCond, DL: dl, VT: MVT::i32));
15678 // vcmp vdup(Y), X, cc -> vcmp X, vdup(Y), reversed(cc)
15679 if (Op0->getOpcode() == ARMISD::VDUP && Op1->getOpcode() != ARMISD::VDUP)
15680 return DAG.getNode(Opcode: ARMISD::VCMP, DL: dl, VT, N1: Op1, N2: Op0,
15681 N3: DAG.getConstant(Val: SwappedCond, DL: dl, VT: MVT::i32));
15682 }
15683
15684 return SDValue();
15685}
15686
15687/// PerformInsertEltCombine - Target-specific dag combine xforms for
15688/// ISD::INSERT_VECTOR_ELT.
15689static SDValue PerformInsertEltCombine(SDNode *N,
15690 TargetLowering::DAGCombinerInfo &DCI) {
15691 // Bitcast an i64 load inserted into a vector to f64.
15692 // Otherwise, the i64 value will be legalized to a pair of i32 values.
15693 EVT VT = N->getValueType(ResNo: 0);
15694 SDNode *Elt = N->getOperand(Num: 1).getNode();
15695 if (VT.getVectorElementType() != MVT::i64 ||
15696 !ISD::isNormalLoad(N: Elt) || cast<LoadSDNode>(Val: Elt)->isVolatile())
15697 return SDValue();
15698
15699 SelectionDAG &DAG = DCI.DAG;
15700 SDLoc dl(N);
15701 EVT FloatVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::f64,
15702 NumElements: VT.getVectorNumElements());
15703 SDValue Vec = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: FloatVT, Operand: N->getOperand(Num: 0));
15704 SDValue V = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::f64, Operand: N->getOperand(Num: 1));
15705 // Make the DAGCombiner fold the bitcasts.
15706 DCI.AddToWorklist(N: Vec.getNode());
15707 DCI.AddToWorklist(N: V.getNode());
15708 SDValue InsElt = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: FloatVT,
15709 N1: Vec, N2: V, N3: N->getOperand(Num: 2));
15710 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: InsElt);
15711}
15712
15713// Convert a pair of extracts from the same base vector to a VMOVRRD. Either
15714// directly or bitcast to an integer if the original is a float vector.
15715// extract(x, n); extract(x, n+1) -> VMOVRRD(extract v2f64 x, n/2)
15716// bitcast(extract(x, n)); bitcast(extract(x, n+1)) -> VMOVRRD(extract x, n/2)
15717static SDValue
15718PerformExtractEltToVMOVRRD(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
15719 EVT VT = N->getValueType(ResNo: 0);
15720 SDLoc dl(N);
15721
15722 if (!DCI.isAfterLegalizeDAG() || VT != MVT::i32 ||
15723 !DCI.DAG.getTargetLoweringInfo().isTypeLegal(VT: MVT::f64))
15724 return SDValue();
15725
15726 SDValue Ext = SDValue(N, 0);
15727 if (Ext.getOpcode() == ISD::BITCAST &&
15728 Ext.getOperand(i: 0).getValueType() == MVT::f32)
15729 Ext = Ext.getOperand(i: 0);
15730 if (Ext.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
15731 !isa<ConstantSDNode>(Val: Ext.getOperand(i: 1)) ||
15732 Ext.getConstantOperandVal(i: 1) % 2 != 0)
15733 return SDValue();
15734 if (Ext->hasOneUse() && (Ext->user_begin()->getOpcode() == ISD::SINT_TO_FP ||
15735 Ext->user_begin()->getOpcode() == ISD::UINT_TO_FP))
15736 return SDValue();
15737
15738 SDValue Op0 = Ext.getOperand(i: 0);
15739 EVT VecVT = Op0.getValueType();
15740 unsigned ResNo = Op0.getResNo();
15741 unsigned Lane = Ext.getConstantOperandVal(i: 1);
15742 if (VecVT.getVectorNumElements() != 4)
15743 return SDValue();
15744
15745 // Find another extract, of Lane + 1
15746 auto OtherIt = find_if(Range: Op0->users(), P: [&](SDNode *V) {
15747 return V->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
15748 isa<ConstantSDNode>(Val: V->getOperand(Num: 1)) &&
15749 V->getConstantOperandVal(Num: 1) == Lane + 1 &&
15750 V->getOperand(Num: 0).getResNo() == ResNo;
15751 });
15752 if (OtherIt == Op0->users().end())
15753 return SDValue();
15754
15755 // For float extracts, we need to be converting to a i32 for both vector
15756 // lanes.
15757 SDValue OtherExt(*OtherIt, 0);
15758 if (OtherExt.getValueType() != MVT::i32) {
15759 if (!OtherExt->hasOneUse() ||
15760 OtherExt->user_begin()->getOpcode() != ISD::BITCAST ||
15761 OtherExt->user_begin()->getValueType(ResNo: 0) != MVT::i32)
15762 return SDValue();
15763 OtherExt = SDValue(*OtherExt->user_begin(), 0);
15764 }
15765
15766 // Convert the type to a f64 and extract with a VMOVRRD.
15767 SDValue F64 = DCI.DAG.getNode(
15768 Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: MVT::f64,
15769 N1: DCI.DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT: MVT::v2f64, Operand: Op0),
15770 N2: DCI.DAG.getConstant(Val: Ext.getConstantOperandVal(i: 1) / 2, DL: dl, VT: MVT::i32));
15771 SDValue VMOVRRD =
15772 DCI.DAG.getNode(Opcode: ARMISD::VMOVRRD, DL: dl, ResultTys: {MVT::i32, MVT::i32}, Ops: F64);
15773
15774 DCI.CombineTo(N: OtherExt.getNode(), Res: SDValue(VMOVRRD.getNode(), 1));
15775 return VMOVRRD;
15776}
15777
15778static SDValue PerformExtractEltCombine(SDNode *N,
15779 TargetLowering::DAGCombinerInfo &DCI,
15780 const ARMSubtarget *ST) {
15781 SDValue Op0 = N->getOperand(Num: 0);
15782 EVT VT = N->getValueType(ResNo: 0);
15783 SDLoc dl(N);
15784
15785 // extract (vdup x) -> x
15786 if (Op0->getOpcode() == ARMISD::VDUP) {
15787 SDValue X = Op0->getOperand(Num: 0);
15788 if (VT == MVT::f16 && X.getValueType() == MVT::i32)
15789 return DCI.DAG.getNode(Opcode: ARMISD::VMOVhr, DL: dl, VT, Operand: X);
15790 if (VT == MVT::i32 && X.getValueType() == MVT::f16)
15791 return DCI.DAG.getNode(Opcode: ARMISD::VMOVrh, DL: dl, VT, Operand: X);
15792 if (VT == MVT::f32 && X.getValueType() == MVT::i32)
15793 return DCI.DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: X);
15794
15795 while (X.getValueType() != VT && X->getOpcode() == ISD::BITCAST)
15796 X = X->getOperand(Num: 0);
15797 if (X.getValueType() == VT)
15798 return X;
15799 }
15800
15801 // extract ARM_BUILD_VECTOR -> x
15802 if (Op0->getOpcode() == ARMISD::BUILD_VECTOR &&
15803 isa<ConstantSDNode>(Val: N->getOperand(Num: 1)) &&
15804 N->getConstantOperandVal(Num: 1) < Op0.getNumOperands()) {
15805 return Op0.getOperand(i: N->getConstantOperandVal(Num: 1));
15806 }
15807
15808 // extract(bitcast(BUILD_VECTOR(VMOVDRR(a, b), ..))) -> a or b
15809 if (Op0.getValueType() == MVT::v4i32 &&
15810 isa<ConstantSDNode>(Val: N->getOperand(Num: 1)) &&
15811 Op0.getOpcode() == ISD::BITCAST &&
15812 Op0.getOperand(i: 0).getOpcode() == ISD::BUILD_VECTOR &&
15813 Op0.getOperand(i: 0).getValueType() == MVT::v2f64) {
15814 SDValue BV = Op0.getOperand(i: 0);
15815 unsigned Offset = N->getConstantOperandVal(Num: 1);
15816 SDValue MOV = BV.getOperand(i: Offset < 2 ? 0 : 1);
15817 if (MOV.getOpcode() == ARMISD::VMOVDRR)
15818 return MOV.getOperand(i: ST->isLittle() ? Offset % 2 : 1 - Offset % 2);
15819 }
15820
15821 // extract x, n; extract x, n+1 -> VMOVRRD x
15822 if (SDValue R = PerformExtractEltToVMOVRRD(N, DCI))
15823 return R;
15824
15825 // extract (MVETrunc(x)) -> extract x
15826 if (Op0->getOpcode() == ARMISD::MVETRUNC) {
15827 unsigned Idx = N->getConstantOperandVal(Num: 1);
15828 unsigned Vec =
15829 Idx / Op0->getOperand(Num: 0).getValueType().getVectorNumElements();
15830 unsigned SubIdx =
15831 Idx % Op0->getOperand(Num: 0).getValueType().getVectorNumElements();
15832 return DCI.DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT, N1: Op0.getOperand(i: Vec),
15833 N2: DCI.DAG.getConstant(Val: SubIdx, DL: dl, VT: MVT::i32));
15834 }
15835
15836 // extract(bitcast(BUILD_VECTOR(extract(bitcast(a)), ..))) -> extract(a)
15837 if (ST->isLittle() && Op0.getOpcode() == ISD::BITCAST &&
15838 Op0.getOperand(i: 0).getOpcode() == ARMISD::BUILD_VECTOR &&
15839 isa<ConstantSDNode>(Val: N->getOperand(Num: 1)) &&
15840 Op0.getScalarValueSizeInBits() <=
15841 Op0.getOperand(i: 0).getScalarValueSizeInBits()) {
15842 unsigned Lane = N->getConstantOperandVal(Num: 1);
15843 EVT ExtVT = Op0.getValueType();
15844 EVT BVVT = Op0.getOperand(i: 0).getValueType();
15845 unsigned BVLane =
15846 (Lane * BVVT.getVectorNumElements()) / ExtVT.getVectorNumElements();
15847 assert(BVLane < Op0.getOperand(0).getNumOperands());
15848 SDValue Ext = Op0.getOperand(i: 0).getOperand(i: BVLane);
15849 if (Ext.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
15850 Ext.getOperand(i: 0).getOpcode() == ISD::BITCAST &&
15851 isa<ConstantSDNode>(Val: Ext.getOperand(i: 1)) &&
15852 Ext.getOperand(i: 0).getOperand(i: 0).getValueType() == ExtVT) {
15853 unsigned InnerLane = Ext.getConstantOperandVal(i: 1);
15854 unsigned BVSubLane = Lane - (BVLane * ExtVT.getVectorNumElements()) /
15855 BVVT.getVectorNumElements();
15856 unsigned FinalLane = (InnerLane * ExtVT.getVectorNumElements()) /
15857 BVVT.getVectorNumElements() +
15858 BVSubLane;
15859 return DCI.DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT,
15860 N1: Ext.getOperand(i: 0).getOperand(i: 0),
15861 N2: DCI.DAG.getConstant(Val: FinalLane, DL: dl, VT: MVT::i32));
15862 }
15863 }
15864
15865 return SDValue();
15866}
15867
15868static SDValue PerformSignExtendInregCombine(SDNode *N, SelectionDAG &DAG) {
15869 SDValue Op = N->getOperand(Num: 0);
15870 EVT VT = N->getValueType(ResNo: 0);
15871
15872 // sext_inreg(VGETLANEu) -> VGETLANEs
15873 if (Op.getOpcode() == ARMISD::VGETLANEu &&
15874 cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT() ==
15875 Op.getOperand(i: 0).getValueType().getScalarType())
15876 return DAG.getNode(Opcode: ARMISD::VGETLANEs, DL: SDLoc(N), VT, N1: Op.getOperand(i: 0),
15877 N2: Op.getOperand(i: 1));
15878
15879 return SDValue();
15880}
15881
15882static SDValue
15883PerformInsertSubvectorCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
15884 SDValue Vec = N->getOperand(Num: 0);
15885 SDValue SubVec = N->getOperand(Num: 1);
15886 uint64_t IdxVal = N->getConstantOperandVal(Num: 2);
15887 EVT VecVT = Vec.getValueType();
15888 EVT SubVT = SubVec.getValueType();
15889
15890 // Only do this for legal fixed vector types.
15891 if (!VecVT.isFixedLengthVector() ||
15892 !DCI.DAG.getTargetLoweringInfo().isTypeLegal(VT: VecVT) ||
15893 !DCI.DAG.getTargetLoweringInfo().isTypeLegal(VT: SubVT))
15894 return SDValue();
15895
15896 // Ignore widening patterns.
15897 if (IdxVal == 0 && Vec.isUndef())
15898 return SDValue();
15899
15900 // Subvector must be half the width and an "aligned" insertion.
15901 unsigned NumSubElts = SubVT.getVectorNumElements();
15902 if ((SubVT.getSizeInBits() * 2) != VecVT.getSizeInBits() ||
15903 (IdxVal != 0 && IdxVal != NumSubElts))
15904 return SDValue();
15905
15906 // Fold insert_subvector -> concat_vectors
15907 // insert_subvector(Vec,Sub,lo) -> concat_vectors(Sub,extract(Vec,hi))
15908 // insert_subvector(Vec,Sub,hi) -> concat_vectors(extract(Vec,lo),Sub)
15909 SDLoc DL(N);
15910 SDValue Lo, Hi;
15911 if (IdxVal == 0) {
15912 Lo = SubVec;
15913 Hi = DCI.DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: SubVT, N1: Vec,
15914 N2: DCI.DAG.getVectorIdxConstant(Val: NumSubElts, DL));
15915 } else {
15916 Lo = DCI.DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: SubVT, N1: Vec,
15917 N2: DCI.DAG.getVectorIdxConstant(Val: 0, DL));
15918 Hi = SubVec;
15919 }
15920 return DCI.DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: VecVT, N1: Lo, N2: Hi);
15921}
15922
15923// shuffle(MVETrunc(x, y)) -> VMOVN(x, y)
15924static SDValue PerformShuffleVMOVNCombine(ShuffleVectorSDNode *N,
15925 SelectionDAG &DAG) {
15926 SDValue Trunc = N->getOperand(Num: 0);
15927 EVT VT = Trunc.getValueType();
15928 if (Trunc.getOpcode() != ARMISD::MVETRUNC || !N->getOperand(Num: 1).isUndef())
15929 return SDValue();
15930
15931 SDLoc DL(Trunc);
15932 if (isVMOVNTruncMask(M: N->getMask(), ToVT: VT, rev: false))
15933 return DAG.getNode(
15934 Opcode: ARMISD::VMOVN, DL, VT,
15935 N1: DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL, VT, Operand: Trunc.getOperand(i: 0)),
15936 N2: DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL, VT, Operand: Trunc.getOperand(i: 1)),
15937 N3: DAG.getConstant(Val: 1, DL, VT: MVT::i32));
15938 else if (isVMOVNTruncMask(M: N->getMask(), ToVT: VT, rev: true))
15939 return DAG.getNode(
15940 Opcode: ARMISD::VMOVN, DL, VT,
15941 N1: DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL, VT, Operand: Trunc.getOperand(i: 1)),
15942 N2: DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL, VT, Operand: Trunc.getOperand(i: 0)),
15943 N3: DAG.getConstant(Val: 1, DL, VT: MVT::i32));
15944 return SDValue();
15945}
15946
15947/// PerformVECTOR_SHUFFLECombine - Target-specific dag combine xforms for
15948/// ISD::VECTOR_SHUFFLE.
15949static SDValue PerformVECTOR_SHUFFLECombine(SDNode *N, SelectionDAG &DAG) {
15950 if (SDValue R = PerformShuffleVMOVNCombine(N: cast<ShuffleVectorSDNode>(Val: N), DAG))
15951 return R;
15952
15953 // The LLVM shufflevector instruction does not require the shuffle mask
15954 // length to match the operand vector length, but ISD::VECTOR_SHUFFLE does
15955 // have that requirement. When translating to ISD::VECTOR_SHUFFLE, if the
15956 // operands do not match the mask length, they are extended by concatenating
15957 // them with undef vectors. That is probably the right thing for other
15958 // targets, but for NEON it is better to concatenate two double-register
15959 // size vector operands into a single quad-register size vector. Do that
15960 // transformation here:
15961 // shuffle(concat(v1, undef), concat(v2, undef)) ->
15962 // shuffle(concat(v1, v2), undef)
15963 SDValue Op0 = N->getOperand(Num: 0);
15964 SDValue Op1 = N->getOperand(Num: 1);
15965 if (Op0.getOpcode() != ISD::CONCAT_VECTORS ||
15966 Op1.getOpcode() != ISD::CONCAT_VECTORS ||
15967 Op0.getNumOperands() != 2 ||
15968 Op1.getNumOperands() != 2)
15969 return SDValue();
15970 SDValue Concat0Op1 = Op0.getOperand(i: 1);
15971 SDValue Concat1Op1 = Op1.getOperand(i: 1);
15972 if (!Concat0Op1.isUndef() || !Concat1Op1.isUndef())
15973 return SDValue();
15974 // Skip the transformation if any of the types are illegal.
15975 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
15976 EVT VT = N->getValueType(ResNo: 0);
15977 if (!TLI.isTypeLegal(VT) ||
15978 !TLI.isTypeLegal(VT: Concat0Op1.getValueType()) ||
15979 !TLI.isTypeLegal(VT: Concat1Op1.getValueType()))
15980 return SDValue();
15981
15982 SDValue NewConcat = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(N), VT,
15983 N1: Op0.getOperand(i: 0), N2: Op1.getOperand(i: 0));
15984 // Translate the shuffle mask.
15985 SmallVector<int, 16> NewMask;
15986 unsigned NumElts = VT.getVectorNumElements();
15987 unsigned HalfElts = NumElts/2;
15988 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val: N);
15989 for (unsigned n = 0; n < NumElts; ++n) {
15990 int MaskElt = SVN->getMaskElt(Idx: n);
15991 int NewElt = -1;
15992 if (MaskElt < (int)HalfElts)
15993 NewElt = MaskElt;
15994 else if (MaskElt >= (int)NumElts && MaskElt < (int)(NumElts + HalfElts))
15995 NewElt = HalfElts + MaskElt - NumElts;
15996 NewMask.push_back(Elt: NewElt);
15997 }
15998 return DAG.getVectorShuffle(VT, dl: SDLoc(N), N1: NewConcat,
15999 N2: DAG.getUNDEF(VT), Mask: NewMask);
16000}
16001
16002/// Load/store instruction that can be merged with a base address
16003/// update
16004struct BaseUpdateTarget {
16005 SDNode *N;
16006 bool isIntrinsic;
16007 bool isStore;
16008 unsigned AddrOpIdx;
16009};
16010
16011struct BaseUpdateUser {
16012 /// Instruction that updates a pointer
16013 SDNode *N;
16014 /// Pointer increment operand
16015 SDValue Inc;
16016 /// Pointer increment value if it is a constant, or 0 otherwise
16017 unsigned ConstInc;
16018};
16019
16020static bool isValidBaseUpdate(SDNode *N, SDNode *User) {
16021 // Check that the add is independent of the load/store.
16022 // Otherwise, folding it would create a cycle. Search through Addr
16023 // as well, since the User may not be a direct user of Addr and
16024 // only share a base pointer.
16025 SmallPtrSet<const SDNode *, 32> Visited;
16026 SmallVector<const SDNode *, 16> Worklist;
16027 Worklist.push_back(Elt: N);
16028 Worklist.push_back(Elt: User);
16029 const unsigned MaxSteps = 1024;
16030 if (SDNode::hasPredecessorHelper(N, Visited, Worklist, MaxSteps) ||
16031 SDNode::hasPredecessorHelper(N: User, Visited, Worklist, MaxSteps))
16032 return false;
16033 return true;
16034}
16035
16036static bool TryCombineBaseUpdate(struct BaseUpdateTarget &Target,
16037 struct BaseUpdateUser &User,
16038 bool SimpleConstIncOnly,
16039 TargetLowering::DAGCombinerInfo &DCI) {
16040 SelectionDAG &DAG = DCI.DAG;
16041 SDNode *N = Target.N;
16042 MemSDNode *MemN = cast<MemSDNode>(Val: N);
16043 SDLoc dl(N);
16044
16045 // Find the new opcode for the updating load/store.
16046 bool isLoadOp = true;
16047 bool isLaneOp = false;
16048 // Workaround for vst1x and vld1x intrinsics which do not have alignment
16049 // as an operand.
16050 bool hasAlignment = true;
16051 unsigned NewOpc = 0;
16052 unsigned NumVecs = 0;
16053 if (Target.isIntrinsic) {
16054 unsigned IntNo = N->getConstantOperandVal(Num: 1);
16055 switch (IntNo) {
16056 default:
16057 llvm_unreachable("unexpected intrinsic for Neon base update");
16058 case Intrinsic::arm_neon_vld1:
16059 NewOpc = ARMISD::VLD1_UPD;
16060 NumVecs = 1;
16061 break;
16062 case Intrinsic::arm_neon_vld2:
16063 NewOpc = ARMISD::VLD2_UPD;
16064 NumVecs = 2;
16065 break;
16066 case Intrinsic::arm_neon_vld3:
16067 NewOpc = ARMISD::VLD3_UPD;
16068 NumVecs = 3;
16069 break;
16070 case Intrinsic::arm_neon_vld4:
16071 NewOpc = ARMISD::VLD4_UPD;
16072 NumVecs = 4;
16073 break;
16074 case Intrinsic::arm_neon_vld1x2:
16075 NewOpc = ARMISD::VLD1x2_UPD;
16076 NumVecs = 2;
16077 hasAlignment = false;
16078 break;
16079 case Intrinsic::arm_neon_vld1x3:
16080 NewOpc = ARMISD::VLD1x3_UPD;
16081 NumVecs = 3;
16082 hasAlignment = false;
16083 break;
16084 case Intrinsic::arm_neon_vld1x4:
16085 NewOpc = ARMISD::VLD1x4_UPD;
16086 NumVecs = 4;
16087 hasAlignment = false;
16088 break;
16089 case Intrinsic::arm_neon_vld2dup:
16090 NewOpc = ARMISD::VLD2DUP_UPD;
16091 NumVecs = 2;
16092 break;
16093 case Intrinsic::arm_neon_vld3dup:
16094 NewOpc = ARMISD::VLD3DUP_UPD;
16095 NumVecs = 3;
16096 break;
16097 case Intrinsic::arm_neon_vld4dup:
16098 NewOpc = ARMISD::VLD4DUP_UPD;
16099 NumVecs = 4;
16100 break;
16101 case Intrinsic::arm_neon_vld2lane:
16102 NewOpc = ARMISD::VLD2LN_UPD;
16103 NumVecs = 2;
16104 isLaneOp = true;
16105 break;
16106 case Intrinsic::arm_neon_vld3lane:
16107 NewOpc = ARMISD::VLD3LN_UPD;
16108 NumVecs = 3;
16109 isLaneOp = true;
16110 break;
16111 case Intrinsic::arm_neon_vld4lane:
16112 NewOpc = ARMISD::VLD4LN_UPD;
16113 NumVecs = 4;
16114 isLaneOp = true;
16115 break;
16116 case Intrinsic::arm_neon_vst1:
16117 NewOpc = ARMISD::VST1_UPD;
16118 NumVecs = 1;
16119 isLoadOp = false;
16120 break;
16121 case Intrinsic::arm_neon_vst2:
16122 NewOpc = ARMISD::VST2_UPD;
16123 NumVecs = 2;
16124 isLoadOp = false;
16125 break;
16126 case Intrinsic::arm_neon_vst3:
16127 NewOpc = ARMISD::VST3_UPD;
16128 NumVecs = 3;
16129 isLoadOp = false;
16130 break;
16131 case Intrinsic::arm_neon_vst4:
16132 NewOpc = ARMISD::VST4_UPD;
16133 NumVecs = 4;
16134 isLoadOp = false;
16135 break;
16136 case Intrinsic::arm_neon_vst2lane:
16137 NewOpc = ARMISD::VST2LN_UPD;
16138 NumVecs = 2;
16139 isLoadOp = false;
16140 isLaneOp = true;
16141 break;
16142 case Intrinsic::arm_neon_vst3lane:
16143 NewOpc = ARMISD::VST3LN_UPD;
16144 NumVecs = 3;
16145 isLoadOp = false;
16146 isLaneOp = true;
16147 break;
16148 case Intrinsic::arm_neon_vst4lane:
16149 NewOpc = ARMISD::VST4LN_UPD;
16150 NumVecs = 4;
16151 isLoadOp = false;
16152 isLaneOp = true;
16153 break;
16154 case Intrinsic::arm_neon_vst1x2:
16155 NewOpc = ARMISD::VST1x2_UPD;
16156 NumVecs = 2;
16157 isLoadOp = false;
16158 hasAlignment = false;
16159 break;
16160 case Intrinsic::arm_neon_vst1x3:
16161 NewOpc = ARMISD::VST1x3_UPD;
16162 NumVecs = 3;
16163 isLoadOp = false;
16164 hasAlignment = false;
16165 break;
16166 case Intrinsic::arm_neon_vst1x4:
16167 NewOpc = ARMISD::VST1x4_UPD;
16168 NumVecs = 4;
16169 isLoadOp = false;
16170 hasAlignment = false;
16171 break;
16172 }
16173 } else {
16174 isLaneOp = true;
16175 switch (N->getOpcode()) {
16176 default:
16177 llvm_unreachable("unexpected opcode for Neon base update");
16178 case ARMISD::VLD1DUP:
16179 NewOpc = ARMISD::VLD1DUP_UPD;
16180 NumVecs = 1;
16181 break;
16182 case ARMISD::VLD2DUP:
16183 NewOpc = ARMISD::VLD2DUP_UPD;
16184 NumVecs = 2;
16185 break;
16186 case ARMISD::VLD3DUP:
16187 NewOpc = ARMISD::VLD3DUP_UPD;
16188 NumVecs = 3;
16189 break;
16190 case ARMISD::VLD4DUP:
16191 NewOpc = ARMISD::VLD4DUP_UPD;
16192 NumVecs = 4;
16193 break;
16194 case ISD::LOAD:
16195 NewOpc = ARMISD::VLD1_UPD;
16196 NumVecs = 1;
16197 isLaneOp = false;
16198 break;
16199 case ISD::STORE:
16200 NewOpc = ARMISD::VST1_UPD;
16201 NumVecs = 1;
16202 isLaneOp = false;
16203 isLoadOp = false;
16204 break;
16205 }
16206 }
16207
16208 // Find the size of memory referenced by the load/store.
16209 EVT VecTy;
16210 if (isLoadOp) {
16211 VecTy = N->getValueType(ResNo: 0);
16212 } else if (Target.isIntrinsic) {
16213 VecTy = N->getOperand(Num: Target.AddrOpIdx + 1).getValueType();
16214 } else {
16215 assert(Target.isStore &&
16216 "Node has to be a load, a store, or an intrinsic!");
16217 VecTy = N->getOperand(Num: 1).getValueType();
16218 }
16219
16220 bool isVLDDUPOp =
16221 NewOpc == ARMISD::VLD1DUP_UPD || NewOpc == ARMISD::VLD2DUP_UPD ||
16222 NewOpc == ARMISD::VLD3DUP_UPD || NewOpc == ARMISD::VLD4DUP_UPD;
16223
16224 unsigned NumBytes = NumVecs * VecTy.getSizeInBits() / 8;
16225 if (isLaneOp || isVLDDUPOp)
16226 NumBytes /= VecTy.getVectorNumElements();
16227
16228 if (NumBytes >= 3 * 16 && User.ConstInc != NumBytes) {
16229 // VLD3/4 and VST3/4 for 128-bit vectors are implemented with two
16230 // separate instructions that make it harder to use a non-constant update.
16231 return false;
16232 }
16233
16234 if (SimpleConstIncOnly && User.ConstInc != NumBytes)
16235 return false;
16236
16237 if (!isValidBaseUpdate(N, User: User.N))
16238 return false;
16239
16240 // OK, we found an ADD we can fold into the base update.
16241 // Now, create a _UPD node, taking care of not breaking alignment.
16242
16243 EVT AlignedVecTy = VecTy;
16244 Align Alignment = MemN->getAlign();
16245
16246 // If this is a less-than-standard-aligned load/store, change the type to
16247 // match the standard alignment.
16248 // The alignment is overlooked when selecting _UPD variants; and it's
16249 // easier to introduce bitcasts here than fix that.
16250 // There are 3 ways to get to this base-update combine:
16251 // - intrinsics: they are assumed to be properly aligned (to the standard
16252 // alignment of the memory type), so we don't need to do anything.
16253 // - ARMISD::VLDx nodes: they are only generated from the aforementioned
16254 // intrinsics, so, likewise, there's nothing to do.
16255 // - generic load/store instructions: the alignment is specified as an
16256 // explicit operand, rather than implicitly as the standard alignment
16257 // of the memory type (like the intrinsics). We need to change the
16258 // memory type to match the explicit alignment. That way, we don't
16259 // generate non-standard-aligned ARMISD::VLDx nodes.
16260 if (isa<LSBaseSDNode>(Val: N)) {
16261 if (Alignment.value() < VecTy.getScalarSizeInBits() / 8) {
16262 MVT EltTy = MVT::getIntegerVT(BitWidth: Alignment.value() * 8);
16263 assert(NumVecs == 1 && "Unexpected multi-element generic load/store.");
16264 assert(!isLaneOp && "Unexpected generic load/store lane.");
16265 unsigned NumElts = NumBytes / (EltTy.getSizeInBits() / 8);
16266 AlignedVecTy = MVT::getVectorVT(VT: EltTy, NumElements: NumElts);
16267 }
16268 // Don't set an explicit alignment on regular load/stores that we want
16269 // to transform to VLD/VST 1_UPD nodes.
16270 // This matches the behavior of regular load/stores, which only get an
16271 // explicit alignment if the MMO alignment is larger than the standard
16272 // alignment of the memory type.
16273 // Intrinsics, however, always get an explicit alignment, set to the
16274 // alignment of the MMO.
16275 Alignment = Align(1);
16276 }
16277
16278 // Create the new updating load/store node.
16279 // First, create an SDVTList for the new updating node's results.
16280 EVT Tys[6];
16281 unsigned NumResultVecs = (isLoadOp ? NumVecs : 0);
16282 unsigned n;
16283 for (n = 0; n < NumResultVecs; ++n)
16284 Tys[n] = AlignedVecTy;
16285 Tys[n++] = MVT::i32;
16286 Tys[n] = MVT::Other;
16287 SDVTList SDTys = DAG.getVTList(VTs: ArrayRef(Tys, NumResultVecs + 2));
16288
16289 // Then, gather the new node's operands.
16290 SmallVector<SDValue, 8> Ops;
16291 Ops.push_back(Elt: N->getOperand(Num: 0)); // incoming chain
16292 Ops.push_back(Elt: N->getOperand(Num: Target.AddrOpIdx));
16293 Ops.push_back(Elt: User.Inc);
16294
16295 if (StoreSDNode *StN = dyn_cast<StoreSDNode>(Val: N)) {
16296 // Try to match the intrinsic's signature
16297 Ops.push_back(Elt: StN->getValue());
16298 } else {
16299 // Loads (and of course intrinsics) match the intrinsics' signature,
16300 // so just add all but the alignment operand.
16301 unsigned LastOperand =
16302 hasAlignment ? N->getNumOperands() - 1 : N->getNumOperands();
16303 for (unsigned i = Target.AddrOpIdx + 1; i < LastOperand; ++i)
16304 Ops.push_back(Elt: N->getOperand(Num: i));
16305 }
16306
16307 // For all node types, the alignment operand is always the last one.
16308 Ops.push_back(Elt: DAG.getConstant(Val: Alignment.value(), DL: dl, VT: MVT::i32));
16309
16310 // If this is a non-standard-aligned STORE, the penultimate operand is the
16311 // stored value. Bitcast it to the aligned type.
16312 if (AlignedVecTy != VecTy && N->getOpcode() == ISD::STORE) {
16313 SDValue &StVal = Ops[Ops.size() - 2];
16314 StVal = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: AlignedVecTy, Operand: StVal);
16315 }
16316
16317 EVT LoadVT = isLaneOp ? VecTy.getVectorElementType() : AlignedVecTy;
16318 SDValue UpdN = DAG.getMemIntrinsicNode(Opcode: NewOpc, dl, VTList: SDTys, Ops, MemVT: LoadVT,
16319 MMO: MemN->getMemOperand());
16320
16321 // Update the uses.
16322 SmallVector<SDValue, 5> NewResults;
16323 for (unsigned i = 0; i < NumResultVecs; ++i)
16324 NewResults.push_back(Elt: SDValue(UpdN.getNode(), i));
16325
16326 // If this is an non-standard-aligned LOAD, the first result is the loaded
16327 // value. Bitcast it to the expected result type.
16328 if (AlignedVecTy != VecTy && N->getOpcode() == ISD::LOAD) {
16329 SDValue &LdVal = NewResults[0];
16330 LdVal = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: VecTy, Operand: LdVal);
16331 }
16332
16333 NewResults.push_back(Elt: SDValue(UpdN.getNode(), NumResultVecs + 1)); // chain
16334 DCI.CombineTo(N, To: NewResults);
16335 DCI.CombineTo(N: User.N, Res: SDValue(UpdN.getNode(), NumResultVecs));
16336
16337 return true;
16338}
16339
16340// If (opcode ptr inc) is and ADD-like instruction, return the
16341// increment value. Otherwise return 0.
16342static unsigned getPointerConstIncrement(unsigned Opcode, SDValue Ptr,
16343 SDValue Inc, const SelectionDAG &DAG) {
16344 ConstantSDNode *CInc = dyn_cast<ConstantSDNode>(Val: Inc.getNode());
16345 if (!CInc)
16346 return 0;
16347
16348 switch (Opcode) {
16349 case ARMISD::VLD1_UPD:
16350 case ISD::ADD:
16351 return CInc->getZExtValue();
16352 case ISD::OR: {
16353 if (DAG.haveNoCommonBitsSet(A: Ptr, B: Inc)) {
16354 // (OR ptr inc) is the same as (ADD ptr inc)
16355 return CInc->getZExtValue();
16356 }
16357 return 0;
16358 }
16359 default:
16360 return 0;
16361 }
16362}
16363
16364static bool findPointerConstIncrement(SDNode *N, SDValue *Ptr, SDValue *CInc) {
16365 switch (N->getOpcode()) {
16366 case ISD::ADD:
16367 case ISD::OR: {
16368 if (isa<ConstantSDNode>(Val: N->getOperand(Num: 1))) {
16369 *Ptr = N->getOperand(Num: 0);
16370 *CInc = N->getOperand(Num: 1);
16371 return true;
16372 }
16373 return false;
16374 }
16375 case ARMISD::VLD1_UPD: {
16376 if (isa<ConstantSDNode>(Val: N->getOperand(Num: 2))) {
16377 *Ptr = N->getOperand(Num: 1);
16378 *CInc = N->getOperand(Num: 2);
16379 return true;
16380 }
16381 return false;
16382 }
16383 default:
16384 return false;
16385 }
16386}
16387
16388/// CombineBaseUpdate - Target-specific DAG combine function for VLDDUP,
16389/// NEON load/store intrinsics, and generic vector load/stores, to merge
16390/// base address updates.
16391/// For generic load/stores, the memory type is assumed to be a vector.
16392/// The caller is assumed to have checked legality.
16393static SDValue CombineBaseUpdate(SDNode *N,
16394 TargetLowering::DAGCombinerInfo &DCI) {
16395 const bool isIntrinsic = (N->getOpcode() == ISD::INTRINSIC_VOID ||
16396 N->getOpcode() == ISD::INTRINSIC_W_CHAIN);
16397 const bool isStore = N->getOpcode() == ISD::STORE;
16398 const unsigned AddrOpIdx = ((isIntrinsic || isStore) ? 2 : 1);
16399 BaseUpdateTarget Target = {.N: N, .isIntrinsic: isIntrinsic, .isStore: isStore, .AddrOpIdx: AddrOpIdx};
16400
16401 // Limit the number of possible base-updates we look at to prevent degenerate
16402 // cases.
16403 unsigned MaxBaseUpdates = ArmMaxBaseUpdatesToCheck;
16404
16405 SDValue Addr = N->getOperand(Num: AddrOpIdx);
16406
16407 SmallVector<BaseUpdateUser, 8> BaseUpdates;
16408
16409 // Search for a use of the address operand that is an increment.
16410 for (SDUse &Use : Addr->uses()) {
16411 SDNode *User = Use.getUser();
16412 if (Use.getResNo() != Addr.getResNo() || User->getNumOperands() != 2)
16413 continue;
16414
16415 SDValue Inc = User->getOperand(Num: Use.getOperandNo() == 1 ? 0 : 1);
16416 unsigned ConstInc =
16417 getPointerConstIncrement(Opcode: User->getOpcode(), Ptr: Addr, Inc, DAG: DCI.DAG);
16418
16419 if (ConstInc || User->getOpcode() == ISD::ADD) {
16420 BaseUpdates.push_back(Elt: {.N: User, .Inc: Inc, .ConstInc: ConstInc});
16421 if (BaseUpdates.size() >= MaxBaseUpdates)
16422 break;
16423 }
16424 }
16425
16426 // If the address is a constant pointer increment itself, find
16427 // another constant increment that has the same base operand
16428 SDValue Base;
16429 SDValue CInc;
16430 if (findPointerConstIncrement(N: Addr.getNode(), Ptr: &Base, CInc: &CInc)) {
16431 unsigned Offset =
16432 getPointerConstIncrement(Opcode: Addr->getOpcode(), Ptr: Base, Inc: CInc, DAG: DCI.DAG);
16433 if (Offset) {
16434 for (SDUse &Use : Base->uses()) {
16435
16436 SDNode *User = Use.getUser();
16437 if (Use.getResNo() != Base.getResNo() || User == Addr.getNode() ||
16438 User->getNumOperands() != 2)
16439 continue;
16440
16441 SDValue UserInc = User->getOperand(Num: Use.getOperandNo() == 0 ? 1 : 0);
16442 unsigned UserOffset =
16443 getPointerConstIncrement(Opcode: User->getOpcode(), Ptr: Base, Inc: UserInc, DAG: DCI.DAG);
16444
16445 if (!UserOffset || UserOffset <= Offset)
16446 continue;
16447
16448 unsigned NewConstInc = UserOffset - Offset;
16449 SDValue NewInc = DCI.DAG.getConstant(Val: NewConstInc, DL: SDLoc(N), VT: MVT::i32);
16450 BaseUpdates.push_back(Elt: {.N: User, .Inc: NewInc, .ConstInc: NewConstInc});
16451 if (BaseUpdates.size() >= MaxBaseUpdates)
16452 break;
16453 }
16454 }
16455 }
16456
16457 // Try to fold the load/store with an update that matches memory
16458 // access size. This should work well for sequential loads.
16459 unsigned NumValidUpd = BaseUpdates.size();
16460 for (unsigned I = 0; I < NumValidUpd; I++) {
16461 BaseUpdateUser &User = BaseUpdates[I];
16462 if (TryCombineBaseUpdate(Target, User, /*SimpleConstIncOnly=*/true, DCI))
16463 return SDValue();
16464 }
16465
16466 // Try to fold with other users. Non-constant updates are considered
16467 // first, and constant updates are sorted to not break a sequence of
16468 // strided accesses (if there is any).
16469 llvm::stable_sort(Range&: BaseUpdates,
16470 C: [](const BaseUpdateUser &LHS, const BaseUpdateUser &RHS) {
16471 return LHS.ConstInc < RHS.ConstInc;
16472 });
16473 for (BaseUpdateUser &User : BaseUpdates) {
16474 if (TryCombineBaseUpdate(Target, User, /*SimpleConstIncOnly=*/false, DCI))
16475 return SDValue();
16476 }
16477 return SDValue();
16478}
16479
16480static SDValue PerformVLDCombine(SDNode *N,
16481 TargetLowering::DAGCombinerInfo &DCI) {
16482 if (DCI.isBeforeLegalize() || DCI.isCalledByLegalizer())
16483 return SDValue();
16484
16485 return CombineBaseUpdate(N, DCI);
16486}
16487
16488static SDValue PerformMVEVLDCombine(SDNode *N,
16489 TargetLowering::DAGCombinerInfo &DCI) {
16490 if (DCI.isBeforeLegalize() || DCI.isCalledByLegalizer())
16491 return SDValue();
16492
16493 SelectionDAG &DAG = DCI.DAG;
16494 SDValue Addr = N->getOperand(Num: 2);
16495 MemSDNode *MemN = cast<MemSDNode>(Val: N);
16496 SDLoc dl(N);
16497
16498 // For the stores, where there are multiple intrinsics we only actually want
16499 // to post-inc the last of the them.
16500 unsigned IntNo = N->getConstantOperandVal(Num: 1);
16501 if (IntNo == Intrinsic::arm_mve_vst2q && N->getConstantOperandVal(Num: 5) != 1)
16502 return SDValue();
16503 if (IntNo == Intrinsic::arm_mve_vst4q && N->getConstantOperandVal(Num: 7) != 3)
16504 return SDValue();
16505
16506 // Search for a use of the address operand that is an increment.
16507 for (SDUse &Use : Addr->uses()) {
16508 SDNode *User = Use.getUser();
16509 if (User->getOpcode() != ISD::ADD || Use.getResNo() != Addr.getResNo())
16510 continue;
16511
16512 // Check that the add is independent of the load/store. Otherwise, folding
16513 // it would create a cycle. We can avoid searching through Addr as it's a
16514 // predecessor to both.
16515 SmallPtrSet<const SDNode *, 32> Visited;
16516 SmallVector<const SDNode *, 16> Worklist;
16517 Visited.insert(Ptr: Addr.getNode());
16518 Worklist.push_back(Elt: N);
16519 Worklist.push_back(Elt: User);
16520 const unsigned MaxSteps = 1024;
16521 if (SDNode::hasPredecessorHelper(N, Visited, Worklist, MaxSteps) ||
16522 SDNode::hasPredecessorHelper(N: User, Visited, Worklist, MaxSteps))
16523 continue;
16524
16525 // Find the new opcode for the updating load/store.
16526 bool isLoadOp = true;
16527 unsigned NewOpc = 0;
16528 unsigned NumVecs = 0;
16529 switch (IntNo) {
16530 default:
16531 llvm_unreachable("unexpected intrinsic for MVE VLDn combine");
16532 case Intrinsic::arm_mve_vld2q:
16533 NewOpc = ARMISD::VLD2_UPD;
16534 NumVecs = 2;
16535 break;
16536 case Intrinsic::arm_mve_vld4q:
16537 NewOpc = ARMISD::VLD4_UPD;
16538 NumVecs = 4;
16539 break;
16540 case Intrinsic::arm_mve_vst2q:
16541 NewOpc = ARMISD::VST2_UPD;
16542 NumVecs = 2;
16543 isLoadOp = false;
16544 break;
16545 case Intrinsic::arm_mve_vst4q:
16546 NewOpc = ARMISD::VST4_UPD;
16547 NumVecs = 4;
16548 isLoadOp = false;
16549 break;
16550 }
16551
16552 // Find the size of memory referenced by the load/store.
16553 EVT VecTy;
16554 if (isLoadOp) {
16555 VecTy = N->getValueType(ResNo: 0);
16556 } else {
16557 VecTy = N->getOperand(Num: 3).getValueType();
16558 }
16559
16560 unsigned NumBytes = NumVecs * VecTy.getSizeInBits() / 8;
16561
16562 // If the increment is a constant, it must match the memory ref size.
16563 SDValue Inc = User->getOperand(Num: User->getOperand(Num: 0) == Addr ? 1 : 0);
16564 ConstantSDNode *CInc = dyn_cast<ConstantSDNode>(Val: Inc.getNode());
16565 if (!CInc || CInc->getZExtValue() != NumBytes)
16566 continue;
16567
16568 // Create the new updating load/store node.
16569 // First, create an SDVTList for the new updating node's results.
16570 EVT Tys[6];
16571 unsigned NumResultVecs = (isLoadOp ? NumVecs : 0);
16572 unsigned n;
16573 for (n = 0; n < NumResultVecs; ++n)
16574 Tys[n] = VecTy;
16575 Tys[n++] = MVT::i32;
16576 Tys[n] = MVT::Other;
16577 SDVTList SDTys = DAG.getVTList(VTs: ArrayRef(Tys, NumResultVecs + 2));
16578
16579 // Then, gather the new node's operands.
16580 SmallVector<SDValue, 8> Ops;
16581 Ops.push_back(Elt: N->getOperand(Num: 0)); // incoming chain
16582 Ops.push_back(Elt: N->getOperand(Num: 2)); // ptr
16583 Ops.push_back(Elt: Inc);
16584
16585 for (unsigned i = 3; i < N->getNumOperands(); ++i)
16586 Ops.push_back(Elt: N->getOperand(Num: i));
16587
16588 SDValue UpdN = DAG.getMemIntrinsicNode(Opcode: NewOpc, dl, VTList: SDTys, Ops, MemVT: VecTy,
16589 MMO: MemN->getMemOperand());
16590
16591 // Update the uses.
16592 SmallVector<SDValue, 5> NewResults;
16593 for (unsigned i = 0; i < NumResultVecs; ++i)
16594 NewResults.push_back(Elt: SDValue(UpdN.getNode(), i));
16595
16596 NewResults.push_back(Elt: SDValue(UpdN.getNode(), NumResultVecs + 1)); // chain
16597 DCI.CombineTo(N, To: NewResults);
16598 DCI.CombineTo(N: User, Res: SDValue(UpdN.getNode(), NumResultVecs));
16599
16600 break;
16601 }
16602
16603 return SDValue();
16604}
16605
16606/// CombineVLDDUP - For a VDUPLANE node N, check if its source operand is a
16607/// vldN-lane (N > 1) intrinsic, and if all the other uses of that intrinsic
16608/// are also VDUPLANEs. If so, combine them to a vldN-dup operation and
16609/// return true.
16610static bool CombineVLDDUP(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
16611 SelectionDAG &DAG = DCI.DAG;
16612 EVT VT = N->getValueType(ResNo: 0);
16613 // vldN-dup instructions only support 64-bit vectors for N > 1.
16614 if (!VT.is64BitVector())
16615 return false;
16616
16617 // Check if the VDUPLANE operand is a vldN-dup intrinsic.
16618 SDNode *VLD = N->getOperand(Num: 0).getNode();
16619 if (VLD->getOpcode() != ISD::INTRINSIC_W_CHAIN)
16620 return false;
16621 unsigned NumVecs = 0;
16622 unsigned NewOpc = 0;
16623 unsigned IntNo = VLD->getConstantOperandVal(Num: 1);
16624 if (IntNo == Intrinsic::arm_neon_vld2lane) {
16625 NumVecs = 2;
16626 NewOpc = ARMISD::VLD2DUP;
16627 } else if (IntNo == Intrinsic::arm_neon_vld3lane) {
16628 NumVecs = 3;
16629 NewOpc = ARMISD::VLD3DUP;
16630 } else if (IntNo == Intrinsic::arm_neon_vld4lane) {
16631 NumVecs = 4;
16632 NewOpc = ARMISD::VLD4DUP;
16633 } else {
16634 return false;
16635 }
16636
16637 // First check that all the vldN-lane uses are VDUPLANEs and that the lane
16638 // numbers match the load.
16639 unsigned VLDLaneNo = VLD->getConstantOperandVal(Num: NumVecs + 3);
16640 for (SDUse &Use : VLD->uses()) {
16641 // Ignore uses of the chain result.
16642 if (Use.getResNo() == NumVecs)
16643 continue;
16644 SDNode *User = Use.getUser();
16645 if (User->getOpcode() != ARMISD::VDUPLANE ||
16646 VLDLaneNo != User->getConstantOperandVal(Num: 1))
16647 return false;
16648 }
16649
16650 // Create the vldN-dup node.
16651 EVT Tys[5];
16652 unsigned n;
16653 for (n = 0; n < NumVecs; ++n)
16654 Tys[n] = VT;
16655 Tys[n] = MVT::Other;
16656 SDVTList SDTys = DAG.getVTList(VTs: ArrayRef(Tys, NumVecs + 1));
16657 SDValue Ops[] = { VLD->getOperand(Num: 0), VLD->getOperand(Num: 2) };
16658 MemIntrinsicSDNode *VLDMemInt = cast<MemIntrinsicSDNode>(Val: VLD);
16659 SDValue VLDDup = DAG.getMemIntrinsicNode(Opcode: NewOpc, dl: SDLoc(VLD), VTList: SDTys,
16660 Ops, MemVT: VLDMemInt->getMemoryVT(),
16661 MMO: VLDMemInt->getMemOperand());
16662
16663 // Update the uses.
16664 for (SDUse &Use : VLD->uses()) {
16665 unsigned ResNo = Use.getResNo();
16666 // Ignore uses of the chain result.
16667 if (ResNo == NumVecs)
16668 continue;
16669 DCI.CombineTo(N: Use.getUser(), Res: SDValue(VLDDup.getNode(), ResNo));
16670 }
16671
16672 // Now the vldN-lane intrinsic is dead except for its chain result.
16673 // Update uses of the chain.
16674 std::vector<SDValue> VLDDupResults;
16675 for (unsigned n = 0; n < NumVecs; ++n)
16676 VLDDupResults.push_back(x: SDValue(VLDDup.getNode(), n));
16677 VLDDupResults.push_back(x: SDValue(VLDDup.getNode(), NumVecs));
16678 DCI.CombineTo(N: VLD, To: VLDDupResults);
16679
16680 return true;
16681}
16682
16683/// PerformVDUPLANECombine - Target-specific dag combine xforms for
16684/// ARMISD::VDUPLANE.
16685static SDValue PerformVDUPLANECombine(SDNode *N,
16686 TargetLowering::DAGCombinerInfo &DCI,
16687 const ARMSubtarget *Subtarget) {
16688 SDValue Op = N->getOperand(Num: 0);
16689 EVT VT = N->getValueType(ResNo: 0);
16690
16691 // On MVE, we just convert the VDUPLANE to a VDUP with an extract.
16692 if (Subtarget->hasMVEIntegerOps()) {
16693 EVT ExtractVT = VT.getVectorElementType();
16694 // We need to ensure we are creating a legal type.
16695 if (!DCI.DAG.getTargetLoweringInfo().isTypeLegal(VT: ExtractVT))
16696 ExtractVT = MVT::i32;
16697 SDValue Extract = DCI.DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N), VT: ExtractVT,
16698 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1));
16699 return DCI.DAG.getNode(Opcode: ARMISD::VDUP, DL: SDLoc(N), VT, Operand: Extract);
16700 }
16701
16702 // If the source is a vldN-lane (N > 1) intrinsic, and all the other uses
16703 // of that intrinsic are also VDUPLANEs, combine them to a vldN-dup operation.
16704 if (CombineVLDDUP(N, DCI))
16705 return SDValue(N, 0);
16706
16707 // If the source is already a VMOVIMM or VMVNIMM splat, the VDUPLANE is
16708 // redundant. Ignore bit_converts for now; element sizes are checked below.
16709 while (Op.getOpcode() == ISD::BITCAST)
16710 Op = Op.getOperand(i: 0);
16711 if (Op.getOpcode() != ARMISD::VMOVIMM && Op.getOpcode() != ARMISD::VMVNIMM)
16712 return SDValue();
16713
16714 // Make sure the VMOV element size is not bigger than the VDUPLANE elements.
16715 unsigned EltSize = Op.getScalarValueSizeInBits();
16716 // The canonical VMOV for a zero vector uses a 32-bit element size.
16717 unsigned Imm = Op.getConstantOperandVal(i: 0);
16718 unsigned EltBits;
16719 if (ARM_AM::decodeVMOVModImm(ModImm: Imm, EltBits) == 0)
16720 EltSize = 8;
16721 if (EltSize > VT.getScalarSizeInBits())
16722 return SDValue();
16723
16724 return DCI.DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT, Operand: Op);
16725}
16726
16727/// PerformVDUPCombine - Target-specific dag combine xforms for ARMISD::VDUP.
16728static SDValue PerformVDUPCombine(SDNode *N, SelectionDAG &DAG,
16729 const ARMSubtarget *Subtarget) {
16730 SDValue Op = N->getOperand(Num: 0);
16731 SDLoc dl(N);
16732
16733 if (Subtarget->hasMVEIntegerOps()) {
16734 // Convert VDUP f32 -> VDUP BITCAST i32 under MVE, as we know the value will
16735 // need to come from a GPR.
16736 if (Op.getValueType() == MVT::f32)
16737 return DAG.getNode(Opcode: ARMISD::VDUP, DL: dl, VT: N->getValueType(ResNo: 0),
16738 Operand: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: Op));
16739 else if (Op.getValueType() == MVT::f16)
16740 return DAG.getNode(Opcode: ARMISD::VDUP, DL: dl, VT: N->getValueType(ResNo: 0),
16741 Operand: DAG.getNode(Opcode: ARMISD::VMOVrh, DL: dl, VT: MVT::i32, Operand: Op));
16742 }
16743
16744 if (!Subtarget->hasNEON())
16745 return SDValue();
16746
16747 // Match VDUP(LOAD) -> VLD1DUP.
16748 // We match this pattern here rather than waiting for isel because the
16749 // transform is only legal for unindexed loads.
16750 LoadSDNode *LD = dyn_cast<LoadSDNode>(Val: Op.getNode());
16751 if (LD && Op.hasOneUse() && LD->isUnindexed() &&
16752 LD->getMemoryVT() == N->getValueType(ResNo: 0).getVectorElementType()) {
16753 SDValue Ops[] = {LD->getOperand(Num: 0), LD->getOperand(Num: 1),
16754 DAG.getConstant(Val: LD->getAlign().value(), DL: SDLoc(N), VT: MVT::i32)};
16755 SDVTList SDTys = DAG.getVTList(VT1: N->getValueType(ResNo: 0), VT2: MVT::Other);
16756 SDValue VLDDup =
16757 DAG.getMemIntrinsicNode(Opcode: ARMISD::VLD1DUP, dl: SDLoc(N), VTList: SDTys, Ops,
16758 MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand());
16759 DAG.ReplaceAllUsesOfValueWith(From: SDValue(LD, 1), To: VLDDup.getValue(R: 1));
16760 return VLDDup;
16761 }
16762
16763 return SDValue();
16764}
16765
16766static SDValue PerformLOADCombine(SDNode *N,
16767 TargetLowering::DAGCombinerInfo &DCI,
16768 const ARMSubtarget *Subtarget) {
16769 EVT VT = N->getValueType(ResNo: 0);
16770 // Let the deinterleave combine form a structured load first.
16771 if (DCI.getDAGCombineLevel() >= AfterLegalizeTypes && ISD::isNormalLoad(N))
16772 for (SDUse &Use : N->uses())
16773 if (Use.getUser()->getOpcode() == ISD::VECTOR_DEINTERLEAVE)
16774 return SDValue();
16775 // If this is a legal vector load, try to combine it into a VLD1_UPD.
16776 if (Subtarget->hasNEON() && ISD::isNormalLoad(N) && VT.isVector() &&
16777 DCI.DAG.getTargetLoweringInfo().isTypeLegal(VT))
16778 return CombineBaseUpdate(N, DCI);
16779
16780 return SDValue();
16781}
16782
16783// Optimize trunc store (of multiple scalars) to shuffle and store. First,
16784// pack all of the elements in one place. Next, store to memory in fewer
16785// chunks.
16786static SDValue PerformTruncatingStoreCombine(StoreSDNode *St,
16787 SelectionDAG &DAG) {
16788 SDValue StVal = St->getValue();
16789 EVT VT = StVal.getValueType();
16790 if (!St->isTruncatingStore() || !VT.isVector())
16791 return SDValue();
16792 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
16793 EVT StVT = St->getMemoryVT();
16794 unsigned NumElems = VT.getVectorNumElements();
16795 assert(StVT != VT && "Cannot truncate to the same type");
16796 unsigned FromEltSz = VT.getScalarSizeInBits();
16797 unsigned ToEltSz = StVT.getScalarSizeInBits();
16798
16799 // From, To sizes and ElemCount must be pow of two
16800 if (!isPowerOf2_32(Value: NumElems * FromEltSz * ToEltSz))
16801 return SDValue();
16802
16803 // We are going to use the original vector elt for storing.
16804 // Accumulated smaller vector elements must be a multiple of the store size.
16805 if (0 != (NumElems * FromEltSz) % ToEltSz)
16806 return SDValue();
16807
16808 unsigned SizeRatio = FromEltSz / ToEltSz;
16809 assert(SizeRatio * NumElems * ToEltSz == VT.getSizeInBits());
16810
16811 // Create a type on which we perform the shuffle.
16812 EVT WideVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: StVT.getScalarType(),
16813 NumElements: NumElems * SizeRatio);
16814 assert(WideVecVT.getSizeInBits() == VT.getSizeInBits());
16815
16816 SDLoc DL(St);
16817 SDValue WideVec = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: WideVecVT, Operand: StVal);
16818 SmallVector<int, 8> ShuffleVec(NumElems * SizeRatio, -1);
16819 for (unsigned i = 0; i < NumElems; ++i)
16820 ShuffleVec[i] = DAG.getDataLayout().isBigEndian() ? (i + 1) * SizeRatio - 1
16821 : i * SizeRatio;
16822
16823 // Can't shuffle using an illegal type.
16824 if (!TLI.isTypeLegal(VT: WideVecVT))
16825 return SDValue();
16826
16827 SDValue Shuff = DAG.getVectorShuffle(
16828 VT: WideVecVT, dl: DL, N1: WideVec, N2: DAG.getUNDEF(VT: WideVec.getValueType()), Mask: ShuffleVec);
16829 // At this point all of the data is stored at the bottom of the
16830 // register. We now need to save it to mem.
16831
16832 // Find the largest store unit
16833 MVT StoreType = MVT::i8;
16834 for (MVT Tp : MVT::integer_valuetypes()) {
16835 if (TLI.isTypeLegal(VT: Tp) && Tp.getSizeInBits() <= NumElems * ToEltSz)
16836 StoreType = Tp;
16837 }
16838 // Didn't find a legal store type.
16839 if (!TLI.isTypeLegal(VT: StoreType))
16840 return SDValue();
16841
16842 // Bitcast the original vector into a vector of store-size units
16843 EVT StoreVecVT =
16844 EVT::getVectorVT(Context&: *DAG.getContext(), VT: StoreType,
16845 NumElements: VT.getSizeInBits() / EVT(StoreType).getSizeInBits());
16846 assert(StoreVecVT.getSizeInBits() == VT.getSizeInBits());
16847 SDValue ShuffWide = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: StoreVecVT, Operand: Shuff);
16848 SmallVector<SDValue, 8> Chains;
16849 SDValue Increment = DAG.getConstant(Val: StoreType.getSizeInBits() / 8, DL,
16850 VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
16851 SDValue BasePtr = St->getBasePtr();
16852
16853 // Perform one or more big stores into memory.
16854 unsigned E = (ToEltSz * NumElems) / StoreType.getSizeInBits();
16855 for (unsigned I = 0; I < E; I++) {
16856 SDValue SubVec = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: StoreType,
16857 N1: ShuffWide, N2: DAG.getIntPtrConstant(Val: I, DL));
16858 SDValue Ch =
16859 DAG.getStore(Chain: St->getChain(), dl: DL, Val: SubVec, Ptr: BasePtr, PtrInfo: St->getPointerInfo(),
16860 Alignment: St->getAlign(), MMOFlags: St->getMemOperand()->getFlags());
16861 BasePtr =
16862 DAG.getNode(Opcode: ISD::ADD, DL, VT: BasePtr.getValueType(), N1: BasePtr, N2: Increment);
16863 Chains.push_back(Elt: Ch);
16864 }
16865 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: Chains);
16866}
16867
16868// Try taking a single vector store from an fpround (which would otherwise turn
16869// into an expensive buildvector) and splitting it into a series of narrowing
16870// stores.
16871static SDValue PerformSplittingToNarrowingStores(StoreSDNode *St,
16872 SelectionDAG &DAG) {
16873 if (!St->isSimple() || St->isTruncatingStore() || !St->isUnindexed())
16874 return SDValue();
16875 SDValue Trunc = St->getValue();
16876 if (Trunc->getOpcode() != ISD::FP_ROUND)
16877 return SDValue();
16878 EVT FromVT = Trunc->getOperand(Num: 0).getValueType();
16879 EVT ToVT = Trunc.getValueType();
16880 if (!ToVT.isVector())
16881 return SDValue();
16882 assert(FromVT.getVectorNumElements() == ToVT.getVectorNumElements());
16883 EVT ToEltVT = ToVT.getVectorElementType();
16884 EVT FromEltVT = FromVT.getVectorElementType();
16885
16886 if (FromEltVT != MVT::f32 || ToEltVT != MVT::f16)
16887 return SDValue();
16888
16889 unsigned NumElements = 4;
16890 if (FromVT.getVectorNumElements() % NumElements != 0)
16891 return SDValue();
16892
16893 // Test if the Trunc will be convertible to a VMOVN with a shuffle, and if so
16894 // use the VMOVN over splitting the store. We are looking for patterns of:
16895 // !rev: 0 N 1 N+1 2 N+2 ...
16896 // rev: N 0 N+1 1 N+2 2 ...
16897 // The shuffle may either be a single source (in which case N = NumElts/2) or
16898 // two inputs extended with concat to the same size (in which case N =
16899 // NumElts).
16900 auto isVMOVNShuffle = [&](ShuffleVectorSDNode *SVN, bool Rev) {
16901 ArrayRef<int> M = SVN->getMask();
16902 unsigned NumElts = ToVT.getVectorNumElements();
16903 if (SVN->getOperand(Num: 1).isUndef())
16904 NumElts /= 2;
16905
16906 unsigned Off0 = Rev ? NumElts : 0;
16907 unsigned Off1 = Rev ? 0 : NumElts;
16908
16909 for (unsigned I = 0; I < NumElts; I += 2) {
16910 if (M[I] >= 0 && M[I] != (int)(Off0 + I / 2))
16911 return false;
16912 if (M[I + 1] >= 0 && M[I + 1] != (int)(Off1 + I / 2))
16913 return false;
16914 }
16915
16916 return true;
16917 };
16918
16919 if (auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Val: Trunc.getOperand(i: 0)))
16920 if (isVMOVNShuffle(Shuffle, false) || isVMOVNShuffle(Shuffle, true))
16921 return SDValue();
16922
16923 LLVMContext &C = *DAG.getContext();
16924 SDLoc DL(St);
16925 // Details about the old store
16926 SDValue Ch = St->getChain();
16927 SDValue BasePtr = St->getBasePtr();
16928 Align Alignment = St->getBaseAlign();
16929 MachineMemOperand::Flags MMOFlags = St->getMemOperand()->getFlags();
16930 AAMDNodes AAInfo = St->getAAInfo();
16931
16932 // We split the store into slices of NumElements. fp16 trunc stores are vcvt
16933 // and then stored as truncating integer stores.
16934 EVT NewFromVT = EVT::getVectorVT(Context&: C, VT: FromEltVT, NumElements);
16935 EVT NewToVT = EVT::getVectorVT(
16936 Context&: C, VT: EVT::getIntegerVT(Context&: C, BitWidth: ToEltVT.getSizeInBits()), NumElements);
16937
16938 SmallVector<SDValue, 4> Stores;
16939 for (unsigned i = 0; i < FromVT.getVectorNumElements() / NumElements; i++) {
16940 unsigned NewOffset = i * NumElements * ToEltVT.getSizeInBits() / 8;
16941 SDValue NewPtr =
16942 DAG.getObjectPtrOffset(SL: DL, Ptr: BasePtr, Offset: TypeSize::getFixed(ExactSize: NewOffset));
16943
16944 SDValue Extract =
16945 DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: NewFromVT, N1: Trunc.getOperand(i: 0),
16946 N2: DAG.getConstant(Val: i * NumElements, DL, VT: MVT::i32));
16947
16948 SDValue FPTrunc =
16949 DAG.getNode(Opcode: ARMISD::VCVTN, DL, VT: MVT::v8f16, N1: DAG.getUNDEF(VT: MVT::v8f16),
16950 N2: Extract, N3: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
16951 Extract = DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL, VT: MVT::v4i32, Operand: FPTrunc);
16952
16953 SDValue Store = DAG.getTruncStore(
16954 Chain: Ch, dl: DL, Val: Extract, Ptr: NewPtr, PtrInfo: St->getPointerInfo().getWithOffset(O: NewOffset),
16955 SVT: NewToVT, Alignment, MMOFlags, Metadata: AAInfo);
16956 Stores.push_back(Elt: Store);
16957 }
16958 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: Stores);
16959}
16960
16961// Try taking a single vector store from an MVETRUNC (which would otherwise turn
16962// into an expensive buildvector) and splitting it into a series of narrowing
16963// stores.
16964static SDValue PerformSplittingMVETruncToNarrowingStores(StoreSDNode *St,
16965 SelectionDAG &DAG) {
16966 if (!St->isSimple() || St->isTruncatingStore() || !St->isUnindexed())
16967 return SDValue();
16968 SDValue Trunc = St->getValue();
16969 if (Trunc->getOpcode() != ARMISD::MVETRUNC)
16970 return SDValue();
16971 EVT FromVT = Trunc->getOperand(Num: 0).getValueType();
16972 EVT ToVT = Trunc.getValueType();
16973
16974 LLVMContext &C = *DAG.getContext();
16975 SDLoc DL(St);
16976 // Details about the old store
16977 SDValue Ch = St->getChain();
16978 SDValue BasePtr = St->getBasePtr();
16979 Align Alignment = St->getBaseAlign();
16980 MachineMemOperand::Flags MMOFlags = St->getMemOperand()->getFlags();
16981 AAMDNodes AAInfo = St->getAAInfo();
16982
16983 EVT NewToVT = EVT::getVectorVT(Context&: C, VT: ToVT.getVectorElementType(),
16984 NumElements: FromVT.getVectorNumElements());
16985
16986 SmallVector<SDValue, 4> Stores;
16987 for (unsigned i = 0; i < Trunc.getNumOperands(); i++) {
16988 unsigned NewOffset =
16989 i * FromVT.getVectorNumElements() * ToVT.getScalarSizeInBits() / 8;
16990 SDValue NewPtr =
16991 DAG.getObjectPtrOffset(SL: DL, Ptr: BasePtr, Offset: TypeSize::getFixed(ExactSize: NewOffset));
16992
16993 SDValue Extract = Trunc.getOperand(i);
16994 SDValue Store = DAG.getTruncStore(
16995 Chain: Ch, dl: DL, Val: Extract, Ptr: NewPtr, PtrInfo: St->getPointerInfo().getWithOffset(O: NewOffset),
16996 SVT: NewToVT, Alignment, MMOFlags, Metadata: AAInfo);
16997 Stores.push_back(Elt: Store);
16998 }
16999 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: Stores);
17000}
17001
17002// Given a floating point store from an extracted vector, with an integer
17003// VGETLANE that already exists, store the existing VGETLANEu directly. This can
17004// help reduce fp register pressure, doesn't require the fp extract and allows
17005// use of more integer post-inc stores not available with vstr.
17006static SDValue PerformExtractFpToIntStores(StoreSDNode *St, SelectionDAG &DAG) {
17007 if (!St->isSimple() || St->isTruncatingStore() || !St->isUnindexed())
17008 return SDValue();
17009 SDValue Extract = St->getValue();
17010 EVT VT = Extract.getValueType();
17011 // For now only uses f16. This may be useful for f32 too, but that will
17012 // be bitcast(extract), not the VGETLANEu we currently check here.
17013 if (VT != MVT::f16 || Extract->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
17014 return SDValue();
17015
17016 SDNode *GetLane =
17017 DAG.getNodeIfExists(Opcode: ARMISD::VGETLANEu, VTList: DAG.getVTList(VT: MVT::i32),
17018 Ops: {Extract.getOperand(i: 0), Extract.getOperand(i: 1)});
17019 if (!GetLane)
17020 return SDValue();
17021
17022 LLVMContext &C = *DAG.getContext();
17023 SDLoc DL(St);
17024 // Create a new integer store to replace the existing floating point version.
17025 SDValue Ch = St->getChain();
17026 SDValue BasePtr = St->getBasePtr();
17027 Align Alignment = St->getBaseAlign();
17028 MachineMemOperand::Flags MMOFlags = St->getMemOperand()->getFlags();
17029 AAMDNodes AAInfo = St->getAAInfo();
17030 EVT NewToVT = EVT::getIntegerVT(Context&: C, BitWidth: VT.getSizeInBits());
17031 SDValue Store = DAG.getTruncStore(Chain: Ch, dl: DL, Val: SDValue(GetLane, 0), Ptr: BasePtr,
17032 PtrInfo: St->getPointerInfo(), SVT: NewToVT, Alignment,
17033 MMOFlags, Metadata: AAInfo);
17034
17035 return Store;
17036}
17037
17038// Combine stores of all VECTOR_INTERLEAVE results into a structured store
17039// (vstN) when they access consecutive memory. Running this after type
17040// legalization let wider vector types to be split into legal types, so each
17041// split can form a structured store.
17042static SDValue PerformLegalizedInterleavedStoreCombine(
17043 StoreSDNode *ST, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG,
17044 const ARMSubtarget *Subtarget) {
17045 if (DCI.getDAGCombineLevel() < AfterLegalizeTypes)
17046 return SDValue();
17047
17048 if (!ISD::isNormalStore(N: ST) || !ST->isSimple() || !ST->getOffset().isUndef())
17049 return SDValue();
17050
17051 SDValue StoredValue = ST->getValue();
17052 SDNode *Interleave = StoredValue.getNode();
17053 if (Interleave->getOpcode() != ISD::VECTOR_INTERLEAVE)
17054 return SDValue();
17055
17056 unsigned NumParts = Interleave->getNumOperands();
17057 if (NumParts < 2 || NumParts > 4)
17058 return SDValue();
17059
17060 if (NumParts == 3 && Subtarget->hasMVEIntegerOps())
17061 return SDValue();
17062
17063 EVT SubVecTy = Interleave->getValueType(ResNo: 0);
17064 if (Subtarget->hasNEON() && !SubVecTy.is64BitVector() &&
17065 !SubVecTy.is128BitVector())
17066 return SDValue();
17067 if (Subtarget->hasMVEIntegerOps() && !SubVecTy.is128BitVector())
17068 return SDValue();
17069 unsigned EltBits = SubVecTy.getScalarSizeInBits();
17070 if (EltBits != 8 && EltBits != 16 && EltBits != 32)
17071 return SDValue();
17072
17073 SmallVector<StoreSDNode *, 4> Stores(NumParts, nullptr);
17074 for (SDUse &Use : Interleave->uses()) {
17075 unsigned ResNo = Use.getResNo();
17076 auto *Store = dyn_cast<StoreSDNode>(Val: Use.getUser());
17077 if (Stores[ResNo] || !Store)
17078 return SDValue();
17079 Stores[ResNo] = Store;
17080 }
17081
17082 if (llvm::is_contained(Range&: Stores, Element: nullptr))
17083 return SDValue();
17084
17085 StoreSDNode *BaseStore = Stores[0];
17086 if (Subtarget->hasMVEIntegerOps() && BaseStore->getAlign() < EltBits / 8)
17087 return SDValue();
17088 unsigned Bytes = SubVecTy.getStoreSize().getFixedValue();
17089 for (auto [Idx, Store] : enumerate(First&: Stores)) {
17090 if (!DAG.areNonVolatileConsecutiveStores(ST: Store, Base: BaseStore, Bytes, Dist: Idx))
17091 return SDValue();
17092 }
17093
17094 static constexpr Intrinsic::ID NEONStores[] = {Intrinsic::arm_neon_vst2,
17095 Intrinsic::arm_neon_vst3,
17096 Intrinsic::arm_neon_vst4};
17097 SDLoc DL(Interleave);
17098 SmallVector<SDValue, 8> Ops;
17099
17100 Ops.push_back(Elt: BaseStore->getChain());
17101 Intrinsic::ID IID = 0;
17102 if (Subtarget->hasNEON())
17103 IID = NEONStores[NumParts - 2];
17104 else
17105 IID = NumParts == 2 ? Intrinsic::arm_mve_vst2q : Intrinsic::arm_mve_vst4q;
17106
17107 Ops.push_back(Elt: DAG.getTargetConstant(Val: IID, DL, VT: MVT::i64));
17108
17109 Ops.push_back(Elt: BaseStore->getBasePtr());
17110 Ops.append(in_start: Interleave->op_begin(), in_end: Interleave->op_end());
17111 Ops.push_back(Elt: DAG.getConstant(Val: BaseStore->getAlign().value(), DL, VT: MVT::i32));
17112
17113 EVT MemVT =
17114 EVT::getVectorVT(Context&: *DAG.getContext(), VT: SubVecTy.getVectorElementType(),
17115 EC: SubVecTy.getVectorElementCount() * NumParts);
17116 MachineFunction &MF = DAG.getMachineFunction();
17117 MachineMemOperand *MMO =
17118 MF.getMachineMemOperand(MMO: BaseStore->getMemOperand(), Offset: 0, Size: NumParts * Bytes);
17119 SDValue NewStore;
17120 if (Subtarget->hasNEON()) {
17121 NewStore = DAG.getMemIntrinsicNode(
17122 Opcode: ISD::INTRINSIC_VOID, dl: DL, VTList: DAG.getVTList(VT: MVT::Other), Ops, MemVT, MMO);
17123 } else {
17124 // MVE
17125 for (unsigned Stage = 0; Stage < NumParts; Stage++) {
17126 Ops.back() = DAG.getConstant(Val: Stage, DL, VT: MVT::i32);
17127 NewStore = DAG.getMemIntrinsicNode(
17128 Opcode: ISD::INTRINSIC_VOID, dl: DL, VTList: DAG.getVTList(VT: MVT::Other), Ops, MemVT, MMO);
17129 Ops.front() = NewStore;
17130 }
17131 }
17132 for (StoreSDNode *Store : Stores) {
17133 DAG.makeEquivalentMemoryOrdering(OldChain: SDValue(Store, 0), NewMemOpChain: NewStore);
17134 if (Store != ST)
17135 DCI.CombineTo(N: Store, Res: NewStore);
17136 }
17137 return NewStore;
17138}
17139
17140static bool
17141isSequentialConcatOfVectorInterleave(SDNode *N, SmallVectorImpl<SDValue> &Ops) {
17142 if (N->getOpcode() != ISD::CONCAT_VECTORS)
17143 return false;
17144
17145 unsigned NumParts = N->getNumOperands();
17146
17147 // We should be concatenating each sequential result from a
17148 // VECTOR_INTERLEAVE.
17149 SDNode *InterleaveOp = N->getOperand(Num: 0).getNode();
17150 if (InterleaveOp->getOpcode() != ISD::VECTOR_INTERLEAVE ||
17151 InterleaveOp->getNumOperands() != NumParts)
17152 return false;
17153
17154 for (unsigned I = 0; I < NumParts; I++)
17155 if (N->getOperand(Num: I) != SDValue(InterleaveOp, I))
17156 return false;
17157
17158 Ops.append(in_start: InterleaveOp->op_begin(), in_end: InterleaveOp->op_end());
17159 return true;
17160}
17161
17162static SDValue PerformInterleavedStoreCombine(
17163 StoreSDNode *ST, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG,
17164 const ARMSubtarget *Subtarget) {
17165 if (!Subtarget->hasNEON() && !Subtarget->hasMVEIntegerOps())
17166 return SDValue();
17167 if (SDValue Res =
17168 PerformLegalizedInterleavedStoreCombine(ST, DCI, DAG, Subtarget))
17169 return Res;
17170
17171 if (!DCI.isBeforeLegalize())
17172 return SDValue();
17173
17174 if (!ISD::isNormalStore(N: ST) || !ST->isSimple() || !ST->getOffset().isUndef())
17175 return SDValue();
17176
17177 SDValue WideValue = ST->getValue();
17178 SmallVector<SDValue, 4> ValueInterleaveOps;
17179 if (!isSequentialConcatOfVectorInterleave(N: WideValue.getNode(),
17180 Ops&: ValueInterleaveOps))
17181 return SDValue();
17182
17183 if (!WideValue.hasOneUse())
17184 return SDValue();
17185
17186 unsigned NumParts = ValueInterleaveOps.size();
17187 if (NumParts != 2 && NumParts != 3 && NumParts != 4)
17188 return SDValue();
17189
17190 if (NumParts == 3 && Subtarget->hasMVEIntegerOps())
17191 return SDValue();
17192
17193 EVT SubVecTy = ValueInterleaveOps[0].getValueType();
17194 if (Subtarget->hasNEON() && !SubVecTy.is64BitVector() &&
17195 !SubVecTy.is128BitVector())
17196 return SDValue();
17197 if (Subtarget->hasMVEIntegerOps() && !SubVecTy.is128BitVector())
17198 return SDValue();
17199 unsigned EltBits = SubVecTy.getScalarSizeInBits();
17200 if (EltBits != 8 && EltBits != 16 && EltBits != 32)
17201 return SDValue();
17202 if (Subtarget->hasMVEIntegerOps() && ST->getAlign() < EltBits / 8)
17203 return SDValue();
17204 static constexpr Intrinsic::ID NEONStores[] = {Intrinsic::arm_neon_vst2,
17205 Intrinsic::arm_neon_vst3,
17206 Intrinsic::arm_neon_vst4};
17207 SDLoc DL(ST);
17208 SmallVector<SDValue, 8> Ops;
17209 Ops.push_back(Elt: ST->getChain());
17210 Intrinsic::ID IID = 0;
17211 if (Subtarget->hasNEON())
17212 IID = NEONStores[NumParts - 2];
17213 else
17214 IID = NumParts == 2 ? Intrinsic::arm_mve_vst2q : Intrinsic::arm_mve_vst4q;
17215
17216 Ops.push_back(Elt: DAG.getTargetConstant(Val: IID, DL, VT: MVT::i64));
17217
17218 Ops.push_back(Elt: ST->getBasePtr());
17219 Ops.append(RHS: ValueInterleaveOps);
17220
17221 Ops.push_back(Elt: DAG.getConstant(Val: ST->getAlign().value(), DL, VT: MVT::i32));
17222 if (Subtarget->hasNEON())
17223 return DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_VOID, dl: DL,
17224 VTList: DAG.getVTList(VT: MVT::Other), Ops,
17225 MemVT: ST->getMemoryVT(), MMO: ST->getMemOperand());
17226 // MVE
17227 SDValue NewStore;
17228 for (unsigned Stage = 0; Stage < NumParts; Stage++) {
17229 Ops.back() = DAG.getConstant(Val: Stage, DL, VT: MVT::i32);
17230 NewStore = DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_VOID, dl: DL,
17231 VTList: DAG.getVTList(VT: MVT::Other), Ops,
17232 MemVT: ST->getMemoryVT(), MMO: ST->getMemOperand());
17233 Ops.front() = NewStore;
17234 }
17235 return NewStore;
17236}
17237
17238/// PerformSTORECombine - Target-specific dag combine xforms for
17239/// ISD::STORE.
17240static SDValue PerformSTORECombine(SDNode *N,
17241 TargetLowering::DAGCombinerInfo &DCI,
17242 const ARMSubtarget *Subtarget) {
17243 StoreSDNode *St = cast<StoreSDNode>(Val: N);
17244 if (St->isVolatile())
17245 return SDValue();
17246 SDValue StVal = St->getValue();
17247 EVT VT = StVal.getValueType();
17248 if (SDValue Store =
17249 PerformInterleavedStoreCombine(ST: St, DCI, DAG&: DCI.DAG, Subtarget))
17250 return Store;
17251
17252 if (Subtarget->hasNEON())
17253 if (SDValue Store = PerformTruncatingStoreCombine(St, DAG&: DCI.DAG))
17254 return Store;
17255
17256 if (Subtarget->hasMVEFloatOps())
17257 if (SDValue NewToken = PerformSplittingToNarrowingStores(St, DAG&: DCI.DAG))
17258 return NewToken;
17259
17260 if (Subtarget->hasMVEIntegerOps()) {
17261 if (SDValue NewChain = PerformExtractFpToIntStores(St, DAG&: DCI.DAG))
17262 return NewChain;
17263 if (SDValue NewToken =
17264 PerformSplittingMVETruncToNarrowingStores(St, DAG&: DCI.DAG))
17265 return NewToken;
17266 }
17267
17268 if (!ISD::isNormalStore(N: St))
17269 return SDValue();
17270
17271 // Split a store of a VMOVDRR into two integer stores to avoid mixing NEON and
17272 // ARM stores of arguments in the same cache line.
17273 if (StVal.getOpcode() == ARMISD::VMOVDRR && StVal->hasOneUse()) {
17274 SelectionDAG &DAG = DCI.DAG;
17275 bool isBigEndian = DAG.getDataLayout().isBigEndian();
17276 SDLoc DL(St);
17277 SDValue BasePtr = St->getBasePtr();
17278 SDValue NewST1 =
17279 DAG.getStore(Chain: St->getChain(), dl: DL, Val: StVal.getOperand(i: isBigEndian ? 1 : 0),
17280 Ptr: BasePtr, PtrInfo: St->getPointerInfo(), Alignment: St->getBaseAlign(),
17281 MMOFlags: St->getMemOperand()->getFlags());
17282
17283 SDValue OffsetPtr = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i32, N1: BasePtr,
17284 N2: DAG.getConstant(Val: 4, DL, VT: MVT::i32));
17285 return DAG.getStore(Chain: NewST1.getValue(R: 0), dl: DL,
17286 Val: StVal.getOperand(i: isBigEndian ? 0 : 1), Ptr: OffsetPtr,
17287 PtrInfo: St->getPointerInfo().getWithOffset(O: 4),
17288 Alignment: St->getBaseAlign(), MMOFlags: St->getMemOperand()->getFlags());
17289 }
17290
17291 if (StVal.getValueType() == MVT::i64 &&
17292 StVal.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
17293 // Bitcast an i64 store extracted from a vector to f64.
17294 // Otherwise, the i64 value will be legalized to a pair of i32 values.
17295 SelectionDAG &DAG = DCI.DAG;
17296 SDLoc dl(StVal);
17297 SDValue IntVec = StVal.getOperand(i: 0);
17298 EVT FloatVT =
17299 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::f64,
17300 NumElements: IntVec.getValueType().getVectorNumElements());
17301 SDValue Vec = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: FloatVT, Operand: IntVec);
17302 SDValue ExtElt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: MVT::f64, N1: Vec,
17303 N2: StVal.getOperand(i: 1));
17304 dl = SDLoc(N);
17305 SDValue V = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i64, Operand: ExtElt);
17306 // Make the DAGCombiner fold the bitcasts.
17307 DCI.AddToWorklist(N: Vec.getNode());
17308 DCI.AddToWorklist(N: ExtElt.getNode());
17309 DCI.AddToWorklist(N: V.getNode());
17310 return DAG.getStore(Chain: St->getChain(), dl, Val: V, Ptr: St->getBasePtr(),
17311 PtrInfo: St->getPointerInfo(), Alignment: St->getAlign(),
17312 MMOFlags: St->getMemOperand()->getFlags(), Metadata: St->getAAInfo());
17313 }
17314
17315 // If this is a legal vector store, try to combine it into a VST1_UPD.
17316 if (Subtarget->hasNEON() && ISD::isNormalStore(N) && VT.isVector() &&
17317 DCI.DAG.getTargetLoweringInfo().isTypeLegal(VT))
17318 return CombineBaseUpdate(N, DCI);
17319
17320 return SDValue();
17321}
17322
17323/// PerformVCVTCombine - VCVT (floating-point to fixed-point, Advanced SIMD)
17324/// can replace combinations of VMUL and VCVT (floating-point to integer)
17325/// when the VMUL has a constant operand that is a power of 2.
17326///
17327/// Example (assume d17 = <float 8.000000e+00, float 8.000000e+00>):
17328/// vmul.f32 d16, d17, d16
17329/// vcvt.s32.f32 d16, d16
17330/// becomes:
17331/// vcvt.s32.f32 d16, d16, #3
17332static SDValue PerformVCVTCombine(SDNode *N, SelectionDAG &DAG,
17333 const ARMSubtarget *Subtarget) {
17334 if (!Subtarget->hasNEON())
17335 return SDValue();
17336
17337 SDValue Op = N->getOperand(Num: 0);
17338 if (!Op.getValueType().isVector() || !Op.getValueType().isSimple() ||
17339 Op.getOpcode() != ISD::FMUL)
17340 return SDValue();
17341
17342 SDValue ConstVec = Op->getOperand(Num: 1);
17343 if (!isa<BuildVectorSDNode>(Val: ConstVec))
17344 return SDValue();
17345
17346 MVT FloatTy = Op.getSimpleValueType().getVectorElementType();
17347 uint32_t FloatBits = FloatTy.getSizeInBits();
17348 MVT IntTy = N->getSimpleValueType(ResNo: 0).getVectorElementType();
17349 uint32_t IntBits = IntTy.getSizeInBits();
17350 unsigned NumLanes = Op.getValueType().getVectorNumElements();
17351 if (FloatBits != 32 || IntBits > 32 || (NumLanes != 4 && NumLanes != 2)) {
17352 // These instructions only exist converting from f32 to i32. We can handle
17353 // smaller integers by generating an extra truncate, but larger ones would
17354 // be lossy. We also can't handle anything other than 2 or 4 lanes, since
17355 // these instructions only support v2i32/v4i32 types.
17356 return SDValue();
17357 }
17358
17359 BitVector UndefElements;
17360 BuildVectorSDNode *BV = cast<BuildVectorSDNode>(Val&: ConstVec);
17361 int32_t C = BV->getConstantFPSplatPow2ToLog2Int(UndefElements: &UndefElements, BitWidth: 33);
17362 if (C == -1 || C == 0 || C > 32)
17363 return SDValue();
17364
17365 SDLoc dl(N);
17366 bool isSigned = N->getOpcode() == ISD::FP_TO_SINT;
17367 unsigned IntrinsicOpcode = isSigned ? Intrinsic::arm_neon_vcvtfp2fxs :
17368 Intrinsic::arm_neon_vcvtfp2fxu;
17369 SDValue FixConv = DAG.getNode(
17370 Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: NumLanes == 2 ? MVT::v2i32 : MVT::v4i32,
17371 N1: DAG.getConstant(Val: IntrinsicOpcode, DL: dl, VT: MVT::i32), N2: Op->getOperand(Num: 0),
17372 N3: DAG.getConstant(Val: C, DL: dl, VT: MVT::i32));
17373
17374 if (IntBits < FloatBits)
17375 FixConv = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: N->getValueType(ResNo: 0), Operand: FixConv);
17376
17377 return FixConv;
17378}
17379
17380static SDValue PerformFAddVSelectCombine(SDNode *N, SelectionDAG &DAG,
17381 const ARMSubtarget *Subtarget) {
17382 if (!Subtarget->hasMVEFloatOps())
17383 return SDValue();
17384
17385 // Turn (fadd x, (vselect c, y, -0.0)) into (vselect c, (fadd x, y), x)
17386 // The second form can be more easily turned into a predicated vadd, and
17387 // possibly combined into a fma to become a predicated vfma.
17388 SDValue Op0 = N->getOperand(Num: 0);
17389 SDValue Op1 = N->getOperand(Num: 1);
17390 EVT VT = N->getValueType(ResNo: 0);
17391 SDLoc DL(N);
17392
17393 // The identity element for a fadd is -0.0 or +0.0 when the nsz flag is set,
17394 // which these VMOV's represent.
17395 auto isIdentitySplat = [&](SDValue Op, bool NSZ) {
17396 if (Op.getOpcode() != ISD::BITCAST ||
17397 Op.getOperand(i: 0).getOpcode() != ARMISD::VMOVIMM)
17398 return false;
17399 uint64_t ImmVal = Op.getOperand(i: 0).getConstantOperandVal(i: 0);
17400 if (VT == MVT::v4f32 && (ImmVal == 1664 || (ImmVal == 0 && NSZ)))
17401 return true;
17402 if (VT == MVT::v8f16 && (ImmVal == 2688 || (ImmVal == 0 && NSZ)))
17403 return true;
17404 return false;
17405 };
17406
17407 if (Op0.getOpcode() == ISD::VSELECT && Op1.getOpcode() != ISD::VSELECT)
17408 std::swap(a&: Op0, b&: Op1);
17409
17410 if (Op1.getOpcode() != ISD::VSELECT)
17411 return SDValue();
17412
17413 SDNodeFlags FaddFlags = N->getFlags();
17414 bool NSZ = FaddFlags.hasNoSignedZeros();
17415 if (!isIdentitySplat(Op1.getOperand(i: 2), NSZ))
17416 return SDValue();
17417
17418 SDValue FAdd =
17419 DAG.getNode(Opcode: ISD::FADD, DL, VT, N1: Op0, N2: Op1.getOperand(i: 1), Flags: FaddFlags);
17420 return DAG.getNode(Opcode: ISD::VSELECT, DL, VT, N1: Op1.getOperand(i: 0), N2: FAdd, N3: Op0, Flags: FaddFlags);
17421}
17422
17423static SDValue PerformFADDVCMLACombine(SDNode *N, SelectionDAG &DAG) {
17424 SDValue LHS = N->getOperand(Num: 0);
17425 SDValue RHS = N->getOperand(Num: 1);
17426 EVT VT = N->getValueType(ResNo: 0);
17427 SDLoc DL(N);
17428
17429 if (!N->getFlags().hasAllowReassociation())
17430 return SDValue();
17431
17432 // Combine fadd(a, vcmla(b, c, d)) -> vcmla(fadd(a, b), b, c)
17433 auto ReassocComplex = [&](SDValue A, SDValue B) {
17434 if (A.getOpcode() != ISD::INTRINSIC_WO_CHAIN)
17435 return SDValue();
17436 unsigned Opc = A.getConstantOperandVal(i: 0);
17437 if (Opc != Intrinsic::arm_mve_vcmlaq)
17438 return SDValue();
17439 SDValue VCMLA = DAG.getNode(
17440 Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT, N1: A.getOperand(i: 0), N2: A.getOperand(i: 1),
17441 N3: DAG.getNode(Opcode: ISD::FADD, DL, VT, N1: A.getOperand(i: 2), N2: B, Flags: N->getFlags()),
17442 N4: A.getOperand(i: 3), N5: A.getOperand(i: 4));
17443 VCMLA->setFlags(A->getFlags());
17444 return VCMLA;
17445 };
17446 if (SDValue R = ReassocComplex(LHS, RHS))
17447 return R;
17448 if (SDValue R = ReassocComplex(RHS, LHS))
17449 return R;
17450
17451 return SDValue();
17452}
17453
17454static SDValue PerformFADDCombine(SDNode *N, SelectionDAG &DAG,
17455 const ARMSubtarget *Subtarget) {
17456 if (SDValue S = PerformFAddVSelectCombine(N, DAG, Subtarget))
17457 return S;
17458 if (SDValue S = PerformFADDVCMLACombine(N, DAG))
17459 return S;
17460 return SDValue();
17461}
17462
17463/// PerformVMulVCTPCombine - VCVT (fixed-point to floating-point, Advanced SIMD)
17464/// can replace combinations of VCVT (integer to floating-point) and VMUL
17465/// when the VMUL has a constant operand that is a power of 2.
17466///
17467/// Example (assume d17 = <float 0.125, float 0.125>):
17468/// vcvt.f32.s32 d16, d16
17469/// vmul.f32 d16, d16, d17
17470/// becomes:
17471/// vcvt.f32.s32 d16, d16, #3
17472static SDValue PerformVMulVCTPCombine(SDNode *N, SelectionDAG &DAG,
17473 const ARMSubtarget *Subtarget) {
17474 if (!Subtarget->hasNEON())
17475 return SDValue();
17476
17477 SDValue Op = N->getOperand(Num: 0);
17478 unsigned OpOpcode = Op.getNode()->getOpcode();
17479 if (!N->getValueType(ResNo: 0).isVector() || !N->getValueType(ResNo: 0).isSimple() ||
17480 (OpOpcode != ISD::SINT_TO_FP && OpOpcode != ISD::UINT_TO_FP))
17481 return SDValue();
17482
17483 SDValue ConstVec = N->getOperand(Num: 1);
17484 if (!isa<BuildVectorSDNode>(Val: ConstVec))
17485 return SDValue();
17486
17487 MVT FloatTy = N->getSimpleValueType(ResNo: 0).getVectorElementType();
17488 uint32_t FloatBits = FloatTy.getSizeInBits();
17489 MVT IntTy = Op.getOperand(i: 0).getSimpleValueType().getVectorElementType();
17490 uint32_t IntBits = IntTy.getSizeInBits();
17491 unsigned NumLanes = Op.getValueType().getVectorNumElements();
17492 if (FloatBits != 32 || IntBits > 32 || (NumLanes != 4 && NumLanes != 2)) {
17493 // These instructions only exist converting from i32 to f32. We can handle
17494 // smaller integers by generating an extra extend, but larger ones would
17495 // be lossy. We also can't handle anything other than 2 or 4 lanes, since
17496 // these instructions only support v2i32/v4i32 types.
17497 return SDValue();
17498 }
17499
17500 ConstantFPSDNode *CN = isConstOrConstSplatFP(N: ConstVec, AllowUndefs: true);
17501 APFloat Recip(0.0f);
17502 if (!CN || !CN->getValueAPF().getExactInverse(Inv: &Recip))
17503 return SDValue();
17504
17505 bool IsExact;
17506 APSInt IntVal(33);
17507 if (Recip.convertToInteger(Result&: IntVal, RM: APFloat::rmTowardZero, IsExact: &IsExact) !=
17508 APFloat::opOK ||
17509 !IsExact)
17510 return SDValue();
17511
17512 int32_t C = IntVal.exactLogBase2();
17513 if (C == -1 || C == 0 || C > 32)
17514 return SDValue();
17515
17516 SDLoc DL(N);
17517 bool isSigned = OpOpcode == ISD::SINT_TO_FP;
17518 SDValue ConvInput = Op.getOperand(i: 0);
17519 if (IntBits < FloatBits)
17520 ConvInput = DAG.getNode(Opcode: isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL,
17521 VT: NumLanes == 2 ? MVT::v2i32 : MVT::v4i32, Operand: ConvInput);
17522
17523 unsigned IntrinsicOpcode = isSigned ? Intrinsic::arm_neon_vcvtfxs2fp
17524 : Intrinsic::arm_neon_vcvtfxu2fp;
17525 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: Op.getValueType(),
17526 N1: DAG.getConstant(Val: IntrinsicOpcode, DL, VT: MVT::i32), N2: ConvInput,
17527 N3: DAG.getConstant(Val: C, DL, VT: MVT::i32));
17528}
17529
17530static SDValue PerformVECREDUCE_ADDCombine(SDNode *N, SelectionDAG &DAG,
17531 const ARMSubtarget *ST) {
17532 if (!ST->hasMVEIntegerOps())
17533 return SDValue();
17534
17535 assert(N->getOpcode() == ISD::VECREDUCE_ADD);
17536 EVT ResVT = N->getValueType(ResNo: 0);
17537 SDValue N0 = N->getOperand(Num: 0);
17538 SDLoc dl(N);
17539
17540 // Try to turn vecreduce_add(add(x, y)) into vecreduce(x) + vecreduce(y)
17541 if (ResVT == MVT::i32 && N0.getOpcode() == ISD::ADD &&
17542 (N0.getValueType() == MVT::v4i32 || N0.getValueType() == MVT::v8i16 ||
17543 N0.getValueType() == MVT::v16i8)) {
17544 SDValue Red0 = DAG.getNode(Opcode: ISD::VECREDUCE_ADD, DL: dl, VT: ResVT, Operand: N0.getOperand(i: 0));
17545 SDValue Red1 = DAG.getNode(Opcode: ISD::VECREDUCE_ADD, DL: dl, VT: ResVT, Operand: N0.getOperand(i: 1));
17546 return DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ResVT, N1: Red0, N2: Red1);
17547 }
17548
17549 // We are looking for something that will have illegal types if left alone,
17550 // but that we can convert to a single instruction under MVE. For example
17551 // vecreduce_add(sext(A, v8i32)) => VADDV.s16 A
17552 // or
17553 // vecreduce_add(mul(zext(A, v16i32), zext(B, v16i32))) => VMLADAV.u8 A, B
17554
17555 // The legal cases are:
17556 // VADDV u/s 8/16/32
17557 // VMLAV u/s 8/16/32
17558 // VADDLV u/s 32
17559 // VMLALV u/s 16/32
17560
17561 // If the input vector is smaller than legal (v4i8/v4i16 for example) we can
17562 // extend it and use v4i32 instead.
17563 auto ExtTypeMatches = [](SDValue A, ArrayRef<MVT> ExtTypes) {
17564 EVT AVT = A.getValueType();
17565 return any_of(Range&: ExtTypes, P: [&](MVT Ty) {
17566 return AVT.getVectorNumElements() == Ty.getVectorNumElements() &&
17567 AVT.bitsLE(VT: Ty);
17568 });
17569 };
17570 auto ExtendIfNeeded = [&](SDValue A, unsigned ExtendCode) {
17571 EVT AVT = A.getValueType();
17572 if (!AVT.is128BitVector())
17573 A = DAG.getNode(
17574 Opcode: ExtendCode, DL: dl,
17575 VT: AVT.changeVectorElementType(
17576 Context&: *DAG.getContext(),
17577 EltVT: MVT::getIntegerVT(BitWidth: 128 / AVT.getVectorMinNumElements())),
17578 Operand: A);
17579 return A;
17580 };
17581 auto IsVADDV = [&](MVT RetTy, unsigned ExtendCode, ArrayRef<MVT> ExtTypes) {
17582 if (ResVT != RetTy || N0->getOpcode() != ExtendCode)
17583 return SDValue();
17584 SDValue A = N0->getOperand(Num: 0);
17585 if (ExtTypeMatches(A, ExtTypes))
17586 return ExtendIfNeeded(A, ExtendCode);
17587 return SDValue();
17588 };
17589 auto IsPredVADDV = [&](MVT RetTy, unsigned ExtendCode, ArrayRef<MVT> ExtTypes,
17590 SDValue &Mask) {
17591 if (ResVT != RetTy || N0->getOpcode() != ISD::VSELECT ||
17592 !ISD::isBuildVectorAllZeros(V: N0->getOperand(Num: 2)))
17593 return SDValue();
17594 Mask = N0->getOperand(Num: 0);
17595 SDValue Ext = N0->getOperand(Num: 1);
17596 if (Ext->getOpcode() != ExtendCode)
17597 return SDValue();
17598 SDValue A = Ext->getOperand(Num: 0);
17599 if (ExtTypeMatches(A, ExtTypes))
17600 return ExtendIfNeeded(A, ExtendCode);
17601 return SDValue();
17602 };
17603 auto IsVMLAV = [&](MVT RetTy, unsigned ExtendCode, ArrayRef<MVT> ExtTypes,
17604 SDValue &A, SDValue &B) {
17605 // For a vmla we are trying to match a larger pattern:
17606 // ExtA = sext/zext A
17607 // ExtB = sext/zext B
17608 // Mul = mul ExtA, ExtB
17609 // vecreduce.add Mul
17610 // There might also be en extra extend between the mul and the addreduce, so
17611 // long as the bitwidth is high enough to make them equivalent (for example
17612 // original v8i16 might be mul at v8i32 and the reduce happens at v8i64).
17613 if (ResVT != RetTy)
17614 return false;
17615 SDValue Mul = N0;
17616 if (Mul->getOpcode() == ExtendCode &&
17617 Mul->getOperand(Num: 0).getScalarValueSizeInBits() * 2 >=
17618 ResVT.getScalarSizeInBits())
17619 Mul = Mul->getOperand(Num: 0);
17620 if (Mul->getOpcode() != ISD::MUL)
17621 return false;
17622 SDValue ExtA = Mul->getOperand(Num: 0);
17623 SDValue ExtB = Mul->getOperand(Num: 1);
17624 if (ExtA->getOpcode() != ExtendCode || ExtB->getOpcode() != ExtendCode)
17625 return false;
17626 A = ExtA->getOperand(Num: 0);
17627 B = ExtB->getOperand(Num: 0);
17628 if (ExtTypeMatches(A, ExtTypes) && ExtTypeMatches(B, ExtTypes)) {
17629 A = ExtendIfNeeded(A, ExtendCode);
17630 B = ExtendIfNeeded(B, ExtendCode);
17631 return true;
17632 }
17633 return false;
17634 };
17635 auto IsPredVMLAV = [&](MVT RetTy, unsigned ExtendCode, ArrayRef<MVT> ExtTypes,
17636 SDValue &A, SDValue &B, SDValue &Mask) {
17637 // Same as the pattern above with a select for the zero predicated lanes
17638 // ExtA = sext/zext A
17639 // ExtB = sext/zext B
17640 // Mul = mul ExtA, ExtB
17641 // N0 = select Mask, Mul, 0
17642 // vecreduce.add N0
17643 if (ResVT != RetTy || N0->getOpcode() != ISD::VSELECT ||
17644 !ISD::isBuildVectorAllZeros(V: N0->getOperand(Num: 2)))
17645 return false;
17646 Mask = N0->getOperand(Num: 0);
17647 SDValue Mul = N0->getOperand(Num: 1);
17648 if (Mul->getOpcode() == ExtendCode &&
17649 Mul->getOperand(Num: 0).getScalarValueSizeInBits() * 2 >=
17650 ResVT.getScalarSizeInBits())
17651 Mul = Mul->getOperand(Num: 0);
17652 if (Mul->getOpcode() != ISD::MUL)
17653 return false;
17654 SDValue ExtA = Mul->getOperand(Num: 0);
17655 SDValue ExtB = Mul->getOperand(Num: 1);
17656 if (ExtA->getOpcode() != ExtendCode || ExtB->getOpcode() != ExtendCode)
17657 return false;
17658 A = ExtA->getOperand(Num: 0);
17659 B = ExtB->getOperand(Num: 0);
17660 if (ExtTypeMatches(A, ExtTypes) && ExtTypeMatches(B, ExtTypes)) {
17661 A = ExtendIfNeeded(A, ExtendCode);
17662 B = ExtendIfNeeded(B, ExtendCode);
17663 return true;
17664 }
17665 return false;
17666 };
17667 auto Create64bitNode = [&](unsigned Opcode, ArrayRef<SDValue> Ops) {
17668 // Split illegal MVT::v16i8->i64 vector reductions into two legal v8i16->i64
17669 // reductions. The operands are extended with MVEEXT, but as they are
17670 // reductions the lane orders do not matter. MVEEXT may be combined with
17671 // loads to produce two extending loads, or else they will be expanded to
17672 // VREV/VMOVL.
17673 EVT VT = Ops[0].getValueType();
17674 if (VT == MVT::v16i8) {
17675 assert((Opcode == ARMISD::VMLALVs || Opcode == ARMISD::VMLALVu) &&
17676 "Unexpected illegal long reduction opcode");
17677 bool IsUnsigned = Opcode == ARMISD::VMLALVu;
17678
17679 SDValue Ext0 =
17680 DAG.getNode(Opcode: IsUnsigned ? ARMISD::MVEZEXT : ARMISD::MVESEXT, DL: dl,
17681 VTList: DAG.getVTList(VT1: MVT::v8i16, VT2: MVT::v8i16), N: Ops[0]);
17682 SDValue Ext1 =
17683 DAG.getNode(Opcode: IsUnsigned ? ARMISD::MVEZEXT : ARMISD::MVESEXT, DL: dl,
17684 VTList: DAG.getVTList(VT1: MVT::v8i16, VT2: MVT::v8i16), N: Ops[1]);
17685
17686 SDValue MLA0 = DAG.getNode(Opcode, DL: dl, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
17687 N1: Ext0, N2: Ext1);
17688 SDValue MLA1 =
17689 DAG.getNode(Opcode: IsUnsigned ? ARMISD::VMLALVAu : ARMISD::VMLALVAs, DL: dl,
17690 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N1: MLA0, N2: MLA0.getValue(R: 1),
17691 N3: Ext0.getValue(R: 1), N4: Ext1.getValue(R: 1));
17692 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: MVT::i64, N1: MLA1, N2: MLA1.getValue(R: 1));
17693 }
17694 SDValue Node = DAG.getNode(Opcode, DL: dl, ResultTys: {MVT::i32, MVT::i32}, Ops);
17695 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: MVT::i64, N1: Node,
17696 N2: SDValue(Node.getNode(), 1));
17697 };
17698
17699 SDValue A, B;
17700 SDValue Mask;
17701 if (IsVMLAV(MVT::i32, ISD::SIGN_EXTEND, {MVT::v8i16, MVT::v16i8}, A, B))
17702 return DAG.getNode(Opcode: ARMISD::VMLAVs, DL: dl, VT: ResVT, N1: A, N2: B);
17703 if (IsVMLAV(MVT::i32, ISD::ZERO_EXTEND, {MVT::v8i16, MVT::v16i8}, A, B))
17704 return DAG.getNode(Opcode: ARMISD::VMLAVu, DL: dl, VT: ResVT, N1: A, N2: B);
17705 if (IsVMLAV(MVT::i64, ISD::SIGN_EXTEND, {MVT::v16i8, MVT::v8i16, MVT::v4i32},
17706 A, B))
17707 return Create64bitNode(ARMISD::VMLALVs, {A, B});
17708 if (IsVMLAV(MVT::i64, ISD::ZERO_EXTEND, {MVT::v16i8, MVT::v8i16, MVT::v4i32},
17709 A, B))
17710 return Create64bitNode(ARMISD::VMLALVu, {A, B});
17711 if (IsVMLAV(MVT::i16, ISD::SIGN_EXTEND, {MVT::v16i8}, A, B))
17712 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: ResVT,
17713 Operand: DAG.getNode(Opcode: ARMISD::VMLAVs, DL: dl, VT: MVT::i32, N1: A, N2: B));
17714 if (IsVMLAV(MVT::i16, ISD::ZERO_EXTEND, {MVT::v16i8}, A, B))
17715 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: ResVT,
17716 Operand: DAG.getNode(Opcode: ARMISD::VMLAVu, DL: dl, VT: MVT::i32, N1: A, N2: B));
17717
17718 if (IsPredVMLAV(MVT::i32, ISD::SIGN_EXTEND, {MVT::v8i16, MVT::v16i8}, A, B,
17719 Mask))
17720 return DAG.getNode(Opcode: ARMISD::VMLAVps, DL: dl, VT: ResVT, N1: A, N2: B, N3: Mask);
17721 if (IsPredVMLAV(MVT::i32, ISD::ZERO_EXTEND, {MVT::v8i16, MVT::v16i8}, A, B,
17722 Mask))
17723 return DAG.getNode(Opcode: ARMISD::VMLAVpu, DL: dl, VT: ResVT, N1: A, N2: B, N3: Mask);
17724 if (IsPredVMLAV(MVT::i64, ISD::SIGN_EXTEND, {MVT::v8i16, MVT::v4i32}, A, B,
17725 Mask))
17726 return Create64bitNode(ARMISD::VMLALVps, {A, B, Mask});
17727 if (IsPredVMLAV(MVT::i64, ISD::ZERO_EXTEND, {MVT::v8i16, MVT::v4i32}, A, B,
17728 Mask))
17729 return Create64bitNode(ARMISD::VMLALVpu, {A, B, Mask});
17730 if (IsPredVMLAV(MVT::i16, ISD::SIGN_EXTEND, {MVT::v16i8}, A, B, Mask))
17731 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: ResVT,
17732 Operand: DAG.getNode(Opcode: ARMISD::VMLAVps, DL: dl, VT: MVT::i32, N1: A, N2: B, N3: Mask));
17733 if (IsPredVMLAV(MVT::i16, ISD::ZERO_EXTEND, {MVT::v16i8}, A, B, Mask))
17734 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: ResVT,
17735 Operand: DAG.getNode(Opcode: ARMISD::VMLAVpu, DL: dl, VT: MVT::i32, N1: A, N2: B, N3: Mask));
17736
17737 if (SDValue A = IsVADDV(MVT::i32, ISD::SIGN_EXTEND, {MVT::v8i16, MVT::v16i8}))
17738 return DAG.getNode(Opcode: ARMISD::VADDVs, DL: dl, VT: ResVT, Operand: A);
17739 if (SDValue A = IsVADDV(MVT::i32, ISD::ZERO_EXTEND, {MVT::v8i16, MVT::v16i8}))
17740 return DAG.getNode(Opcode: ARMISD::VADDVu, DL: dl, VT: ResVT, Operand: A);
17741 if (SDValue A = IsVADDV(MVT::i64, ISD::SIGN_EXTEND, {MVT::v4i32}))
17742 return Create64bitNode(ARMISD::VADDLVs, {A});
17743 if (SDValue A = IsVADDV(MVT::i64, ISD::ZERO_EXTEND, {MVT::v4i32}))
17744 return Create64bitNode(ARMISD::VADDLVu, {A});
17745 if (SDValue A = IsVADDV(MVT::i16, ISD::SIGN_EXTEND, {MVT::v16i8}))
17746 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: ResVT,
17747 Operand: DAG.getNode(Opcode: ARMISD::VADDVs, DL: dl, VT: MVT::i32, Operand: A));
17748 if (SDValue A = IsVADDV(MVT::i16, ISD::ZERO_EXTEND, {MVT::v16i8}))
17749 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: ResVT,
17750 Operand: DAG.getNode(Opcode: ARMISD::VADDVu, DL: dl, VT: MVT::i32, Operand: A));
17751
17752 if (SDValue A = IsPredVADDV(MVT::i32, ISD::SIGN_EXTEND, {MVT::v8i16, MVT::v16i8}, Mask))
17753 return DAG.getNode(Opcode: ARMISD::VADDVps, DL: dl, VT: ResVT, N1: A, N2: Mask);
17754 if (SDValue A = IsPredVADDV(MVT::i32, ISD::ZERO_EXTEND, {MVT::v8i16, MVT::v16i8}, Mask))
17755 return DAG.getNode(Opcode: ARMISD::VADDVpu, DL: dl, VT: ResVT, N1: A, N2: Mask);
17756 if (SDValue A = IsPredVADDV(MVT::i64, ISD::SIGN_EXTEND, {MVT::v4i32}, Mask))
17757 return Create64bitNode(ARMISD::VADDLVps, {A, Mask});
17758 if (SDValue A = IsPredVADDV(MVT::i64, ISD::ZERO_EXTEND, {MVT::v4i32}, Mask))
17759 return Create64bitNode(ARMISD::VADDLVpu, {A, Mask});
17760 if (SDValue A = IsPredVADDV(MVT::i16, ISD::SIGN_EXTEND, {MVT::v16i8}, Mask))
17761 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: ResVT,
17762 Operand: DAG.getNode(Opcode: ARMISD::VADDVps, DL: dl, VT: MVT::i32, N1: A, N2: Mask));
17763 if (SDValue A = IsPredVADDV(MVT::i16, ISD::ZERO_EXTEND, {MVT::v16i8}, Mask))
17764 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: ResVT,
17765 Operand: DAG.getNode(Opcode: ARMISD::VADDVpu, DL: dl, VT: MVT::i32, N1: A, N2: Mask));
17766
17767 // Some complications. We can get a case where the two inputs of the mul are
17768 // the same, then the output sext will have been helpfully converted to a
17769 // zext. Turn it back.
17770 SDValue Op = N0;
17771 if (Op->getOpcode() == ISD::VSELECT)
17772 Op = Op->getOperand(Num: 1);
17773 if (Op->getOpcode() == ISD::ZERO_EXTEND &&
17774 Op->getOperand(Num: 0)->getOpcode() == ISD::MUL) {
17775 SDValue Mul = Op->getOperand(Num: 0);
17776 if (Mul->getOperand(Num: 0) == Mul->getOperand(Num: 1) &&
17777 Mul->getOperand(Num: 0)->getOpcode() == ISD::SIGN_EXTEND) {
17778 SDValue Ext = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: N0->getValueType(ResNo: 0), Operand: Mul);
17779 if (Op != N0)
17780 Ext = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: N0->getValueType(ResNo: 0),
17781 N1: N0->getOperand(Num: 0), N2: Ext, N3: N0->getOperand(Num: 2));
17782 return DAG.getNode(Opcode: ISD::VECREDUCE_ADD, DL: dl, VT: ResVT, Operand: Ext);
17783 }
17784 }
17785
17786 return SDValue();
17787}
17788
17789// Looks for vaddv(shuffle) or vmlav(shuffle, shuffle), with a shuffle where all
17790// the lanes are used. Due to the reduction being commutative the shuffle can be
17791// removed.
17792static SDValue PerformReduceShuffleCombine(SDNode *N, SelectionDAG &DAG) {
17793 unsigned VecOp = N->getOperand(Num: 0).getValueType().isVector() ? 0 : 2;
17794 auto *Shuf = dyn_cast<ShuffleVectorSDNode>(Val: N->getOperand(Num: VecOp));
17795 if (!Shuf || !Shuf->getOperand(Num: 1).isUndef())
17796 return SDValue();
17797
17798 // Check all elements are used once in the mask.
17799 ArrayRef<int> Mask = Shuf->getMask();
17800 APInt SetElts(Mask.size(), 0);
17801 for (int E : Mask) {
17802 if (E < 0 || E >= (int)Mask.size())
17803 return SDValue();
17804 SetElts.setBit(E);
17805 }
17806 if (!SetElts.isAllOnes())
17807 return SDValue();
17808
17809 if (N->getNumOperands() != VecOp + 1) {
17810 auto *Shuf2 = dyn_cast<ShuffleVectorSDNode>(Val: N->getOperand(Num: VecOp + 1));
17811 if (!Shuf2 || !Shuf2->getOperand(Num: 1).isUndef() || Shuf2->getMask() != Mask)
17812 return SDValue();
17813 }
17814
17815 SmallVector<SDValue> Ops;
17816 for (SDValue Op : N->ops()) {
17817 if (Op.getValueType().isVector())
17818 Ops.push_back(Elt: Op.getOperand(i: 0));
17819 else
17820 Ops.push_back(Elt: Op);
17821 }
17822 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VTList: N->getVTList(), Ops);
17823}
17824
17825static SDValue PerformVMOVNCombine(SDNode *N,
17826 TargetLowering::DAGCombinerInfo &DCI) {
17827 SDValue Op0 = N->getOperand(Num: 0);
17828 SDValue Op1 = N->getOperand(Num: 1);
17829 unsigned IsTop = N->getConstantOperandVal(Num: 2);
17830
17831 // VMOVNT a undef -> a
17832 // VMOVNB a undef -> a
17833 // VMOVNB undef a -> a
17834 if (Op1->isUndef())
17835 return Op0;
17836 if (Op0->isUndef() && !IsTop)
17837 return Op1;
17838
17839 // VMOVNt(c, VQMOVNb(a, b)) => VQMOVNt(c, b)
17840 // VMOVNb(c, VQMOVNb(a, b)) => VQMOVNb(c, b)
17841 if ((Op1->getOpcode() == ARMISD::VQMOVNs ||
17842 Op1->getOpcode() == ARMISD::VQMOVNu) &&
17843 Op1->getConstantOperandVal(Num: 2) == 0)
17844 return DCI.DAG.getNode(Opcode: Op1->getOpcode(), DL: SDLoc(Op1), VT: N->getValueType(ResNo: 0),
17845 N1: Op0, N2: Op1->getOperand(Num: 1), N3: N->getOperand(Num: 2));
17846
17847 // Only the bottom lanes from Qm (Op1) and either the top or bottom lanes from
17848 // Qd (Op0) are demanded from a VMOVN, depending on whether we are inserting
17849 // into the top or bottom lanes.
17850 unsigned NumElts = N->getValueType(ResNo: 0).getVectorNumElements();
17851 APInt Op1DemandedElts = APInt::getSplat(NewLen: NumElts, V: APInt::getLowBitsSet(numBits: 2, loBitsSet: 1));
17852 APInt Op0DemandedElts =
17853 IsTop ? Op1DemandedElts
17854 : APInt::getSplat(NewLen: NumElts, V: APInt::getHighBitsSet(numBits: 2, hiBitsSet: 1));
17855
17856 const TargetLowering &TLI = DCI.DAG.getTargetLoweringInfo();
17857 if (TLI.SimplifyDemandedVectorElts(Op: Op0, DemandedElts: Op0DemandedElts, DCI))
17858 return SDValue(N, 0);
17859 if (TLI.SimplifyDemandedVectorElts(Op: Op1, DemandedElts: Op1DemandedElts, DCI))
17860 return SDValue(N, 0);
17861
17862 return SDValue();
17863}
17864
17865static SDValue PerformVQMOVNCombine(SDNode *N,
17866 TargetLowering::DAGCombinerInfo &DCI) {
17867 SDValue Op0 = N->getOperand(Num: 0);
17868 unsigned IsTop = N->getConstantOperandVal(Num: 2);
17869
17870 unsigned NumElts = N->getValueType(ResNo: 0).getVectorNumElements();
17871 APInt Op0DemandedElts =
17872 APInt::getSplat(NewLen: NumElts, V: IsTop ? APInt::getLowBitsSet(numBits: 2, loBitsSet: 1)
17873 : APInt::getHighBitsSet(numBits: 2, hiBitsSet: 1));
17874
17875 const TargetLowering &TLI = DCI.DAG.getTargetLoweringInfo();
17876 if (TLI.SimplifyDemandedVectorElts(Op: Op0, DemandedElts: Op0DemandedElts, DCI))
17877 return SDValue(N, 0);
17878 return SDValue();
17879}
17880
17881static SDValue PerformVQDMULHCombine(SDNode *N,
17882 TargetLowering::DAGCombinerInfo &DCI) {
17883 EVT VT = N->getValueType(ResNo: 0);
17884 SDValue LHS = N->getOperand(Num: 0);
17885 SDValue RHS = N->getOperand(Num: 1);
17886
17887 auto *Shuf0 = dyn_cast<ShuffleVectorSDNode>(Val&: LHS);
17888 auto *Shuf1 = dyn_cast<ShuffleVectorSDNode>(Val&: RHS);
17889 // Turn VQDMULH(shuffle, shuffle) -> shuffle(VQDMULH)
17890 if (Shuf0 && Shuf1 && Shuf0->getMask().equals(RHS: Shuf1->getMask()) &&
17891 LHS.getOperand(i: 1).isUndef() && RHS.getOperand(i: 1).isUndef() &&
17892 (LHS.hasOneUse() || RHS.hasOneUse() || LHS == RHS)) {
17893 SDLoc DL(N);
17894 SDValue NewBinOp = DCI.DAG.getNode(Opcode: N->getOpcode(), DL, VT,
17895 N1: LHS.getOperand(i: 0), N2: RHS.getOperand(i: 0));
17896 SDValue UndefV = LHS.getOperand(i: 1);
17897 return DCI.DAG.getVectorShuffle(VT, dl: DL, N1: NewBinOp, N2: UndefV, Mask: Shuf0->getMask());
17898 }
17899 return SDValue();
17900}
17901
17902static SDValue PerformLongShiftCombine(SDNode *N, SelectionDAG &DAG) {
17903 SDLoc DL(N);
17904 SDValue Op0 = N->getOperand(Num: 0);
17905 SDValue Op1 = N->getOperand(Num: 1);
17906
17907 // Turn X << -C -> X >> C and viceversa. The negative shifts can come up from
17908 // uses of the intrinsics.
17909 if (auto C = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 2))) {
17910 int ShiftAmt = C->getSExtValue();
17911 if (ShiftAmt == 0) {
17912 SDValue Merge = DAG.getMergeValues(Ops: {Op0, Op1}, dl: DL);
17913 DAG.ReplaceAllUsesWith(From: N, To: Merge.getNode());
17914 return SDValue();
17915 }
17916
17917 if (ShiftAmt >= -32 && ShiftAmt < 0) {
17918 unsigned NewOpcode =
17919 N->getOpcode() == ARMISD::LSLL ? ARMISD::LSRL : ARMISD::LSLL;
17920 SDValue NewShift = DAG.getNode(Opcode: NewOpcode, DL, VTList: N->getVTList(), N1: Op0, N2: Op1,
17921 N3: DAG.getConstant(Val: -ShiftAmt, DL, VT: MVT::i32));
17922 DAG.ReplaceAllUsesWith(From: N, To: NewShift.getNode());
17923 return NewShift;
17924 }
17925 }
17926
17927 return SDValue();
17928}
17929
17930/// PerformIntrinsicCombine - ARM-specific DAG combining for intrinsics.
17931SDValue ARMTargetLowering::PerformIntrinsicCombine(SDNode *N,
17932 DAGCombinerInfo &DCI) const {
17933 SelectionDAG &DAG = DCI.DAG;
17934 unsigned IntNo = N->getConstantOperandVal(Num: 0);
17935 switch (IntNo) {
17936 default:
17937 // Don't do anything for most intrinsics.
17938 break;
17939
17940 // Vector shifts: check for immediate versions and lower them.
17941 // Note: This is done during DAG combining instead of DAG legalizing because
17942 // the build_vectors for 64-bit vector element shift counts are generally
17943 // not legal, and it is hard to see their values after they get legalized to
17944 // loads from a constant pool.
17945 case Intrinsic::arm_neon_vshifts:
17946 case Intrinsic::arm_neon_vshiftu:
17947 case Intrinsic::arm_neon_vrshifts:
17948 case Intrinsic::arm_neon_vrshiftu:
17949 case Intrinsic::arm_neon_vrshiftn:
17950 case Intrinsic::arm_neon_vqshifts:
17951 case Intrinsic::arm_neon_vqshiftu:
17952 case Intrinsic::arm_neon_vqshiftsu:
17953 case Intrinsic::arm_neon_vqshiftns:
17954 case Intrinsic::arm_neon_vqshiftnu:
17955 case Intrinsic::arm_neon_vqshiftnsu:
17956 case Intrinsic::arm_neon_vqrshiftns:
17957 case Intrinsic::arm_neon_vqrshiftnu:
17958 case Intrinsic::arm_neon_vqrshiftnsu: {
17959 EVT VT = N->getOperand(Num: 1).getValueType();
17960 int64_t Cnt;
17961 unsigned VShiftOpc = 0;
17962
17963 switch (IntNo) {
17964 case Intrinsic::arm_neon_vshifts:
17965 case Intrinsic::arm_neon_vshiftu:
17966 if (isVShiftLImm(Op: N->getOperand(Num: 2), VT, isLong: false, Cnt)) {
17967 VShiftOpc = ARMISD::VSHLIMM;
17968 break;
17969 }
17970 if (isVShiftRImm(Op: N->getOperand(Num: 2), VT, isNarrow: false, isIntrinsic: true, Cnt)) {
17971 VShiftOpc = (IntNo == Intrinsic::arm_neon_vshifts ? ARMISD::VSHRsIMM
17972 : ARMISD::VSHRuIMM);
17973 break;
17974 }
17975 return SDValue();
17976
17977 case Intrinsic::arm_neon_vrshifts:
17978 case Intrinsic::arm_neon_vrshiftu:
17979 if (isVShiftRImm(Op: N->getOperand(Num: 2), VT, isNarrow: false, isIntrinsic: true, Cnt))
17980 break;
17981 return SDValue();
17982
17983 case Intrinsic::arm_neon_vqshifts:
17984 case Intrinsic::arm_neon_vqshiftu:
17985 if (isVShiftLImm(Op: N->getOperand(Num: 2), VT, isLong: false, Cnt))
17986 break;
17987 return SDValue();
17988
17989 case Intrinsic::arm_neon_vqshiftsu:
17990 if (isVShiftLImm(Op: N->getOperand(Num: 2), VT, isLong: false, Cnt))
17991 break;
17992 llvm_unreachable("invalid shift count for vqshlu intrinsic");
17993
17994 case Intrinsic::arm_neon_vrshiftn:
17995 case Intrinsic::arm_neon_vqshiftns:
17996 case Intrinsic::arm_neon_vqshiftnu:
17997 case Intrinsic::arm_neon_vqshiftnsu:
17998 case Intrinsic::arm_neon_vqrshiftns:
17999 case Intrinsic::arm_neon_vqrshiftnu:
18000 case Intrinsic::arm_neon_vqrshiftnsu:
18001 // Narrowing shifts require an immediate right shift.
18002 if (isVShiftRImm(Op: N->getOperand(Num: 2), VT, isNarrow: true, isIntrinsic: true, Cnt))
18003 break;
18004 llvm_unreachable("invalid shift count for narrowing vector shift "
18005 "intrinsic");
18006
18007 default:
18008 llvm_unreachable("unhandled vector shift");
18009 }
18010
18011 switch (IntNo) {
18012 case Intrinsic::arm_neon_vshifts:
18013 case Intrinsic::arm_neon_vshiftu:
18014 // Opcode already set above.
18015 break;
18016 case Intrinsic::arm_neon_vrshifts:
18017 VShiftOpc = ARMISD::VRSHRsIMM;
18018 break;
18019 case Intrinsic::arm_neon_vrshiftu:
18020 VShiftOpc = ARMISD::VRSHRuIMM;
18021 break;
18022 case Intrinsic::arm_neon_vrshiftn:
18023 VShiftOpc = ARMISD::VRSHRNIMM;
18024 break;
18025 case Intrinsic::arm_neon_vqshifts:
18026 VShiftOpc = ARMISD::VQSHLsIMM;
18027 break;
18028 case Intrinsic::arm_neon_vqshiftu:
18029 VShiftOpc = ARMISD::VQSHLuIMM;
18030 break;
18031 case Intrinsic::arm_neon_vqshiftsu:
18032 VShiftOpc = ARMISD::VQSHLsuIMM;
18033 break;
18034 case Intrinsic::arm_neon_vqshiftns:
18035 VShiftOpc = ARMISD::VQSHRNsIMM;
18036 break;
18037 case Intrinsic::arm_neon_vqshiftnu:
18038 VShiftOpc = ARMISD::VQSHRNuIMM;
18039 break;
18040 case Intrinsic::arm_neon_vqshiftnsu:
18041 VShiftOpc = ARMISD::VQSHRNsuIMM;
18042 break;
18043 case Intrinsic::arm_neon_vqrshiftns:
18044 VShiftOpc = ARMISD::VQRSHRNsIMM;
18045 break;
18046 case Intrinsic::arm_neon_vqrshiftnu:
18047 VShiftOpc = ARMISD::VQRSHRNuIMM;
18048 break;
18049 case Intrinsic::arm_neon_vqrshiftnsu:
18050 VShiftOpc = ARMISD::VQRSHRNsuIMM;
18051 break;
18052 }
18053
18054 SDLoc dl(N);
18055 return DAG.getNode(Opcode: VShiftOpc, DL: dl, VT: N->getValueType(ResNo: 0),
18056 N1: N->getOperand(Num: 1), N2: DAG.getConstant(Val: Cnt, DL: dl, VT: MVT::i32));
18057 }
18058
18059 case Intrinsic::arm_neon_vshiftins: {
18060 EVT VT = N->getOperand(Num: 1).getValueType();
18061 int64_t Cnt;
18062 unsigned VShiftOpc = 0;
18063
18064 if (isVShiftLImm(Op: N->getOperand(Num: 3), VT, isLong: false, Cnt))
18065 VShiftOpc = ARMISD::VSLIIMM;
18066 else if (isVShiftRImm(Op: N->getOperand(Num: 3), VT, isNarrow: false, isIntrinsic: true, Cnt))
18067 VShiftOpc = ARMISD::VSRIIMM;
18068 else {
18069 llvm_unreachable("invalid shift count for vsli/vsri intrinsic");
18070 }
18071
18072 SDLoc dl(N);
18073 return DAG.getNode(Opcode: VShiftOpc, DL: dl, VT: N->getValueType(ResNo: 0),
18074 N1: N->getOperand(Num: 1), N2: N->getOperand(Num: 2),
18075 N3: DAG.getConstant(Val: Cnt, DL: dl, VT: MVT::i32));
18076 }
18077
18078 case Intrinsic::arm_neon_vqrshifts:
18079 case Intrinsic::arm_neon_vqrshiftu:
18080 // No immediate versions of these to check for.
18081 break;
18082
18083 case Intrinsic::arm_neon_vbsl: {
18084 SDLoc dl(N);
18085 return DAG.getNode(Opcode: ARMISD::VBSP, DL: dl, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
18086 N2: N->getOperand(Num: 2), N3: N->getOperand(Num: 3));
18087 }
18088 case Intrinsic::arm_mve_vqdmlah:
18089 case Intrinsic::arm_mve_vqdmlash:
18090 case Intrinsic::arm_mve_vqrdmlah:
18091 case Intrinsic::arm_mve_vqrdmlash:
18092 case Intrinsic::arm_mve_vmla_n_predicated:
18093 case Intrinsic::arm_mve_vmlas_n_predicated:
18094 case Intrinsic::arm_mve_vqdmlah_predicated:
18095 case Intrinsic::arm_mve_vqdmlash_predicated:
18096 case Intrinsic::arm_mve_vqrdmlah_predicated:
18097 case Intrinsic::arm_mve_vqrdmlash_predicated: {
18098 // These intrinsics all take an i32 scalar operand which is narrowed to the
18099 // size of a single lane of the vector type they return. So we don't need
18100 // any bits of that operand above that point, which allows us to eliminate
18101 // uxth/sxth.
18102 unsigned BitWidth = N->getValueType(ResNo: 0).getScalarSizeInBits();
18103 APInt DemandedMask = APInt::getLowBitsSet(numBits: 32, loBitsSet: BitWidth);
18104 if (SimplifyDemandedBits(Op: N->getOperand(Num: 3), DemandedBits: DemandedMask, DCI))
18105 return SDValue();
18106 break;
18107 }
18108
18109 case Intrinsic::arm_mve_minv:
18110 case Intrinsic::arm_mve_maxv:
18111 case Intrinsic::arm_mve_minav:
18112 case Intrinsic::arm_mve_maxav:
18113 case Intrinsic::arm_mve_minv_predicated:
18114 case Intrinsic::arm_mve_maxv_predicated:
18115 case Intrinsic::arm_mve_minav_predicated:
18116 case Intrinsic::arm_mve_maxav_predicated: {
18117 // These intrinsics all take an i32 scalar operand which is narrowed to the
18118 // size of a single lane of the vector type they take as the other input.
18119 unsigned BitWidth = N->getOperand(Num: 2)->getValueType(ResNo: 0).getScalarSizeInBits();
18120 APInt DemandedMask = APInt::getLowBitsSet(numBits: 32, loBitsSet: BitWidth);
18121 if (SimplifyDemandedBits(Op: N->getOperand(Num: 1), DemandedBits: DemandedMask, DCI))
18122 return SDValue();
18123 break;
18124 }
18125
18126 case Intrinsic::arm_mve_addv: {
18127 // Turn this intrinsic straight into the appropriate ARMISD::VADDV node,
18128 // which allow PerformADDVecReduce to turn it into VADDLV when possible.
18129 bool Unsigned = N->getConstantOperandVal(Num: 2);
18130 unsigned Opc = Unsigned ? ARMISD::VADDVu : ARMISD::VADDVs;
18131 return DAG.getNode(Opcode: Opc, DL: SDLoc(N), VTList: N->getVTList(), N: N->getOperand(Num: 1));
18132 }
18133
18134 case Intrinsic::arm_mve_addlv:
18135 case Intrinsic::arm_mve_addlv_predicated: {
18136 // Same for these, but ARMISD::VADDLV has to be followed by a BUILD_PAIR
18137 // which recombines the two outputs into an i64
18138 bool Unsigned = N->getConstantOperandVal(Num: 2);
18139 unsigned Opc = IntNo == Intrinsic::arm_mve_addlv ?
18140 (Unsigned ? ARMISD::VADDLVu : ARMISD::VADDLVs) :
18141 (Unsigned ? ARMISD::VADDLVpu : ARMISD::VADDLVps);
18142
18143 SmallVector<SDValue, 4> Ops;
18144 for (unsigned i = 1, e = N->getNumOperands(); i < e; i++)
18145 if (i != 2) // skip the unsigned flag
18146 Ops.push_back(Elt: N->getOperand(Num: i));
18147
18148 SDLoc dl(N);
18149 SDValue val = DAG.getNode(Opcode: Opc, DL: dl, ResultTys: {MVT::i32, MVT::i32}, Ops);
18150 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: MVT::i64, N1: val.getValue(R: 0),
18151 N2: val.getValue(R: 1));
18152 }
18153 }
18154
18155 return SDValue();
18156}
18157
18158bool ARMTargetLowering::hasAndNot(SDValue Y) const {
18159 EVT VT = Y.getValueType();
18160 if (!VT.isVector())
18161 return hasAndNotCompare(V: Y);
18162 if (Subtarget->hasMVEIntegerOps())
18163 return VT.is128BitVector();
18164 if (Subtarget->hasNEON())
18165 return VT.is64BitVector() || VT.is128BitVector();
18166 return false;
18167}
18168
18169/// PerformShiftCombine - Checks for immediate versions of vector shifts and
18170/// lowers them. As with the vector shift intrinsics, this is done during DAG
18171/// combining instead of DAG legalizing because the build_vectors for 64-bit
18172/// vector element shift counts are generally not legal, and it is hard to see
18173/// their values after they get legalized to loads from a constant pool.
18174static SDValue PerformShiftCombine(SDNode *N,
18175 TargetLowering::DAGCombinerInfo &DCI,
18176 const ARMSubtarget *ST) {
18177 SelectionDAG &DAG = DCI.DAG;
18178 EVT VT = N->getValueType(ResNo: 0);
18179
18180 if (ST->isThumb1Only() && N->getOpcode() == ISD::SHL && VT == MVT::i32 &&
18181 N->getOperand(Num: 0)->getOpcode() == ISD::AND &&
18182 N->getOperand(Num: 0)->hasOneUse()) {
18183 if (DCI.isBeforeLegalize() || DCI.isCalledByLegalizer())
18184 return SDValue();
18185 // Look for the pattern (shl (and x, AndMask), ShiftAmt). This doesn't
18186 // usually show up because instcombine prefers to canonicalize it to
18187 // (and (shl x, ShiftAmt) (shl AndMask, ShiftAmt)), but the shift can come
18188 // out of GEP lowering in some cases.
18189 SDValue N0 = N->getOperand(Num: 0);
18190 ConstantSDNode *ShiftAmtNode = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
18191 if (!ShiftAmtNode)
18192 return SDValue();
18193 uint32_t ShiftAmt = static_cast<uint32_t>(ShiftAmtNode->getZExtValue());
18194 ConstantSDNode *AndMaskNode = dyn_cast<ConstantSDNode>(Val: N0->getOperand(Num: 1));
18195 if (!AndMaskNode)
18196 return SDValue();
18197 uint32_t AndMask = static_cast<uint32_t>(AndMaskNode->getZExtValue());
18198 // Don't transform uxtb/uxth.
18199 if (AndMask == 255 || AndMask == 65535)
18200 return SDValue();
18201 if (isMask_32(Value: AndMask)) {
18202 uint32_t MaskedBits = llvm::countl_zero(Val: AndMask);
18203 if (MaskedBits > ShiftAmt) {
18204 SDLoc DL(N);
18205 SDValue SHL = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i32, N1: N0->getOperand(Num: 0),
18206 N2: DAG.getConstant(Val: MaskedBits, DL, VT: MVT::i32));
18207 return DAG.getNode(
18208 Opcode: ISD::SRL, DL, VT: MVT::i32, N1: SHL,
18209 N2: DAG.getConstant(Val: MaskedBits - ShiftAmt, DL, VT: MVT::i32));
18210 }
18211 }
18212 }
18213
18214 // Nothing to be done for scalar shifts.
18215 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
18216 if (!VT.isVector() || !TLI.isTypeLegal(VT))
18217 return SDValue();
18218 if (ST->hasMVEIntegerOps())
18219 return SDValue();
18220
18221 int64_t Cnt;
18222
18223 switch (N->getOpcode()) {
18224 default: llvm_unreachable("unexpected shift opcode");
18225
18226 case ISD::SHL:
18227 if (isVShiftLImm(Op: N->getOperand(Num: 1), VT, isLong: false, Cnt)) {
18228 SDLoc dl(N);
18229 return DAG.getNode(Opcode: ARMISD::VSHLIMM, DL: dl, VT, N1: N->getOperand(Num: 0),
18230 N2: DAG.getConstant(Val: Cnt, DL: dl, VT: MVT::i32));
18231 }
18232 break;
18233
18234 case ISD::SRA:
18235 case ISD::SRL:
18236 if (isVShiftRImm(Op: N->getOperand(Num: 1), VT, isNarrow: false, isIntrinsic: false, Cnt)) {
18237 unsigned VShiftOpc =
18238 (N->getOpcode() == ISD::SRA ? ARMISD::VSHRsIMM : ARMISD::VSHRuIMM);
18239 SDLoc dl(N);
18240 return DAG.getNode(Opcode: VShiftOpc, DL: dl, VT, N1: N->getOperand(Num: 0),
18241 N2: DAG.getConstant(Val: Cnt, DL: dl, VT: MVT::i32));
18242 }
18243 }
18244 return SDValue();
18245}
18246
18247// Look for a sign/zero/fpextend extend of a larger than legal load. This can be
18248// split into multiple extending loads, which are simpler to deal with than an
18249// arbitrary extend. For fp extends we use an integer extending load and a VCVTL
18250// to convert the type to an f32.
18251static SDValue PerformSplittingToWideningLoad(SDNode *N, SelectionDAG &DAG) {
18252 SDValue N0 = N->getOperand(Num: 0);
18253 if (N0.getOpcode() != ISD::LOAD)
18254 return SDValue();
18255 LoadSDNode *LD = cast<LoadSDNode>(Val: N0.getNode());
18256 if (!LD->isSimple() || !N0.hasOneUse() || LD->isIndexed() ||
18257 LD->getExtensionType() != ISD::NON_EXTLOAD)
18258 return SDValue();
18259 EVT FromVT = LD->getValueType(ResNo: 0);
18260 EVT ToVT = N->getValueType(ResNo: 0);
18261 if (!ToVT.isVector())
18262 return SDValue();
18263 assert(FromVT.getVectorNumElements() == ToVT.getVectorNumElements());
18264 EVT ToEltVT = ToVT.getVectorElementType();
18265 EVT FromEltVT = FromVT.getVectorElementType();
18266
18267 unsigned NumElements = 0;
18268 if (ToEltVT == MVT::i32 && FromEltVT == MVT::i8)
18269 NumElements = 4;
18270 if (ToEltVT == MVT::f32 && FromEltVT == MVT::f16)
18271 NumElements = 4;
18272 if (NumElements == 0 ||
18273 (FromEltVT != MVT::f16 && FromVT.getVectorNumElements() == NumElements) ||
18274 FromVT.getVectorNumElements() % NumElements != 0 ||
18275 !isPowerOf2_32(Value: NumElements))
18276 return SDValue();
18277
18278 LLVMContext &C = *DAG.getContext();
18279 SDLoc DL(LD);
18280 // Details about the old load
18281 SDValue Ch = LD->getChain();
18282 SDValue BasePtr = LD->getBasePtr();
18283 Align Alignment = LD->getBaseAlign();
18284 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
18285 AAMDNodes AAInfo = LD->getAAInfo();
18286
18287 ISD::LoadExtType NewExtType =
18288 N->getOpcode() == ISD::SIGN_EXTEND ? ISD::SEXTLOAD : ISD::ZEXTLOAD;
18289 SDValue Offset = DAG.getPOISON(VT: BasePtr.getValueType());
18290 EVT NewFromVT = EVT::getVectorVT(
18291 Context&: C, VT: EVT::getIntegerVT(Context&: C, BitWidth: FromEltVT.getScalarSizeInBits()), NumElements);
18292 EVT NewToVT = EVT::getVectorVT(
18293 Context&: C, VT: EVT::getIntegerVT(Context&: C, BitWidth: ToEltVT.getScalarSizeInBits()), NumElements);
18294
18295 SmallVector<SDValue, 4> Loads;
18296 SmallVector<SDValue, 4> Chains;
18297 for (unsigned i = 0; i < FromVT.getVectorNumElements() / NumElements; i++) {
18298 unsigned NewOffset = (i * NewFromVT.getSizeInBits()) / 8;
18299 SDValue NewPtr =
18300 DAG.getObjectPtrOffset(SL: DL, Ptr: BasePtr, Offset: TypeSize::getFixed(ExactSize: NewOffset));
18301
18302 SDValue NewLoad =
18303 DAG.getLoad(AM: ISD::UNINDEXED, ExtType: NewExtType, VT: NewToVT, dl: DL, Chain: Ch, Ptr: NewPtr, Offset,
18304 PtrInfo: LD->getPointerInfo().getWithOffset(O: NewOffset), MemVT: NewFromVT,
18305 Alignment, MMOFlags, Metadata: AAInfo);
18306 Loads.push_back(Elt: NewLoad);
18307 Chains.push_back(Elt: SDValue(NewLoad.getNode(), 1));
18308 }
18309
18310 // Float truncs need to extended with VCVTB's into their floating point types.
18311 if (FromEltVT == MVT::f16) {
18312 SmallVector<SDValue, 4> Extends;
18313
18314 for (unsigned i = 0; i < Loads.size(); i++) {
18315 SDValue LoadBC =
18316 DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL, VT: MVT::v8f16, Operand: Loads[i]);
18317 SDValue FPExt = DAG.getNode(Opcode: ARMISD::VCVTL, DL, VT: MVT::v4f32, N1: LoadBC,
18318 N2: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
18319 Extends.push_back(Elt: FPExt);
18320 }
18321
18322 Loads = Extends;
18323 }
18324
18325 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: Chains);
18326 DAG.ReplaceAllUsesOfValueWith(From: SDValue(LD, 1), To: NewChain);
18327 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: ToVT, Ops: Loads);
18328}
18329
18330/// PerformExtendCombine - Target-specific DAG combining for ISD::SIGN_EXTEND,
18331/// ISD::ZERO_EXTEND, and ISD::ANY_EXTEND.
18332static SDValue PerformExtendCombine(SDNode *N, SelectionDAG &DAG,
18333 const ARMSubtarget *ST) {
18334 SDValue N0 = N->getOperand(Num: 0);
18335 EVT VT = N->getValueType(ResNo: 0);
18336 SDLoc DL(N);
18337
18338 // Check for sign- and zero-extensions of vector extract operations of 8- and
18339 // 16-bit vector elements. NEON and MVE support these directly. They are
18340 // handled during DAG combining because type legalization will promote them
18341 // to 32-bit types and it is messy to recognize the operations after that.
18342 if ((ST->hasNEON() || ST->hasMVEIntegerOps()) &&
18343 N0.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
18344 SDValue Vec = N0.getOperand(i: 0);
18345 SDValue Lane = N0.getOperand(i: 1);
18346 EVT EltVT = N0.getValueType();
18347 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
18348
18349 if (VT == MVT::i32 &&
18350 (EltVT == MVT::i8 || EltVT == MVT::i16) &&
18351 TLI.isTypeLegal(VT: Vec.getValueType()) &&
18352 isa<ConstantSDNode>(Val: Lane)) {
18353
18354 unsigned Opc = 0;
18355 switch (N->getOpcode()) {
18356 default: llvm_unreachable("unexpected opcode");
18357 case ISD::SIGN_EXTEND:
18358 Opc = ARMISD::VGETLANEs;
18359 break;
18360 case ISD::ZERO_EXTEND:
18361 case ISD::ANY_EXTEND:
18362 Opc = ARMISD::VGETLANEu;
18363 break;
18364 }
18365 return DAG.getNode(Opcode: Opc, DL, VT, N1: Vec, N2: Lane);
18366 }
18367 }
18368
18369 if (ST->hasMVEIntegerOps())
18370 if (SDValue NewLoad = PerformSplittingToWideningLoad(N, DAG))
18371 return NewLoad;
18372
18373 // Combine sext(buildvector(..)) to buildvector(sext(..)) to help avoid
18374 // difficult to lower i1 buildvector.
18375 if (ST->hasMVEIntegerOps() && N0.getValueType().getScalarSizeInBits() == 1 &&
18376 N0.getOpcode() == ISD::BUILD_VECTOR && VT.getScalarSizeInBits() <= 32) {
18377 SmallVector<SDValue> Ops;
18378 for (unsigned I = 0; I < N0.getNumOperands(); I++) {
18379 SDValue InReg = N0.getOperand(i: I);
18380 if (N->getOpcode() == ISD::ZERO_EXTEND)
18381 InReg = DAG.getNode(Opcode: ISD::AND, DL, VT: InReg.getValueType(), N1: InReg,
18382 N2: DAG.getConstant(Val: 1, DL, VT: InReg.getValueType()));
18383 else if (N->getOpcode() == ISD::SIGN_EXTEND)
18384 InReg = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: InReg.getValueType(),
18385 N1: InReg, N2: DAG.getValueType(MVT::i1));
18386 SDValue Ext = DAG.getNode(Opcode: N->getOpcode(), DL, VT: MVT::i32, Operand: InReg);
18387 Ops.push_back(Elt: Ext);
18388 }
18389 return DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT, Ops);
18390 }
18391
18392 return SDValue();
18393}
18394
18395static SDValue PerformFPExtendCombine(SDNode *N, SelectionDAG &DAG,
18396 const ARMSubtarget *ST) {
18397 if (ST->hasMVEFloatOps())
18398 if (SDValue NewLoad = PerformSplittingToWideningLoad(N, DAG))
18399 return NewLoad;
18400
18401 return SDValue();
18402}
18403
18404// Lower smin(smax(x, C1), C2) to ssat or usat, if they have saturating
18405// constant bounds.
18406static SDValue PerformMinMaxToSatCombine(SDValue Op, SelectionDAG &DAG,
18407 const ARMSubtarget *Subtarget) {
18408 if ((Subtarget->isThumb() || !Subtarget->hasV6Ops()) &&
18409 !Subtarget->isThumb2())
18410 return SDValue();
18411
18412 EVT VT = Op.getValueType();
18413 SDValue Op0 = Op.getOperand(i: 0);
18414
18415 if (VT != MVT::i32 ||
18416 (Op0.getOpcode() != ISD::SMIN && Op0.getOpcode() != ISD::SMAX) ||
18417 !isa<ConstantSDNode>(Val: Op.getOperand(i: 1)) ||
18418 !isa<ConstantSDNode>(Val: Op0.getOperand(i: 1)))
18419 return SDValue();
18420
18421 SDValue Min = Op;
18422 SDValue Max = Op0;
18423 SDValue Input = Op0.getOperand(i: 0);
18424 if (Min.getOpcode() == ISD::SMAX)
18425 std::swap(a&: Min, b&: Max);
18426
18427 if (Min.getOpcode() != ISD::SMIN || Max.getOpcode() != ISD::SMAX)
18428 return SDValue();
18429
18430 APInt MinC = Min.getConstantOperandAPInt(i: 1);
18431 APInt MaxC = Max.getConstantOperandAPInt(i: 1);
18432 if (MaxC.sgt(RHS: MinC))
18433 return SDValue();
18434
18435 SDLoc DL(Op);
18436
18437 // A clamp whose bounds are already a saturation range maps to a single
18438 // SSAT / USAT.
18439 if ((MinC + 1).isPowerOf2()) {
18440 if (MinC == ~MaxC)
18441 return DAG.getNode(Opcode: ARMISD::SSAT, DL, VT, N1: Input,
18442 N2: DAG.getConstant(Val: MinC.countr_one(), DL, VT));
18443 if (MaxC == 0)
18444 return DAG.getNode(Opcode: ARMISD::USAT, DL, VT, N1: Input,
18445 N2: DAG.getConstant(Val: MinC.countr_one(), DL, VT));
18446 }
18447
18448 // For power-of-two clamp widths, convert the range to be zero-centered,
18449 // apply SSAT, and convert the result back.
18450 //
18451 // Width = Hi - Lo + 1
18452 // Center = Lo + Width / 2
18453 // Result = ssat(X - Center) + Center
18454 //
18455 // The idea is to shift the input so that the clamp range is centered
18456 // around zero, apply ssat, and then shift the result back.
18457 //
18458 // For example clamp(X, -118, 137) -> Width = 256, Center = 10, so it becomes
18459 // ssat(X - 10, 8) + 10
18460
18461 APInt Width = MinC - MaxC + 1;
18462 if (!Width.isPowerOf2() || Width.isOne())
18463 return SDValue();
18464 unsigned SatBit = Width.logBase2() - 1; // ssat to SatBit + 1 signed bits
18465 APInt Center = MaxC + Width.lshr(shiftAmt: 1);
18466
18467 // The rewrite is only valid when X - Center does not overflow;
18468 SDValue NegC = DAG.getConstant(Val: -Center, DL, VT);
18469 if (DAG.computeOverflowForSignedAdd(N0: Input, N1: NegC) != SelectionDAG::OFK_Never)
18470 return SDValue();
18471
18472 SDValue Shifted = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Input, N2: NegC);
18473 SDValue Sat = DAG.getNode(Opcode: ARMISD::SSAT, DL, VT, N1: Shifted,
18474 N2: DAG.getConstant(Val: SatBit, DL, VT));
18475 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Sat, N2: DAG.getConstant(Val: Center, DL, VT));
18476}
18477
18478/// PerformMinMaxCombine - Target-specific DAG combining for creating truncating
18479/// saturates.
18480static SDValue PerformMinMaxCombine(SDNode *N, SelectionDAG &DAG,
18481 const ARMSubtarget *ST) {
18482 EVT VT = N->getValueType(ResNo: 0);
18483 SDValue N0 = N->getOperand(Num: 0);
18484
18485 if (VT == MVT::i32)
18486 return PerformMinMaxToSatCombine(Op: SDValue(N, 0), DAG, Subtarget: ST);
18487
18488 if (!ST->hasMVEIntegerOps())
18489 return SDValue();
18490
18491 if (SDValue V = PerformVQDMULHCombine(N, DAG))
18492 return V;
18493
18494 if (VT != MVT::v4i32 && VT != MVT::v8i16)
18495 return SDValue();
18496
18497 auto IsSignedSaturate = [&](SDNode *Min, SDNode *Max) {
18498 // Check one is a smin and the other is a smax
18499 if (Min->getOpcode() != ISD::SMIN)
18500 std::swap(a&: Min, b&: Max);
18501 if (Min->getOpcode() != ISD::SMIN || Max->getOpcode() != ISD::SMAX)
18502 return false;
18503
18504 APInt SaturateC;
18505 if (VT == MVT::v4i32)
18506 SaturateC = APInt(32, (1 << 15) - 1, true);
18507 else //if (VT == MVT::v8i16)
18508 SaturateC = APInt(16, (1 << 7) - 1, true);
18509
18510 APInt MinC, MaxC;
18511 if (!ISD::isConstantSplatVector(V: Min->getOperand(Num: 1), SplatValue&: MinC) ||
18512 MinC != SaturateC)
18513 return false;
18514 if (!ISD::isConstantSplatVector(V: Max->getOperand(Num: 1), SplatValue&: MaxC) ||
18515 MaxC != ~SaturateC)
18516 return false;
18517 return true;
18518 };
18519
18520 if (IsSignedSaturate(N, N0.getNode())) {
18521 SDLoc DL(N);
18522 MVT ExtVT, HalfVT;
18523 if (VT == MVT::v4i32) {
18524 HalfVT = MVT::v8i16;
18525 ExtVT = MVT::v4i16;
18526 } else { // if (VT == MVT::v8i16)
18527 HalfVT = MVT::v16i8;
18528 ExtVT = MVT::v8i8;
18529 }
18530
18531 // Create a VQMOVNB with undef top lanes, then signed extended into the top
18532 // half. That extend will hopefully be removed if only the bottom bits are
18533 // demanded (though a truncating store, for example).
18534 SDValue VQMOVN =
18535 DAG.getNode(Opcode: ARMISD::VQMOVNs, DL, VT: HalfVT, N1: DAG.getUNDEF(VT: HalfVT),
18536 N2: N0->getOperand(Num: 0), N3: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
18537 SDValue Bitcast = DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL, VT, Operand: VQMOVN);
18538 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT, N1: Bitcast,
18539 N2: DAG.getValueType(ExtVT));
18540 }
18541
18542 auto IsUnsignedSaturate = [&](SDNode *Min) {
18543 // For unsigned, we just need to check for <= 0xffff
18544 if (Min->getOpcode() != ISD::UMIN)
18545 return false;
18546
18547 APInt SaturateC;
18548 if (VT == MVT::v4i32)
18549 SaturateC = APInt(32, (1 << 16) - 1, true);
18550 else //if (VT == MVT::v8i16)
18551 SaturateC = APInt(16, (1 << 8) - 1, true);
18552
18553 APInt MinC;
18554 if (!ISD::isConstantSplatVector(V: Min->getOperand(Num: 1), SplatValue&: MinC) ||
18555 MinC != SaturateC)
18556 return false;
18557 return true;
18558 };
18559
18560 if (IsUnsignedSaturate(N)) {
18561 SDLoc DL(N);
18562 MVT HalfVT;
18563 unsigned ExtConst;
18564 if (VT == MVT::v4i32) {
18565 HalfVT = MVT::v8i16;
18566 ExtConst = 0x0000FFFF;
18567 } else { //if (VT == MVT::v8i16)
18568 HalfVT = MVT::v16i8;
18569 ExtConst = 0x00FF;
18570 }
18571
18572 // Create a VQMOVNB with undef top lanes, then ZExt into the top half with
18573 // an AND. That extend will hopefully be removed if only the bottom bits are
18574 // demanded (though a truncating store, for example).
18575 SDValue VQMOVN =
18576 DAG.getNode(Opcode: ARMISD::VQMOVNu, DL, VT: HalfVT, N1: DAG.getUNDEF(VT: HalfVT), N2: N0,
18577 N3: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
18578 SDValue Bitcast = DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL, VT, Operand: VQMOVN);
18579 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Bitcast,
18580 N2: DAG.getConstant(Val: ExtConst, DL, VT));
18581 }
18582
18583 return SDValue();
18584}
18585
18586static const APInt *isPowerOf2Constant(SDValue V) {
18587 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: V);
18588 if (!C)
18589 return nullptr;
18590 const APInt *CV = &C->getAPIntValue();
18591 return CV->isPowerOf2() ? CV : nullptr;
18592}
18593
18594SDValue ARMTargetLowering::PerformCMOVToBFICombine(SDNode *CMOV, SelectionDAG &DAG) const {
18595 // If we have a CMOV, OR and AND combination such as:
18596 // if (x & CN)
18597 // y |= CM;
18598 //
18599 // And:
18600 // * CN is a single bit;
18601 // * All bits covered by CM are known zero in y
18602 //
18603 // Then we can convert this into a sequence of BFI instructions. This will
18604 // always be a win if CM is a single bit, will always be no worse than the
18605 // TST&OR sequence if CM is two bits, and for thumb will be no worse if CM is
18606 // three bits (due to the extra IT instruction).
18607
18608 SDValue Op0 = CMOV->getOperand(Num: 0);
18609 SDValue Op1 = CMOV->getOperand(Num: 1);
18610 auto CC = CMOV->getConstantOperandAPInt(Num: 2).getLimitedValue();
18611 SDValue CmpZ = CMOV->getOperand(Num: 3);
18612
18613 // The compare must be against zero.
18614 if (!isNullConstant(V: CmpZ->getOperand(Num: 1)))
18615 return SDValue();
18616
18617 assert(CmpZ->getOpcode() == ARMISD::CMPZ);
18618 SDValue And = CmpZ->getOperand(Num: 0);
18619 if (And->getOpcode() != ISD::AND)
18620 return SDValue();
18621 const APInt *AndC = isPowerOf2Constant(V: And->getOperand(Num: 1));
18622 if (!AndC)
18623 return SDValue();
18624 SDValue X = And->getOperand(Num: 0);
18625
18626 if (CC == ARMCC::EQ) {
18627 // We're performing an "equal to zero" compare. Swap the operands so we
18628 // canonicalize on a "not equal to zero" compare.
18629 std::swap(a&: Op0, b&: Op1);
18630 } else {
18631 assert(CC == ARMCC::NE && "How can a CMPZ node not be EQ or NE?");
18632 }
18633
18634 if (Op1->getOpcode() != ISD::OR)
18635 return SDValue();
18636
18637 ConstantSDNode *OrC = dyn_cast<ConstantSDNode>(Val: Op1->getOperand(Num: 1));
18638 if (!OrC)
18639 return SDValue();
18640 SDValue Y = Op1->getOperand(Num: 0);
18641
18642 if (Op0 != Y)
18643 return SDValue();
18644
18645 // Now, is it profitable to continue?
18646 APInt OrCI = OrC->getAPIntValue();
18647 unsigned Heuristic = Subtarget->isThumb() ? 3 : 2;
18648 if (OrCI.popcount() > Heuristic)
18649 return SDValue();
18650
18651 // Lastly, can we determine that the bits defined by OrCI
18652 // are zero in Y?
18653 KnownBits Known = DAG.computeKnownBits(Op: Y);
18654 if ((OrCI & Known.Zero) != OrCI)
18655 return SDValue();
18656
18657 // OK, we can do the combine.
18658 SDValue V = Y;
18659 SDLoc dl(X);
18660 EVT VT = X.getValueType();
18661 unsigned BitInX = AndC->logBase2();
18662
18663 if (BitInX != 0) {
18664 // We must shift X first.
18665 X = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: X,
18666 N2: DAG.getConstant(Val: BitInX, DL: dl, VT));
18667 }
18668
18669 for (unsigned BitInY = 0, NumActiveBits = OrCI.getActiveBits();
18670 BitInY < NumActiveBits; ++BitInY) {
18671 if (OrCI[BitInY] == 0)
18672 continue;
18673 APInt Mask(VT.getSizeInBits(), 0);
18674 Mask.setBit(BitInY);
18675 V = DAG.getNode(Opcode: ARMISD::BFI, DL: dl, VT, N1: V, N2: X,
18676 // Confusingly, the operand is an *inverted* mask.
18677 N3: DAG.getConstant(Val: ~Mask, DL: dl, VT));
18678 }
18679
18680 return V;
18681}
18682
18683// Given N, the value controlling the conditional branch, search for the loop
18684// intrinsic, returning it, along with how the value is used. We need to handle
18685// patterns such as the following:
18686// (brcond (xor (setcc (loop.decrement), 0, ne), 1), exit)
18687// (brcond (setcc (loop.decrement), 0, eq), exit)
18688// (brcond (setcc (loop.decrement), 0, ne), header)
18689static SDValue SearchLoopIntrinsic(SDValue N, ISD::CondCode &CC, int &Imm,
18690 bool &Negate) {
18691 switch (N->getOpcode()) {
18692 default:
18693 break;
18694 case ISD::XOR: {
18695 if (!isa<ConstantSDNode>(Val: N.getOperand(i: 1)))
18696 return SDValue();
18697 if (!cast<ConstantSDNode>(Val: N.getOperand(i: 1))->isOne())
18698 return SDValue();
18699 Negate = !Negate;
18700 return SearchLoopIntrinsic(N: N.getOperand(i: 0), CC, Imm, Negate);
18701 }
18702 case ISD::SETCC: {
18703 auto *Const = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1));
18704 if (!Const)
18705 return SDValue();
18706 if (Const->isZero())
18707 Imm = 0;
18708 else if (Const->isOne())
18709 Imm = 1;
18710 else
18711 return SDValue();
18712 CC = N.getOperandCondCode(i: 2);
18713 return SearchLoopIntrinsic(N: N->getOperand(Num: 0), CC, Imm, Negate);
18714 }
18715 case ISD::INTRINSIC_W_CHAIN: {
18716 unsigned IntOp = N.getConstantOperandVal(i: 1);
18717 if (IntOp != Intrinsic::test_start_loop_iterations &&
18718 IntOp != Intrinsic::loop_decrement_reg)
18719 return SDValue();
18720 return N;
18721 }
18722 }
18723 return SDValue();
18724}
18725
18726static SDValue PerformHWLoopCombine(SDNode *N,
18727 TargetLowering::DAGCombinerInfo &DCI,
18728 const ARMSubtarget *ST) {
18729
18730 // The hwloop intrinsics that we're interested are used for control-flow,
18731 // either for entering or exiting the loop:
18732 // - test.start.loop.iterations will test whether its operand is zero. If it
18733 // is zero, the proceeding branch should not enter the loop.
18734 // - loop.decrement.reg also tests whether its operand is zero. If it is
18735 // zero, the proceeding branch should not branch back to the beginning of
18736 // the loop.
18737 // So here, we need to check that how the brcond is using the result of each
18738 // of the intrinsics to ensure that we're branching to the right place at the
18739 // right time.
18740
18741 ISD::CondCode CC;
18742 SDValue Cond;
18743 int Imm = 1;
18744 bool Negate = false;
18745 SDValue Chain = N->getOperand(Num: 0);
18746 SDValue Dest;
18747
18748 if (N->getOpcode() == ISD::BRCOND) {
18749 CC = ISD::SETEQ;
18750 Cond = N->getOperand(Num: 1);
18751 Dest = N->getOperand(Num: 2);
18752 } else {
18753 assert(N->getOpcode() == ISD::BR_CC && "Expected BRCOND or BR_CC!");
18754 CC = N->getOperandCondCode(Num: 1);
18755 Cond = N->getOperand(Num: 2);
18756 Dest = N->getOperand(Num: 4);
18757 if (auto *Const = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 3))) {
18758 if (!Const->isOne() && !Const->isZero())
18759 return SDValue();
18760 Imm = Const->getZExtValue();
18761 } else
18762 return SDValue();
18763 }
18764
18765 SDValue Int = SearchLoopIntrinsic(N: Cond, CC, Imm, Negate);
18766 if (!Int)
18767 return SDValue();
18768
18769 if (Negate)
18770 CC = ISD::getSetCCInverse(Operation: CC, /* Integer inverse */ Type: MVT::i32);
18771
18772 auto IsTrueIfZero = [](ISD::CondCode CC, int Imm) {
18773 return (CC == ISD::SETEQ && Imm == 0) ||
18774 (CC == ISD::SETNE && Imm == 1) ||
18775 (CC == ISD::SETLT && Imm == 1) ||
18776 (CC == ISD::SETULT && Imm == 1);
18777 };
18778
18779 auto IsFalseIfZero = [](ISD::CondCode CC, int Imm) {
18780 return (CC == ISD::SETEQ && Imm == 1) ||
18781 (CC == ISD::SETNE && Imm == 0) ||
18782 (CC == ISD::SETGT && Imm == 0) ||
18783 (CC == ISD::SETUGT && Imm == 0) ||
18784 (CC == ISD::SETGE && Imm == 1) ||
18785 (CC == ISD::SETUGE && Imm == 1);
18786 };
18787
18788 assert((IsTrueIfZero(CC, Imm) || IsFalseIfZero(CC, Imm)) &&
18789 "unsupported condition");
18790
18791 SDLoc dl(Int);
18792 SelectionDAG &DAG = DCI.DAG;
18793 SDValue Elements = Int.getOperand(i: 2);
18794 unsigned IntOp = Int->getConstantOperandVal(Num: 1);
18795 assert((N->hasOneUse() && N->user_begin()->getOpcode() == ISD::BR) &&
18796 "expected single br user");
18797 SDNode *Br = *N->user_begin();
18798 SDValue OtherTarget = Br->getOperand(Num: 1);
18799
18800 // Update the unconditional branch to branch to the given Dest.
18801 auto UpdateUncondBr = [](SDNode *Br, SDValue Dest, SelectionDAG &DAG) {
18802 SDValue NewBrOps[] = { Br->getOperand(Num: 0), Dest };
18803 SDValue NewBr = DAG.getNode(Opcode: ISD::BR, DL: SDLoc(Br), VT: MVT::Other, Ops: NewBrOps);
18804 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Br, 0), To: NewBr);
18805 };
18806
18807 if (IntOp == Intrinsic::test_start_loop_iterations) {
18808 SDValue Res;
18809 SDValue Setup = DAG.getNode(Opcode: ARMISD::WLSSETUP, DL: dl, VT: MVT::i32, Operand: Elements);
18810 // We expect this 'instruction' to branch when the counter is zero.
18811 if (IsTrueIfZero(CC, Imm)) {
18812 SDValue Ops[] = {Chain, Setup, Dest};
18813 Res = DAG.getNode(Opcode: ARMISD::WLS, DL: dl, VT: MVT::Other, Ops);
18814 } else {
18815 // The logic is the reverse of what we need for WLS, so find the other
18816 // basic block target: the target of the proceeding br.
18817 UpdateUncondBr(Br, Dest, DAG);
18818
18819 SDValue Ops[] = {Chain, Setup, OtherTarget};
18820 Res = DAG.getNode(Opcode: ARMISD::WLS, DL: dl, VT: MVT::Other, Ops);
18821 }
18822 // Update LR count to the new value
18823 DAG.ReplaceAllUsesOfValueWith(From: Int.getValue(R: 0), To: Setup);
18824 // Update chain
18825 DAG.ReplaceAllUsesOfValueWith(From: Int.getValue(R: 2), To: Int.getOperand(i: 0));
18826 return Res;
18827 } else {
18828 SDValue Size =
18829 DAG.getTargetConstant(Val: Int.getConstantOperandVal(i: 3), DL: dl, VT: MVT::i32);
18830 SDValue Args[] = { Int.getOperand(i: 0), Elements, Size, };
18831 SDValue LoopDec = DAG.getNode(Opcode: ARMISD::LOOP_DEC, DL: dl,
18832 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other), Ops: Args);
18833 DAG.ReplaceAllUsesWith(From: Int.getNode(), To: LoopDec.getNode());
18834
18835 // We expect this instruction to branch when the count is not zero.
18836 SDValue Target = IsFalseIfZero(CC, Imm) ? Dest : OtherTarget;
18837
18838 // Update the unconditional branch to target the loop preheader if we've
18839 // found the condition has been reversed.
18840 if (Target == OtherTarget)
18841 UpdateUncondBr(Br, Dest, DAG);
18842
18843 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other,
18844 N1: SDValue(LoopDec.getNode(), 1), N2: Chain);
18845
18846 SDValue EndArgs[] = { Chain, SDValue(LoopDec.getNode(), 0), Target };
18847 return DAG.getNode(Opcode: ARMISD::LE, DL: dl, VT: MVT::Other, Ops: EndArgs);
18848 }
18849 return SDValue();
18850}
18851
18852/// PerformBRCONDCombine - Target-specific DAG combining for ARMISD::BRCOND.
18853SDValue
18854ARMTargetLowering::PerformBRCONDCombine(SDNode *N, SelectionDAG &DAG) const {
18855 SDValue Cmp = N->getOperand(Num: 3);
18856 if (Cmp.getOpcode() != ARMISD::CMPZ)
18857 // Only looking at NE cases.
18858 return SDValue();
18859
18860 SDLoc dl(N);
18861 SDValue LHS = Cmp.getOperand(i: 0);
18862 SDValue RHS = Cmp.getOperand(i: 1);
18863 SDValue Chain = N->getOperand(Num: 0);
18864 SDValue BB = N->getOperand(Num: 1);
18865 SDValue ARMcc = N->getOperand(Num: 2);
18866 ARMCC::CondCodes CC = (ARMCC::CondCodes)ARMcc->getAsZExtVal();
18867
18868 // (brcond Chain BB ne (cmpz (and (cmov 0 1 CC Flags) 1) 0))
18869 // -> (brcond Chain BB CC Flags)
18870 if (CC == ARMCC::NE && LHS.getOpcode() == ISD::AND && LHS->hasOneUse() &&
18871 LHS->getOperand(Num: 0)->getOpcode() == ARMISD::CMOV &&
18872 LHS->getOperand(Num: 0)->hasOneUse() &&
18873 isNullConstant(V: LHS->getOperand(Num: 0)->getOperand(Num: 0)) &&
18874 isOneConstant(V: LHS->getOperand(Num: 0)->getOperand(Num: 1)) &&
18875 isOneConstant(V: LHS->getOperand(Num: 1)) && isNullConstant(V: RHS)) {
18876 return DAG.getNode(Opcode: ARMISD::BRCOND, DL: dl, VT: MVT::Other, N1: Chain, N2: BB,
18877 N3: LHS->getOperand(Num: 0)->getOperand(Num: 2),
18878 N4: LHS->getOperand(Num: 0)->getOperand(Num: 3));
18879 }
18880
18881 return SDValue();
18882}
18883
18884/// PerformCMOVCombine - Target-specific DAG combining for ARMISD::CMOV.
18885SDValue
18886ARMTargetLowering::PerformCMOVCombine(SDNode *N, SelectionDAG &DAG) const {
18887 SDLoc dl(N);
18888 EVT VT = N->getValueType(ResNo: 0);
18889 SDValue FalseVal = N->getOperand(Num: 0);
18890 SDValue TrueVal = N->getOperand(Num: 1);
18891 SDValue ARMcc = N->getOperand(Num: 2);
18892 SDValue Cmp = N->getOperand(Num: 3);
18893
18894 // Try to form CSINV etc.
18895 unsigned Opcode;
18896 bool InvertCond;
18897 if (SDValue CSetOp =
18898 matchCSET(Opcode, InvertCond, TrueVal, FalseVal, Subtarget)) {
18899 if (InvertCond) {
18900 ARMCC::CondCodes CondCode =
18901 (ARMCC::CondCodes)cast<const ConstantSDNode>(Val&: ARMcc)->getZExtValue();
18902 CondCode = ARMCC::getOppositeCondition(CC: CondCode);
18903 ARMcc = DAG.getConstant(Val: CondCode, DL: SDLoc(ARMcc), VT: MVT::i32);
18904 }
18905 return DAG.getNode(Opcode, DL: dl, VT, N1: CSetOp, N2: CSetOp, N3: ARMcc, N4: Cmp);
18906 }
18907
18908 if (Cmp.getOpcode() != ARMISD::CMPZ)
18909 // Only looking at EQ and NE cases.
18910 return SDValue();
18911
18912 SDValue LHS = Cmp.getOperand(i: 0);
18913 SDValue RHS = Cmp.getOperand(i: 1);
18914 ARMCC::CondCodes CC = (ARMCC::CondCodes)ARMcc->getAsZExtVal();
18915
18916 // BFI is only available on V6T2+.
18917 if (!Subtarget->isThumb1Only() && Subtarget->hasV6T2Ops()) {
18918 SDValue R = PerformCMOVToBFICombine(CMOV: N, DAG);
18919 if (R)
18920 return R;
18921 }
18922
18923 // Simplify
18924 // mov r1, r0
18925 // cmp r1, x
18926 // mov r0, y
18927 // moveq r0, x
18928 // to
18929 // cmp r0, x
18930 // movne r0, y
18931 //
18932 // mov r1, r0
18933 // cmp r1, x
18934 // mov r0, x
18935 // movne r0, y
18936 // to
18937 // cmp r0, x
18938 // movne r0, y
18939 /// FIXME: Turn this into a target neutral optimization?
18940 SDValue Res;
18941 if (CC == ARMCC::NE && FalseVal == RHS && FalseVal != LHS) {
18942 Res = DAG.getNode(Opcode: ARMISD::CMOV, DL: dl, VT, N1: LHS, N2: TrueVal, N3: ARMcc, N4: Cmp);
18943 } else if (CC == ARMCC::EQ && TrueVal == RHS) {
18944 SDValue ARMcc;
18945 SDValue NewCmp = getARMCmp(LHS, RHS, CC: ISD::SETNE, ARMcc, DAG, dl);
18946 Res = DAG.getNode(Opcode: ARMISD::CMOV, DL: dl, VT, N1: LHS, N2: FalseVal, N3: ARMcc, N4: NewCmp);
18947 }
18948
18949 // (cmov F T ne (cmpz (cmov 0 1 CC Flags) 0))
18950 // -> (cmov F T CC Flags)
18951 if (CC == ARMCC::NE && LHS.getOpcode() == ARMISD::CMOV && LHS->hasOneUse() &&
18952 isNullConstant(V: LHS->getOperand(Num: 0)) && isOneConstant(V: LHS->getOperand(Num: 1)) &&
18953 isNullConstant(V: RHS)) {
18954 return DAG.getNode(Opcode: ARMISD::CMOV, DL: dl, VT, N1: FalseVal, N2: TrueVal,
18955 N3: LHS->getOperand(Num: 2), N4: LHS->getOperand(Num: 3));
18956 }
18957
18958 if (!VT.isInteger())
18959 return SDValue();
18960
18961 // Fold away an unnecessary CMPZ/CMOV
18962 // CMOV A, B, C1, (CMPZ (CMOV 1, 0, C2, D), 0) ->
18963 // if C1==EQ -> CMOV A, B, C2, D
18964 // if C1==NE -> CMOV A, B, NOT(C2), D
18965 if (N->getConstantOperandVal(Num: 2) == ARMCC::EQ ||
18966 N->getConstantOperandVal(Num: 2) == ARMCC::NE) {
18967 ARMCC::CondCodes Cond;
18968 if (SDValue C = IsCMPZCSINC(Cmp: N->getOperand(Num: 3).getNode(), CC&: Cond)) {
18969 if (N->getConstantOperandVal(Num: 2) == ARMCC::NE)
18970 Cond = ARMCC::getOppositeCondition(CC: Cond);
18971 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: MVT::i32, N1: N->getOperand(Num: 0),
18972 N2: N->getOperand(Num: 1),
18973 N3: DAG.getConstant(Val: Cond, DL: SDLoc(N), VT: MVT::i32), N4: C);
18974 }
18975 }
18976
18977 // Materialize a boolean comparison for integers so we can avoid branching.
18978 if (isNullConstant(V: FalseVal)) {
18979 if (CC == ARMCC::EQ && isOneConstant(V: TrueVal)) {
18980 if (!Subtarget->isThumb1Only() && Subtarget->hasV5TOps()) {
18981 // If x == y then x - y == 0 and ARM's CLZ will return 32, shifting it
18982 // right 5 bits will make that 32 be 1, otherwise it will be 0.
18983 // CMOV 0, 1, ==, (CMPZ x, y) -> SRL (CTLZ (SUB x, y)), 5
18984 SDValue Value;
18985 if (VT == MVT::i32 && Subtarget->isThumb2() && isNullConstant(V: RHS) &&
18986 Cmp.hasOneUse())
18987 Value = getShiftedLowMaskZeroTest(Value: LHS, DAG, DL: dl);
18988 if (!Value)
18989 Value = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: LHS, N2: RHS);
18990 Res =
18991 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: DAG.getNode(Opcode: ISD::CTLZ, DL: dl, VT, Operand: Value),
18992 N2: DAG.getConstant(Val: 5, DL: dl, VT: MVT::i32));
18993 } else {
18994 // CMOV 0, 1, ==, (CMPZ x, y) ->
18995 // (UADDO_CARRY (SUB x, y), t:0, t:1)
18996 // where t = (USUBO_CARRY 0, (SUB x, y), 0)
18997 //
18998 // The USUBO_CARRY computes 0 - (x - y) and this will give a borrow when
18999 // x != y. In other words, a carry C == 1 when x == y, C == 0
19000 // otherwise.
19001 // The final UADDO_CARRY computes
19002 // x - y + (0 - (x - y)) + C == C
19003 SDValue Sub = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: LHS, N2: RHS);
19004 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: MVT::i32);
19005 SDValue Neg = DAG.getNode(Opcode: ISD::USUBO, DL: dl, VTList: VTs, N1: FalseVal, N2: Sub);
19006 // ISD::USUBO_CARRY returns a borrow but we want the carry here
19007 // actually.
19008 SDValue Carry =
19009 DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32,
19010 N1: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32), N2: Neg.getValue(R: 1));
19011 Res = DAG.getNode(Opcode: ISD::UADDO_CARRY, DL: dl, VTList: VTs, N1: Sub, N2: Neg, N3: Carry);
19012 }
19013 } else if (CC == ARMCC::NE && !isNullConstant(V: RHS) &&
19014 (!Subtarget->isThumb1Only() || isPowerOf2Constant(V: TrueVal))) {
19015 // This seems pointless but will allow us to combine it further below.
19016 // CMOV 0, z, !=, (CMPZ x, y) -> CMOV (SUBC x, y), z, !=, (SUBC x, y):1
19017 SDValue Sub =
19018 DAG.getNode(Opcode: ARMISD::SUBC, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: MVT::i32), N1: LHS, N2: RHS);
19019 Res = DAG.getNode(Opcode: ARMISD::CMOV, DL: dl, VT, N1: Sub, N2: TrueVal, N3: ARMcc,
19020 N4: Sub.getValue(R: 1));
19021 FalseVal = Sub;
19022 }
19023 } else if (isNullConstant(V: TrueVal)) {
19024 if (CC == ARMCC::EQ && !isNullConstant(V: RHS) &&
19025 (!Subtarget->isThumb1Only() || isPowerOf2Constant(V: FalseVal))) {
19026 // This seems pointless but will allow us to combine it further below
19027 // Note that we change == for != as this is the dual for the case above.
19028 // CMOV z, 0, ==, (CMPZ x, y) -> CMOV (SUBC x, y), z, !=, (SUBC x, y):1
19029 SDValue Sub =
19030 DAG.getNode(Opcode: ARMISD::SUBC, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: MVT::i32), N1: LHS, N2: RHS);
19031 Res = DAG.getNode(Opcode: ARMISD::CMOV, DL: dl, VT, N1: Sub, N2: FalseVal,
19032 N3: DAG.getConstant(Val: ARMCC::NE, DL: dl, VT: MVT::i32),
19033 N4: Sub.getValue(R: 1));
19034 FalseVal = Sub;
19035 }
19036 }
19037
19038 // On Thumb1, the DAG above may be further combined if z is a power of 2
19039 // (z == 2 ^ K).
19040 // CMOV (SUBC x, y), z, !=, (SUBC x, y):1 ->
19041 // t1 = (USUBO (SUB x, y), 1)
19042 // t2 = (USUBO_CARRY (SUB x, y), t1:0, t1:1)
19043 // Result = if K != 0 then (SHL t2:0, K) else t2:0
19044 //
19045 // This also handles the special case of comparing against zero; it's
19046 // essentially, the same pattern, except there's no SUBC:
19047 // CMOV x, z, !=, (CMPZ x, 0) ->
19048 // t1 = (USUBO x, 1)
19049 // t2 = (USUBO_CARRY x, t1:0, t1:1)
19050 // Result = if K != 0 then (SHL t2:0, K) else t2:0
19051 const APInt *TrueConst;
19052 if (Subtarget->isThumb1Only() && CC == ARMCC::NE &&
19053 ((FalseVal.getOpcode() == ARMISD::SUBC && FalseVal.getOperand(i: 0) == LHS &&
19054 FalseVal.getOperand(i: 1) == RHS) ||
19055 (FalseVal == LHS && isNullConstant(V: RHS))) &&
19056 (TrueConst = isPowerOf2Constant(V: TrueVal))) {
19057 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: MVT::i32);
19058 unsigned ShiftAmount = TrueConst->logBase2();
19059 if (ShiftAmount)
19060 TrueVal = DAG.getConstant(Val: 1, DL: dl, VT);
19061 SDValue Subc = DAG.getNode(Opcode: ISD::USUBO, DL: dl, VTList: VTs, N1: FalseVal, N2: TrueVal);
19062 Res = DAG.getNode(Opcode: ISD::USUBO_CARRY, DL: dl, VTList: VTs, N1: FalseVal, N2: Subc,
19063 N3: Subc.getValue(R: 1));
19064
19065 if (ShiftAmount)
19066 Res = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Res,
19067 N2: DAG.getConstant(Val: ShiftAmount, DL: dl, VT: MVT::i32));
19068 }
19069
19070 if (Res.getNode()) {
19071 KnownBits Known = DAG.computeKnownBits(Op: SDValue(N,0));
19072 // Capture demanded bits information that would be otherwise lost.
19073 if (Known.Zero == 0xfffffffe)
19074 Res = DAG.getNode(Opcode: ISD::AssertZext, DL: dl, VT: MVT::i32, N1: Res,
19075 N2: DAG.getValueType(MVT::i1));
19076 else if (Known.Zero == 0xffffff00)
19077 Res = DAG.getNode(Opcode: ISD::AssertZext, DL: dl, VT: MVT::i32, N1: Res,
19078 N2: DAG.getValueType(MVT::i8));
19079 else if (Known.Zero == 0xffff0000)
19080 Res = DAG.getNode(Opcode: ISD::AssertZext, DL: dl, VT: MVT::i32, N1: Res,
19081 N2: DAG.getValueType(MVT::i16));
19082 }
19083
19084 return Res;
19085}
19086
19087static SDValue PerformBITCASTCombine(SDNode *N,
19088 TargetLowering::DAGCombinerInfo &DCI,
19089 const ARMSubtarget *ST) {
19090 SelectionDAG &DAG = DCI.DAG;
19091 SDValue Src = N->getOperand(Num: 0);
19092 EVT DstVT = N->getValueType(ResNo: 0);
19093
19094 // Convert v4f32 bitcast (v4i32 vdup (i32)) -> v4f32 vdup (i32) under MVE.
19095 if (ST->hasMVEIntegerOps() && Src.getOpcode() == ARMISD::VDUP) {
19096 EVT SrcVT = Src.getValueType();
19097 if (SrcVT.getScalarSizeInBits() == DstVT.getScalarSizeInBits())
19098 return DAG.getNode(Opcode: ARMISD::VDUP, DL: SDLoc(N), VT: DstVT, Operand: Src.getOperand(i: 0));
19099 }
19100
19101 // We may have a bitcast of something that has already had this bitcast
19102 // combine performed on it, so skip past any VECTOR_REG_CASTs.
19103 if (Src.getOpcode() == ARMISD::VECTOR_REG_CAST &&
19104 Src.getOperand(i: 0).getValueType().getScalarSizeInBits() <=
19105 Src.getValueType().getScalarSizeInBits())
19106 Src = Src.getOperand(i: 0);
19107
19108 // Bitcast from element-wise VMOV or VMVN doesn't need VREV if the VREV that
19109 // would be generated is at least the width of the element type.
19110 EVT SrcVT = Src.getValueType();
19111 if ((Src.getOpcode() == ARMISD::VMOVIMM ||
19112 Src.getOpcode() == ARMISD::VMVNIMM ||
19113 Src.getOpcode() == ARMISD::VMOVFPIMM) &&
19114 SrcVT.getScalarSizeInBits() <= DstVT.getScalarSizeInBits() &&
19115 DAG.getDataLayout().isBigEndian())
19116 return DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: SDLoc(N), VT: DstVT, Operand: Src);
19117
19118 // bitcast(extract(x, n)); bitcast(extract(x, n+1)) -> VMOVRRD x
19119 if (SDValue R = PerformExtractEltToVMOVRRD(N, DCI))
19120 return R;
19121
19122 return SDValue();
19123}
19124
19125// Some combines for the MVETrunc truncations legalizer helper. Also lowers the
19126// node into stack operations after legalizeOps.
19127SDValue ARMTargetLowering::PerformMVETruncCombine(
19128 SDNode *N, TargetLowering::DAGCombinerInfo &DCI) const {
19129 SelectionDAG &DAG = DCI.DAG;
19130 EVT VT = N->getValueType(ResNo: 0);
19131 SDLoc DL(N);
19132
19133 // MVETrunc(Undef, Undef) -> Undef
19134 if (all_of(Range: N->ops(), P: [](SDValue Op) { return Op.isUndef(); }))
19135 return DAG.getUNDEF(VT);
19136
19137 // MVETrunc(MVETrunc a b, MVETrunc c, d) -> MVETrunc
19138 if (N->getNumOperands() == 2 &&
19139 N->getOperand(Num: 0).getOpcode() == ARMISD::MVETRUNC &&
19140 N->getOperand(Num: 1).getOpcode() == ARMISD::MVETRUNC)
19141 return DAG.getNode(Opcode: ARMISD::MVETRUNC, DL, VT, N1: N->getOperand(Num: 0).getOperand(i: 0),
19142 N2: N->getOperand(Num: 0).getOperand(i: 1),
19143 N3: N->getOperand(Num: 1).getOperand(i: 0),
19144 N4: N->getOperand(Num: 1).getOperand(i: 1));
19145
19146 // MVETrunc(shuffle, shuffle) -> VMOVN
19147 if (N->getNumOperands() == 2 &&
19148 N->getOperand(Num: 0).getOpcode() == ISD::VECTOR_SHUFFLE &&
19149 N->getOperand(Num: 1).getOpcode() == ISD::VECTOR_SHUFFLE) {
19150 auto *S0 = cast<ShuffleVectorSDNode>(Val: N->getOperand(Num: 0).getNode());
19151 auto *S1 = cast<ShuffleVectorSDNode>(Val: N->getOperand(Num: 1).getNode());
19152
19153 if (S0->getOperand(Num: 0) == S1->getOperand(Num: 0) &&
19154 S0->getOperand(Num: 1) == S1->getOperand(Num: 1)) {
19155 // Construct complete shuffle mask
19156 SmallVector<int, 8> Mask(S0->getMask());
19157 Mask.append(in_start: S1->getMask().begin(), in_end: S1->getMask().end());
19158
19159 if (isVMOVNTruncMask(M: Mask, ToVT: VT, rev: false))
19160 return DAG.getNode(
19161 Opcode: ARMISD::VMOVN, DL, VT,
19162 N1: DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL, VT, Operand: S0->getOperand(Num: 0)),
19163 N2: DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL, VT, Operand: S0->getOperand(Num: 1)),
19164 N3: DAG.getConstant(Val: 1, DL, VT: MVT::i32));
19165 if (isVMOVNTruncMask(M: Mask, ToVT: VT, rev: true))
19166 return DAG.getNode(
19167 Opcode: ARMISD::VMOVN, DL, VT,
19168 N1: DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL, VT, Operand: S0->getOperand(Num: 1)),
19169 N2: DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL, VT, Operand: S0->getOperand(Num: 0)),
19170 N3: DAG.getConstant(Val: 1, DL, VT: MVT::i32));
19171 }
19172 }
19173
19174 // For MVETrunc of a buildvector or shuffle, it can be beneficial to lower the
19175 // truncate to a buildvector to allow the generic optimisations to kick in.
19176 if (all_of(Range: N->ops(), P: [](SDValue Op) {
19177 return Op.getOpcode() == ISD::BUILD_VECTOR ||
19178 Op.getOpcode() == ISD::VECTOR_SHUFFLE ||
19179 (Op.getOpcode() == ISD::BITCAST &&
19180 Op.getOperand(i: 0).getOpcode() == ISD::BUILD_VECTOR);
19181 })) {
19182 SmallVector<SDValue, 8> Extracts;
19183 for (unsigned Op = 0; Op < N->getNumOperands(); Op++) {
19184 SDValue O = N->getOperand(Num: Op);
19185 for (unsigned i = 0; i < O.getValueType().getVectorNumElements(); i++) {
19186 SDValue Ext = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: O,
19187 N2: DAG.getConstant(Val: i, DL, VT: MVT::i32));
19188 Extracts.push_back(Elt: Ext);
19189 }
19190 }
19191 return DAG.getBuildVector(VT, DL, Ops: Extracts);
19192 }
19193
19194 // If we are late in the legalization process and nothing has optimised
19195 // the trunc to anything better, lower it to a stack store and reload,
19196 // performing the truncation whilst keeping the lanes in the correct order:
19197 // VSTRH.32 a, stack; VSTRH.32 b, stack+8; VLDRW.32 stack;
19198 if (!DCI.isAfterLegalizeDAG())
19199 return SDValue();
19200
19201 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: TypeSize::getFixed(ExactSize: 16), Alignment: Align(4));
19202 int SPFI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
19203 int NumIns = N->getNumOperands();
19204 assert((NumIns == 2 || NumIns == 4) &&
19205 "Expected 2 or 4 inputs to an MVETrunc");
19206 EVT StoreVT = VT.getHalfNumVectorElementsVT(Context&: *DAG.getContext());
19207 if (N->getNumOperands() == 4)
19208 StoreVT = StoreVT.getHalfNumVectorElementsVT(Context&: *DAG.getContext());
19209
19210 SmallVector<SDValue> Chains;
19211 for (int I = 0; I < NumIns; I++) {
19212 SDValue Ptr = DAG.getNode(
19213 Opcode: ISD::ADD, DL, VT: StackPtr.getValueType(), N1: StackPtr,
19214 N2: DAG.getConstant(Val: I * 16 / NumIns, DL, VT: StackPtr.getValueType()));
19215 MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(
19216 MF&: DAG.getMachineFunction(), FI: SPFI, Offset: I * 16 / NumIns);
19217 SDValue Ch = DAG.getTruncStore(Chain: DAG.getEntryNode(), dl: DL, Val: N->getOperand(Num: I),
19218 Ptr, PtrInfo: MPI, SVT: StoreVT, Alignment: Align(4));
19219 Chains.push_back(Elt: Ch);
19220 }
19221
19222 SDValue Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: Chains);
19223 MachinePointerInfo MPI =
19224 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI, Offset: 0);
19225 return DAG.getLoad(VT, dl: DL, Chain, Ptr: StackPtr, PtrInfo: MPI, Alignment: Align(4));
19226}
19227
19228// Take a MVEEXT(load x) and split that into (extload x, extload x+8)
19229static SDValue PerformSplittingMVEEXTToWideningLoad(SDNode *N,
19230 SelectionDAG &DAG) {
19231 SDValue N0 = N->getOperand(Num: 0);
19232 LoadSDNode *LD = dyn_cast<LoadSDNode>(Val: N0.getNode());
19233 if (!LD || !LD->isSimple() || !N0.hasOneUse() || LD->isIndexed())
19234 return SDValue();
19235
19236 EVT FromVT = LD->getMemoryVT();
19237 EVT ToVT = N->getValueType(ResNo: 0);
19238 if (!ToVT.isVector())
19239 return SDValue();
19240 assert(FromVT.getVectorNumElements() == ToVT.getVectorNumElements() * 2);
19241 EVT ToEltVT = ToVT.getVectorElementType();
19242 EVT FromEltVT = FromVT.getVectorElementType();
19243
19244 unsigned NumElements = 0;
19245 if (ToEltVT == MVT::i32 && (FromEltVT == MVT::i16 || FromEltVT == MVT::i8))
19246 NumElements = 4;
19247 if (ToEltVT == MVT::i16 && FromEltVT == MVT::i8)
19248 NumElements = 8;
19249 assert(NumElements != 0);
19250
19251 ISD::LoadExtType NewExtType =
19252 N->getOpcode() == ARMISD::MVESEXT ? ISD::SEXTLOAD : ISD::ZEXTLOAD;
19253 if (LD->getExtensionType() != ISD::NON_EXTLOAD &&
19254 LD->getExtensionType() != ISD::EXTLOAD &&
19255 LD->getExtensionType() != NewExtType)
19256 return SDValue();
19257
19258 LLVMContext &C = *DAG.getContext();
19259 SDLoc DL(LD);
19260 // Details about the old load
19261 SDValue Ch = LD->getChain();
19262 SDValue BasePtr = LD->getBasePtr();
19263 Align Alignment = LD->getBaseAlign();
19264 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
19265 AAMDNodes AAInfo = LD->getAAInfo();
19266
19267 SDValue Offset = DAG.getPOISON(VT: BasePtr.getValueType());
19268 EVT NewFromVT = EVT::getVectorVT(
19269 Context&: C, VT: EVT::getIntegerVT(Context&: C, BitWidth: FromEltVT.getScalarSizeInBits()), NumElements);
19270 EVT NewToVT = EVT::getVectorVT(
19271 Context&: C, VT: EVT::getIntegerVT(Context&: C, BitWidth: ToEltVT.getScalarSizeInBits()), NumElements);
19272
19273 SmallVector<SDValue, 4> Loads;
19274 SmallVector<SDValue, 4> Chains;
19275 for (unsigned i = 0; i < FromVT.getVectorNumElements() / NumElements; i++) {
19276 unsigned NewOffset = (i * NewFromVT.getSizeInBits()) / 8;
19277 SDValue NewPtr =
19278 DAG.getObjectPtrOffset(SL: DL, Ptr: BasePtr, Offset: TypeSize::getFixed(ExactSize: NewOffset));
19279
19280 SDValue NewLoad =
19281 DAG.getLoad(AM: ISD::UNINDEXED, ExtType: NewExtType, VT: NewToVT, dl: DL, Chain: Ch, Ptr: NewPtr, Offset,
19282 PtrInfo: LD->getPointerInfo().getWithOffset(O: NewOffset), MemVT: NewFromVT,
19283 Alignment, MMOFlags, Metadata: AAInfo);
19284 Loads.push_back(Elt: NewLoad);
19285 Chains.push_back(Elt: SDValue(NewLoad.getNode(), 1));
19286 }
19287
19288 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: Chains);
19289 DAG.ReplaceAllUsesOfValueWith(From: SDValue(LD, 1), To: NewChain);
19290 return DAG.getMergeValues(Ops: Loads, dl: DL);
19291}
19292
19293// Perform combines for MVEEXT. If it has not be optimized to anything better
19294// before lowering, it gets converted to stack store and extloads performing the
19295// extend whilst still keeping the same lane ordering.
19296SDValue ARMTargetLowering::PerformMVEExtCombine(
19297 SDNode *N, TargetLowering::DAGCombinerInfo &DCI) const {
19298 SelectionDAG &DAG = DCI.DAG;
19299 EVT VT = N->getValueType(ResNo: 0);
19300 SDLoc DL(N);
19301 assert(N->getNumValues() == 2 && "Expected MVEEXT with 2 elements");
19302 assert((VT == MVT::v4i32 || VT == MVT::v8i16) && "Unexpected MVEEXT type");
19303
19304 EVT ExtVT = N->getOperand(Num: 0).getValueType().getHalfNumVectorElementsVT(
19305 Context&: *DAG.getContext());
19306 auto Extend = [&](SDValue V) {
19307 SDValue VVT = DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL, VT, Operand: V);
19308 return N->getOpcode() == ARMISD::MVESEXT
19309 ? DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT, N1: VVT,
19310 N2: DAG.getValueType(ExtVT))
19311 : DAG.getZeroExtendInReg(Op: VVT, DL, VT: ExtVT);
19312 };
19313
19314 // MVEEXT(VDUP) -> SIGN_EXTEND_INREG(VDUP)
19315 if (N->getOperand(Num: 0).getOpcode() == ARMISD::VDUP) {
19316 SDValue Ext = Extend(N->getOperand(Num: 0));
19317 return DAG.getMergeValues(Ops: {Ext, Ext}, dl: DL);
19318 }
19319
19320 // MVEEXT(shuffle) -> SIGN_EXTEND_INREG/ZERO_EXTEND_INREG
19321 if (auto *SVN = dyn_cast<ShuffleVectorSDNode>(Val: N->getOperand(Num: 0))) {
19322 ArrayRef<int> Mask = SVN->getMask();
19323 assert(Mask.size() == 2 * VT.getVectorNumElements());
19324 assert(Mask.size() == SVN->getValueType(0).getVectorNumElements());
19325 unsigned Rev = VT == MVT::v4i32 ? ARMISD::VREV32 : ARMISD::VREV16;
19326 SDValue Op0 = SVN->getOperand(Num: 0);
19327 SDValue Op1 = SVN->getOperand(Num: 1);
19328
19329 auto CheckInregMask = [&](int Start, int Offset) {
19330 for (int Idx = 0, E = VT.getVectorNumElements(); Idx < E; ++Idx)
19331 if (Mask[Start + Idx] >= 0 && Mask[Start + Idx] != Idx * 2 + Offset)
19332 return false;
19333 return true;
19334 };
19335 SDValue V0 = SDValue(N, 0);
19336 SDValue V1 = SDValue(N, 1);
19337 if (CheckInregMask(0, 0))
19338 V0 = Extend(Op0);
19339 else if (CheckInregMask(0, 1))
19340 V0 = Extend(DAG.getNode(Opcode: Rev, DL, VT: SVN->getValueType(ResNo: 0), Operand: Op0));
19341 else if (CheckInregMask(0, Mask.size()))
19342 V0 = Extend(Op1);
19343 else if (CheckInregMask(0, Mask.size() + 1))
19344 V0 = Extend(DAG.getNode(Opcode: Rev, DL, VT: SVN->getValueType(ResNo: 0), Operand: Op1));
19345
19346 if (CheckInregMask(VT.getVectorNumElements(), Mask.size()))
19347 V1 = Extend(Op1);
19348 else if (CheckInregMask(VT.getVectorNumElements(), Mask.size() + 1))
19349 V1 = Extend(DAG.getNode(Opcode: Rev, DL, VT: SVN->getValueType(ResNo: 0), Operand: Op1));
19350 else if (CheckInregMask(VT.getVectorNumElements(), 0))
19351 V1 = Extend(Op0);
19352 else if (CheckInregMask(VT.getVectorNumElements(), 1))
19353 V1 = Extend(DAG.getNode(Opcode: Rev, DL, VT: SVN->getValueType(ResNo: 0), Operand: Op0));
19354
19355 if (V0.getNode() != N || V1.getNode() != N)
19356 return DAG.getMergeValues(Ops: {V0, V1}, dl: DL);
19357 }
19358
19359 // MVEEXT(load) -> extload, extload
19360 if (N->getOperand(Num: 0)->getOpcode() == ISD::LOAD)
19361 if (SDValue L = PerformSplittingMVEEXTToWideningLoad(N, DAG))
19362 return L;
19363
19364 if (!DCI.isAfterLegalizeDAG())
19365 return SDValue();
19366
19367 // Lower to a stack store and reload:
19368 // VSTRW.32 a, stack; VLDRH.32 stack; VLDRH.32 stack+8;
19369 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: TypeSize::getFixed(ExactSize: 16), Alignment: Align(4));
19370 int SPFI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
19371 int NumOuts = N->getNumValues();
19372 assert((NumOuts == 2 || NumOuts == 4) &&
19373 "Expected 2 or 4 outputs to an MVEEXT");
19374 EVT LoadVT = N->getOperand(Num: 0).getValueType().getHalfNumVectorElementsVT(
19375 Context&: *DAG.getContext());
19376 if (N->getNumOperands() == 4)
19377 LoadVT = LoadVT.getHalfNumVectorElementsVT(Context&: *DAG.getContext());
19378
19379 MachinePointerInfo MPI =
19380 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI, Offset: 0);
19381 SDValue Chain = DAG.getStore(Chain: DAG.getEntryNode(), dl: DL, Val: N->getOperand(Num: 0),
19382 Ptr: StackPtr, PtrInfo: MPI, Alignment: Align(4));
19383
19384 SmallVector<SDValue> Loads;
19385 for (int I = 0; I < NumOuts; I++) {
19386 SDValue Ptr = DAG.getNode(
19387 Opcode: ISD::ADD, DL, VT: StackPtr.getValueType(), N1: StackPtr,
19388 N2: DAG.getConstant(Val: I * 16 / NumOuts, DL, VT: StackPtr.getValueType()));
19389 MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(
19390 MF&: DAG.getMachineFunction(), FI: SPFI, Offset: I * 16 / NumOuts);
19391 SDValue Load = DAG.getExtLoad(
19392 ExtType: N->getOpcode() == ARMISD::MVESEXT ? ISD::SEXTLOAD : ISD::ZEXTLOAD, dl: DL,
19393 VT, Chain, Ptr, PtrInfo: MPI, MemVT: LoadVT, Alignment: Align(4));
19394 Loads.push_back(Elt: Load);
19395 }
19396
19397 return DAG.getMergeValues(Ops: Loads, dl: DL);
19398}
19399
19400SDValue ARMTargetLowering::PerformDAGCombine(SDNode *N,
19401 DAGCombinerInfo &DCI) const {
19402 switch (N->getOpcode()) {
19403 default: break;
19404 case ISD::SELECT_CC:
19405 case ISD::SELECT: return PerformSELECTCombine(N, DCI, Subtarget);
19406 case ISD::VSELECT: return PerformVSELECTCombine(N, DCI, Subtarget);
19407 case ISD::SETCC: return PerformVSetCCToVCTPCombine(N, DCI, Subtarget);
19408 case ARMISD::ADDE: return PerformADDECombine(N, DCI, Subtarget);
19409 case ARMISD::UMLAL: return PerformUMLALCombine(N, DAG&: DCI.DAG, Subtarget);
19410 case ISD::ADD: return PerformADDCombine(N, DCI, Subtarget);
19411 case ISD::SUB: return PerformSUBCombine(N, DCI, Subtarget);
19412 case ISD::MUL: return PerformMULCombine(N, DCI, Subtarget);
19413 case ISD::OR: return PerformORCombine(N, DCI, Subtarget);
19414 case ISD::XOR: return PerformXORCombine(N, DCI, Subtarget);
19415 case ISD::AND: return PerformANDCombine(N, DCI, Subtarget);
19416 case ISD::BRCOND:
19417 case ISD::BR_CC: return PerformHWLoopCombine(N, DCI, ST: Subtarget);
19418 case ARMISD::ADDC:
19419 case ARMISD::SUBC: return PerformAddcSubcCombine(N, DCI, Subtarget);
19420 case ARMISD::SUBE: return PerformAddeSubeCombine(N, DCI, Subtarget);
19421 case ARMISD::BFI: return PerformBFICombine(N, DAG&: DCI.DAG);
19422 case ARMISD::VMOVRRD: return PerformVMOVRRDCombine(N, DCI, Subtarget);
19423 case ARMISD::VMOVDRR: return PerformVMOVDRRCombine(N, DAG&: DCI.DAG);
19424 case ARMISD::VMOVhr: return PerformVMOVhrCombine(N, DCI);
19425 case ARMISD::VMOVrh: return PerformVMOVrhCombine(N, DAG&: DCI.DAG);
19426 case ARMISD::VMOVrs:
19427 return PerformVMOVrsCombine(N, DAG&: DCI.DAG);
19428 case ISD::STORE: return PerformSTORECombine(N, DCI, Subtarget);
19429 case ISD::BUILD_VECTOR: return PerformBUILD_VECTORCombine(N, DCI, Subtarget);
19430 case ISD::INSERT_VECTOR_ELT: return PerformInsertEltCombine(N, DCI);
19431 case ISD::EXTRACT_VECTOR_ELT:
19432 return PerformExtractEltCombine(N, DCI, ST: Subtarget);
19433 case ISD::SIGN_EXTEND_INREG: return PerformSignExtendInregCombine(N, DAG&: DCI.DAG);
19434 case ISD::INSERT_SUBVECTOR: return PerformInsertSubvectorCombine(N, DCI);
19435 case ISD::VECTOR_SHUFFLE: return PerformVECTOR_SHUFFLECombine(N, DAG&: DCI.DAG);
19436 case ARMISD::VDUPLANE: return PerformVDUPLANECombine(N, DCI, Subtarget);
19437 case ARMISD::VDUP: return PerformVDUPCombine(N, DAG&: DCI.DAG, Subtarget);
19438 case ISD::FP_TO_SINT:
19439 case ISD::FP_TO_UINT:
19440 return PerformVCVTCombine(N, DAG&: DCI.DAG, Subtarget);
19441 case ISD::FADD:
19442 return PerformFADDCombine(N, DAG&: DCI.DAG, Subtarget);
19443 case ISD::FMUL:
19444 return PerformVMulVCTPCombine(N, DAG&: DCI.DAG, Subtarget);
19445 case ISD::INTRINSIC_WO_CHAIN:
19446 return PerformIntrinsicCombine(N, DCI);
19447 case ISD::SHL:
19448 case ISD::SRA:
19449 case ISD::SRL:
19450 return PerformShiftCombine(N, DCI, ST: Subtarget);
19451 case ISD::SIGN_EXTEND:
19452 case ISD::ZERO_EXTEND:
19453 case ISD::ANY_EXTEND:
19454 return PerformExtendCombine(N, DAG&: DCI.DAG, ST: Subtarget);
19455 case ISD::FP_EXTEND:
19456 return PerformFPExtendCombine(N, DAG&: DCI.DAG, ST: Subtarget);
19457 case ISD::SMIN:
19458 case ISD::UMIN:
19459 case ISD::SMAX:
19460 case ISD::UMAX:
19461 return PerformMinMaxCombine(N, DAG&: DCI.DAG, ST: Subtarget);
19462 case ARMISD::CMOV:
19463 return PerformCMOVCombine(N, DAG&: DCI.DAG);
19464 case ARMISD::BRCOND:
19465 return PerformBRCONDCombine(N, DAG&: DCI.DAG);
19466 case ARMISD::CMPZ:
19467 return PerformCMPZCombine(N, DAG&: DCI.DAG);
19468 case ARMISD::CSINC:
19469 case ARMISD::CSINV:
19470 case ARMISD::CSNEG:
19471 return PerformCSETCombine(N, DAG&: DCI.DAG);
19472 case ISD::LOAD:
19473 return PerformLOADCombine(N, DCI, Subtarget);
19474 case ISD::VECTOR_DEINTERLEAVE:
19475 return PerformVECTOR_DEINTERLEAVECombine(N, DCI, Subtarget);
19476 case ARMISD::VLD1DUP:
19477 case ARMISD::VLD2DUP:
19478 case ARMISD::VLD3DUP:
19479 case ARMISD::VLD4DUP:
19480 return PerformVLDCombine(N, DCI);
19481 case ARMISD::BUILD_VECTOR:
19482 return PerformARMBUILD_VECTORCombine(N, DCI);
19483 case ISD::BITCAST:
19484 return PerformBITCASTCombine(N, DCI, ST: Subtarget);
19485 case ARMISD::PREDICATE_CAST:
19486 return PerformPREDICATE_CASTCombine(N, DCI);
19487 case ARMISD::VECTOR_REG_CAST:
19488 return PerformVECTOR_REG_CASTCombine(N, DAG&: DCI.DAG, ST: Subtarget);
19489 case ARMISD::MVETRUNC:
19490 return PerformMVETruncCombine(N, DCI);
19491 case ARMISD::MVESEXT:
19492 case ARMISD::MVEZEXT:
19493 return PerformMVEExtCombine(N, DCI);
19494 case ARMISD::VCMP:
19495 return PerformVCMPCombine(N, DAG&: DCI.DAG, Subtarget);
19496 case ISD::VECREDUCE_ADD:
19497 return PerformVECREDUCE_ADDCombine(N, DAG&: DCI.DAG, ST: Subtarget);
19498 case ARMISD::VADDVs:
19499 case ARMISD::VADDVu:
19500 case ARMISD::VADDLVs:
19501 case ARMISD::VADDLVu:
19502 case ARMISD::VADDLVAs:
19503 case ARMISD::VADDLVAu:
19504 case ARMISD::VMLAVs:
19505 case ARMISD::VMLAVu:
19506 case ARMISD::VMLALVs:
19507 case ARMISD::VMLALVu:
19508 case ARMISD::VMLALVAs:
19509 case ARMISD::VMLALVAu:
19510 return PerformReduceShuffleCombine(N, DAG&: DCI.DAG);
19511 case ARMISD::VMOVN:
19512 return PerformVMOVNCombine(N, DCI);
19513 case ARMISD::VQMOVNs:
19514 case ARMISD::VQMOVNu:
19515 return PerformVQMOVNCombine(N, DCI);
19516 case ARMISD::VQDMULH:
19517 return PerformVQDMULHCombine(N, DCI);
19518 case ARMISD::ASRL:
19519 case ARMISD::LSRL:
19520 case ARMISD::LSLL:
19521 return PerformLongShiftCombine(N, DAG&: DCI.DAG);
19522 case ARMISD::SMULWB: {
19523 unsigned BitWidth = N->getValueType(ResNo: 0).getSizeInBits();
19524 APInt DemandedMask = APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: 16);
19525 if (SimplifyDemandedBits(Op: N->getOperand(Num: 1), DemandedBits: DemandedMask, DCI))
19526 return SDValue();
19527 break;
19528 }
19529 case ARMISD::SMULWT: {
19530 unsigned BitWidth = N->getValueType(ResNo: 0).getSizeInBits();
19531 APInt DemandedMask = APInt::getHighBitsSet(numBits: BitWidth, hiBitsSet: 16);
19532 if (SimplifyDemandedBits(Op: N->getOperand(Num: 1), DemandedBits: DemandedMask, DCI))
19533 return SDValue();
19534 break;
19535 }
19536 case ARMISD::SMLALBB:
19537 case ARMISD::QADD16b:
19538 case ARMISD::QSUB16b:
19539 case ARMISD::UQADD16b:
19540 case ARMISD::UQSUB16b: {
19541 unsigned BitWidth = N->getValueType(ResNo: 0).getSizeInBits();
19542 APInt DemandedMask = APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: 16);
19543 if ((SimplifyDemandedBits(Op: N->getOperand(Num: 0), DemandedBits: DemandedMask, DCI)) ||
19544 (SimplifyDemandedBits(Op: N->getOperand(Num: 1), DemandedBits: DemandedMask, DCI)))
19545 return SDValue();
19546 break;
19547 }
19548 case ARMISD::SMLALBT: {
19549 unsigned LowWidth = N->getOperand(Num: 0).getValueType().getSizeInBits();
19550 APInt LowMask = APInt::getLowBitsSet(numBits: LowWidth, loBitsSet: 16);
19551 unsigned HighWidth = N->getOperand(Num: 1).getValueType().getSizeInBits();
19552 APInt HighMask = APInt::getHighBitsSet(numBits: HighWidth, hiBitsSet: 16);
19553 if ((SimplifyDemandedBits(Op: N->getOperand(Num: 0), DemandedBits: LowMask, DCI)) ||
19554 (SimplifyDemandedBits(Op: N->getOperand(Num: 1), DemandedBits: HighMask, DCI)))
19555 return SDValue();
19556 break;
19557 }
19558 case ARMISD::SMLALTB: {
19559 unsigned HighWidth = N->getOperand(Num: 0).getValueType().getSizeInBits();
19560 APInt HighMask = APInt::getHighBitsSet(numBits: HighWidth, hiBitsSet: 16);
19561 unsigned LowWidth = N->getOperand(Num: 1).getValueType().getSizeInBits();
19562 APInt LowMask = APInt::getLowBitsSet(numBits: LowWidth, loBitsSet: 16);
19563 if ((SimplifyDemandedBits(Op: N->getOperand(Num: 0), DemandedBits: HighMask, DCI)) ||
19564 (SimplifyDemandedBits(Op: N->getOperand(Num: 1), DemandedBits: LowMask, DCI)))
19565 return SDValue();
19566 break;
19567 }
19568 case ARMISD::SMLALTT: {
19569 unsigned BitWidth = N->getValueType(ResNo: 0).getSizeInBits();
19570 APInt DemandedMask = APInt::getHighBitsSet(numBits: BitWidth, hiBitsSet: 16);
19571 if ((SimplifyDemandedBits(Op: N->getOperand(Num: 0), DemandedBits: DemandedMask, DCI)) ||
19572 (SimplifyDemandedBits(Op: N->getOperand(Num: 1), DemandedBits: DemandedMask, DCI)))
19573 return SDValue();
19574 break;
19575 }
19576 case ARMISD::QADD8b:
19577 case ARMISD::QSUB8b:
19578 case ARMISD::UQADD8b:
19579 case ARMISD::UQSUB8b: {
19580 unsigned BitWidth = N->getValueType(ResNo: 0).getSizeInBits();
19581 APInt DemandedMask = APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: 8);
19582 if ((SimplifyDemandedBits(Op: N->getOperand(Num: 0), DemandedBits: DemandedMask, DCI)) ||
19583 (SimplifyDemandedBits(Op: N->getOperand(Num: 1), DemandedBits: DemandedMask, DCI)))
19584 return SDValue();
19585 break;
19586 }
19587 case ARMISD::VBSP:
19588 if (N->getOperand(Num: 1) == N->getOperand(Num: 2))
19589 return N->getOperand(Num: 1);
19590 return SDValue();
19591 case ISD::INTRINSIC_VOID:
19592 case ISD::INTRINSIC_W_CHAIN:
19593 switch (N->getConstantOperandVal(Num: 1)) {
19594 case Intrinsic::arm_neon_vld1:
19595 case Intrinsic::arm_neon_vld1x2:
19596 case Intrinsic::arm_neon_vld1x3:
19597 case Intrinsic::arm_neon_vld1x4:
19598 case Intrinsic::arm_neon_vld2:
19599 case Intrinsic::arm_neon_vld3:
19600 case Intrinsic::arm_neon_vld4:
19601 case Intrinsic::arm_neon_vld2lane:
19602 case Intrinsic::arm_neon_vld3lane:
19603 case Intrinsic::arm_neon_vld4lane:
19604 case Intrinsic::arm_neon_vld2dup:
19605 case Intrinsic::arm_neon_vld3dup:
19606 case Intrinsic::arm_neon_vld4dup:
19607 case Intrinsic::arm_neon_vst1:
19608 case Intrinsic::arm_neon_vst1x2:
19609 case Intrinsic::arm_neon_vst1x3:
19610 case Intrinsic::arm_neon_vst1x4:
19611 case Intrinsic::arm_neon_vst2:
19612 case Intrinsic::arm_neon_vst3:
19613 case Intrinsic::arm_neon_vst4:
19614 case Intrinsic::arm_neon_vst2lane:
19615 case Intrinsic::arm_neon_vst3lane:
19616 case Intrinsic::arm_neon_vst4lane:
19617 return PerformVLDCombine(N, DCI);
19618 case Intrinsic::arm_mve_vld2q:
19619 case Intrinsic::arm_mve_vld4q:
19620 case Intrinsic::arm_mve_vst2q:
19621 case Intrinsic::arm_mve_vst4q:
19622 return PerformMVEVLDCombine(N, DCI);
19623 default: break;
19624 }
19625 break;
19626 }
19627 return SDValue();
19628}
19629
19630bool ARMTargetLowering::isDesirableToTransformToIntegerOp(unsigned Opc,
19631 EVT VT) const {
19632 return (VT == MVT::f32) && (Opc == ISD::LOAD || Opc == ISD::STORE);
19633}
19634
19635bool ARMTargetLowering::allowsMisalignedMemoryAccesses(EVT VT, unsigned,
19636 Align Alignment,
19637 MachineMemOperand::Flags,
19638 unsigned *Fast) const {
19639 // Depends what it gets converted into if the type is weird.
19640 if (!VT.isSimple())
19641 return false;
19642
19643 // The AllowsUnaligned flag models the SCTLR.A setting in ARM cpus
19644 bool AllowsUnaligned = Subtarget->allowsUnalignedMem();
19645 auto Ty = VT.getSimpleVT().SimpleTy;
19646
19647 if (Ty == MVT::i8 || Ty == MVT::i16 || Ty == MVT::i32) {
19648 // Unaligned access can use (for example) LRDB, LRDH, LDR
19649 if (AllowsUnaligned) {
19650 if (Fast)
19651 *Fast = Subtarget->hasV7Ops();
19652 return true;
19653 }
19654 }
19655
19656 if (Ty == MVT::f64 || Ty == MVT::v2f64) {
19657 // For any little-endian targets with neon, we can support unaligned ld/st
19658 // of D and Q (e.g. {D0,D1}) registers by using vld1.i8/vst1.i8.
19659 // A big-endian target may also explicitly support unaligned accesses
19660 if (Subtarget->hasNEON() && (AllowsUnaligned || Subtarget->isLittle())) {
19661 if (Fast)
19662 *Fast = 1;
19663 return true;
19664 }
19665 }
19666
19667 if (!Subtarget->hasMVEIntegerOps())
19668 return false;
19669
19670 // These are for predicates
19671 if ((Ty == MVT::v16i1 || Ty == MVT::v8i1 || Ty == MVT::v4i1 ||
19672 Ty == MVT::v2i1)) {
19673 if (Fast)
19674 *Fast = 1;
19675 return true;
19676 }
19677
19678 // These are for truncated stores/narrowing loads. They are fine so long as
19679 // the alignment is at least the size of the item being loaded
19680 if ((Ty == MVT::v4i8 || Ty == MVT::v8i8 || Ty == MVT::v4i16) &&
19681 Alignment >= VT.getScalarSizeInBits() / 8) {
19682 if (Fast)
19683 *Fast = true;
19684 return true;
19685 }
19686
19687 // In little-endian MVE, the store instructions VSTRB.U8, VSTRH.U16 and
19688 // VSTRW.U32 all store the vector register in exactly the same format, and
19689 // differ only in the range of their immediate offset field and the required
19690 // alignment. So there is always a store that can be used, regardless of
19691 // actual type.
19692 //
19693 // For big endian, that is not the case. But can still emit a (VSTRB.U8;
19694 // VREV64.8) pair and get the same effect. This will likely be better than
19695 // aligning the vector through the stack.
19696 if (Ty == MVT::v16i8 || Ty == MVT::v8i16 || Ty == MVT::v8f16 ||
19697 Ty == MVT::v4i32 || Ty == MVT::v4f32 || Ty == MVT::v2i64 ||
19698 Ty == MVT::v2f64) {
19699 if (Fast)
19700 *Fast = 1;
19701 return true;
19702 }
19703
19704 return false;
19705}
19706
19707EVT ARMTargetLowering::getOptimalMemOpType(
19708 LLVMContext &Context, const MemOp &Op,
19709 const AttributeList &FuncAttributes) const {
19710 // See if we can use NEON instructions for this...
19711 if ((Op.isMemcpyOrMemmove() || Op.isZeroMemset()) && Subtarget->hasNEON() &&
19712 !FuncAttributes.hasFnAttr(Kind: Attribute::NoImplicitFloat)) {
19713 unsigned Fast;
19714 if (Op.size() >= 16 &&
19715 (Op.isAligned(AlignCheck: Align(16)) ||
19716 (allowsMisalignedMemoryAccesses(VT: MVT::v2f64, 0, Alignment: Align(1),
19717 MachineMemOperand::MONone, Fast: &Fast) &&
19718 Fast))) {
19719 return MVT::v2f64;
19720 } else if (Op.size() >= 8 &&
19721 (Op.isAligned(AlignCheck: Align(8)) ||
19722 (allowsMisalignedMemoryAccesses(
19723 VT: MVT::f64, 0, Alignment: Align(1), MachineMemOperand::MONone, Fast: &Fast) &&
19724 Fast))) {
19725 return MVT::f64;
19726 }
19727 }
19728
19729 // Let the target-independent logic figure it out.
19730 return MVT::Other;
19731}
19732
19733// 64-bit integers are split into their high and low parts and held in two
19734// different registers, so the trunc is free since the low register can just
19735// be used.
19736bool ARMTargetLowering::isTruncateFree(Type *SrcTy, Type *DstTy) const {
19737 if (!SrcTy->isIntegerTy() || !DstTy->isIntegerTy())
19738 return false;
19739 unsigned SrcBits = SrcTy->getPrimitiveSizeInBits();
19740 unsigned DestBits = DstTy->getPrimitiveSizeInBits();
19741 return (SrcBits == 64 && DestBits == 32);
19742}
19743
19744bool ARMTargetLowering::isTruncateFree(EVT SrcVT, EVT DstVT) const {
19745 if (SrcVT.isVector() || DstVT.isVector() || !SrcVT.isInteger() ||
19746 !DstVT.isInteger())
19747 return false;
19748 unsigned SrcBits = SrcVT.getSizeInBits();
19749 unsigned DestBits = DstVT.getSizeInBits();
19750 return (SrcBits == 64 && DestBits == 32);
19751}
19752
19753bool ARMTargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
19754 if (Val.getOpcode() != ISD::LOAD)
19755 return false;
19756
19757 EVT VT1 = Val.getValueType();
19758 if (!VT1.isSimple() || !VT1.isInteger() ||
19759 !VT2.isSimple() || !VT2.isInteger())
19760 return false;
19761
19762 switch (VT1.getSimpleVT().SimpleTy) {
19763 default: break;
19764 case MVT::i1:
19765 case MVT::i8:
19766 case MVT::i16:
19767 // 8-bit and 16-bit loads implicitly zero-extend to 32-bits.
19768 return true;
19769 }
19770
19771 return false;
19772}
19773
19774bool ARMTargetLowering::isFNegFree(EVT VT) const {
19775 if (!VT.isSimple())
19776 return false;
19777
19778 // There are quite a few FP16 instructions (e.g. VNMLA, VNMLS, etc.) that
19779 // negate values directly (fneg is free). So, we don't want to let the DAG
19780 // combiner rewrite fneg into xors and some other instructions. For f16 and
19781 // FullFP16 argument passing, some bitcast nodes may be introduced,
19782 // triggering this DAG combine rewrite, so we are avoiding that with this.
19783 switch (VT.getSimpleVT().SimpleTy) {
19784 default: break;
19785 case MVT::f16:
19786 return Subtarget->hasFullFP16();
19787 }
19788
19789 return false;
19790}
19791
19792Type *ARMTargetLowering::shouldConvertSplatType(ShuffleVectorInst *SVI) const {
19793 if (!Subtarget->hasMVEIntegerOps())
19794 return nullptr;
19795 Type *SVIType = SVI->getType();
19796 Type *ScalarType = SVIType->getScalarType();
19797
19798 if (ScalarType->isFloatTy())
19799 return Type::getInt32Ty(C&: SVIType->getContext());
19800 if (ScalarType->isHalfTy())
19801 return Type::getInt16Ty(C&: SVIType->getContext());
19802 return nullptr;
19803}
19804
19805bool ARMTargetLowering::isVectorLoadExtDesirable(SDValue ExtVal) const {
19806 EVT VT = ExtVal.getValueType();
19807
19808 if (!isTypeLegal(VT))
19809 return false;
19810
19811 if (auto *Ld = dyn_cast<MaskedLoadSDNode>(Val: ExtVal.getOperand(i: 0))) {
19812 if (Ld->isExpandingLoad())
19813 return false;
19814 }
19815
19816 if (Subtarget->hasMVEIntegerOps())
19817 return true;
19818
19819 // Don't create a loadext if we can fold the extension into a wide/long
19820 // instruction.
19821 // If there's more than one user instruction, the loadext is desirable no
19822 // matter what. There can be two uses by the same instruction.
19823 if (ExtVal->use_empty() ||
19824 !ExtVal->user_begin()->isOnlyUserOf(N: ExtVal.getNode()))
19825 return true;
19826
19827 SDNode *U = *ExtVal->user_begin();
19828 if ((U->getOpcode() == ISD::ADD || U->getOpcode() == ISD::SUB ||
19829 U->getOpcode() == ISD::SHL || U->getOpcode() == ARMISD::VSHLIMM))
19830 return false;
19831
19832 return true;
19833}
19834
19835bool ARMTargetLowering::allowTruncateForTailCall(Type *Ty1, Type *Ty2) const {
19836 if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy())
19837 return false;
19838
19839 if (!isTypeLegal(VT: EVT::getEVT(Ty: Ty1)))
19840 return false;
19841
19842 assert(Ty1->getPrimitiveSizeInBits() <= 64 && "i128 is probably not a noop");
19843
19844 // Assuming the caller doesn't have a zeroext or signext return parameter,
19845 // truncation all the way down to i1 is valid.
19846 return true;
19847}
19848
19849/// isFMAFasterThanFMulAndFAdd - Return true if an FMA operation is faster
19850/// than a pair of fmul and fadd instructions. fmuladd intrinsics will be
19851/// expanded to FMAs when this method returns true, otherwise fmuladd is
19852/// expanded to fmul + fadd.
19853///
19854/// ARM supports both fused and unfused multiply-add operations; we already
19855/// lower a pair of fmul and fadd to the latter so it's not clear that there
19856/// would be a gain or that the gain would be worthwhile enough to risk
19857/// correctness bugs.
19858///
19859/// For MVE, we set this to true as it helps simplify the need for some
19860/// patterns (and we don't have the non-fused floating point instruction).
19861bool ARMTargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
19862 EVT VT) const {
19863 if (Subtarget->useSoftFloat())
19864 return false;
19865
19866 if (!VT.isSimple())
19867 return false;
19868
19869 switch (VT.getSimpleVT().SimpleTy) {
19870 case MVT::v4f32:
19871 case MVT::v8f16:
19872 return Subtarget->hasMVEFloatOps();
19873 case MVT::f16:
19874 return Subtarget->useFPVFMx16();
19875 case MVT::f32:
19876 return Subtarget->useFPVFMx();
19877 case MVT::f64:
19878 return Subtarget->useFPVFMx64();
19879 default:
19880 break;
19881 }
19882
19883 return false;
19884}
19885
19886static bool isLegalT1AddressImmediate(int64_t V, EVT VT) {
19887 if (V < 0)
19888 return false;
19889
19890 unsigned Scale = 1;
19891 switch (VT.getSimpleVT().SimpleTy) {
19892 case MVT::i1:
19893 case MVT::i8:
19894 // Scale == 1;
19895 break;
19896 case MVT::i16:
19897 // Scale == 2;
19898 Scale = 2;
19899 break;
19900 default:
19901 // On thumb1 we load most things (i32, i64, floats, etc) with a LDR
19902 // Scale == 4;
19903 Scale = 4;
19904 break;
19905 }
19906
19907 if ((V & (Scale - 1)) != 0)
19908 return false;
19909 return isUInt<5>(x: V / Scale);
19910}
19911
19912static bool isLegalT2AddressImmediate(int64_t V, EVT VT,
19913 const ARMSubtarget *Subtarget) {
19914 if (!VT.isInteger() && !VT.isFloatingPoint())
19915 return false;
19916 if (VT.isVector() && Subtarget->hasNEON())
19917 return false;
19918 if (VT.isVector() && VT.isFloatingPoint() && Subtarget->hasMVEIntegerOps() &&
19919 !Subtarget->hasMVEFloatOps())
19920 return false;
19921
19922 bool IsNeg = false;
19923 if (V < 0) {
19924 IsNeg = true;
19925 V = -V;
19926 }
19927
19928 unsigned NumBytes = std::max(a: (unsigned)VT.getSizeInBits() / 8, b: 1U);
19929
19930 // MVE: size * imm7
19931 if (VT.isVector() && Subtarget->hasMVEIntegerOps()) {
19932 switch (VT.getSimpleVT().getVectorElementType().SimpleTy) {
19933 case MVT::i32:
19934 case MVT::f32:
19935 return isShiftedUInt<7,2>(x: V);
19936 case MVT::i16:
19937 case MVT::f16:
19938 return isShiftedUInt<7,1>(x: V);
19939 case MVT::i8:
19940 return isUInt<7>(x: V);
19941 default:
19942 return false;
19943 }
19944 }
19945
19946 // half VLDR: 2 * imm8
19947 if (VT.isFloatingPoint() && NumBytes == 2 && Subtarget->hasFPRegs16())
19948 return isShiftedUInt<8, 1>(x: V);
19949 // VLDR and LDRD: 4 * imm8
19950 if ((VT.isFloatingPoint() && Subtarget->hasVFP2Base()) || NumBytes == 8)
19951 return isShiftedUInt<8, 2>(x: V);
19952
19953 if (NumBytes == 1 || NumBytes == 2 || NumBytes == 4) {
19954 // + imm12 or - imm8
19955 if (IsNeg)
19956 return isUInt<8>(x: V);
19957 return isUInt<12>(x: V);
19958 }
19959
19960 return false;
19961}
19962
19963/// isLegalAddressImmediate - Return true if the integer value can be used
19964/// as the offset of the target addressing mode for load / store of the
19965/// given type.
19966static bool isLegalAddressImmediate(int64_t V, EVT VT,
19967 const ARMSubtarget *Subtarget) {
19968 if (V == 0)
19969 return true;
19970
19971 if (!VT.isSimple())
19972 return false;
19973
19974 if (Subtarget->isThumb1Only())
19975 return isLegalT1AddressImmediate(V, VT);
19976 else if (Subtarget->isThumb2())
19977 return isLegalT2AddressImmediate(V, VT, Subtarget);
19978
19979 // ARM mode.
19980 if (V < 0)
19981 V = - V;
19982 switch (VT.getSimpleVT().SimpleTy) {
19983 default: return false;
19984 case MVT::i1:
19985 case MVT::i8:
19986 case MVT::i32:
19987 // +- imm12
19988 return isUInt<12>(x: V);
19989 case MVT::i16:
19990 // +- imm8
19991 return isUInt<8>(x: V);
19992 case MVT::f32:
19993 case MVT::f64:
19994 if (!Subtarget->hasVFP2Base()) // FIXME: NEON?
19995 return false;
19996 return isShiftedUInt<8, 2>(x: V);
19997 }
19998}
19999
20000bool ARMTargetLowering::isLegalT2ScaledAddressingMode(const AddrMode &AM,
20001 EVT VT) const {
20002 int Scale = AM.Scale;
20003 if (Scale < 0)
20004 return false;
20005
20006 switch (VT.getSimpleVT().SimpleTy) {
20007 default: return false;
20008 case MVT::i1:
20009 case MVT::i8:
20010 case MVT::i16:
20011 case MVT::i32:
20012 if (Scale == 1)
20013 return true;
20014 // r + r << imm
20015 Scale = Scale & ~1;
20016 return Scale == 2 || Scale == 4 || Scale == 8;
20017 case MVT::i64:
20018 // FIXME: What are we trying to model here? ldrd doesn't have an r + r
20019 // version in Thumb mode.
20020 // r + r
20021 if (Scale == 1)
20022 return true;
20023 // r * 2 (this can be lowered to r + r).
20024 if (!AM.HasBaseReg && Scale == 2)
20025 return true;
20026 return false;
20027 case MVT::isVoid:
20028 // Note, we allow "void" uses (basically, uses that aren't loads or
20029 // stores), because arm allows folding a scale into many arithmetic
20030 // operations. This should be made more precise and revisited later.
20031
20032 // Allow r << imm, but the imm has to be a multiple of two.
20033 if (Scale & 1) return false;
20034 return isPowerOf2_32(Value: Scale);
20035 }
20036}
20037
20038bool ARMTargetLowering::isLegalT1ScaledAddressingMode(const AddrMode &AM,
20039 EVT VT) const {
20040 const int Scale = AM.Scale;
20041
20042 // Negative scales are not supported in Thumb1.
20043 if (Scale < 0)
20044 return false;
20045
20046 // Thumb1 addressing modes do not support register scaling excepting the
20047 // following cases:
20048 // 1. Scale == 1 means no scaling.
20049 // 2. Scale == 2 this can be lowered to r + r if there is no base register.
20050 return (Scale == 1) || (!AM.HasBaseReg && Scale == 2);
20051}
20052
20053/// isLegalAddressingMode - Return true if the addressing mode represented
20054/// by AM is legal for this target, for a load/store of the specified type.
20055bool ARMTargetLowering::isLegalAddressingMode(const DataLayout &DL,
20056 const AddrMode &AM, Type *Ty,
20057 unsigned AS, Instruction *I) const {
20058 EVT VT = getValueType(DL, Ty, AllowUnknown: true);
20059 if (!isLegalAddressImmediate(V: AM.BaseOffs, VT, Subtarget))
20060 return false;
20061
20062 // Can never fold addr of global into load/store.
20063 if (AM.BaseGV)
20064 return false;
20065
20066 switch (AM.Scale) {
20067 case 0: // no scale reg, must be "r+i" or "r", or "i".
20068 break;
20069 default:
20070 // ARM doesn't support any R+R*scale+imm addr modes.
20071 if (AM.BaseOffs)
20072 return false;
20073
20074 if (!VT.isSimple())
20075 return false;
20076
20077 if (Subtarget->isThumb1Only())
20078 return isLegalT1ScaledAddressingMode(AM, VT);
20079
20080 if (Subtarget->isThumb2())
20081 return isLegalT2ScaledAddressingMode(AM, VT);
20082
20083 int Scale = AM.Scale;
20084 switch (VT.getSimpleVT().SimpleTy) {
20085 default: return false;
20086 case MVT::i1:
20087 case MVT::i8:
20088 case MVT::i32:
20089 if (Scale < 0) Scale = -Scale;
20090 if (Scale == 1)
20091 return true;
20092 // r + r << imm
20093 return isPowerOf2_32(Value: Scale & ~1);
20094 case MVT::i16:
20095 case MVT::i64:
20096 // r +/- r
20097 if (Scale == 1 || (AM.HasBaseReg && Scale == -1))
20098 return true;
20099 // r * 2 (this can be lowered to r + r).
20100 if (!AM.HasBaseReg && Scale == 2)
20101 return true;
20102 return false;
20103
20104 case MVT::isVoid:
20105 // Note, we allow "void" uses (basically, uses that aren't loads or
20106 // stores), because arm allows folding a scale into many arithmetic
20107 // operations. This should be made more precise and revisited later.
20108
20109 // Allow r << imm, but the imm has to be a multiple of two.
20110 if (Scale & 1) return false;
20111 return isPowerOf2_32(Value: Scale);
20112 }
20113 }
20114 return true;
20115}
20116
20117/// isLegalICmpImmediate - Return true if the specified immediate is legal
20118/// icmp immediate, that is the target has icmp instructions which can compare
20119/// a register against the immediate without having to materialize the
20120/// immediate into a register.
20121bool ARMTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
20122 // Thumb2 and ARM modes can use cmn for negative immediates.
20123 if (!Subtarget->isThumb())
20124 return ARM_AM::getSOImmVal(Arg: (uint32_t)Imm) != -1 ||
20125 ARM_AM::getSOImmVal(Arg: -(uint32_t)Imm) != -1;
20126 if (Subtarget->isThumb2())
20127 return ARM_AM::getT2SOImmVal(Arg: (uint32_t)Imm) != -1 ||
20128 ARM_AM::getT2SOImmVal(Arg: -(uint32_t)Imm) != -1;
20129 // Thumb1 doesn't have cmn, and only 8-bit immediates.
20130 return Imm >= 0 && Imm <= 255;
20131}
20132
20133/// isLegalAddImmediate - Return true if the specified immediate is a legal add
20134/// *or sub* immediate, that is the target has add or sub instructions which can
20135/// add a register with the immediate without having to materialize the
20136/// immediate into a register.
20137bool ARMTargetLowering::isLegalAddImmediate(int64_t Imm) const {
20138 // Same encoding for add/sub, just flip the sign.
20139 uint64_t AbsImm = AbsoluteValue(X: Imm);
20140 if (!Subtarget->isThumb())
20141 return ARM_AM::getSOImmVal(Arg: AbsImm) != -1;
20142 if (Subtarget->isThumb2())
20143 return ARM_AM::getT2SOImmVal(Arg: AbsImm) != -1;
20144 // Thumb1 only has 8-bit unsigned immediate.
20145 return AbsImm <= 255;
20146}
20147
20148// Return false to prevent folding
20149// (mul (add r, c0), c1) -> (add (mul r, c1), c0*c1) in DAGCombine,
20150// if the folding leads to worse code.
20151bool ARMTargetLowering::isMulAddWithConstProfitable(SDValue AddNode,
20152 SDValue ConstNode) const {
20153 // Let the DAGCombiner decide for vector types and large types.
20154 const EVT VT = AddNode.getValueType();
20155 if (VT.isVector() || VT.getScalarSizeInBits() > 32)
20156 return true;
20157
20158 // It is worse if c0 is legal add immediate, while c1*c0 is not
20159 // and has to be composed by at least two instructions.
20160 const ConstantSDNode *C0Node = cast<ConstantSDNode>(Val: AddNode.getOperand(i: 1));
20161 const ConstantSDNode *C1Node = cast<ConstantSDNode>(Val&: ConstNode);
20162 const int64_t C0 = C0Node->getSExtValue();
20163 APInt CA = C0Node->getAPIntValue() * C1Node->getAPIntValue();
20164 if (!isLegalAddImmediate(Imm: C0) || isLegalAddImmediate(Imm: CA.getSExtValue()))
20165 return true;
20166 if (ConstantMaterializationCost(Val: (unsigned)CA.getZExtValue(), Subtarget) > 1)
20167 return false;
20168
20169 // Default to true and let the DAGCombiner decide.
20170 return true;
20171}
20172
20173static bool getARMIndexedAddressParts(SDNode *Ptr, EVT VT,
20174 bool isSEXTLoad, SDValue &Base,
20175 SDValue &Offset, bool &isInc,
20176 SelectionDAG &DAG) {
20177 if (Ptr->getOpcode() != ISD::ADD && Ptr->getOpcode() != ISD::SUB)
20178 return false;
20179
20180 if (VT == MVT::i16 || ((VT == MVT::i8 || VT == MVT::i1) && isSEXTLoad)) {
20181 // AddressingMode 3
20182 Base = Ptr->getOperand(Num: 0);
20183 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Val: Ptr->getOperand(Num: 1))) {
20184 int RHSC = (int)RHS->getZExtValue();
20185 if (RHSC < 0 && RHSC > -256) {
20186 assert(Ptr->getOpcode() == ISD::ADD);
20187 isInc = false;
20188 Offset = DAG.getConstant(Val: -RHSC, DL: SDLoc(Ptr), VT: RHS->getValueType(ResNo: 0));
20189 return true;
20190 }
20191 }
20192 isInc = (Ptr->getOpcode() == ISD::ADD);
20193 Offset = Ptr->getOperand(Num: 1);
20194 return true;
20195 } else if (VT == MVT::i32 || VT == MVT::i8 || VT == MVT::i1) {
20196 // AddressingMode 2
20197 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Val: Ptr->getOperand(Num: 1))) {
20198 int RHSC = (int)RHS->getZExtValue();
20199 if (RHSC < 0 && RHSC > -0x1000) {
20200 assert(Ptr->getOpcode() == ISD::ADD);
20201 isInc = false;
20202 Offset = DAG.getConstant(Val: -RHSC, DL: SDLoc(Ptr), VT: RHS->getValueType(ResNo: 0));
20203 Base = Ptr->getOperand(Num: 0);
20204 return true;
20205 }
20206 }
20207
20208 if (Ptr->getOpcode() == ISD::ADD) {
20209 isInc = true;
20210 ARM_AM::ShiftOpc ShOpcVal=
20211 ARM_AM::getShiftOpcForNode(Opcode: Ptr->getOperand(Num: 0).getOpcode());
20212 if (ShOpcVal != ARM_AM::no_shift) {
20213 Base = Ptr->getOperand(Num: 1);
20214 Offset = Ptr->getOperand(Num: 0);
20215 } else {
20216 Base = Ptr->getOperand(Num: 0);
20217 Offset = Ptr->getOperand(Num: 1);
20218 }
20219 return true;
20220 }
20221
20222 isInc = (Ptr->getOpcode() == ISD::ADD);
20223 Base = Ptr->getOperand(Num: 0);
20224 Offset = Ptr->getOperand(Num: 1);
20225 return true;
20226 }
20227
20228 // FIXME: Use VLDM / VSTM to emulate indexed FP load / store.
20229 return false;
20230}
20231
20232static bool getT2IndexedAddressParts(SDNode *Ptr, EVT VT,
20233 bool isSEXTLoad, SDValue &Base,
20234 SDValue &Offset, bool &isInc,
20235 SelectionDAG &DAG) {
20236 if (Ptr->getOpcode() != ISD::ADD && Ptr->getOpcode() != ISD::SUB)
20237 return false;
20238
20239 Base = Ptr->getOperand(Num: 0);
20240 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Val: Ptr->getOperand(Num: 1))) {
20241 int RHSC = (int)RHS->getZExtValue();
20242 if (RHSC < 0 && RHSC > -0x100) { // 8 bits.
20243 assert(Ptr->getOpcode() == ISD::ADD);
20244 isInc = false;
20245 Offset = DAG.getConstant(Val: -RHSC, DL: SDLoc(Ptr), VT: RHS->getValueType(ResNo: 0));
20246 return true;
20247 } else if (RHSC > 0 && RHSC < 0x100) { // 8 bit, no zero.
20248 isInc = Ptr->getOpcode() == ISD::ADD;
20249 Offset = DAG.getConstant(Val: RHSC, DL: SDLoc(Ptr), VT: RHS->getValueType(ResNo: 0));
20250 return true;
20251 }
20252 }
20253
20254 return false;
20255}
20256
20257static bool getMVEIndexedAddressParts(SDNode *Ptr, EVT VT, Align Alignment,
20258 bool isSEXTLoad, bool IsMasked, bool isLE,
20259 SDValue &Base, SDValue &Offset,
20260 bool &isInc, SelectionDAG &DAG) {
20261 if (Ptr->getOpcode() != ISD::ADD && Ptr->getOpcode() != ISD::SUB)
20262 return false;
20263 if (!isa<ConstantSDNode>(Val: Ptr->getOperand(Num: 1)))
20264 return false;
20265
20266 // We allow LE non-masked loads to change the type (for example use a vldrb.8
20267 // as opposed to a vldrw.32). This can allow extra addressing modes or
20268 // alignments for what is otherwise an equivalent instruction.
20269 bool CanChangeType = isLE && !IsMasked;
20270
20271 ConstantSDNode *RHS = cast<ConstantSDNode>(Val: Ptr->getOperand(Num: 1));
20272 int RHSC = (int)RHS->getZExtValue();
20273
20274 auto IsInRange = [&](int RHSC, int Limit, int Scale) {
20275 if (RHSC < 0 && RHSC > -Limit * Scale && RHSC % Scale == 0) {
20276 assert(Ptr->getOpcode() == ISD::ADD);
20277 isInc = false;
20278 Offset = DAG.getConstant(Val: -RHSC, DL: SDLoc(Ptr), VT: RHS->getValueType(ResNo: 0));
20279 return true;
20280 } else if (RHSC > 0 && RHSC < Limit * Scale && RHSC % Scale == 0) {
20281 isInc = Ptr->getOpcode() == ISD::ADD;
20282 Offset = DAG.getConstant(Val: RHSC, DL: SDLoc(Ptr), VT: RHS->getValueType(ResNo: 0));
20283 return true;
20284 }
20285 return false;
20286 };
20287
20288 // Try to find a matching instruction based on s/zext, Alignment, Offset and
20289 // (in BE/masked) type.
20290 Base = Ptr->getOperand(Num: 0);
20291 if (VT == MVT::v4i16) {
20292 if (Alignment >= 2 && IsInRange(RHSC, 0x80, 2))
20293 return true;
20294 } else if (VT == MVT::v4i8 || VT == MVT::v8i8) {
20295 if (IsInRange(RHSC, 0x80, 1))
20296 return true;
20297 } else if (Alignment >= 4 &&
20298 (CanChangeType || VT == MVT::v4i32 || VT == MVT::v4f32) &&
20299 IsInRange(RHSC, 0x80, 4))
20300 return true;
20301 else if (Alignment >= 2 &&
20302 (CanChangeType || VT == MVT::v8i16 || VT == MVT::v8f16) &&
20303 IsInRange(RHSC, 0x80, 2))
20304 return true;
20305 else if ((CanChangeType || VT == MVT::v16i8) && IsInRange(RHSC, 0x80, 1))
20306 return true;
20307 return false;
20308}
20309
20310/// getPreIndexedAddressParts - returns true by value, base pointer and
20311/// offset pointer and addressing mode by reference if the node's address
20312/// can be legally represented as pre-indexed load / store address.
20313bool
20314ARMTargetLowering::getPreIndexedAddressParts(SDNode *N, SDValue &Base,
20315 SDValue &Offset,
20316 ISD::MemIndexedMode &AM,
20317 SelectionDAG &DAG) const {
20318 if (Subtarget->isThumb1Only())
20319 return false;
20320
20321 EVT VT;
20322 SDValue Ptr;
20323 Align Alignment;
20324 unsigned AS = 0;
20325 bool isSEXTLoad = false;
20326 bool IsMasked = false;
20327 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Val: N)) {
20328 Ptr = LD->getBasePtr();
20329 VT = LD->getMemoryVT();
20330 Alignment = LD->getAlign();
20331 AS = LD->getAddressSpace();
20332 isSEXTLoad = LD->getExtensionType() == ISD::SEXTLOAD;
20333 } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(Val: N)) {
20334 Ptr = ST->getBasePtr();
20335 VT = ST->getMemoryVT();
20336 Alignment = ST->getAlign();
20337 AS = ST->getAddressSpace();
20338 } else if (MaskedLoadSDNode *LD = dyn_cast<MaskedLoadSDNode>(Val: N)) {
20339 Ptr = LD->getBasePtr();
20340 VT = LD->getMemoryVT();
20341 Alignment = LD->getAlign();
20342 AS = LD->getAddressSpace();
20343 isSEXTLoad = LD->getExtensionType() == ISD::SEXTLOAD;
20344 IsMasked = true;
20345 } else if (MaskedStoreSDNode *ST = dyn_cast<MaskedStoreSDNode>(Val: N)) {
20346 Ptr = ST->getBasePtr();
20347 VT = ST->getMemoryVT();
20348 Alignment = ST->getAlign();
20349 AS = ST->getAddressSpace();
20350 IsMasked = true;
20351 } else
20352 return false;
20353
20354 unsigned Fast = 0;
20355 if (!allowsMisalignedMemoryAccesses(VT, AS, Alignment,
20356 MachineMemOperand::MONone, Fast: &Fast)) {
20357 // Only generate post-increment or pre-increment forms when a real
20358 // hardware instruction exists for them. Do not emit postinc/preinc
20359 // if the operation will end up as a libcall.
20360 return false;
20361 }
20362
20363 bool isInc;
20364 bool isLegal = false;
20365 if (VT.isVector())
20366 isLegal = Subtarget->hasMVEIntegerOps() &&
20367 getMVEIndexedAddressParts(
20368 Ptr: Ptr.getNode(), VT, Alignment, isSEXTLoad, IsMasked,
20369 isLE: Subtarget->isLittle(), Base, Offset, isInc, DAG);
20370 else {
20371 if (Subtarget->isThumb2())
20372 isLegal = getT2IndexedAddressParts(Ptr: Ptr.getNode(), VT, isSEXTLoad, Base,
20373 Offset, isInc, DAG);
20374 else
20375 isLegal = getARMIndexedAddressParts(Ptr: Ptr.getNode(), VT, isSEXTLoad, Base,
20376 Offset, isInc, DAG);
20377 }
20378 if (!isLegal)
20379 return false;
20380
20381 AM = isInc ? ISD::PRE_INC : ISD::PRE_DEC;
20382 return true;
20383}
20384
20385/// getPostIndexedAddressParts - returns true by value, base pointer and
20386/// offset pointer and addressing mode by reference if this node can be
20387/// combined with a load / store to form a post-indexed load / store.
20388bool ARMTargetLowering::getPostIndexedAddressParts(SDNode *N, SDNode *Op,
20389 SDValue &Base,
20390 SDValue &Offset,
20391 ISD::MemIndexedMode &AM,
20392 SelectionDAG &DAG) const {
20393 EVT VT;
20394 SDValue Ptr;
20395 Align Alignment;
20396 bool isSEXTLoad = false, isNonExt;
20397 bool IsMasked = false;
20398 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Val: N)) {
20399 VT = LD->getMemoryVT();
20400 Ptr = LD->getBasePtr();
20401 Alignment = LD->getAlign();
20402 isSEXTLoad = LD->getExtensionType() == ISD::SEXTLOAD;
20403 isNonExt = LD->getExtensionType() == ISD::NON_EXTLOAD;
20404 } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(Val: N)) {
20405 VT = ST->getMemoryVT();
20406 Ptr = ST->getBasePtr();
20407 Alignment = ST->getAlign();
20408 isNonExt = !ST->isTruncatingStore();
20409 } else if (MaskedLoadSDNode *LD = dyn_cast<MaskedLoadSDNode>(Val: N)) {
20410 VT = LD->getMemoryVT();
20411 Ptr = LD->getBasePtr();
20412 Alignment = LD->getAlign();
20413 isSEXTLoad = LD->getExtensionType() == ISD::SEXTLOAD;
20414 isNonExt = LD->getExtensionType() == ISD::NON_EXTLOAD;
20415 IsMasked = true;
20416 } else if (MaskedStoreSDNode *ST = dyn_cast<MaskedStoreSDNode>(Val: N)) {
20417 VT = ST->getMemoryVT();
20418 Ptr = ST->getBasePtr();
20419 Alignment = ST->getAlign();
20420 isNonExt = !ST->isTruncatingStore();
20421 IsMasked = true;
20422 } else
20423 return false;
20424
20425 if (Subtarget->isThumb1Only()) {
20426 // Thumb-1 can do a limited post-inc load or store as an updating LDM. It
20427 // must be non-extending/truncating, i32, with an offset of 4.
20428 assert(Op->getValueType(0) == MVT::i32 && "Non-i32 post-inc op?!");
20429 if (Op->getOpcode() != ISD::ADD || !isNonExt)
20430 return false;
20431 auto *RHS = dyn_cast<ConstantSDNode>(Val: Op->getOperand(Num: 1));
20432 if (!RHS || RHS->getZExtValue() != 4)
20433 return false;
20434 if (Alignment < Align(4))
20435 return false;
20436
20437 Offset = Op->getOperand(Num: 1);
20438 Base = Op->getOperand(Num: 0);
20439 AM = ISD::POST_INC;
20440 return true;
20441 }
20442
20443 bool isInc;
20444 bool isLegal = false;
20445 if (VT.isVector())
20446 isLegal = Subtarget->hasMVEIntegerOps() &&
20447 getMVEIndexedAddressParts(Ptr: Op, VT, Alignment, isSEXTLoad, IsMasked,
20448 isLE: Subtarget->isLittle(), Base, Offset,
20449 isInc, DAG);
20450 else {
20451 if (Subtarget->isThumb2())
20452 isLegal = getT2IndexedAddressParts(Ptr: Op, VT, isSEXTLoad, Base, Offset,
20453 isInc, DAG);
20454 else
20455 isLegal = getARMIndexedAddressParts(Ptr: Op, VT, isSEXTLoad, Base, Offset,
20456 isInc, DAG);
20457 }
20458 if (!isLegal)
20459 return false;
20460
20461 if (Ptr != Base) {
20462 // Swap base ptr and offset to catch more post-index load / store when
20463 // it's legal. In Thumb2 mode, offset must be an immediate.
20464 if (Ptr == Offset && Op->getOpcode() == ISD::ADD &&
20465 !Subtarget->isThumb2())
20466 std::swap(a&: Base, b&: Offset);
20467
20468 // Post-indexed load / store update the base pointer.
20469 if (Ptr != Base)
20470 return false;
20471 }
20472
20473 AM = isInc ? ISD::POST_INC : ISD::POST_DEC;
20474 return true;
20475}
20476
20477void ARMTargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
20478 KnownBits &Known,
20479 const APInt &DemandedElts,
20480 const SelectionDAG &DAG,
20481 unsigned Depth) const {
20482 unsigned BitWidth = Known.getBitWidth();
20483 Known.resetAll();
20484 switch (Op.getOpcode()) {
20485 default: break;
20486 case ARMISD::ADDC:
20487 case ARMISD::ADDE:
20488 case ARMISD::SUBC:
20489 case ARMISD::SUBE:
20490 // Special cases when we convert a carry to a boolean.
20491 if (Op.getResNo() == 0) {
20492 SDValue LHS = Op.getOperand(i: 0);
20493 SDValue RHS = Op.getOperand(i: 1);
20494 // (ADDE 0, 0, C) will give us a single bit.
20495 if (Op->getOpcode() == ARMISD::ADDE && isNullConstant(V: LHS) &&
20496 isNullConstant(V: RHS)) {
20497 Known.Zero |= APInt::getHighBitsSet(numBits: BitWidth, hiBitsSet: BitWidth - 1);
20498 return;
20499 }
20500 }
20501 break;
20502 case ARMISD::CMOV: {
20503 // Bits are known zero/one if known on the LHS and RHS.
20504 Known = DAG.computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth+1);
20505 if (Known.isUnknown())
20506 return;
20507
20508 KnownBits KnownRHS = DAG.computeKnownBits(Op: Op.getOperand(i: 1), Depth: Depth+1);
20509 Known = Known.intersectWith(RHS: KnownRHS);
20510 return;
20511 }
20512 case ISD::INTRINSIC_W_CHAIN: {
20513 Intrinsic::ID IntID =
20514 static_cast<Intrinsic::ID>(Op->getConstantOperandVal(Num: 1));
20515 switch (IntID) {
20516 default: return;
20517 case Intrinsic::arm_ldaex:
20518 case Intrinsic::arm_ldrex: {
20519 EVT VT = cast<MemIntrinsicSDNode>(Val: Op)->getMemoryVT();
20520 unsigned MemBits = VT.getScalarSizeInBits();
20521 Known.Zero |= APInt::getHighBitsSet(numBits: BitWidth, hiBitsSet: BitWidth - MemBits);
20522 return;
20523 }
20524 }
20525 }
20526 case ARMISD::BFI: {
20527 // Conservatively, we can recurse down the first operand
20528 // and just mask out all affected bits.
20529 Known = DAG.computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
20530
20531 // The operand to BFI is already a mask suitable for removing the bits it
20532 // sets.
20533 const APInt &Mask = Op.getConstantOperandAPInt(i: 2);
20534 Known.Zero &= Mask;
20535 Known.One &= Mask;
20536 return;
20537 }
20538 case ARMISD::VGETLANEs:
20539 case ARMISD::VGETLANEu: {
20540 const SDValue &SrcSV = Op.getOperand(i: 0);
20541 EVT VecVT = SrcSV.getValueType();
20542 assert(VecVT.isVector() && "VGETLANE expected a vector type");
20543 const unsigned NumSrcElts = VecVT.getVectorNumElements();
20544 ConstantSDNode *Pos = cast<ConstantSDNode>(Val: Op.getOperand(i: 1).getNode());
20545 assert(Pos->getAPIntValue().ult(NumSrcElts) &&
20546 "VGETLANE index out of bounds");
20547 unsigned Idx = Pos->getZExtValue();
20548 APInt DemandedElt = APInt::getOneBitSet(numBits: NumSrcElts, BitNo: Idx);
20549 Known = DAG.computeKnownBits(Op: SrcSV, DemandedElts: DemandedElt, Depth: Depth + 1);
20550
20551 EVT VT = Op.getValueType();
20552 const unsigned DstSz = VT.getScalarSizeInBits();
20553 const unsigned SrcSz = VecVT.getVectorElementType().getSizeInBits();
20554 (void)SrcSz;
20555 assert(SrcSz == Known.getBitWidth());
20556 assert(DstSz > SrcSz);
20557 if (Op.getOpcode() == ARMISD::VGETLANEs)
20558 Known = Known.sext(BitWidth: DstSz);
20559 else {
20560 Known = Known.zext(BitWidth: DstSz);
20561 }
20562 assert(DstSz == Known.getBitWidth());
20563 break;
20564 }
20565 case ARMISD::VMOVrh: {
20566 KnownBits KnownOp = DAG.computeKnownBits(Op: Op->getOperand(Num: 0), Depth: Depth + 1);
20567 assert(KnownOp.getBitWidth() == 16);
20568 Known = KnownOp.zext(BitWidth: 32);
20569 break;
20570 }
20571 case ARMISD::CSINC:
20572 case ARMISD::CSINV:
20573 case ARMISD::CSNEG: {
20574 KnownBits KnownOp0 = DAG.computeKnownBits(Op: Op->getOperand(Num: 0), Depth: Depth + 1);
20575 KnownBits KnownOp1 = DAG.computeKnownBits(Op: Op->getOperand(Num: 1), Depth: Depth + 1);
20576
20577 // The result is either:
20578 // CSINC: KnownOp0 or KnownOp1 + 1
20579 // CSINV: KnownOp0 or ~KnownOp1
20580 // CSNEG: KnownOp0 or KnownOp1 * -1
20581 if (Op.getOpcode() == ARMISD::CSINC)
20582 KnownOp1 =
20583 KnownBits::add(LHS: KnownOp1, RHS: KnownBits::makeConstant(C: APInt(32, 1)));
20584 else if (Op.getOpcode() == ARMISD::CSINV)
20585 std::swap(a&: KnownOp1.Zero, b&: KnownOp1.One);
20586 else if (Op.getOpcode() == ARMISD::CSNEG)
20587 KnownOp1 = KnownBits::mul(LHS: KnownOp1,
20588 RHS: KnownBits::makeConstant(C: APInt::getAllOnes(numBits: 32)));
20589
20590 Known = KnownOp0.intersectWith(RHS: KnownOp1);
20591 break;
20592 }
20593 case ARMISD::VORRIMM:
20594 case ARMISD::VBICIMM: {
20595 unsigned Encoded = Op.getConstantOperandVal(i: 1);
20596 unsigned DecEltBits = 0;
20597 uint64_t DecodedVal = ARM_AM::decodeVMOVModImm(ModImm: Encoded, EltBits&: DecEltBits);
20598
20599 unsigned EltBits = Op.getScalarValueSizeInBits();
20600 if (EltBits != DecEltBits) {
20601 // Be conservative: only update Known when EltBits == DecEltBits.
20602 // This is believed to always be true for VORRIMM/VBICIMM today, but if
20603 // that changes in the future, doing nothing here is safer than risking
20604 // subtle bugs.
20605 break;
20606 }
20607
20608 KnownBits KnownLHS = DAG.computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
20609 bool IsVORR = Op.getOpcode() == ARMISD::VORRIMM;
20610 APInt Imm(DecEltBits, DecodedVal);
20611
20612 Known.One = IsVORR ? (KnownLHS.One | Imm) : (KnownLHS.One & ~Imm);
20613 Known.Zero = IsVORR ? (KnownLHS.Zero & ~Imm) : (KnownLHS.Zero | Imm);
20614 break;
20615 }
20616 }
20617}
20618
20619static bool isLegalLogicalImmediate(unsigned Imm,
20620 const ARMSubtarget *Subtarget) {
20621 if (!Subtarget->isThumb())
20622 return ARM_AM::getSOImmVal(Arg: Imm) != -1;
20623 if (Subtarget->isThumb2())
20624 return ARM_AM::getT2SOImmVal(Arg: Imm) != -1;
20625 // Thumb1 only has 8-bit unsigned immediate.
20626 return Imm <= 255;
20627}
20628
20629/// Refine i32 AND/OR/XOR with a constant RHS using demanded bits: replace the
20630/// immediate with an equivalent constant that ARM/Thumb can encode as a
20631/// logical immediate (or that selects better lowering), without changing the
20632/// computed result on those demanded bits.
20633static bool optimizeLogicalImm(SDValue Op, unsigned Imm,
20634 const APInt &DemandedBits,
20635 const ARMSubtarget *Subtarget,
20636 TargetLowering::TargetLoweringOpt &TLO) {
20637
20638 if (Imm == 0 || Imm == ~0U)
20639 return false;
20640
20641 unsigned Opc = Op.getOpcode();
20642 unsigned Demanded = DemandedBits.getZExtValue();
20643 EVT VT = Op.getValueType();
20644
20645 unsigned ShrunkImm = Imm & Demanded;
20646 unsigned ExpandedImm = Imm | ~Demanded;
20647
20648 auto IsLegalImm = [ShrunkImm, ExpandedImm](unsigned CandidateImm) -> bool {
20649 return (ShrunkImm & CandidateImm) == ShrunkImm &&
20650 (~ExpandedImm & CandidateImm) == 0;
20651 };
20652 auto UseImm = [Imm, Opc, Op, VT, &TLO](unsigned NewImm) -> bool {
20653 if (NewImm == Imm)
20654 return true;
20655 SDLoc DL(Op);
20656 SDValue NewC = TLO.DAG.getConstant(Val: NewImm, DL, VT);
20657 SDValue NewOp =
20658 TLO.DAG.getNode(Opcode: Opc, DL, VT, N1: Op.getOperand(i: 0), N2: NewC, Flags: Op->getFlags());
20659 return TLO.CombineTo(O: Op, N: NewOp);
20660 };
20661
20662 // Shrunk immediate is 0: AND becomes zero; OR/XOR with 0 leaves the other
20663 // operand (still valid on demanded bits).
20664 if (ShrunkImm == 0) {
20665 ++NumOptimizedImms;
20666 return UseImm(ShrunkImm);
20667 }
20668
20669 // If the immediate is all ones: for AND this removes the operation; for
20670 // OR/XOR it remains a transform valid on demanded bits. (Target-independent
20671 // shrink may not fold this, so keep it to avoid obscure combine loops.)
20672 if (ExpandedImm == ~0U) {
20673 ++NumOptimizedImms;
20674 return UseImm(ExpandedImm);
20675 }
20676
20677 // Thumb1: prefer 0xFF / 0xFFFF when they fit the demanded-bit envelope so
20678 // lowering can match uxtb / uxth (AND immediates only; OR/XOR do not use
20679 // that). Run this before strict ShrunkImm: a tight 8-bit ShrunkImm can be
20680 // legal while 0xFF still matches the envelope and yields better isel (uxtb).
20681 if (Opc == ISD::AND && Subtarget->hasV6Ops()) {
20682 if (IsLegalImm(0xFF)) {
20683 ++NumOptimizedImms;
20684 return UseImm(0xFF);
20685 }
20686
20687 if (IsLegalImm(0xFFFF)) {
20688 ++NumOptimizedImms;
20689 return UseImm(0xFFFF);
20690 }
20691 }
20692
20693 // Don't optimize if it is legal.
20694 if (isLegalLogicalImmediate(Imm, Subtarget))
20695 return false;
20696
20697 // FIXME: Check for BIC being legal causes infinite loop due to target
20698 // independent DAG combine undoing this.
20699
20700 // Prefer strict shrink when ShrunkImm encodes for this target, before
20701 // complement expansion.
20702 if (isLegalLogicalImmediate(Imm: ShrunkImm, Subtarget)) {
20703 ++NumOptimizedImms;
20704 return UseImm(ShrunkImm);
20705 }
20706
20707 // Complement expansion: if all undemanded bits are already one, ExpandedImm
20708 // is Imm with every non-demanded bit set. When (~ExpandedImm) < 256, the
20709 // complement fits in an 8-bit unsigned value, i.e. bits 8–31 of ExpandedImm
20710 // are all ones; only the low byte may differ from ~0. Use that expanded
20711 // constant so isel sees a mask shape that fits logical-immediate patterns.
20712 if ((~ExpandedImm) < 256) {
20713 ++NumOptimizedImms;
20714 return UseImm(ExpandedImm);
20715 }
20716
20717 // FIXME: The check for v6 is because this interferes with some ubfx
20718 // optimizations.
20719 if (Opc == ISD::AND && isLegalLogicalImmediate(Imm: ~ExpandedImm, Subtarget) &&
20720 !Subtarget->hasV6Ops()) {
20721 ++NumOptimizedImms;
20722 return UseImm(ExpandedImm);
20723 }
20724
20725 // Potential improvements:
20726 //
20727 // We could try to recognize lsls+lsrs or lsrs+lsls pairs here.
20728 // We could try to prefer Thumb1 immediates which can be lowered to a
20729 // two-instruction sequence.
20730
20731 return false;
20732}
20733
20734bool ARMTargetLowering::targetShrinkDemandedConstant(
20735 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
20736 TargetLoweringOpt &TLO) const {
20737 // Delay this optimization to as late as possible.
20738 if (!TLO.LegalOps)
20739 return false;
20740
20741 EVT VT = Op.getValueType();
20742
20743 // Ignore vectors.
20744 if (VT.isVector())
20745 return false;
20746
20747 unsigned Size = VT.getSizeInBits();
20748
20749 if (Size != 32)
20750 return false;
20751
20752 // Exit early if we demand all bits.
20753 if (DemandedBits.isAllOnes())
20754 return false;
20755
20756 switch (Op.getOpcode()) {
20757 default:
20758 return false;
20759 case ISD::AND:
20760 case ISD::OR:
20761 case ISD::XOR:
20762 break;
20763 }
20764 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1));
20765 if (!C)
20766 return false;
20767 unsigned Imm = C->getZExtValue();
20768 return optimizeLogicalImm(Op, Imm, DemandedBits, Subtarget, TLO);
20769}
20770
20771bool ARMTargetLowering::SimplifyDemandedBitsForTargetNode(
20772 SDValue Op, const APInt &OriginalDemandedBits,
20773 const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO,
20774 unsigned Depth) const {
20775 unsigned Opc = Op.getOpcode();
20776
20777 switch (Opc) {
20778 case ARMISD::ASRL:
20779 case ARMISD::LSRL: {
20780 // If this is result 0 and the other result is unused, see if the demand
20781 // bits allow us to shrink this long shift into a standard small shift in
20782 // the opposite direction.
20783 if (Op.getResNo() == 0 && !Op->hasAnyUseOfValue(Value: 1) &&
20784 isa<ConstantSDNode>(Val: Op->getOperand(Num: 2))) {
20785 unsigned ShAmt = Op->getConstantOperandVal(Num: 2);
20786 if (ShAmt < 32 && OriginalDemandedBits.isSubsetOf(RHS: APInt::getAllOnes(numBits: 32)
20787 << (32 - ShAmt)))
20788 return TLO.CombineTo(
20789 O: Op, N: TLO.DAG.getNode(
20790 Opcode: ISD::SHL, DL: SDLoc(Op), VT: MVT::i32, N1: Op.getOperand(i: 1),
20791 N2: TLO.DAG.getConstant(Val: 32 - ShAmt, DL: SDLoc(Op), VT: MVT::i32)));
20792 }
20793 break;
20794 }
20795 case ARMISD::VBICIMM: {
20796 SDValue Op0 = Op.getOperand(i: 0);
20797 unsigned ModImm = Op.getConstantOperandVal(i: 1);
20798 unsigned EltBits = 0;
20799 uint64_t Mask = ARM_AM::decodeVMOVModImm(ModImm, EltBits);
20800 if ((OriginalDemandedBits & Mask) == 0)
20801 return TLO.CombineTo(O: Op, N: Op0);
20802 }
20803 }
20804
20805 return TargetLowering::SimplifyDemandedBitsForTargetNode(
20806 Op, DemandedBits: OriginalDemandedBits, DemandedElts: OriginalDemandedElts, Known, TLO, Depth);
20807}
20808
20809//===----------------------------------------------------------------------===//
20810// ARM Inline Assembly Support
20811//===----------------------------------------------------------------------===//
20812
20813const char *ARMTargetLowering::LowerXConstraint(EVT ConstraintVT) const {
20814 // At this point, we have to lower this constraint to something else, so we
20815 // lower it to an "r" or "w". However, by doing this we will force the result
20816 // to be in register, while the X constraint is much more permissive.
20817 //
20818 // Although we are correct (we are free to emit anything, without
20819 // constraints), we might break use cases that would expect us to be more
20820 // efficient and emit something else.
20821 if (!Subtarget->hasVFP2Base())
20822 return "r";
20823 if (ConstraintVT.isFloatingPoint())
20824 return "w";
20825 if (ConstraintVT.isVector() && Subtarget->hasNEON() &&
20826 (ConstraintVT.getSizeInBits() == 64 ||
20827 ConstraintVT.getSizeInBits() == 128))
20828 return "w";
20829
20830 return "r";
20831}
20832
20833/// getConstraintType - Given a constraint letter, return the type of
20834/// constraint it is for this target.
20835ARMTargetLowering::ConstraintType
20836ARMTargetLowering::getConstraintType(StringRef Constraint) const {
20837 unsigned S = Constraint.size();
20838 if (S == 1) {
20839 switch (Constraint[0]) {
20840 default: break;
20841 case 'l': return C_RegisterClass;
20842 case 'w': return C_RegisterClass;
20843 case 'h': return C_RegisterClass;
20844 case 'x': return C_RegisterClass;
20845 case 't': return C_RegisterClass;
20846 case 'j': return C_Immediate; // Constant for movw.
20847 // An address with a single base register. Due to the way we
20848 // currently handle addresses it is the same as an 'r' memory constraint.
20849 case 'Q': return C_Memory;
20850 }
20851 } else if (S == 2) {
20852 switch (Constraint[0]) {
20853 default: break;
20854 case 'T': return C_RegisterClass;
20855 // All 'U+' constraints are addresses.
20856 case 'U': return C_Memory;
20857 }
20858 }
20859 return TargetLowering::getConstraintType(Constraint);
20860}
20861
20862/// Examine constraint type and operand type and determine a weight value.
20863/// This object must already have been set up with the operand type
20864/// and the current alternative constraint selected.
20865TargetLowering::ConstraintWeight
20866ARMTargetLowering::getSingleConstraintMatchWeight(
20867 AsmOperandInfo &info, const char *constraint) const {
20868 ConstraintWeight weight = CW_Invalid;
20869 Value *CallOperandVal = info.CallOperandVal;
20870 // If we don't have a value, we can't do a match,
20871 // but allow it at the lowest weight.
20872 if (!CallOperandVal)
20873 return CW_Default;
20874 Type *type = CallOperandVal->getType();
20875 // Look at the constraint type.
20876 switch (*constraint) {
20877 default:
20878 weight = TargetLowering::getSingleConstraintMatchWeight(info, constraint);
20879 break;
20880 case 'l':
20881 if (type->isIntegerTy()) {
20882 if (Subtarget->isThumb())
20883 weight = CW_SpecificReg;
20884 else
20885 weight = CW_Register;
20886 }
20887 break;
20888 case 'w':
20889 if (type->isFloatingPointTy())
20890 weight = CW_Register;
20891 break;
20892 }
20893 return weight;
20894}
20895
20896static bool isIncompatibleReg(const MCPhysReg &PR, MVT VT) {
20897 if (PR == 0 || VT == MVT::Other)
20898 return false;
20899 if (ARM::SPRRegClass.contains(Reg: PR))
20900 return VT != MVT::f32 && VT != MVT::f16 && VT != MVT::i32;
20901 if (ARM::DPRRegClass.contains(Reg: PR))
20902 return VT != MVT::f64 && !VT.is64BitVector();
20903 return false;
20904}
20905
20906using RCPair = std::pair<unsigned, const TargetRegisterClass *>;
20907
20908RCPair ARMTargetLowering::getRegForInlineAsmConstraint(
20909 const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const {
20910 switch (Constraint.size()) {
20911 case 1:
20912 // GCC ARM Constraint Letters
20913 switch (Constraint[0]) {
20914 case 'l': // Low regs or general regs.
20915 if (Subtarget->isThumb())
20916 return RCPair(0U, &ARM::tGPRRegClass);
20917 return RCPair(0U, &ARM::GPRRegClass);
20918 case 'h': // High regs or no regs.
20919 if (Subtarget->isThumb())
20920 return RCPair(0U, &ARM::hGPRRegClass);
20921 break;
20922 case 'r':
20923 if (Subtarget->isThumb1Only())
20924 return RCPair(0U, &ARM::tGPRRegClass);
20925 return RCPair(0U, &ARM::GPRRegClass);
20926 case 'w':
20927 if (VT == MVT::Other)
20928 break;
20929 if (VT == MVT::f32 || VT == MVT::f16 || VT == MVT::bf16)
20930 return RCPair(0U, &ARM::SPRRegClass);
20931 if (VT.getSizeInBits() == 64)
20932 return RCPair(0U, &ARM::DPRRegClass);
20933 if (VT.getSizeInBits() == 128)
20934 return RCPair(0U, &ARM::QPRRegClass);
20935 break;
20936 case 'x':
20937 if (VT == MVT::Other)
20938 break;
20939 if (VT == MVT::f32 || VT == MVT::f16 || VT == MVT::bf16)
20940 return RCPair(0U, &ARM::SPR_8RegClass);
20941 if (VT.getSizeInBits() == 64)
20942 return RCPair(0U, &ARM::DPR_8RegClass);
20943 if (VT.getSizeInBits() == 128)
20944 return RCPair(0U, &ARM::QPR_8RegClass);
20945 break;
20946 case 't':
20947 if (VT == MVT::Other)
20948 break;
20949 if (VT == MVT::f32 || VT == MVT::i32 || VT == MVT::f16 || VT == MVT::bf16)
20950 return RCPair(0U, &ARM::SPRRegClass);
20951 if (VT.getSizeInBits() == 64)
20952 return RCPair(0U, &ARM::DPR_VFP2RegClass);
20953 if (VT.getSizeInBits() == 128)
20954 return RCPair(0U, &ARM::QPR_VFP2RegClass);
20955 break;
20956 }
20957 break;
20958
20959 case 2:
20960 if (Constraint[0] == 'T') {
20961 switch (Constraint[1]) {
20962 default:
20963 break;
20964 case 'e':
20965 return RCPair(0U, &ARM::tGPREvenRegClass);
20966 case 'o':
20967 return RCPair(0U, &ARM::tGPROddRegClass);
20968 }
20969 }
20970 break;
20971
20972 default:
20973 break;
20974 }
20975
20976 if (StringRef("{cc}").equals_insensitive(RHS: Constraint))
20977 return std::make_pair(x: unsigned(ARM::CPSR), y: &ARM::CCRRegClass);
20978
20979 // r14 is an alias of lr.
20980 if (StringRef("{r14}").equals_insensitive(RHS: Constraint))
20981 Constraint = "{lr}";
20982
20983 auto RCP = TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
20984 if (isIncompatibleReg(PR: RCP.first, VT))
20985 return {0, nullptr};
20986 return RCP;
20987}
20988
20989/// LowerAsmOperandForConstraint - Lower the specified operand into the Ops
20990/// vector. If it is invalid, don't add anything to Ops.
20991void ARMTargetLowering::LowerAsmOperandForConstraint(SDValue Op,
20992 StringRef Constraint,
20993 std::vector<SDValue> &Ops,
20994 SelectionDAG &DAG) const {
20995 SDValue Result;
20996
20997 // Currently only support length 1 constraints.
20998 if (Constraint.size() != 1)
20999 return;
21000
21001 char ConstraintLetter = Constraint[0];
21002 switch (ConstraintLetter) {
21003 default: break;
21004 case 'j':
21005 case 'I': case 'J': case 'K': case 'L':
21006 case 'M': case 'N': case 'O':
21007 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: Op);
21008 if (!C)
21009 return;
21010
21011 int64_t CVal64 = C->getSExtValue();
21012 int CVal = (int) CVal64;
21013 // None of these constraints allow values larger than 32 bits. Check
21014 // that the value fits in an int.
21015 if (CVal != CVal64)
21016 return;
21017
21018 switch (ConstraintLetter) {
21019 case 'j':
21020 // Constant suitable for movw, must be between 0 and
21021 // 65535.
21022 if (Subtarget->hasV6T2Ops() || (Subtarget->hasV8MBaselineOps()))
21023 if (CVal >= 0 && CVal <= 65535)
21024 break;
21025 return;
21026 case 'I':
21027 if (Subtarget->isThumb1Only()) {
21028 // This must be a constant between 0 and 255, for ADD
21029 // immediates.
21030 if (CVal >= 0 && CVal <= 255)
21031 break;
21032 } else if (Subtarget->isThumb2()) {
21033 // A constant that can be used as an immediate value in a
21034 // data-processing instruction.
21035 if (ARM_AM::getT2SOImmVal(Arg: CVal) != -1)
21036 break;
21037 } else {
21038 // A constant that can be used as an immediate value in a
21039 // data-processing instruction.
21040 if (ARM_AM::getSOImmVal(Arg: CVal) != -1)
21041 break;
21042 }
21043 return;
21044
21045 case 'J':
21046 if (Subtarget->isThumb1Only()) {
21047 // This must be a constant between -255 and -1, for negated ADD
21048 // immediates. This can be used in GCC with an "n" modifier that
21049 // prints the negated value, for use with SUB instructions. It is
21050 // not useful otherwise but is implemented for compatibility.
21051 if (CVal >= -255 && CVal <= -1)
21052 break;
21053 } else {
21054 // This must be a constant between -4095 and 4095. This is suitable
21055 // for use as the immediate offset field in LDR and STR instructions
21056 // such as LDR r0,[r1,#offset].
21057 if (CVal >= -4095 && CVal <= 4095)
21058 break;
21059 }
21060 return;
21061
21062 case 'K':
21063 if (Subtarget->isThumb1Only()) {
21064 // A 32-bit value where only one byte has a nonzero value. Exclude
21065 // zero to match GCC. This constraint is used by GCC internally for
21066 // constants that can be loaded with a move/shift combination.
21067 // It is not useful otherwise but is implemented for compatibility.
21068 if (CVal != 0 && ARM_AM::isThumbImmShiftedVal(V: CVal))
21069 break;
21070 } else if (Subtarget->isThumb2()) {
21071 // A constant whose bitwise inverse can be used as an immediate
21072 // value in a data-processing instruction. This can be used in GCC
21073 // with a "B" modifier that prints the inverted value, for use with
21074 // BIC and MVN instructions. It is not useful otherwise but is
21075 // implemented for compatibility.
21076 if (ARM_AM::getT2SOImmVal(Arg: ~CVal) != -1)
21077 break;
21078 } else {
21079 // A constant whose bitwise inverse can be used as an immediate
21080 // value in a data-processing instruction. This can be used in GCC
21081 // with a "B" modifier that prints the inverted value, for use with
21082 // BIC and MVN instructions. It is not useful otherwise but is
21083 // implemented for compatibility.
21084 if (ARM_AM::getSOImmVal(Arg: ~CVal) != -1)
21085 break;
21086 }
21087 return;
21088
21089 case 'L':
21090 if (Subtarget->isThumb1Only()) {
21091 // This must be a constant between -7 and 7,
21092 // for 3-operand ADD/SUB immediate instructions.
21093 if (CVal >= -7 && CVal < 7)
21094 break;
21095 } else if (Subtarget->isThumb2()) {
21096 // A constant whose negation can be used as an immediate value in a
21097 // data-processing instruction. This can be used in GCC with an "n"
21098 // modifier that prints the negated value, for use with SUB
21099 // instructions. It is not useful otherwise but is implemented for
21100 // compatibility.
21101 if (ARM_AM::getT2SOImmVal(Arg: -CVal) != -1)
21102 break;
21103 } else {
21104 // A constant whose negation can be used as an immediate value in a
21105 // data-processing instruction. This can be used in GCC with an "n"
21106 // modifier that prints the negated value, for use with SUB
21107 // instructions. It is not useful otherwise but is implemented for
21108 // compatibility.
21109 if (ARM_AM::getSOImmVal(Arg: -CVal) != -1)
21110 break;
21111 }
21112 return;
21113
21114 case 'M':
21115 if (Subtarget->isThumb1Only()) {
21116 // This must be a multiple of 4 between 0 and 1020, for
21117 // ADD sp + immediate.
21118 if ((CVal >= 0 && CVal <= 1020) && ((CVal & 3) == 0))
21119 break;
21120 } else {
21121 // A power of two or a constant between 0 and 32. This is used in
21122 // GCC for the shift amount on shifted register operands, but it is
21123 // useful in general for any shift amounts.
21124 if ((CVal >= 0 && CVal <= 32) || ((CVal & (CVal - 1)) == 0))
21125 break;
21126 }
21127 return;
21128
21129 case 'N':
21130 if (Subtarget->isThumb1Only()) {
21131 // This must be a constant between 0 and 31, for shift amounts.
21132 if (CVal >= 0 && CVal <= 31)
21133 break;
21134 }
21135 return;
21136
21137 case 'O':
21138 if (Subtarget->isThumb1Only()) {
21139 // This must be a multiple of 4 between -508 and 508, for
21140 // ADD/SUB sp = sp + immediate.
21141 if ((CVal >= -508 && CVal <= 508) && ((CVal & 3) == 0))
21142 break;
21143 }
21144 return;
21145 }
21146 Result = DAG.getSignedTargetConstant(Val: CVal, DL: SDLoc(Op), VT: Op.getValueType());
21147 break;
21148 }
21149
21150 if (Result.getNode()) {
21151 Ops.push_back(x: Result);
21152 return;
21153 }
21154 return TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
21155}
21156
21157static RTLIB::Libcall getDivRemLibcall(
21158 const SDNode *N, MVT::SimpleValueType SVT) {
21159 assert((N->getOpcode() == ISD::SDIVREM || N->getOpcode() == ISD::UDIVREM ||
21160 N->getOpcode() == ISD::SREM || N->getOpcode() == ISD::UREM) &&
21161 "Unhandled Opcode in getDivRemLibcall");
21162 bool isSigned = N->getOpcode() == ISD::SDIVREM ||
21163 N->getOpcode() == ISD::SREM;
21164 RTLIB::Libcall LC;
21165 switch (SVT) {
21166 default: llvm_unreachable("Unexpected request for libcall!");
21167 case MVT::i8: LC = isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break;
21168 case MVT::i16: LC = isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
21169 case MVT::i32: LC = isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
21170 case MVT::i64: LC = isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
21171 }
21172 return LC;
21173}
21174
21175static TargetLowering::ArgListTy
21176getDivRemArgList(const SDNode *N, FunctionType *FuncTy,
21177 const AttributeList &FuncAttrs, RTLIB::LibcallImpl LCImpl) {
21178 assert((N->getOpcode() == ISD::SDIVREM || N->getOpcode() == ISD::UDIVREM ||
21179 N->getOpcode() == ISD::SREM || N->getOpcode() == ISD::UREM) &&
21180 "Unhandled Opcode in getDivRemArgList");
21181 SDValue Ops[2] = {N->getOperand(Num: 0), N->getOperand(Num: 1)};
21182
21183 // The Windows __rt_*div* helpers take the divisor before the dividend.
21184 switch (LCImpl) {
21185 case RTLIB::impl___rt_sdiv:
21186 case RTLIB::impl___rt_udiv:
21187 case RTLIB::impl___rt_sdiv64:
21188 case RTLIB::impl___rt_udiv64:
21189 std::swap(a&: Ops[0], b&: Ops[1]);
21190 break;
21191 default:
21192 break;
21193 }
21194
21195 return TargetLowering::getArgListForFunctionType(FuncTy, FuncAttrs, Ops);
21196}
21197
21198SDValue ARMTargetLowering::LowerDivRem(SDValue Op, SelectionDAG &DAG) const {
21199 assert((Subtarget->isTargetAEABI() || Subtarget->isTargetAndroid() ||
21200 Subtarget->isTargetGNUAEABI() || Subtarget->isTargetMuslAEABI() ||
21201 Subtarget->isTargetFuchsia() || Subtarget->isTargetWindows()) &&
21202 "Register-based DivRem lowering only");
21203 unsigned Opcode = Op->getOpcode();
21204 assert((Opcode == ISD::SDIVREM || Opcode == ISD::UDIVREM) &&
21205 "Invalid opcode for Div/Rem lowering");
21206 bool isSigned = (Opcode == ISD::SDIVREM);
21207 EVT VT = Op->getValueType(ResNo: 0);
21208 SDLoc dl(Op);
21209
21210 if (VT == MVT::i64 && isa<ConstantSDNode>(Val: Op.getOperand(i: 1))) {
21211 SmallVector<SDValue> Result;
21212 if (expandDIVREMByConstant(N: Op.getNode(), Result, HiLoVT: MVT::i32, DAG)) {
21213 SDValue Res0 =
21214 DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT, N1: Result[0], N2: Result[1]);
21215 SDValue Res1 =
21216 DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT, N1: Result[2], N2: Result[3]);
21217 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: dl, VTList: Op->getVTList(),
21218 Ops: {Res0, Res1});
21219 }
21220 }
21221
21222 // If the target has hardware divide, use divide + multiply + subtract:
21223 // div = a / b
21224 // rem = a - b * div
21225 // return {div, rem}
21226 // This should be lowered into UDIV/SDIV + MLS later on.
21227 bool hasDivide = Subtarget->isThumb() ? Subtarget->hasDivideInThumbMode()
21228 : Subtarget->hasDivideInARMMode();
21229 if (hasDivide && Op->getValueType(ResNo: 0).isSimple() &&
21230 Op->getSimpleValueType(ResNo: 0) == MVT::i32) {
21231 unsigned DivOpcode = isSigned ? ISD::SDIV : ISD::UDIV;
21232 const SDValue Dividend = Op->getOperand(Num: 0);
21233 const SDValue Divisor = Op->getOperand(Num: 1);
21234 SDValue Div = DAG.getNode(Opcode: DivOpcode, DL: dl, VT, N1: Dividend, N2: Divisor);
21235 SDValue Mul = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: Div, N2: Divisor);
21236 SDValue Rem = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Dividend, N2: Mul);
21237
21238 SDValue Values[2] = {Div, Rem};
21239 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: VT), Ops: Values);
21240 }
21241
21242 RTLIB::Libcall LC = getDivRemLibcall(N: Op.getNode(),
21243 SVT: VT.getSimpleVT().SimpleTy);
21244 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
21245 if (LCImpl == RTLIB::Unsupported)
21246 return SDValue();
21247
21248 auto [FuncTy, FuncAttrs] =
21249 DAG.getLibcalls().getRuntimeLibcallsInfo().getFunctionTy(
21250 Ctx&: *DAG.getContext(), TT: getTM().getTargetTriple(), DL: DAG.getDataLayout(),
21251 LibcallImpl: LCImpl);
21252 Type *RetTy = FuncTy->getReturnType();
21253
21254 SDValue InChain = DAG.getEntryNode();
21255
21256 TargetLowering::ArgListTy Args =
21257 getDivRemArgList(N: Op.getNode(), FuncTy, FuncAttrs, LCImpl);
21258
21259 SDValue Callee =
21260 DAG.getExternalSymbol(LCImpl, VT: getPointerTy(DL: DAG.getDataLayout()));
21261
21262 if (getTM().getTargetTriple().isOSWindows())
21263 InChain = WinDBZCheckDenominator(DAG, N: Op.getNode(), InChain);
21264
21265 TargetLowering::CallLoweringInfo CLI(DAG);
21266 CLI.setDebugLoc(dl)
21267 .setChain(InChain)
21268 .setCallee(CC: DAG.getLibcalls().getLibcallImplCallingConv(Call: LCImpl), ResultType: RetTy,
21269 Target: Callee, ArgsList: std::move(Args))
21270 .setInRegister()
21271 .setSExtResult(isSigned)
21272 .setZExtResult(!isSigned);
21273
21274 std::pair<SDValue, SDValue> CallInfo = LowerCallTo(CLI);
21275 return CallInfo.first;
21276}
21277
21278// Lowers REM using divmod helpers
21279// see RTABI section 4.2/4.3
21280SDValue ARMTargetLowering::LowerREM(SDNode *N, SelectionDAG &DAG) const {
21281 EVT VT = N->getValueType(ResNo: 0);
21282
21283 if (VT == MVT::i64 && isa<ConstantSDNode>(Val: N->getOperand(Num: 1))) {
21284 SmallVector<SDValue> Result;
21285 if (expandDIVREMByConstant(N, Result, HiLoVT: MVT::i32, DAG))
21286 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
21287 N1: Result[0], N2: Result[1]);
21288 }
21289
21290 RTLIB::Libcall LC = getDivRemLibcall(N, SVT: N->getValueType(ResNo: 0).getSimpleVT().
21291 SimpleTy);
21292 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
21293 if (LCImpl == RTLIB::Unsupported)
21294 return SDValue();
21295
21296 auto [FuncTy, FuncAttrs] =
21297 DAG.getLibcalls().getRuntimeLibcallsInfo().getFunctionTy(
21298 Ctx&: *DAG.getContext(), TT: getTM().getTargetTriple(), DL: DAG.getDataLayout(),
21299 LibcallImpl: LCImpl);
21300 Type *RetTy = FuncTy->getReturnType();
21301
21302 SDValue InChain = DAG.getEntryNode();
21303 TargetLowering::ArgListTy Args =
21304 getDivRemArgList(N, FuncTy, FuncAttrs, LCImpl);
21305 bool isSigned = N->getOpcode() == ISD::SREM;
21306
21307 SDValue Callee =
21308 DAG.getExternalSymbol(LCImpl, VT: getPointerTy(DL: DAG.getDataLayout()));
21309
21310 if (getTM().getTargetTriple().isOSWindows())
21311 InChain = WinDBZCheckDenominator(DAG, N, InChain);
21312
21313 // Lower call
21314 CallLoweringInfo CLI(DAG);
21315 CLI.setChain(InChain)
21316 .setCallee(CC: DAG.getLibcalls().getLibcallImplCallingConv(Call: LCImpl), ResultType: RetTy,
21317 Target: Callee, ArgsList: std::move(Args))
21318 .setSExtResult(isSigned)
21319 .setZExtResult(!isSigned)
21320 .setDebugLoc(SDLoc(N));
21321 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
21322
21323 // Return second (rem) result operand (first contains div)
21324 SDNode *ResNode = CallResult.first.getNode();
21325 assert(ResNode->getNumOperands() == 2 && "divmod should return two operands");
21326 return ResNode->getOperand(Num: 1);
21327}
21328
21329SDValue
21330ARMTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const {
21331 assert(getTM().getTargetTriple().isOSWindows() &&
21332 "unsupported target platform");
21333 SDLoc DL(Op);
21334
21335 // Get the inputs.
21336 SDValue Chain = Op.getOperand(i: 0);
21337 SDValue Size = Op.getOperand(i: 1);
21338
21339 if (DAG.getMachineFunction().getFunction().hasFnAttribute(
21340 Kind: "no-stack-arg-probe")) {
21341 MaybeAlign Align =
21342 cast<ConstantSDNode>(Val: Op.getOperand(i: 2))->getMaybeAlignValue();
21343 SDValue SP = DAG.getCopyFromReg(Chain, dl: DL, Reg: ARM::SP, VT: MVT::i32);
21344 Chain = SP.getValue(R: 1);
21345 SP = DAG.getNode(Opcode: ISD::SUB, DL, VT: MVT::i32, N1: SP, N2: Size);
21346 if (Align)
21347 SP = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: SP.getValue(R: 0),
21348 N2: DAG.getSignedConstant(Val: -Align->value(), DL, VT: MVT::i32));
21349 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: ARM::SP, N: SP);
21350 SDValue Ops[2] = { SP, Chain };
21351 return DAG.getMergeValues(Ops, dl: DL);
21352 }
21353
21354 SDValue Words = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: Size,
21355 N2: DAG.getConstant(Val: 2, DL, VT: MVT::i32));
21356
21357 SDValue Glue;
21358 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: ARM::R4, N: Words, Glue);
21359 Glue = Chain.getValue(R: 1);
21360
21361 SDVTList NodeTys = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
21362 Chain = DAG.getNode(Opcode: ARMISD::WIN__CHKSTK, DL, VTList: NodeTys, N1: Chain, N2: Glue);
21363
21364 SDValue NewSP = DAG.getCopyFromReg(Chain, dl: DL, Reg: ARM::SP, VT: MVT::i32);
21365 Chain = NewSP.getValue(R: 1);
21366
21367 SDValue Ops[2] = { NewSP, Chain };
21368 return DAG.getMergeValues(Ops, dl: DL);
21369}
21370
21371SDValue ARMTargetLowering::LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const {
21372 bool IsStrict = Op->isStrictFPOpcode();
21373 SDValue SrcVal = Op.getOperand(i: IsStrict ? 1 : 0);
21374 EVT DstTy = Op.getValueType();
21375 const unsigned DstSz = DstTy.getSizeInBits();
21376 EVT SrcTy = SrcVal.getValueType();
21377 const unsigned SrcSz = SrcTy.getSizeInBits();
21378 assert(DstSz > SrcSz && DstSz <= 64 && SrcSz >= 16 &&
21379 "Unexpected type for custom-lowering FP_EXTEND");
21380
21381 // f16->f32 is legal with hasFP16
21382 if (DstTy == MVT::f32 && SrcTy == MVT::f16 && Subtarget->hasFP16())
21383 return Op;
21384 // f16->f64 is legal with hasFP16 and hasFP64
21385 if (DstTy == MVT::f64 && SrcTy == MVT::f16 && Subtarget->hasFP16() &&
21386 Subtarget->hasFP64() && Subtarget->hasFPARMv8Base())
21387 return Op;
21388
21389 // Expand any bf16 ops into a shift and a possible further extend
21390 if (SrcTy == MVT::bf16) {
21391 SDLoc DL(Op);
21392 SDValue Mov = DAG.getNode(Opcode: ARMISD::VMOVrs, DL, VT: MVT::i32, Operand: SrcVal);
21393 SDValue Shft = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i32, N1: Mov,
21394 N2: DAG.getConstant(Val: 16, DL, VT: MVT::i32));
21395 SDValue Res = DAG.getBitcast(VT: MVT::f32, V: Shft);
21396 if (DstTy == MVT::f32)
21397 return IsStrict ? DAG.getMergeValues(Ops: {Res, Op.getOperand(i: 0)}, dl: DL) : Res;
21398
21399 assert(DstTy == MVT::f64 && "Unexpected BF16 convert");
21400 if (IsStrict)
21401 return DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL, ResultTys: {DstTy, MVT::Other},
21402 Ops: {Op.getOperand(i: 0), Res});
21403 return DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: DstTy, Operand: Res);
21404 }
21405
21406 // Converting from 32 -> 64 is valid if we have FP64.
21407 if (SrcTy == MVT::f32 && DstTy == MVT::f64 && Subtarget->hasFP64()) {
21408 // FIXME: Remove this when we have strict fp instruction selection patterns
21409 if (IsStrict) {
21410 SDLoc Loc(Op);
21411 SDValue Result = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: Loc, VT: DstTy, Operand: SrcVal);
21412 return DAG.getMergeValues(Ops: {Result, Op.getOperand(i: 0)}, dl: Loc);
21413 }
21414 return Op;
21415 }
21416
21417 // Either we are converting from 16 -> 64, without FP16 and/or
21418 // FP.double-precision or without Armv8-fp. So we must do it in two
21419 // steps.
21420 // Or we are converting from 32 -> 64 without fp.double-precision or 16 -> 32
21421 // without FP16. So we must do a function call.
21422 SDLoc Loc(Op);
21423 RTLIB::Libcall LC;
21424 MakeLibCallOptions CallOptions;
21425 SDValue Chain = IsStrict ? Op.getOperand(i: 0) : SDValue();
21426 for (unsigned Sz = SrcSz; Sz <= 32 && Sz < DstSz; Sz *= 2) {
21427 bool Supported = (Sz == 16 ? Subtarget->hasFP16() : Subtarget->hasFP64());
21428 MVT SrcVT = (Sz == 16 ? MVT::f16 : MVT::f32);
21429 MVT DstVT = (Sz == 16 ? MVT::f32 : MVT::f64);
21430 if (Supported) {
21431 if (IsStrict) {
21432 SrcVal = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: Loc,
21433 ResultTys: {DstVT, MVT::Other}, Ops: {Chain, SrcVal});
21434 Chain = SrcVal.getValue(R: 1);
21435 } else {
21436 SrcVal = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: Loc, VT: DstVT, Operand: SrcVal);
21437 }
21438 } else {
21439 LC = RTLIB::getFPEXT(OpVT: SrcVT, RetVT: DstVT);
21440 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
21441 "Unexpected type for custom-lowering FP_EXTEND");
21442 std::tie(args&: SrcVal, args&: Chain) = makeLibCall(DAG, LC, RetVT: DstVT, Ops: SrcVal, CallOptions,
21443 dl: Loc, Chain);
21444 }
21445 }
21446
21447 return IsStrict ? DAG.getMergeValues(Ops: {SrcVal, Chain}, dl: Loc) : SrcVal;
21448}
21449
21450SDValue ARMTargetLowering::LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const {
21451 bool IsStrict = Op->isStrictFPOpcode();
21452
21453 SDValue SrcVal = Op.getOperand(i: IsStrict ? 1 : 0);
21454 EVT SrcVT = SrcVal.getValueType();
21455 EVT DstVT = Op.getValueType();
21456
21457 if (DstVT == MVT::bf16) {
21458 if (Subtarget->hasBF16() && SrcVT == MVT::f32)
21459 return Op;
21460 return SDValue();
21461 }
21462
21463 const unsigned DstSz = Op.getValueType().getSizeInBits();
21464 const unsigned SrcSz = SrcVT.getSizeInBits();
21465 (void)DstSz;
21466 assert(DstSz < SrcSz && SrcSz <= 64 && DstSz >= 16 &&
21467 "Unexpected type for custom-lowering FP_ROUND");
21468
21469 assert((!Subtarget->hasFP64() || !Subtarget->hasFPARMv8Base()) &&
21470 "With both FP DP and 16, any FP conversion is legal!");
21471
21472 SDLoc Loc(Op);
21473
21474 // Instruction from 32 -> 16 if hasFP16 is valid
21475 if (SrcSz == 32 && Subtarget->hasFP16())
21476 return Op;
21477
21478 // Lib call from 32 -> 16 / 64 -> [32, 16]
21479 RTLIB::Libcall LC = RTLIB::getFPROUND(OpVT: SrcVT, RetVT: DstVT);
21480 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
21481 "Unexpected type for custom-lowering FP_ROUND");
21482 MakeLibCallOptions CallOptions;
21483 SDValue Chain = IsStrict ? Op.getOperand(i: 0) : SDValue();
21484 SDValue Result;
21485 std::tie(args&: Result, args&: Chain) = makeLibCall(DAG, LC, RetVT: DstVT, Ops: SrcVal, CallOptions,
21486 dl: Loc, Chain);
21487 return IsStrict ? DAG.getMergeValues(Ops: {Result, Chain}, dl: Loc) : Result;
21488}
21489
21490bool
21491ARMTargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const {
21492 // The ARM target isn't yet aware of offsets.
21493 return false;
21494}
21495
21496bool ARM::isBitFieldInvertedMask(unsigned v) {
21497 if (v == 0xffffffff)
21498 return false;
21499
21500 // there can be 1's on either or both "outsides", all the "inside"
21501 // bits must be 0's
21502 return isShiftedMask_32(Value: ~v);
21503}
21504
21505/// Returns how to materialize the specified FP immediate.
21506TargetLowering::FPImmAction
21507ARMTargetLowering::getFPImmAction(const APFloat &Imm, EVT VT,
21508 bool ForCodeSize) const {
21509 if (!Subtarget->hasVFP3Base())
21510 return FPImmAction::ConstantPool;
21511 if (VT == MVT::f16 && Subtarget->hasFullFP16())
21512 return ARM_AM::getFP16Imm(FPImm: Imm) != -1 ? FPImmAction::Legal
21513 : FPImmAction::ConstantPool;
21514 if (VT == MVT::f32 && Subtarget->hasFullFP16() &&
21515 ARM_AM::getFP32FP16Imm(FPImm: Imm) != -1)
21516 return FPImmAction::Legal;
21517 if (VT == MVT::f32)
21518 return ARM_AM::getFP32Imm(FPImm: Imm) != -1 ? FPImmAction::Legal
21519 : FPImmAction::ConstantPool;
21520 if (VT == MVT::f64 && Subtarget->hasFP64())
21521 return ARM_AM::getFP64Imm(FPImm: Imm) != -1 ? FPImmAction::Legal
21522 : FPImmAction::ConstantPool;
21523 return FPImmAction::ConstantPool;
21524}
21525
21526TargetLowering::FPImmAction
21527ARMTargetLowering::getFPImmAction(const APFloat &Imm, LLT Ty,
21528 bool ForCodeSize) const {
21529 if (Subtarget->useSoftFloat() || !Subtarget->hasVFP2Base())
21530 return FPImmAction::Bitcast;
21531 return FPImmAction::ConstantPool;
21532}
21533
21534/// getTgtMemIntrinsic - Represent NEON load and store intrinsics as
21535/// MemIntrinsicNodes. The associated MachineMemOperands record the alignment
21536/// specified in the intrinsic calls.
21537void ARMTargetLowering::getTgtMemIntrinsic(
21538 SmallVectorImpl<IntrinsicInfo> &Infos, const CallBase &I,
21539 MachineFunction &MF, unsigned Intrinsic) const {
21540 IntrinsicInfo Info;
21541 switch (Intrinsic) {
21542 case Intrinsic::arm_neon_vld1:
21543 case Intrinsic::arm_neon_vld2:
21544 case Intrinsic::arm_neon_vld3:
21545 case Intrinsic::arm_neon_vld4:
21546 case Intrinsic::arm_neon_vld2lane:
21547 case Intrinsic::arm_neon_vld3lane:
21548 case Intrinsic::arm_neon_vld4lane:
21549 case Intrinsic::arm_neon_vld2dup:
21550 case Intrinsic::arm_neon_vld3dup:
21551 case Intrinsic::arm_neon_vld4dup: {
21552 Info.opc = ISD::INTRINSIC_W_CHAIN;
21553 // Conservatively set memVT to the entire set of vectors loaded.
21554 auto &DL = I.getDataLayout();
21555 uint64_t NumElts = DL.getTypeSizeInBits(Ty: I.getType()) / 64;
21556 Info.memVT = EVT::getVectorVT(Context&: I.getType()->getContext(), VT: MVT::i64, NumElements: NumElts);
21557 Info.ptrVal = I.getArgOperand(i: 0);
21558 Info.offset = 0;
21559 Value *AlignArg = I.getArgOperand(i: I.arg_size() - 1);
21560 Info.align = cast<ConstantInt>(Val: AlignArg)->getMaybeAlignValue();
21561 // volatile loads with NEON intrinsics not supported
21562 Info.flags = MachineMemOperand::MOLoad;
21563 Infos.push_back(Elt: Info);
21564 return;
21565 }
21566 case Intrinsic::arm_neon_vld1x2:
21567 case Intrinsic::arm_neon_vld1x3:
21568 case Intrinsic::arm_neon_vld1x4: {
21569 Info.opc = ISD::INTRINSIC_W_CHAIN;
21570 // Conservatively set memVT to the entire set of vectors loaded.
21571 auto &DL = I.getDataLayout();
21572 uint64_t NumElts = DL.getTypeSizeInBits(Ty: I.getType()) / 64;
21573 Info.memVT = EVT::getVectorVT(Context&: I.getType()->getContext(), VT: MVT::i64, NumElements: NumElts);
21574 Info.ptrVal = I.getArgOperand(i: I.arg_size() - 1);
21575 Info.offset = 0;
21576 Info.align = I.getParamAlign(ArgNo: I.arg_size() - 1).valueOrOne();
21577 // volatile loads with NEON intrinsics not supported
21578 Info.flags = MachineMemOperand::MOLoad;
21579 Infos.push_back(Elt: Info);
21580 return;
21581 }
21582 case Intrinsic::arm_neon_vst1:
21583 case Intrinsic::arm_neon_vst2:
21584 case Intrinsic::arm_neon_vst3:
21585 case Intrinsic::arm_neon_vst4:
21586 case Intrinsic::arm_neon_vst2lane:
21587 case Intrinsic::arm_neon_vst3lane:
21588 case Intrinsic::arm_neon_vst4lane: {
21589 Info.opc = ISD::INTRINSIC_VOID;
21590 // Conservatively set memVT to the entire set of vectors stored.
21591 auto &DL = I.getDataLayout();
21592 unsigned NumElts = 0;
21593 for (unsigned ArgI = 1, ArgE = I.arg_size(); ArgI < ArgE; ++ArgI) {
21594 Type *ArgTy = I.getArgOperand(i: ArgI)->getType();
21595 if (!ArgTy->isVectorTy())
21596 break;
21597 NumElts += DL.getTypeSizeInBits(Ty: ArgTy) / 64;
21598 }
21599 Info.memVT = EVT::getVectorVT(Context&: I.getType()->getContext(), VT: MVT::i64, NumElements: NumElts);
21600 Info.ptrVal = I.getArgOperand(i: 0);
21601 Info.offset = 0;
21602 Value *AlignArg = I.getArgOperand(i: I.arg_size() - 1);
21603 Info.align = cast<ConstantInt>(Val: AlignArg)->getMaybeAlignValue();
21604 // volatile stores with NEON intrinsics not supported
21605 Info.flags = MachineMemOperand::MOStore;
21606 Infos.push_back(Elt: Info);
21607 return;
21608 }
21609 case Intrinsic::arm_neon_vst1x2:
21610 case Intrinsic::arm_neon_vst1x3:
21611 case Intrinsic::arm_neon_vst1x4: {
21612 Info.opc = ISD::INTRINSIC_VOID;
21613 // Conservatively set memVT to the entire set of vectors stored.
21614 auto &DL = I.getDataLayout();
21615 unsigned NumElts = 0;
21616 for (unsigned ArgI = 1, ArgE = I.arg_size(); ArgI < ArgE; ++ArgI) {
21617 Type *ArgTy = I.getArgOperand(i: ArgI)->getType();
21618 if (!ArgTy->isVectorTy())
21619 break;
21620 NumElts += DL.getTypeSizeInBits(Ty: ArgTy) / 64;
21621 }
21622 Info.memVT = EVT::getVectorVT(Context&: I.getType()->getContext(), VT: MVT::i64, NumElements: NumElts);
21623 Info.ptrVal = I.getArgOperand(i: 0);
21624 Info.offset = 0;
21625 Info.align = I.getParamAlign(ArgNo: 0).valueOrOne();
21626 // volatile stores with NEON intrinsics not supported
21627 Info.flags = MachineMemOperand::MOStore;
21628 Infos.push_back(Elt: Info);
21629 return;
21630 }
21631 case Intrinsic::arm_mve_vld2q:
21632 case Intrinsic::arm_mve_vld4q: {
21633 Info.opc = ISD::INTRINSIC_W_CHAIN;
21634 // Conservatively set memVT to the entire set of vectors loaded.
21635 Type *VecTy = cast<StructType>(Val: I.getType())->getElementType(N: 1);
21636 unsigned Factor = Intrinsic == Intrinsic::arm_mve_vld2q ? 2 : 4;
21637 Info.memVT = EVT::getVectorVT(Context&: VecTy->getContext(), VT: MVT::i64, NumElements: Factor * 2);
21638 Info.ptrVal = I.getArgOperand(i: 0);
21639 Info.offset = 0;
21640 Info.align = Align(VecTy->getScalarSizeInBits() / 8);
21641 // volatile loads with MVE intrinsics not supported
21642 Info.flags = MachineMemOperand::MOLoad;
21643 Infos.push_back(Elt: Info);
21644 return;
21645 }
21646 case Intrinsic::arm_mve_vst2q:
21647 case Intrinsic::arm_mve_vst4q: {
21648 Info.opc = ISD::INTRINSIC_VOID;
21649 // Conservatively set memVT to the entire set of vectors stored.
21650 Type *VecTy = I.getArgOperand(i: 1)->getType();
21651 unsigned Factor = Intrinsic == Intrinsic::arm_mve_vst2q ? 2 : 4;
21652 Info.memVT = EVT::getVectorVT(Context&: VecTy->getContext(), VT: MVT::i64, NumElements: Factor * 2);
21653 Info.ptrVal = I.getArgOperand(i: 0);
21654 Info.offset = 0;
21655 Info.align = Align(VecTy->getScalarSizeInBits() / 8);
21656 // volatile stores with MVE intrinsics not supported
21657 Info.flags = MachineMemOperand::MOStore;
21658 Infos.push_back(Elt: Info);
21659 return;
21660 }
21661 case Intrinsic::arm_mve_vldr_gather_base:
21662 case Intrinsic::arm_mve_vldr_gather_base_predicated: {
21663 Info.opc = ISD::INTRINSIC_W_CHAIN;
21664 Info.ptrVal = nullptr;
21665 Info.memVT = MVT::getVT(Ty: I.getType());
21666 Info.align = Align(1);
21667 Info.flags |= MachineMemOperand::MOLoad;
21668 Infos.push_back(Elt: Info);
21669 return;
21670 }
21671 case Intrinsic::arm_mve_vldr_gather_base_wb:
21672 case Intrinsic::arm_mve_vldr_gather_base_wb_predicated: {
21673 Info.opc = ISD::INTRINSIC_W_CHAIN;
21674 Info.ptrVal = nullptr;
21675 Info.memVT = MVT::getVT(Ty: I.getType()->getContainedType(i: 0));
21676 Info.align = Align(1);
21677 Info.flags |= MachineMemOperand::MOLoad;
21678 Infos.push_back(Elt: Info);
21679 return;
21680 }
21681 case Intrinsic::arm_mve_vldr_gather_offset:
21682 case Intrinsic::arm_mve_vldr_gather_offset_predicated: {
21683 Info.opc = ISD::INTRINSIC_W_CHAIN;
21684 Info.ptrVal = nullptr;
21685 MVT DataVT = MVT::getVT(Ty: I.getType());
21686 unsigned MemSize = cast<ConstantInt>(Val: I.getArgOperand(i: 2))->getZExtValue();
21687 Info.memVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: MemSize),
21688 NumElements: DataVT.getVectorNumElements());
21689 Info.align = Align(1);
21690 Info.flags |= MachineMemOperand::MOLoad;
21691 Infos.push_back(Elt: Info);
21692 return;
21693 }
21694 case Intrinsic::arm_mve_vstr_scatter_base:
21695 case Intrinsic::arm_mve_vstr_scatter_base_predicated: {
21696 Info.opc = ISD::INTRINSIC_VOID;
21697 Info.ptrVal = nullptr;
21698 Info.memVT = MVT::getVT(Ty: I.getArgOperand(i: 2)->getType());
21699 Info.align = Align(1);
21700 Info.flags |= MachineMemOperand::MOStore;
21701 Infos.push_back(Elt: Info);
21702 return;
21703 }
21704 case Intrinsic::arm_mve_vstr_scatter_base_wb:
21705 case Intrinsic::arm_mve_vstr_scatter_base_wb_predicated: {
21706 Info.opc = ISD::INTRINSIC_W_CHAIN;
21707 Info.ptrVal = nullptr;
21708 Info.memVT = MVT::getVT(Ty: I.getArgOperand(i: 2)->getType());
21709 Info.align = Align(1);
21710 Info.flags |= MachineMemOperand::MOStore;
21711 Infos.push_back(Elt: Info);
21712 return;
21713 }
21714 case Intrinsic::arm_mve_vstr_scatter_offset:
21715 case Intrinsic::arm_mve_vstr_scatter_offset_predicated: {
21716 Info.opc = ISD::INTRINSIC_VOID;
21717 Info.ptrVal = nullptr;
21718 MVT DataVT = MVT::getVT(Ty: I.getArgOperand(i: 2)->getType());
21719 unsigned MemSize = cast<ConstantInt>(Val: I.getArgOperand(i: 3))->getZExtValue();
21720 Info.memVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: MemSize),
21721 NumElements: DataVT.getVectorNumElements());
21722 Info.align = Align(1);
21723 Info.flags |= MachineMemOperand::MOStore;
21724 Infos.push_back(Elt: Info);
21725 return;
21726 }
21727 case Intrinsic::arm_ldaex:
21728 case Intrinsic::arm_ldrex: {
21729 auto &DL = I.getDataLayout();
21730 Type *ValTy = I.getParamElementType(ArgNo: 0);
21731 Info.opc = ISD::INTRINSIC_W_CHAIN;
21732 Info.memVT = MVT::getVT(Ty: ValTy);
21733 Info.ptrVal = I.getArgOperand(i: 0);
21734 Info.offset = 0;
21735 Info.align = DL.getABITypeAlign(Ty: ValTy);
21736 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile;
21737 Infos.push_back(Elt: Info);
21738 return;
21739 }
21740 case Intrinsic::arm_stlex:
21741 case Intrinsic::arm_strex: {
21742 auto &DL = I.getDataLayout();
21743 Type *ValTy = I.getParamElementType(ArgNo: 1);
21744 Info.opc = ISD::INTRINSIC_W_CHAIN;
21745 Info.memVT = MVT::getVT(Ty: ValTy);
21746 Info.ptrVal = I.getArgOperand(i: 1);
21747 Info.offset = 0;
21748 Info.align = DL.getABITypeAlign(Ty: ValTy);
21749 Info.flags = MachineMemOperand::MOStore | MachineMemOperand::MOVolatile;
21750 Infos.push_back(Elt: Info);
21751 return;
21752 }
21753 case Intrinsic::arm_stlexd:
21754 case Intrinsic::arm_strexd:
21755 Info.opc = ISD::INTRINSIC_W_CHAIN;
21756 Info.memVT = MVT::i64;
21757 Info.ptrVal = I.getArgOperand(i: 2);
21758 Info.offset = 0;
21759 Info.align = Align(8);
21760 Info.flags = MachineMemOperand::MOStore | MachineMemOperand::MOVolatile;
21761 Infos.push_back(Elt: Info);
21762 return;
21763
21764 case Intrinsic::arm_ldaexd:
21765 case Intrinsic::arm_ldrexd:
21766 Info.opc = ISD::INTRINSIC_W_CHAIN;
21767 Info.memVT = MVT::i64;
21768 Info.ptrVal = I.getArgOperand(i: 0);
21769 Info.offset = 0;
21770 Info.align = Align(8);
21771 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile;
21772 Infos.push_back(Elt: Info);
21773 return;
21774
21775 default:
21776 break;
21777 }
21778}
21779
21780/// Returns true if it is beneficial to convert a load of a constant
21781/// to just the constant itself.
21782bool ARMTargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
21783 Type *Ty) const {
21784 assert(Ty->isIntegerTy());
21785
21786 unsigned Bits = Ty->getPrimitiveSizeInBits();
21787 if (Bits == 0 || Bits > 32)
21788 return false;
21789 return true;
21790}
21791
21792TargetLowering::ExtractSubvectorCost
21793ARMTargetLowering::getExtractSubvectorCost(EVT ResVT, EVT SrcVT,
21794 unsigned Index) const {
21795 if (!isOperationLegalOrCustom(Op: ISD::EXTRACT_SUBVECTOR, VT: ResVT))
21796 return ExtractSubvectorCost::Expensive;
21797
21798 if (Index == 0 || Index == ResVT.getVectorNumElements())
21799 return ExtractSubvectorCost::Free;
21800 return ExtractSubvectorCost::Expensive;
21801}
21802
21803Instruction *ARMTargetLowering::makeDMB(IRBuilderBase &Builder,
21804 ARM_MB::MemBOpt Domain) const {
21805 // First, if the target has no DMB, see what fallback we can use.
21806 if (!Subtarget->hasDataBarrier()) {
21807 // Some ARMv6 cpus can support data barriers with an mcr instruction.
21808 // Thumb1 and pre-v6 ARM mode use a libcall instead and should never get
21809 // here.
21810 if (Subtarget->hasV6Ops() && !Subtarget->isThumb()) {
21811 Value* args[6] = {Builder.getInt32(C: 15), Builder.getInt32(C: 0),
21812 Builder.getInt32(C: 0), Builder.getInt32(C: 7),
21813 Builder.getInt32(C: 10), Builder.getInt32(C: 5)};
21814 return Builder.CreateIntrinsicWithoutFolding(ID: Intrinsic::arm_mcr, Args: args);
21815 }
21816 // Instead of using barriers, atomic accesses on these subtargets use
21817 // libcalls.
21818 llvm_unreachable("makeDMB on a target so old that it has no barriers");
21819 } else {
21820 // Only a full system barrier exists in the M-class architectures.
21821 Domain = Subtarget->isMClass() ? ARM_MB::SY : Domain;
21822 Constant *CDomain = Builder.getInt32(C: Domain);
21823 return Builder.CreateIntrinsicWithoutFolding(ID: Intrinsic::arm_dmb, Args: CDomain);
21824 }
21825}
21826
21827// Based on http://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html
21828Instruction *ARMTargetLowering::emitLeadingFence(IRBuilderBase &Builder,
21829 Instruction *Inst,
21830 AtomicOrdering Ord) const {
21831 switch (Ord) {
21832 case AtomicOrdering::NotAtomic:
21833 case AtomicOrdering::Unordered:
21834 llvm_unreachable("Invalid fence: unordered/non-atomic");
21835 case AtomicOrdering::Monotonic:
21836 case AtomicOrdering::Acquire:
21837 return nullptr; // Nothing to do
21838 case AtomicOrdering::SequentiallyConsistent:
21839 if (!Inst->hasAtomicStore())
21840 return nullptr; // Nothing to do
21841 [[fallthrough]];
21842 case AtomicOrdering::Release:
21843 case AtomicOrdering::AcquireRelease:
21844 if (Subtarget->preferISHSTBarriers())
21845 return makeDMB(Builder, Domain: ARM_MB::ISHST);
21846 // FIXME: add a comment with a link to documentation justifying this.
21847 else
21848 return makeDMB(Builder, Domain: ARM_MB::ISH);
21849 }
21850 llvm_unreachable("Unknown fence ordering in emitLeadingFence");
21851}
21852
21853Instruction *ARMTargetLowering::emitTrailingFence(IRBuilderBase &Builder,
21854 Instruction *Inst,
21855 AtomicOrdering Ord) const {
21856 switch (Ord) {
21857 case AtomicOrdering::NotAtomic:
21858 case AtomicOrdering::Unordered:
21859 llvm_unreachable("Invalid fence: unordered/not-atomic");
21860 case AtomicOrdering::Monotonic:
21861 case AtomicOrdering::Release:
21862 return nullptr; // Nothing to do
21863 case AtomicOrdering::Acquire:
21864 case AtomicOrdering::AcquireRelease:
21865 case AtomicOrdering::SequentiallyConsistent:
21866 return makeDMB(Builder, Domain: ARM_MB::ISH);
21867 }
21868 llvm_unreachable("Unknown fence ordering in emitTrailingFence");
21869}
21870
21871// Loads and stores less than 64-bits are already atomic; ones above that
21872// are doomed anyway, so defer to the default libcall and blame the OS when
21873// things go wrong. Cortex M doesn't have ldrexd/strexd though, so don't emit
21874// anything for those.
21875TargetLoweringBase::AtomicExpansionKind
21876ARMTargetLowering::shouldExpandAtomicStoreInIR(StoreInst *SI) const {
21877 bool has64BitLLSC;
21878 if (Subtarget->isMClass())
21879 has64BitLLSC = false;
21880 else if (Subtarget->isThumb())
21881 has64BitLLSC = Subtarget->hasV7Ops();
21882 else
21883 has64BitLLSC = Subtarget->hasV6Ops();
21884
21885 unsigned Size = SI->getValueOperand()->getType()->getPrimitiveSizeInBits();
21886 if (Size == 64 && has64BitLLSC)
21887 return AtomicExpansionKind::Expand;
21888
21889 if (Size <= 32 && !hasNativeAtomics(ST: *Subtarget) && !InsertFencesForAtomic &&
21890 isReleaseOrStronger(AO: SI->getOrdering()))
21891 return AtomicExpansionKind::Expand;
21892
21893 return AtomicExpansionKind::None;
21894}
21895
21896// Loads and stores less than 64-bits are already atomic; ones above that
21897// are doomed anyway, so defer to the default libcall and blame the OS when
21898// things go wrong. Cortex M doesn't have ldrexd/strexd though, so don't emit
21899// anything for those.
21900// FIXME: ldrd and strd are atomic if the CPU has LPAE (e.g. A15 has that
21901// guarantee, see DDI0406C ARM architecture reference manual,
21902// sections A8.8.72-74 LDRD)
21903TargetLowering::AtomicExpansionKind
21904ARMTargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const {
21905 bool has64BitLLSC;
21906 if (Subtarget->isMClass())
21907 has64BitLLSC = false;
21908 else if (Subtarget->isThumb())
21909 has64BitLLSC = Subtarget->hasV7Ops();
21910 else
21911 has64BitLLSC = Subtarget->hasV6Ops();
21912
21913 unsigned Size = LI->getType()->getPrimitiveSizeInBits();
21914 if (Size == 64 && has64BitLLSC)
21915 return AtomicExpansionKind::LLOnly;
21916
21917 if (Size <= 32 && !hasNativeAtomics(ST: *Subtarget) && !InsertFencesForAtomic &&
21918 isAcquireOrStronger(AO: LI->getOrdering()))
21919 return AtomicExpansionKind::CmpXChg;
21920
21921 return AtomicExpansionKind::None;
21922}
21923
21924// For the real atomic operations, we have ldrex/strex up to 32 bits,
21925// and up to 64 bits on the non-M profiles
21926TargetLowering::AtomicExpansionKind
21927ARMTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
21928 if (AI->isFloatingPointOperation())
21929 return AtomicExpansionKind::CmpXChg;
21930
21931 unsigned Size = AI->getType()->getPrimitiveSizeInBits();
21932 bool hasAtomicRMW;
21933 if (Subtarget->isMClass())
21934 hasAtomicRMW = Subtarget->hasV8MBaselineOps();
21935 else if (Subtarget->isThumb())
21936 hasAtomicRMW = Subtarget->hasV7Ops();
21937 else
21938 hasAtomicRMW = Subtarget->hasV6Ops();
21939 if (Size <= (Subtarget->isMClass() ? 32U : 64U) && hasAtomicRMW) {
21940 // At -O0, fast-regalloc cannot cope with the live vregs necessary to
21941 // implement atomicrmw without spilling. If the target address is also on
21942 // the stack and close enough to the spill slot, this can lead to a
21943 // situation where the monitor always gets cleared and the atomic operation
21944 // can never succeed. So at -O0 lower this operation to a CAS loop.
21945 if (getTargetMachine().getOptLevel() == CodeGenOptLevel::None)
21946 return AtomicExpansionKind::CmpXChg;
21947 return AtomicExpansionKind::LLSC;
21948 }
21949 return AtomicExpansionKind::None;
21950}
21951
21952// Similar to shouldExpandAtomicRMWInIR, ldrex/strex can be used up to 32
21953// bits, and up to 64 bits on the non-M profiles.
21954TargetLowering::AtomicExpansionKind
21955ARMTargetLowering::shouldExpandAtomicCmpXchgInIR(
21956 const AtomicCmpXchgInst *AI) const {
21957 // At -O0, fast-regalloc cannot cope with the live vregs necessary to
21958 // implement cmpxchg without spilling. If the address being exchanged is also
21959 // on the stack and close enough to the spill slot, this can lead to a
21960 // situation where the monitor always gets cleared and the atomic operation
21961 // can never succeed. So at -O0 we need a late-expanded pseudo-inst instead.
21962 unsigned Size = AI->getOperand(i_nocapture: 1)->getType()->getPrimitiveSizeInBits();
21963 bool HasAtomicCmpXchg;
21964 if (Subtarget->isMClass())
21965 HasAtomicCmpXchg = Subtarget->hasV8MBaselineOps();
21966 else if (Subtarget->isThumb())
21967 HasAtomicCmpXchg = Subtarget->hasV7Ops();
21968 else
21969 HasAtomicCmpXchg = Subtarget->hasV6Ops();
21970 if (getTargetMachine().getOptLevel() != CodeGenOptLevel::None &&
21971 HasAtomicCmpXchg && Size <= (Subtarget->isMClass() ? 32U : 64U))
21972 return AtomicExpansionKind::LLSC;
21973 return AtomicExpansionKind::None;
21974}
21975
21976bool ARMTargetLowering::shouldInsertFencesForAtomic(
21977 const Instruction *I) const {
21978 return InsertFencesForAtomic;
21979}
21980
21981bool ARMTargetLowering::useLoadStackGuardNode(const Module &M) const {
21982 if (Subtarget->isTargetFuchsia())
21983 return TargetLowering::useLoadStackGuardNode(M);
21984 // ROPI/RWPI are not supported currently.
21985 return !Subtarget->isROPI() && !Subtarget->isRWPI();
21986}
21987
21988Value *
21989ARMTargetLowering::getIRStackGuard(IRBuilderBase &IRB,
21990 const LibcallLoweringInfo &Libcalls) const {
21991 // Fuchsia provides a fixed TLS slot for the stack cookie.
21992 // <zircon/tls.h> defines ZX_TLS_STACK_GUARD_OFFSET with this value.
21993 if (Subtarget->isTargetFuchsia()) {
21994 Module *M = IRB.getModule();
21995 Function *ThreadPointerFunc = Intrinsic::getOrInsertDeclaration(
21996 M, id: Intrinsic::thread_pointer, OverloadTys: IRB.getPtrTy());
21997 return IRB.CreateConstGEP1_32(Ty: IRB.getInt8Ty(),
21998 Ptr: IRB.CreateCall(Callee: ThreadPointerFunc), Idx0: -0x10);
21999 }
22000
22001 return TargetLowering::getIRStackGuard(IRB, Libcalls);
22002}
22003
22004void ARMTargetLowering::insertSSPDeclarations(
22005 Module &M, const LibcallLoweringInfo &Libcalls) const {
22006 // MSVC CRT provides functionalities for stack protection.
22007 RTLIB::LibcallImpl SecurityCheckCookieLibcall =
22008 Libcalls.getLibcallImpl(Call: RTLIB::SECURITY_CHECK_COOKIE);
22009
22010 RTLIB::LibcallImpl SecurityCookieVar =
22011 Libcalls.getLibcallImpl(Call: RTLIB::STACK_CHECK_GUARD);
22012 if (SecurityCheckCookieLibcall != RTLIB::Unsupported &&
22013 SecurityCookieVar != RTLIB::Unsupported) {
22014 // MSVC CRT has a global variable holding security cookie.
22015 const RTLIB::RuntimeLibcallsInfo &RTLCI = Libcalls.getRuntimeLibcallsInfo();
22016 RTLCI.getOrInsertImplGlobal(M, LibcallImpl: SecurityCookieVar);
22017
22018 // MSVC CRT has a function to validate security cookie.
22019 FunctionCallee SecurityCheckCookie =
22020 RTLCI.getOrInsertLibcallDecl(M, LibcallImpl: SecurityCheckCookieLibcall);
22021 if (Function *F = dyn_cast<Function>(Val: SecurityCheckCookie.getCallee()))
22022 F->addParamAttr(ArgNo: 0, Kind: Attribute::AttrKind::InReg);
22023 }
22024
22025 TargetLowering::insertSSPDeclarations(M, Libcalls);
22026}
22027
22028bool ARMTargetLowering::canCombineStoreAndExtract(Type *VectorTy, Value *Idx,
22029 unsigned &Cost) const {
22030 // If we do not have NEON, vector types are not natively supported.
22031 if (!Subtarget->hasNEON())
22032 return false;
22033
22034 // Floating point values and vector values map to the same register file.
22035 // Therefore, although we could do a store extract of a vector type, this is
22036 // better to leave at float as we have more freedom in the addressing mode for
22037 // those.
22038 if (VectorTy->isFPOrFPVectorTy())
22039 return false;
22040
22041 // If the index is unknown at compile time, this is very expensive to lower
22042 // and it is not possible to combine the store with the extract.
22043 if (!isa<ConstantInt>(Val: Idx))
22044 return false;
22045
22046 assert(VectorTy->isVectorTy() && "VectorTy is not a vector type");
22047 unsigned BitWidth = VectorTy->getPrimitiveSizeInBits().getFixedValue();
22048 // We can do a store + vector extract on any vector that fits perfectly in a D
22049 // or Q register.
22050 if (BitWidth == 64 || BitWidth == 128) {
22051 Cost = 0;
22052 return true;
22053 }
22054 return false;
22055}
22056
22057bool ARMTargetLowering::canCreateUndefOrPoisonForTargetNode(
22058 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
22059 UndefPoisonKind Kind, bool ConsiderFlags, unsigned Depth) const {
22060 unsigned Opcode = Op.getOpcode();
22061 switch (Opcode) {
22062 case ARMISD::VORRIMM:
22063 case ARMISD::VBICIMM:
22064 return false;
22065 }
22066 return TargetLowering::canCreateUndefOrPoisonForTargetNode(
22067 Op, DemandedElts, DAG, Kind, ConsiderFlags, Depth);
22068}
22069
22070bool ARMTargetLowering::isCheapToSpeculateCttz(Type *Ty) const {
22071 return Subtarget->hasV5TOps() && !Subtarget->isThumb1Only();
22072}
22073
22074bool ARMTargetLowering::isCheapToSpeculateCtlz(Type *Ty) const {
22075 return Subtarget->hasV5TOps() && !Subtarget->isThumb1Only();
22076}
22077
22078bool ARMTargetLowering::isMaskAndCmp0FoldingBeneficial(
22079 const Instruction &AndI) const {
22080 if (!Subtarget->hasV7Ops())
22081 return false;
22082
22083 // Sink the `and` instruction only if the mask would fit into a modified
22084 // immediate operand.
22085 ConstantInt *Mask = dyn_cast<ConstantInt>(Val: AndI.getOperand(i: 1));
22086 if (!Mask || Mask->getValue().getBitWidth() > 32u)
22087 return false;
22088 auto MaskVal = unsigned(Mask->getValue().getZExtValue());
22089 return (Subtarget->isThumb2() ? ARM_AM::getT2SOImmVal(Arg: MaskVal)
22090 : ARM_AM::getSOImmVal(Arg: MaskVal)) != -1;
22091}
22092
22093TargetLowering::ShiftLegalizationStrategy
22094ARMTargetLowering::preferredShiftLegalizationStrategy(
22095 SelectionDAG &DAG, SDNode *N, unsigned ExpansionFactor) const {
22096 if (Subtarget->hasMinSize() && !getTM().getTargetTriple().isOSWindows())
22097 return ShiftLegalizationStrategy::LowerToLibcall;
22098 return TargetLowering::preferredShiftLegalizationStrategy(DAG, N,
22099 ExpansionFactor);
22100}
22101
22102Value *ARMTargetLowering::emitLoadLinked(IRBuilderBase &Builder, Type *ValueTy,
22103 Value *Addr,
22104 AtomicOrdering Ord) const {
22105 Module *M = Builder.getModule();
22106 bool IsAcquire = isAcquireOrStronger(AO: Ord);
22107
22108 // Since i64 isn't legal and intrinsics don't get type-lowered, the ldrexd
22109 // intrinsic must return {i32, i32} and we have to recombine them into a
22110 // single i64 here.
22111 if (ValueTy->getPrimitiveSizeInBits() == 64) {
22112 Intrinsic::ID Int =
22113 IsAcquire ? Intrinsic::arm_ldaexd : Intrinsic::arm_ldrexd;
22114
22115 Value *LoHi =
22116 Builder.CreateIntrinsic(ID: Int, Args: Addr, /*FMFSource=*/nullptr, Name: "lohi");
22117
22118 Value *Lo = Builder.CreateExtractValue(Agg: LoHi, Idxs: 0, Name: "lo");
22119 Value *Hi = Builder.CreateExtractValue(Agg: LoHi, Idxs: 1, Name: "hi");
22120 if (!Subtarget->isLittle())
22121 std::swap (a&: Lo, b&: Hi);
22122 Lo = Builder.CreateZExt(V: Lo, DestTy: ValueTy, Name: "lo64");
22123 Hi = Builder.CreateZExt(V: Hi, DestTy: ValueTy, Name: "hi64");
22124 return Builder.CreateOr(
22125 LHS: Lo, RHS: Builder.CreateShl(LHS: Hi, RHS: ConstantInt::get(Ty: ValueTy, V: 32)), Name: "val64");
22126 }
22127
22128 Type *Tys[] = { Addr->getType() };
22129 Intrinsic::ID Int = IsAcquire ? Intrinsic::arm_ldaex : Intrinsic::arm_ldrex;
22130 CallInst *CI = Builder.CreateIntrinsicWithoutFolding(ID: Int, OverloadTypes: Tys, Args: Addr);
22131
22132 CI->addParamAttr(
22133 ArgNo: 0, Attr: Attribute::get(Context&: M->getContext(), Kind: Attribute::ElementType, Ty: ValueTy));
22134 return Builder.CreateTruncOrBitCast(V: CI, DestTy: ValueTy);
22135}
22136
22137void ARMTargetLowering::emitAtomicCmpXchgNoStoreLLBalance(
22138 IRBuilderBase &Builder) const {
22139 if (!(Subtarget->isThumb() ? Subtarget->hasV7Clrex()
22140 : Subtarget->hasV6KOps()))
22141 return;
22142 Builder.CreateIntrinsic(ID: Intrinsic::arm_clrex, Args: {});
22143}
22144
22145Value *ARMTargetLowering::emitStoreConditional(IRBuilderBase &Builder,
22146 Value *Val, Value *Addr,
22147 AtomicOrdering Ord) const {
22148 Module *M = Builder.getModule();
22149 bool IsRelease = isReleaseOrStronger(AO: Ord);
22150
22151 // Since the intrinsics must have legal type, the i64 intrinsics take two
22152 // parameters: "i32, i32". We must marshal Val into the appropriate form
22153 // before the call.
22154 if (Val->getType()->getPrimitiveSizeInBits() == 64) {
22155 Intrinsic::ID Int =
22156 IsRelease ? Intrinsic::arm_stlexd : Intrinsic::arm_strexd;
22157 Type *Int32Ty = Type::getInt32Ty(C&: M->getContext());
22158
22159 Value *Lo = Builder.CreateTrunc(V: Val, DestTy: Int32Ty, Name: "lo");
22160 Value *Hi = Builder.CreateTrunc(V: Builder.CreateLShr(LHS: Val, RHS: 32), DestTy: Int32Ty, Name: "hi");
22161 if (!Subtarget->isLittle())
22162 std::swap(a&: Lo, b&: Hi);
22163 return Builder.CreateIntrinsic(ID: Int, Args: {Lo, Hi, Addr});
22164 }
22165
22166 Intrinsic::ID Int = IsRelease ? Intrinsic::arm_stlex : Intrinsic::arm_strex;
22167 Type *Tys[] = { Addr->getType() };
22168 Function *Strex = Intrinsic::getOrInsertDeclaration(M, id: Int, OverloadTys: Tys);
22169
22170 CallInst *CI = Builder.CreateCall(
22171 Callee: Strex, Args: {Builder.CreateZExtOrBitCast(
22172 V: Val, DestTy: Strex->getFunctionType()->getParamType(i: 0)),
22173 Addr});
22174 CI->addParamAttr(ArgNo: 1, Attr: Attribute::get(Context&: M->getContext(), Kind: Attribute::ElementType,
22175 Ty: Val->getType()));
22176 return CI;
22177}
22178
22179
22180bool ARMTargetLowering::alignLoopsWithOptSize() const {
22181 return Subtarget->isMClass();
22182}
22183
22184/// A helper function for determining the number of interleaved accesses we
22185/// will generate when lowering accesses of the given type.
22186unsigned
22187ARMTargetLowering::getNumInterleavedAccesses(VectorType *VecTy,
22188 const DataLayout &DL) const {
22189 return (DL.getTypeSizeInBits(Ty: VecTy) + 127) / 128;
22190}
22191
22192bool ARMTargetLowering::isLegalInterleavedAccessType(
22193 unsigned Factor, FixedVectorType *VecTy, Align Alignment,
22194 const DataLayout &DL) const {
22195
22196 unsigned VecSize = DL.getTypeSizeInBits(Ty: VecTy);
22197 unsigned ElSize = DL.getTypeSizeInBits(Ty: VecTy->getElementType());
22198
22199 if (!Subtarget->hasNEON() && !Subtarget->hasMVEIntegerOps())
22200 return false;
22201
22202 // Ensure the vector doesn't have f16 elements. Even though we could do an
22203 // i16 vldN, we can't hold the f16 vectors and will end up converting via
22204 // f32.
22205 if (Subtarget->hasNEON() && VecTy->getElementType()->isHalfTy())
22206 return false;
22207 if (Subtarget->hasMVEIntegerOps() && Factor == 3)
22208 return false;
22209
22210 // Ensure the number of vector elements is greater than 1.
22211 if (VecTy->getNumElements() < 2)
22212 return false;
22213
22214 // Ensure the element type is legal.
22215 if (ElSize != 8 && ElSize != 16 && ElSize != 32)
22216 return false;
22217 // And the alignment if high enough under MVE.
22218 if (Subtarget->hasMVEIntegerOps() && Alignment < ElSize / 8)
22219 return false;
22220
22221 // Ensure the total vector size is 64 or a multiple of 128. Types larger than
22222 // 128 will be split into multiple interleaved accesses.
22223 if (Subtarget->hasNEON() && VecSize == 64)
22224 return true;
22225 return VecSize % 128 == 0;
22226}
22227
22228unsigned ARMTargetLowering::getMaxSupportedInterleaveFactor() const {
22229 if (Subtarget->hasNEON())
22230 return 4;
22231 if (Subtarget->hasMVEIntegerOps())
22232 return MVEMaxSupportedInterleaveFactor;
22233 return TargetLoweringBase::getMaxSupportedInterleaveFactor();
22234}
22235
22236/// Lower an interleaved load into a vldN intrinsic.
22237///
22238/// E.g. Lower an interleaved load (Factor = 2):
22239/// %wide.vec = load <8 x i32>, <8 x i32>* %ptr, align 4
22240/// %v0 = shuffle %wide.vec, undef, <0, 2, 4, 6> ; Extract even elements
22241/// %v1 = shuffle %wide.vec, undef, <1, 3, 5, 7> ; Extract odd elements
22242///
22243/// Into:
22244/// %vld2 = { <4 x i32>, <4 x i32> } call llvm.arm.neon.vld2(%ptr, 4)
22245/// %vec0 = extractelement { <4 x i32>, <4 x i32> } %vld2, i32 0
22246/// %vec1 = extractelement { <4 x i32>, <4 x i32> } %vld2, i32 1
22247bool ARMTargetLowering::lowerInterleavedLoad(
22248 Instruction *Load, Value *Mask, ArrayRef<ShuffleVectorInst *> Shuffles,
22249 ArrayRef<unsigned> Indices, unsigned Factor, const APInt &GapMask) const {
22250 assert(Factor >= 2 && Factor <= getMaxSupportedInterleaveFactor() &&
22251 "Invalid interleave factor");
22252 assert(!Shuffles.empty() && "Empty shufflevector input");
22253 assert(Shuffles.size() == Indices.size() &&
22254 "Unmatched number of shufflevectors and indices");
22255
22256 auto *LI = dyn_cast<LoadInst>(Val: Load);
22257 if (!LI)
22258 return false;
22259 assert(!Mask && GapMask.popcount() == Factor && "Unexpected mask on a load");
22260
22261 auto *VecTy = cast<FixedVectorType>(Val: Shuffles[0]->getType());
22262 Type *EltTy = VecTy->getElementType();
22263
22264 const DataLayout &DL = LI->getDataLayout();
22265 Align Alignment = LI->getAlign();
22266
22267 // Skip if we do not have NEON and skip illegal vector types. We can
22268 // "legalize" wide vector types into multiple interleaved accesses as long as
22269 // the vector types are divisible by 128.
22270 if (!isLegalInterleavedAccessType(Factor, VecTy, Alignment, DL))
22271 return false;
22272
22273 unsigned NumLoads = getNumInterleavedAccesses(VecTy, DL);
22274
22275 // A pointer vector can not be the return type of the ldN intrinsics. Need to
22276 // load integer vectors first and then convert to pointer vectors.
22277 if (EltTy->isPointerTy())
22278 VecTy = FixedVectorType::get(ElementType: DL.getIntPtrType(EltTy), FVTy: VecTy);
22279
22280 IRBuilder<> Builder(LI);
22281 Builder.disableGEPCanonicalization();
22282
22283 // The base address of the load.
22284 Value *BaseAddr = LI->getPointerOperand();
22285
22286 if (NumLoads > 1) {
22287 // If we're going to generate more than one load, reset the sub-vector type
22288 // to something legal.
22289 VecTy = FixedVectorType::get(ElementType: VecTy->getElementType(),
22290 NumElts: VecTy->getNumElements() / NumLoads);
22291 }
22292
22293 assert(isTypeLegal(EVT::getEVT(VecTy)) && "Illegal vldN vector type!");
22294
22295 auto createLoadIntrinsic = [&](Value *BaseAddr) {
22296 if (Subtarget->hasNEON()) {
22297 Type *PtrTy = Builder.getPtrTy(AddrSpace: LI->getPointerAddressSpace());
22298 Type *Tys[] = {VecTy, PtrTy};
22299 static const Intrinsic::ID LoadInts[3] = {Intrinsic::arm_neon_vld2,
22300 Intrinsic::arm_neon_vld3,
22301 Intrinsic::arm_neon_vld4};
22302
22303 SmallVector<Value *, 2> Ops;
22304 Ops.push_back(Elt: BaseAddr);
22305 Ops.push_back(Elt: Builder.getInt32(C: LI->getAlign().value()));
22306
22307 return Builder.CreateIntrinsic(ID: LoadInts[Factor - 2], OverloadTypes: Tys, Args: Ops,
22308 /*FMFSource=*/nullptr, Name: "vldN");
22309 } else {
22310 assert((Factor == 2 || Factor == 4) &&
22311 "expected interleave factor of 2 or 4 for MVE");
22312 Intrinsic::ID LoadInts =
22313 Factor == 2 ? Intrinsic::arm_mve_vld2q : Intrinsic::arm_mve_vld4q;
22314 Type *PtrTy = Builder.getPtrTy(AddrSpace: LI->getPointerAddressSpace());
22315 Type *Tys[] = {VecTy, PtrTy};
22316
22317 SmallVector<Value *, 2> Ops;
22318 Ops.push_back(Elt: BaseAddr);
22319 return Builder.CreateIntrinsic(ID: LoadInts, OverloadTypes: Tys, Args: Ops, /*FMFSource=*/nullptr,
22320 Name: "vldN");
22321 }
22322 };
22323
22324 // Holds sub-vectors extracted from the load intrinsic return values. The
22325 // sub-vectors are associated with the shufflevector instructions they will
22326 // replace.
22327 DenseMap<ShuffleVectorInst *, SmallVector<Value *, 4>> SubVecs;
22328
22329 for (unsigned LoadCount = 0; LoadCount < NumLoads; ++LoadCount) {
22330 // If we're generating more than one load, compute the base address of
22331 // subsequent loads as an offset from the previous.
22332 if (LoadCount > 0)
22333 BaseAddr = Builder.CreateConstGEP1_32(Ty: VecTy->getElementType(), Ptr: BaseAddr,
22334 Idx0: VecTy->getNumElements() * Factor);
22335
22336 Value *VldN = createLoadIntrinsic(BaseAddr);
22337
22338 // Replace uses of each shufflevector with the corresponding vector loaded
22339 // by ldN.
22340 for (unsigned i = 0; i < Shuffles.size(); i++) {
22341 ShuffleVectorInst *SV = Shuffles[i];
22342 unsigned Index = Indices[i];
22343
22344 Value *SubVec = Builder.CreateExtractValue(Agg: VldN, Idxs: Index);
22345
22346 // Convert the integer vector to pointer vector if the element is pointer.
22347 if (EltTy->isPointerTy())
22348 SubVec = Builder.CreateIntToPtr(
22349 V: SubVec,
22350 DestTy: FixedVectorType::get(ElementType: SV->getType()->getElementType(), FVTy: VecTy));
22351
22352 SubVecs[SV].push_back(Elt: SubVec);
22353 }
22354 }
22355
22356 // Replace uses of the shufflevector instructions with the sub-vectors
22357 // returned by the load intrinsic. If a shufflevector instruction is
22358 // associated with more than one sub-vector, those sub-vectors will be
22359 // concatenated into a single wide vector.
22360 for (ShuffleVectorInst *SVI : Shuffles) {
22361 auto &SubVec = SubVecs[SVI];
22362 auto *WideVec =
22363 SubVec.size() > 1 ? concatenateVectors(Builder, Vecs: SubVec) : SubVec[0];
22364 SVI->replaceAllUsesWith(V: WideVec);
22365 }
22366
22367 return true;
22368}
22369
22370/// Lower an interleaved store into a vstN intrinsic.
22371///
22372/// E.g. Lower an interleaved store (Factor = 3):
22373/// %i.vec = shuffle <8 x i32> %v0, <8 x i32> %v1,
22374/// <0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11>
22375/// store <12 x i32> %i.vec, <12 x i32>* %ptr, align 4
22376///
22377/// Into:
22378/// %sub.v0 = shuffle <8 x i32> %v0, <8 x i32> v1, <0, 1, 2, 3>
22379/// %sub.v1 = shuffle <8 x i32> %v0, <8 x i32> v1, <4, 5, 6, 7>
22380/// %sub.v2 = shuffle <8 x i32> %v0, <8 x i32> v1, <8, 9, 10, 11>
22381/// call void llvm.arm.neon.vst3(%ptr, %sub.v0, %sub.v1, %sub.v2, 4)
22382///
22383/// Note that the new shufflevectors will be removed and we'll only generate one
22384/// vst3 instruction in CodeGen.
22385///
22386/// Example for a more general valid mask (Factor 3). Lower:
22387/// %i.vec = shuffle <32 x i32> %v0, <32 x i32> %v1,
22388/// <4, 32, 16, 5, 33, 17, 6, 34, 18, 7, 35, 19>
22389/// store <12 x i32> %i.vec, <12 x i32>* %ptr
22390///
22391/// Into:
22392/// %sub.v0 = shuffle <32 x i32> %v0, <32 x i32> v1, <4, 5, 6, 7>
22393/// %sub.v1 = shuffle <32 x i32> %v0, <32 x i32> v1, <32, 33, 34, 35>
22394/// %sub.v2 = shuffle <32 x i32> %v0, <32 x i32> v1, <16, 17, 18, 19>
22395/// call void llvm.arm.neon.vst3(%ptr, %sub.v0, %sub.v1, %sub.v2, 4)
22396bool ARMTargetLowering::lowerInterleavedStore(Instruction *Store,
22397 Value *LaneMask,
22398 ShuffleVectorInst *SVI,
22399 unsigned Factor,
22400 const APInt &GapMask) const {
22401 assert(Factor >= 2 && Factor <= getMaxSupportedInterleaveFactor() &&
22402 "Invalid interleave factor");
22403 auto *SI = dyn_cast<StoreInst>(Val: Store);
22404 if (!SI)
22405 return false;
22406 assert(!LaneMask && GapMask.popcount() == Factor &&
22407 "Unexpected mask on store");
22408
22409 auto *VecTy = cast<FixedVectorType>(Val: SVI->getType());
22410 assert(VecTy->getNumElements() % Factor == 0 && "Invalid interleaved store");
22411
22412 unsigned LaneLen = VecTy->getNumElements() / Factor;
22413 Type *EltTy = VecTy->getElementType();
22414 auto *SubVecTy = FixedVectorType::get(ElementType: EltTy, NumElts: LaneLen);
22415
22416 const DataLayout &DL = SI->getDataLayout();
22417 Align Alignment = SI->getAlign();
22418
22419 // Skip if we do not have NEON and skip illegal vector types. We can
22420 // "legalize" wide vector types into multiple interleaved accesses as long as
22421 // the vector types are divisible by 128.
22422 if (!isLegalInterleavedAccessType(Factor, VecTy: SubVecTy, Alignment, DL))
22423 return false;
22424
22425 unsigned NumStores = getNumInterleavedAccesses(VecTy: SubVecTy, DL);
22426
22427 Value *Op0 = SVI->getOperand(i_nocapture: 0);
22428 Value *Op1 = SVI->getOperand(i_nocapture: 1);
22429 IRBuilder<> Builder(SI);
22430 Builder.disableGEPCanonicalization();
22431
22432 // StN intrinsics don't support pointer vectors as arguments. Convert pointer
22433 // vectors to integer vectors.
22434 if (EltTy->isPointerTy()) {
22435 Type *IntTy = DL.getIntPtrType(EltTy);
22436
22437 // Convert to the corresponding integer vector.
22438 auto *IntVecTy =
22439 FixedVectorType::get(ElementType: IntTy, FVTy: cast<FixedVectorType>(Val: Op0->getType()));
22440 Op0 = Builder.CreatePtrToInt(V: Op0, DestTy: IntVecTy);
22441 Op1 = Builder.CreatePtrToInt(V: Op1, DestTy: IntVecTy);
22442
22443 SubVecTy = FixedVectorType::get(ElementType: IntTy, NumElts: LaneLen);
22444 }
22445
22446 // The base address of the store.
22447 Value *BaseAddr = SI->getPointerOperand();
22448
22449 if (NumStores > 1) {
22450 // If we're going to generate more than one store, reset the lane length
22451 // and sub-vector type to something legal.
22452 LaneLen /= NumStores;
22453 SubVecTy = FixedVectorType::get(ElementType: SubVecTy->getElementType(), NumElts: LaneLen);
22454 }
22455
22456 assert(isTypeLegal(EVT::getEVT(SubVecTy)) && "Illegal vstN vector type!");
22457
22458 auto Mask = SVI->getShuffleMask();
22459
22460 auto createStoreIntrinsic = [&](Value *BaseAddr,
22461 SmallVectorImpl<Value *> &Shuffles) {
22462 if (Subtarget->hasNEON()) {
22463 static const Intrinsic::ID StoreInts[3] = {Intrinsic::arm_neon_vst2,
22464 Intrinsic::arm_neon_vst3,
22465 Intrinsic::arm_neon_vst4};
22466 Type *PtrTy = Builder.getPtrTy(AddrSpace: SI->getPointerAddressSpace());
22467 Type *Tys[] = {PtrTy, SubVecTy};
22468
22469 SmallVector<Value *, 6> Ops;
22470 Ops.push_back(Elt: BaseAddr);
22471 append_range(C&: Ops, R&: Shuffles);
22472 Ops.push_back(Elt: Builder.getInt32(C: SI->getAlign().value()));
22473 Builder.CreateIntrinsic(ID: StoreInts[Factor - 2], OverloadTypes: Tys, Args: Ops);
22474 } else {
22475 assert((Factor == 2 || Factor == 4) &&
22476 "expected interleave factor of 2 or 4 for MVE");
22477 Intrinsic::ID StoreInts =
22478 Factor == 2 ? Intrinsic::arm_mve_vst2q : Intrinsic::arm_mve_vst4q;
22479 Type *PtrTy = Builder.getPtrTy(AddrSpace: SI->getPointerAddressSpace());
22480 Type *Tys[] = {PtrTy, SubVecTy};
22481
22482 SmallVector<Value *, 6> Ops;
22483 Ops.push_back(Elt: BaseAddr);
22484 append_range(C&: Ops, R&: Shuffles);
22485 for (unsigned F = 0; F < Factor; F++) {
22486 Ops.push_back(Elt: Builder.getInt32(C: F));
22487 Builder.CreateIntrinsic(ID: StoreInts, OverloadTypes: Tys, Args: Ops);
22488 Ops.pop_back();
22489 }
22490 }
22491 };
22492
22493 for (unsigned StoreCount = 0; StoreCount < NumStores; ++StoreCount) {
22494 // If we generating more than one store, we compute the base address of
22495 // subsequent stores as an offset from the previous.
22496 if (StoreCount > 0)
22497 BaseAddr = Builder.CreateConstGEP1_32(Ty: SubVecTy->getElementType(),
22498 Ptr: BaseAddr, Idx0: LaneLen * Factor);
22499
22500 SmallVector<Value *, 4> Shuffles;
22501
22502 // Split the shufflevector operands into sub vectors for the new vstN call.
22503 for (unsigned i = 0; i < Factor; i++) {
22504 unsigned IdxI = StoreCount * LaneLen * Factor + i;
22505 if (Mask[IdxI] >= 0) {
22506 Shuffles.push_back(Elt: Builder.CreateShuffleVector(
22507 V1: Op0, V2: Op1, Mask: createSequentialMask(Start: Mask[IdxI], NumInts: LaneLen, NumUndefs: 0)));
22508 } else {
22509 unsigned StartMask = 0;
22510 for (unsigned j = 1; j < LaneLen; j++) {
22511 unsigned IdxJ = StoreCount * LaneLen * Factor + j;
22512 if (Mask[IdxJ * Factor + IdxI] >= 0) {
22513 StartMask = Mask[IdxJ * Factor + IdxI] - IdxJ;
22514 break;
22515 }
22516 }
22517 // Note: If all elements in a chunk are undefs, StartMask=0!
22518 // Note: Filling undef gaps with random elements is ok, since
22519 // those elements were being written anyway (with undefs).
22520 // In the case of all undefs we're defaulting to using elems from 0
22521 // Note: StartMask cannot be negative, it's checked in
22522 // isReInterleaveMask
22523 Shuffles.push_back(Elt: Builder.CreateShuffleVector(
22524 V1: Op0, V2: Op1, Mask: createSequentialMask(Start: StartMask, NumInts: LaneLen, NumUndefs: 0)));
22525 }
22526 }
22527
22528 createStoreIntrinsic(BaseAddr, Shuffles);
22529 }
22530 return true;
22531}
22532
22533enum HABaseType {
22534 HA_UNKNOWN = 0,
22535 HA_FLOAT,
22536 HA_DOUBLE,
22537 HA_VECT64,
22538 HA_VECT128
22539};
22540
22541static bool isHomogeneousAggregate(Type *Ty, HABaseType &Base,
22542 uint64_t &Members) {
22543 if (auto *ST = dyn_cast<StructType>(Val: Ty)) {
22544 for (unsigned i = 0; i < ST->getNumElements(); ++i) {
22545 uint64_t SubMembers = 0;
22546 if (!isHomogeneousAggregate(Ty: ST->getElementType(N: i), Base, Members&: SubMembers))
22547 return false;
22548 Members += SubMembers;
22549 }
22550 } else if (auto *AT = dyn_cast<ArrayType>(Val: Ty)) {
22551 uint64_t SubMembers = 0;
22552 if (!isHomogeneousAggregate(Ty: AT->getElementType(), Base, Members&: SubMembers))
22553 return false;
22554 Members += SubMembers * AT->getNumElements();
22555 } else if (Ty->isFloatTy()) {
22556 if (Base != HA_UNKNOWN && Base != HA_FLOAT)
22557 return false;
22558 Members = 1;
22559 Base = HA_FLOAT;
22560 } else if (Ty->isDoubleTy()) {
22561 if (Base != HA_UNKNOWN && Base != HA_DOUBLE)
22562 return false;
22563 Members = 1;
22564 Base = HA_DOUBLE;
22565 } else if (auto *VT = dyn_cast<VectorType>(Val: Ty)) {
22566 Members = 1;
22567 switch (Base) {
22568 case HA_FLOAT:
22569 case HA_DOUBLE:
22570 return false;
22571 case HA_VECT64:
22572 return VT->getPrimitiveSizeInBits().getFixedValue() == 64;
22573 case HA_VECT128:
22574 return VT->getPrimitiveSizeInBits().getFixedValue() == 128;
22575 case HA_UNKNOWN:
22576 switch (VT->getPrimitiveSizeInBits().getFixedValue()) {
22577 case 64:
22578 Base = HA_VECT64;
22579 return true;
22580 case 128:
22581 Base = HA_VECT128;
22582 return true;
22583 default:
22584 return false;
22585 }
22586 }
22587 }
22588
22589 return (Members > 0 && Members <= 4);
22590}
22591
22592/// Return the correct alignment for the current calling convention.
22593Align ARMTargetLowering::getABIAlignmentForCallingConv(
22594 Type *ArgTy, const DataLayout &DL) const {
22595 const Align ABITypeAlign = DL.getABITypeAlign(Ty: ArgTy);
22596 if (!ArgTy->isVectorTy())
22597 return ABITypeAlign;
22598
22599 // Avoid over-aligning vector parameters. It would require realigning the
22600 // stack and waste space for no real benefit.
22601 MaybeAlign StackAlign = DL.getStackAlignment();
22602 assert(StackAlign && "data layout string is missing stack alignment");
22603 return std::min(a: ABITypeAlign, b: *StackAlign);
22604}
22605
22606/// Return true if a type is an AAPCS-VFP homogeneous aggregate or one of
22607/// [N x i32] or [N x i64]. This allows front-ends to skip emitting padding when
22608/// passing according to AAPCS rules.
22609bool ARMTargetLowering::functionArgumentNeedsConsecutiveRegisters(
22610 Type *Ty, CallingConv::ID CallConv, bool isVarArg,
22611 const DataLayout &DL) const {
22612 if (getEffectiveCallingConv(CC: CallConv, isVarArg) !=
22613 CallingConv::ARM_AAPCS_VFP)
22614 return false;
22615
22616 HABaseType Base = HA_UNKNOWN;
22617 uint64_t Members = 0;
22618 bool IsHA = isHomogeneousAggregate(Ty, Base, Members);
22619 LLVM_DEBUG(dbgs() << "isHA: " << IsHA << " "; Ty->dump());
22620
22621 bool IsIntArray = Ty->isArrayTy() && Ty->getArrayElementType()->isIntegerTy();
22622 return IsHA || IsIntArray;
22623}
22624
22625Register ARMTargetLowering::getExceptionPointerRegister(
22626 ExceptionHandling EH, const Constant *PersonalityFn) const {
22627 // Platforms which do not use SjLj EH may return values in these registers
22628 // via the personality function.
22629 return EH == ExceptionHandling::SjLj ? Register() : ARM::R0;
22630}
22631
22632Register ARMTargetLowering::getExceptionSelectorRegister(
22633 ExceptionHandling EH, const Constant *PersonalityFn) const {
22634 // Platforms which do not use SjLj EH may return values in these registers
22635 // via the personality function.
22636 return EH == ExceptionHandling::SjLj ? Register() : ARM::R1;
22637}
22638
22639void ARMTargetLowering::initializeSplitCSR(MachineBasicBlock *Entry) const {
22640 // Update IsSplitCSR in ARMFunctionInfo.
22641 ARMFunctionInfo *AFI = Entry->getParent()->getInfo<ARMFunctionInfo>();
22642 AFI->setIsSplitCSR(true);
22643}
22644
22645void ARMTargetLowering::insertCopiesSplitCSR(
22646 MachineBasicBlock *Entry,
22647 const SmallVectorImpl<MachineBasicBlock *> &Exits) const {
22648 const ARMBaseRegisterInfo *TRI = Subtarget->getRegisterInfo();
22649 const MCPhysReg *IStart = TRI->getCalleeSavedRegsViaCopy(MF: Entry->getParent());
22650 if (!IStart)
22651 return;
22652
22653 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
22654 MachineRegisterInfo *MRI = &Entry->getParent()->getRegInfo();
22655 MachineBasicBlock::iterator MBBI = Entry->begin();
22656 for (const MCPhysReg *I = IStart; *I; ++I) {
22657 const TargetRegisterClass *RC = nullptr;
22658 if (ARM::GPRRegClass.contains(Reg: *I))
22659 RC = &ARM::GPRRegClass;
22660 else if (ARM::DPRRegClass.contains(Reg: *I))
22661 RC = &ARM::DPRRegClass;
22662 else
22663 llvm_unreachable("Unexpected register class in CSRsViaCopy!");
22664
22665 Register NewVR = MRI->createVirtualRegister(RegClass: RC);
22666 // Create copy from CSR to a virtual register.
22667 // FIXME: this currently does not emit CFI pseudo-instructions, it works
22668 // fine for CXX_FAST_TLS since the C++-style TLS access functions should be
22669 // nounwind. If we want to generalize this later, we may need to emit
22670 // CFI pseudo-instructions.
22671 assert(Entry->getParent()->getFunction().hasFnAttribute(
22672 Attribute::NoUnwind) &&
22673 "Function should be nounwind in insertCopiesSplitCSR!");
22674 Entry->addLiveIn(PhysReg: *I);
22675 BuildMI(BB&: *Entry, I: MBBI, MIMD: DebugLoc(), MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: NewVR)
22676 .addReg(RegNo: *I);
22677
22678 // Insert the copy-back instructions right before the terminator.
22679 for (auto *Exit : Exits)
22680 BuildMI(BB&: *Exit, I: Exit->getFirstTerminator(), MIMD: DebugLoc(),
22681 MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: *I)
22682 .addReg(RegNo: NewVR);
22683 }
22684}
22685
22686void ARMTargetLowering::finalizeLowering(MachineFunction &MF) const {
22687 MF.getFrameInfo().computeMaxCallFrameSize(MF);
22688 TargetLoweringBase::finalizeLowering(MF);
22689}
22690
22691bool ARMTargetLowering::isComplexDeinterleavingSupported() const {
22692 return Subtarget->hasMVEIntegerOps();
22693}
22694
22695bool ARMTargetLowering::isComplexDeinterleavingOperationSupported(
22696 ComplexDeinterleavingOperation Operation, Type *Ty) const {
22697 auto *VTy = dyn_cast<FixedVectorType>(Val: Ty);
22698 if (!VTy)
22699 return false;
22700
22701 auto *ScalarTy = VTy->getScalarType();
22702 unsigned NumElements = VTy->getNumElements();
22703
22704 unsigned VTyWidth = VTy->getScalarSizeInBits() * NumElements;
22705 if (VTyWidth < 128 || !llvm::isPowerOf2_32(Value: VTyWidth))
22706 return false;
22707
22708 // Both VCADD and VCMUL/VCMLA support the same types, F16 and F32
22709 if (ScalarTy->isHalfTy() || ScalarTy->isFloatTy())
22710 return Subtarget->hasMVEFloatOps();
22711
22712 if (Operation != ComplexDeinterleavingOperation::CAdd)
22713 return false;
22714
22715 return Subtarget->hasMVEIntegerOps() &&
22716 (ScalarTy->isIntegerTy(BitWidth: 8) || ScalarTy->isIntegerTy(BitWidth: 16) ||
22717 ScalarTy->isIntegerTy(BitWidth: 32));
22718}
22719
22720ArrayRef<MCPhysReg> ARMTargetLowering::getRoundingControlRegisters() const {
22721 static const MCPhysReg RCRegs[] = {ARM::FPSCR_RM};
22722 return RCRegs;
22723}
22724
22725Value *ARMTargetLowering::createComplexDeinterleavingIR(
22726 IRBuilderBase &B, ComplexDeinterleavingOperation OperationType,
22727 ComplexDeinterleavingRotation Rotation, Value *InputA, Value *InputB,
22728 Value *Accumulator) const {
22729
22730 FixedVectorType *Ty = cast<FixedVectorType>(Val: InputA->getType());
22731
22732 unsigned TyWidth = Ty->getScalarSizeInBits() * Ty->getNumElements();
22733
22734 assert(TyWidth >= 128 && "Width of vector type must be at least 128 bits");
22735
22736 if (TyWidth > 128) {
22737 int Stride = Ty->getNumElements() / 2;
22738 auto SplitSeq = llvm::seq<int>(Begin: 0, End: Ty->getNumElements());
22739 auto SplitSeqVec = llvm::to_vector(Range&: SplitSeq);
22740 ArrayRef<int> LowerSplitMask(&SplitSeqVec[0], Stride);
22741 ArrayRef<int> UpperSplitMask(&SplitSeqVec[Stride], Stride);
22742
22743 auto *LowerSplitA = B.CreateShuffleVector(V: InputA, Mask: LowerSplitMask);
22744 auto *LowerSplitB = B.CreateShuffleVector(V: InputB, Mask: LowerSplitMask);
22745 auto *UpperSplitA = B.CreateShuffleVector(V: InputA, Mask: UpperSplitMask);
22746 auto *UpperSplitB = B.CreateShuffleVector(V: InputB, Mask: UpperSplitMask);
22747 Value *LowerSplitAcc = nullptr;
22748 Value *UpperSplitAcc = nullptr;
22749
22750 if (Accumulator) {
22751 LowerSplitAcc = B.CreateShuffleVector(V: Accumulator, Mask: LowerSplitMask);
22752 UpperSplitAcc = B.CreateShuffleVector(V: Accumulator, Mask: UpperSplitMask);
22753 }
22754
22755 auto *LowerSplitInt = createComplexDeinterleavingIR(
22756 B, OperationType, Rotation, InputA: LowerSplitA, InputB: LowerSplitB, Accumulator: LowerSplitAcc);
22757 auto *UpperSplitInt = createComplexDeinterleavingIR(
22758 B, OperationType, Rotation, InputA: UpperSplitA, InputB: UpperSplitB, Accumulator: UpperSplitAcc);
22759
22760 ArrayRef<int> JoinMask(&SplitSeqVec[0], Ty->getNumElements());
22761 return B.CreateShuffleVector(V1: LowerSplitInt, V2: UpperSplitInt, Mask: JoinMask);
22762 }
22763
22764 auto *IntTy = Type::getInt32Ty(C&: B.getContext());
22765
22766 ConstantInt *ConstRotation = nullptr;
22767 if (OperationType == ComplexDeinterleavingOperation::CMulPartial) {
22768 ConstRotation = ConstantInt::get(Ty: IntTy, V: (int)Rotation);
22769
22770 if (Accumulator)
22771 return B.CreateIntrinsic(ID: Intrinsic::arm_mve_vcmlaq, OverloadTypes: Ty,
22772 Args: {ConstRotation, Accumulator, InputB, InputA});
22773 return B.CreateIntrinsic(ID: Intrinsic::arm_mve_vcmulq, OverloadTypes: Ty,
22774 Args: {ConstRotation, InputB, InputA});
22775 }
22776
22777 if (OperationType == ComplexDeinterleavingOperation::CAdd) {
22778 // 1 means the value is not halved.
22779 auto *ConstHalving = ConstantInt::get(Ty: IntTy, V: 1);
22780
22781 if (Rotation == ComplexDeinterleavingRotation::Rotation_90)
22782 ConstRotation = ConstantInt::get(Ty: IntTy, V: 0);
22783 else if (Rotation == ComplexDeinterleavingRotation::Rotation_270)
22784 ConstRotation = ConstantInt::get(Ty: IntTy, V: 1);
22785
22786 if (!ConstRotation)
22787 return nullptr; // Invalid rotation for arm_mve_vcaddq
22788
22789 return B.CreateIntrinsic(ID: Intrinsic::arm_mve_vcaddq, OverloadTypes: Ty,
22790 Args: {ConstHalving, ConstRotation, InputA, InputB});
22791 }
22792
22793 return nullptr;
22794}
22795