1//===- ARMISelLowering.cpp - ARM DAG Lowering Implementation --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the interfaces that ARM uses to lower LLVM code into a
10// selection DAG.
11//
12//===----------------------------------------------------------------------===//
13
14#include "ARMISelLowering.h"
15#include "ARMBaseInstrInfo.h"
16#include "ARMBaseRegisterInfo.h"
17#include "ARMCallingConv.h"
18#include "ARMConstantPoolValue.h"
19#include "ARMMachineFunctionInfo.h"
20#include "ARMPerfectShuffle.h"
21#include "ARMRegisterInfo.h"
22#include "ARMSelectionDAGInfo.h"
23#include "ARMSubtarget.h"
24#include "ARMTargetTransformInfo.h"
25#include "MCTargetDesc/ARMAddressingModes.h"
26#include "MCTargetDesc/ARMBaseInfo.h"
27#include "Utils/ARMBaseInfo.h"
28#include "llvm/ADT/APFloat.h"
29#include "llvm/ADT/APInt.h"
30#include "llvm/ADT/ArrayRef.h"
31#include "llvm/ADT/BitVector.h"
32#include "llvm/ADT/DenseMap.h"
33#include "llvm/ADT/STLExtras.h"
34#include "llvm/ADT/SmallPtrSet.h"
35#include "llvm/ADT/SmallVector.h"
36#include "llvm/ADT/Statistic.h"
37#include "llvm/ADT/StringExtras.h"
38#include "llvm/ADT/StringRef.h"
39#include "llvm/ADT/StringSwitch.h"
40#include "llvm/ADT/Twine.h"
41#include "llvm/Analysis/VectorUtils.h"
42#include "llvm/CodeGen/CallingConvLower.h"
43#include "llvm/CodeGen/ComplexDeinterleavingPass.h"
44#include "llvm/CodeGen/ISDOpcodes.h"
45#include "llvm/CodeGen/MachineBasicBlock.h"
46#include "llvm/CodeGen/MachineConstantPool.h"
47#include "llvm/CodeGen/MachineFrameInfo.h"
48#include "llvm/CodeGen/MachineFunction.h"
49#include "llvm/CodeGen/MachineInstr.h"
50#include "llvm/CodeGen/MachineInstrBuilder.h"
51#include "llvm/CodeGen/MachineJumpTableInfo.h"
52#include "llvm/CodeGen/MachineMemOperand.h"
53#include "llvm/CodeGen/MachineOperand.h"
54#include "llvm/CodeGen/MachineRegisterInfo.h"
55#include "llvm/CodeGen/RuntimeLibcallUtil.h"
56#include "llvm/CodeGen/SelectionDAG.h"
57#include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
58#include "llvm/CodeGen/SelectionDAGNodes.h"
59#include "llvm/CodeGen/TargetInstrInfo.h"
60#include "llvm/CodeGen/TargetLowering.h"
61#include "llvm/CodeGen/TargetOpcodes.h"
62#include "llvm/CodeGen/TargetRegisterInfo.h"
63#include "llvm/CodeGen/TargetSubtargetInfo.h"
64#include "llvm/CodeGen/ValueTypes.h"
65#include "llvm/CodeGenTypes/MachineValueType.h"
66#include "llvm/IR/Attributes.h"
67#include "llvm/IR/CallingConv.h"
68#include "llvm/IR/Constant.h"
69#include "llvm/IR/Constants.h"
70#include "llvm/IR/DataLayout.h"
71#include "llvm/IR/DebugLoc.h"
72#include "llvm/IR/DerivedTypes.h"
73#include "llvm/IR/Function.h"
74#include "llvm/IR/GlobalAlias.h"
75#include "llvm/IR/GlobalValue.h"
76#include "llvm/IR/GlobalVariable.h"
77#include "llvm/IR/IRBuilder.h"
78#include "llvm/IR/InlineAsm.h"
79#include "llvm/IR/Instruction.h"
80#include "llvm/IR/Instructions.h"
81#include "llvm/IR/IntrinsicInst.h"
82#include "llvm/IR/Intrinsics.h"
83#include "llvm/IR/IntrinsicsARM.h"
84#include "llvm/IR/Module.h"
85#include "llvm/IR/Type.h"
86#include "llvm/IR/User.h"
87#include "llvm/IR/Value.h"
88#include "llvm/MC/MCInstrDesc.h"
89#include "llvm/MC/MCInstrItineraries.h"
90#include "llvm/MC/MCSchedule.h"
91#include "llvm/Support/AtomicOrdering.h"
92#include "llvm/Support/BranchProbability.h"
93#include "llvm/Support/Casting.h"
94#include "llvm/Support/CodeGen.h"
95#include "llvm/Support/CommandLine.h"
96#include "llvm/Support/Compiler.h"
97#include "llvm/Support/Debug.h"
98#include "llvm/Support/ErrorHandling.h"
99#include "llvm/Support/KnownBits.h"
100#include "llvm/Support/MathExtras.h"
101#include "llvm/Support/raw_ostream.h"
102#include "llvm/Target/TargetMachine.h"
103#include "llvm/Target/TargetOptions.h"
104#include "llvm/TargetParser/Triple.h"
105#include <algorithm>
106#include <cassert>
107#include <cstdint>
108#include <cstdlib>
109#include <iterator>
110#include <limits>
111#include <optional>
112#include <tuple>
113#include <utility>
114#include <vector>
115
116using namespace llvm;
117
118#define DEBUG_TYPE "arm-isel"
119
120STATISTIC(NumTailCalls, "Number of tail calls");
121STATISTIC(NumOptimizedImms, "Number of times immediates were optimized");
122STATISTIC(NumMovwMovt, "Number of GAs materialized with movw + movt");
123STATISTIC(NumLoopByVals, "Number of loops generated for byval arguments");
124STATISTIC(NumConstpoolPromoted,
125 "Number of constants with their storage promoted into constant pools");
126
127static cl::opt<bool>
128ARMInterworking("arm-interworking", cl::Hidden,
129 cl::desc("Enable / disable ARM interworking (for debugging only)"),
130 cl::init(Val: true));
131
132static cl::opt<bool> EnableConstpoolPromotion(
133 "arm-promote-constant", cl::Hidden,
134 cl::desc("Enable / disable promotion of unnamed_addr constants into "
135 "constant pools"),
136 cl::init(Val: false)); // FIXME: set to true by default once PR32780 is fixed
137static cl::opt<unsigned> ConstpoolPromotionMaxSize(
138 "arm-promote-constant-max-size", cl::Hidden,
139 cl::desc("Maximum size of constant to promote into a constant pool"),
140 cl::init(Val: 64));
141static cl::opt<unsigned> ConstpoolPromotionMaxTotal(
142 "arm-promote-constant-max-total", cl::Hidden,
143 cl::desc("Maximum size of ALL constants to promote into a constant pool"),
144 cl::init(Val: 128));
145
146cl::opt<unsigned>
147MVEMaxSupportedInterleaveFactor("mve-max-interleave-factor", cl::Hidden,
148 cl::desc("Maximum interleave factor for MVE VLDn to generate."),
149 cl::init(Val: 2));
150
151cl::opt<unsigned> ArmMaxBaseUpdatesToCheck(
152 "arm-max-base-updates-to-check", cl::Hidden,
153 cl::desc("Maximum number of base-updates to check generating postindex."),
154 cl::init(Val: 64));
155
156/// Value type used for "flags" operands / results (either CPSR or FPSCR_NZCV).
157constexpr MVT FlagsVT = MVT::i32;
158
159// The APCS parameter registers.
160static const MCPhysReg GPRArgRegs[] = {
161 ARM::R0, ARM::R1, ARM::R2, ARM::R3
162};
163
164static SDValue handleCMSEValue(const SDValue &Value, const ISD::InputArg &Arg,
165 SelectionDAG &DAG, const SDLoc &DL) {
166 assert(Arg.ArgVT.isScalarInteger());
167 assert(Arg.ArgVT.bitsLT(MVT::i32));
168 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: Arg.ArgVT, Operand: Value);
169 SDValue Ext =
170 DAG.getNode(Opcode: Arg.Flags.isSExt() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL,
171 VT: MVT::i32, Operand: Trunc);
172 return Ext;
173}
174
175void ARMTargetLowering::addTypeForNEON(MVT VT, MVT PromotedLdStVT) {
176 if (VT != PromotedLdStVT) {
177 setOperationAction(Op: ISD::LOAD, VT, Action: Promote);
178 AddPromotedToType (Opc: ISD::LOAD, OrigVT: VT, DestVT: PromotedLdStVT);
179
180 setOperationAction(Op: ISD::STORE, VT, Action: Promote);
181 AddPromotedToType (Opc: ISD::STORE, OrigVT: VT, DestVT: PromotedLdStVT);
182 }
183
184 MVT ElemTy = VT.getVectorElementType();
185 if (ElemTy != MVT::f64)
186 setOperationAction(Op: ISD::SETCC, VT, Action: Custom);
187 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT, Action: Custom);
188 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT, Action: Custom);
189 if (ElemTy == MVT::i32) {
190 setOperationAction(Op: ISD::SINT_TO_FP, VT, Action: Custom);
191 setOperationAction(Op: ISD::UINT_TO_FP, VT, Action: Custom);
192 setOperationAction(Op: ISD::FP_TO_SINT, VT, Action: Custom);
193 setOperationAction(Op: ISD::FP_TO_UINT, VT, Action: Custom);
194 } else {
195 setOperationAction(Op: ISD::SINT_TO_FP, VT, Action: Expand);
196 setOperationAction(Op: ISD::UINT_TO_FP, VT, Action: Expand);
197 setOperationAction(Op: ISD::FP_TO_SINT, VT, Action: Expand);
198 setOperationAction(Op: ISD::FP_TO_UINT, VT, Action: Expand);
199 }
200 setOperationAction(Op: ISD::BUILD_VECTOR, VT, Action: Custom);
201 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT, Action: Custom);
202 setOperationAction(Op: ISD::CONCAT_VECTORS, VT, Action: Legal);
203 setOperationAction(Op: ISD::EXTRACT_SUBVECTOR, VT, Action: Legal);
204 setOperationAction(Op: ISD::SELECT, VT, Action: Expand);
205 setOperationAction(Op: ISD::SELECT_CC, VT, Action: Expand);
206 setOperationAction(Op: ISD::VSELECT, VT, Action: Expand);
207 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT, Action: Expand);
208 if (VT.isInteger()) {
209 setOperationAction(Op: ISD::SHL, VT, Action: Custom);
210 setOperationAction(Op: ISD::SRA, VT, Action: Custom);
211 setOperationAction(Op: ISD::SRL, VT, Action: Custom);
212 }
213
214 // Neon does not support vector divide/remainder operations.
215 setOperationAction(Op: ISD::SDIV, VT, Action: Expand);
216 setOperationAction(Op: ISD::UDIV, VT, Action: Expand);
217 setOperationAction(Op: ISD::FDIV, VT, Action: Expand);
218 setOperationAction(Op: ISD::SREM, VT, Action: Expand);
219 setOperationAction(Op: ISD::UREM, VT, Action: Expand);
220 setOperationAction(Op: ISD::FREM, VT, Action: Expand);
221 setOperationAction(Op: ISD::SDIVREM, VT, Action: Expand);
222 setOperationAction(Op: ISD::UDIVREM, VT, Action: Expand);
223
224 if (!VT.isFloatingPoint() && VT != MVT::v2i64 && VT != MVT::v1i64)
225 for (auto Opcode : {ISD::ABS, ISD::ABDS, ISD::ABDU, ISD::SMIN, ISD::SMAX,
226 ISD::UMIN, ISD::UMAX, ISD::CTLS})
227 setOperationAction(Op: Opcode, VT, Action: Legal);
228 if (!VT.isFloatingPoint())
229 for (auto Opcode : {ISD::SADDSAT, ISD::UADDSAT, ISD::SSUBSAT, ISD::USUBSAT})
230 setOperationAction(Op: Opcode, VT, Action: Legal);
231}
232
233void ARMTargetLowering::addDRTypeForNEON(MVT VT) {
234 addRegisterClass(VT, RC: &ARM::DPRRegClass);
235 addTypeForNEON(VT, PromotedLdStVT: MVT::f64);
236}
237
238void ARMTargetLowering::addQRTypeForNEON(MVT VT) {
239 addRegisterClass(VT, RC: &ARM::DPairRegClass);
240 addTypeForNEON(VT, PromotedLdStVT: MVT::v2f64);
241}
242
243void ARMTargetLowering::setAllExpand(MVT VT) {
244 for (unsigned Opc = 0; Opc < ISD::BUILTIN_OP_END; ++Opc)
245 setOperationAction(Op: Opc, VT, Action: Expand);
246
247 // We support these really simple operations even on types where all
248 // the actual arithmetic has to be broken down into simpler
249 // operations or turned into library calls.
250 setOperationAction(Op: ISD::BITCAST, VT, Action: Legal);
251 setOperationAction(Op: ISD::LOAD, VT, Action: Legal);
252 setOperationAction(Op: ISD::STORE, VT, Action: Legal);
253 setOperationAction(Ops: {ISD::UNDEF, ISD::POISON}, VT, Action: Legal);
254}
255
256void ARMTargetLowering::addAllExtLoads(const MVT From, const MVT To,
257 LegalizeAction Action) {
258 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: From, MemVT: To, Action);
259 setLoadExtAction(ExtType: ISD::ZEXTLOAD, ValVT: From, MemVT: To, Action);
260 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: From, MemVT: To, Action);
261}
262
263void ARMTargetLowering::addMVEVectorTypes(bool HasMVEFP) {
264 const MVT IntTypes[] = { MVT::v16i8, MVT::v8i16, MVT::v4i32 };
265
266 for (auto VT : IntTypes) {
267 addRegisterClass(VT, RC: &ARM::MQPRRegClass);
268 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT, Action: Custom);
269 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT, Action: Custom);
270 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT, Action: Custom);
271 setOperationAction(Op: ISD::BUILD_VECTOR, VT, Action: Custom);
272 setOperationAction(Op: ISD::SHL, VT, Action: Custom);
273 setOperationAction(Op: ISD::SRA, VT, Action: Custom);
274 setOperationAction(Op: ISD::SRL, VT, Action: Custom);
275 setOperationAction(Op: ISD::SMIN, VT, Action: Legal);
276 setOperationAction(Op: ISD::SMAX, VT, Action: Legal);
277 setOperationAction(Op: ISD::UMIN, VT, Action: Legal);
278 setOperationAction(Op: ISD::UMAX, VT, Action: Legal);
279 setOperationAction(Op: ISD::ABS, VT, Action: Legal);
280 setOperationAction(Op: ISD::CTLS, VT, Action: Legal);
281 setOperationAction(Op: ISD::SETCC, VT, Action: Custom);
282 setOperationAction(Op: ISD::MLOAD, VT, Action: Custom);
283 setOperationAction(Op: ISD::MSTORE, VT, Action: Legal);
284 setOperationAction(Op: ISD::CTLZ, VT, Action: Legal);
285 setOperationAction(Op: ISD::CTTZ, VT, Action: Custom);
286 setOperationAction(Op: ISD::BITREVERSE, VT, Action: Legal);
287 setOperationAction(Op: ISD::BSWAP, VT, Action: Legal);
288 setOperationAction(Op: ISD::SADDSAT, VT, Action: Legal);
289 setOperationAction(Op: ISD::UADDSAT, VT, Action: Legal);
290 setOperationAction(Op: ISD::SSUBSAT, VT, Action: Legal);
291 setOperationAction(Op: ISD::USUBSAT, VT, Action: Legal);
292 setOperationAction(Op: ISD::ABDS, VT, Action: Legal);
293 setOperationAction(Op: ISD::ABDU, VT, Action: Legal);
294 setOperationAction(Op: ISD::AVGFLOORS, VT, Action: Legal);
295 setOperationAction(Op: ISD::AVGFLOORU, VT, Action: Legal);
296 setOperationAction(Op: ISD::AVGCEILS, VT, Action: Legal);
297 setOperationAction(Op: ISD::AVGCEILU, VT, Action: Legal);
298
299 // No native support for these.
300 setOperationAction(Op: ISD::UDIV, VT, Action: Expand);
301 setOperationAction(Op: ISD::SDIV, VT, Action: Expand);
302 setOperationAction(Op: ISD::UREM, VT, Action: Expand);
303 setOperationAction(Op: ISD::SREM, VT, Action: Expand);
304 setOperationAction(Op: ISD::UDIVREM, VT, Action: Expand);
305 setOperationAction(Op: ISD::SDIVREM, VT, Action: Expand);
306 setOperationAction(Op: ISD::CTPOP, VT, Action: Expand);
307 setOperationAction(Op: ISD::SELECT, VT, Action: Expand);
308 setOperationAction(Op: ISD::SELECT_CC, VT, Action: Expand);
309
310 // Vector reductions
311 setOperationAction(Op: ISD::VECREDUCE_ADD, VT, Action: Legal);
312 setOperationAction(Op: ISD::VECREDUCE_SMAX, VT, Action: Legal);
313 setOperationAction(Op: ISD::VECREDUCE_UMAX, VT, Action: Legal);
314 setOperationAction(Op: ISD::VECREDUCE_SMIN, VT, Action: Legal);
315 setOperationAction(Op: ISD::VECREDUCE_UMIN, VT, Action: Legal);
316 setOperationAction(Op: ISD::VECREDUCE_MUL, VT, Action: Custom);
317 setOperationAction(Op: ISD::VECREDUCE_AND, VT, Action: Custom);
318 setOperationAction(Op: ISD::VECREDUCE_OR, VT, Action: Custom);
319 setOperationAction(Op: ISD::VECREDUCE_XOR, VT, Action: Custom);
320
321 if (!HasMVEFP) {
322 setOperationAction(Op: ISD::SINT_TO_FP, VT, Action: Expand);
323 setOperationAction(Op: ISD::UINT_TO_FP, VT, Action: Expand);
324 setOperationAction(Op: ISD::FP_TO_SINT, VT, Action: Expand);
325 setOperationAction(Op: ISD::FP_TO_UINT, VT, Action: Expand);
326 } else {
327 setOperationAction(Op: ISD::FP_TO_SINT_SAT, VT, Action: Custom);
328 setOperationAction(Op: ISD::FP_TO_UINT_SAT, VT, Action: Custom);
329 }
330
331 // Pre and Post inc are supported on loads and stores
332 for (unsigned im = (unsigned)ISD::PRE_INC;
333 im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) {
334 setIndexedLoadAction(IdxModes: im, VT, Action: Legal);
335 setIndexedStoreAction(IdxModes: im, VT, Action: Legal);
336 setIndexedMaskedLoadAction(IdxMode: im, VT, Action: Legal);
337 setIndexedMaskedStoreAction(IdxMode: im, VT, Action: Legal);
338 }
339 }
340
341 const MVT FloatTypes[] = { MVT::v8f16, MVT::v4f32 };
342 for (auto VT : FloatTypes) {
343 addRegisterClass(VT, RC: &ARM::MQPRRegClass);
344 if (!HasMVEFP)
345 setAllExpand(VT);
346
347 // These are legal or custom whether we have MVE.fp or not
348 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT, Action: Custom);
349 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT, Action: Custom);
350 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: VT.getVectorElementType(), Action: Custom);
351 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT, Action: Custom);
352 setOperationAction(Op: ISD::BUILD_VECTOR, VT, Action: Custom);
353 setOperationAction(Op: ISD::BUILD_VECTOR, VT: VT.getVectorElementType(), Action: Custom);
354 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT, Action: Legal);
355 setOperationAction(Op: ISD::SETCC, VT, Action: Custom);
356 setOperationAction(Op: ISD::MLOAD, VT, Action: Custom);
357 setOperationAction(Op: ISD::MSTORE, VT, Action: Legal);
358 setOperationAction(Op: ISD::SELECT, VT, Action: Expand);
359 setOperationAction(Op: ISD::SELECT_CC, VT, Action: Expand);
360
361 // Pre and Post inc are supported on loads and stores
362 for (unsigned im = (unsigned)ISD::PRE_INC;
363 im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) {
364 setIndexedLoadAction(IdxModes: im, VT, Action: Legal);
365 setIndexedStoreAction(IdxModes: im, VT, Action: Legal);
366 setIndexedMaskedLoadAction(IdxMode: im, VT, Action: Legal);
367 setIndexedMaskedStoreAction(IdxMode: im, VT, Action: Legal);
368 }
369
370 if (HasMVEFP) {
371 setOperationAction(Op: ISD::FMINNUM, VT, Action: Legal);
372 setOperationAction(Op: ISD::FMAXNUM, VT, Action: Legal);
373 for (auto Op : {ISD::FROUND, ISD::STRICT_FROUND, ISD::FROUNDEVEN,
374 ISD::STRICT_FROUNDEVEN, ISD::FTRUNC, ISD::STRICT_FTRUNC,
375 ISD::FRINT, ISD::STRICT_FRINT, ISD::FFLOOR,
376 ISD::STRICT_FFLOOR, ISD::FCEIL, ISD::STRICT_FCEIL}) {
377 setOperationAction(Op, VT, Action: Legal);
378 }
379 setOperationAction(Op: ISD::VECREDUCE_FADD, VT, Action: Custom);
380 setOperationAction(Op: ISD::VECREDUCE_FMUL, VT, Action: Custom);
381 setOperationAction(Op: ISD::VECREDUCE_FMIN, VT, Action: Custom);
382 setOperationAction(Op: ISD::VECREDUCE_FMAX, VT, Action: Custom);
383
384 // No native support for these.
385 setOperationAction(Op: ISD::FDIV, VT, Action: Expand);
386 setOperationAction(Op: ISD::FREM, VT, Action: Expand);
387 setOperationAction(Op: ISD::FSQRT, VT, Action: Expand);
388 setOperationAction(Op: ISD::FSIN, VT, Action: Expand);
389 setOperationAction(Op: ISD::FCOS, VT, Action: Expand);
390 setOperationAction(Op: ISD::FTAN, VT, Action: Expand);
391 setOperationAction(Op: ISD::FPOW, VT, Action: Expand);
392 setOperationAction(Op: ISD::FLOG, VT, Action: Expand);
393 setOperationAction(Op: ISD::FLOG2, VT, Action: Expand);
394 setOperationAction(Op: ISD::FLOG10, VT, Action: Expand);
395 setOperationAction(Op: ISD::FEXP, VT, Action: Expand);
396 setOperationAction(Op: ISD::FEXP2, VT, Action: Expand);
397 setOperationAction(Op: ISD::FEXP10, VT, Action: Expand);
398 setOperationAction(Op: ISD::FNEARBYINT, VT, Action: Expand);
399 }
400 }
401
402 // Custom Expand smaller than legal vector reductions to prevent false zero
403 // items being added.
404 setOperationAction(Op: ISD::VECREDUCE_FADD, VT: MVT::v4f16, Action: Custom);
405 setOperationAction(Op: ISD::VECREDUCE_FMUL, VT: MVT::v4f16, Action: Custom);
406 setOperationAction(Op: ISD::VECREDUCE_FMIN, VT: MVT::v4f16, Action: Custom);
407 setOperationAction(Op: ISD::VECREDUCE_FMAX, VT: MVT::v4f16, Action: Custom);
408 setOperationAction(Op: ISD::VECREDUCE_FADD, VT: MVT::v2f16, Action: Custom);
409 setOperationAction(Op: ISD::VECREDUCE_FMUL, VT: MVT::v2f16, Action: Custom);
410 setOperationAction(Op: ISD::VECREDUCE_FMIN, VT: MVT::v2f16, Action: Custom);
411 setOperationAction(Op: ISD::VECREDUCE_FMAX, VT: MVT::v2f16, Action: Custom);
412
413 // We 'support' these types up to bitcast/load/store level, regardless of
414 // MVE integer-only / float support. Only doing FP data processing on the FP
415 // vector types is inhibited at integer-only level.
416 const MVT LongTypes[] = { MVT::v2i64, MVT::v2f64 };
417 for (auto VT : LongTypes) {
418 addRegisterClass(VT, RC: &ARM::MQPRRegClass);
419 setAllExpand(VT);
420 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT, Action: Custom);
421 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT, Action: Custom);
422 setOperationAction(Op: ISD::BUILD_VECTOR, VT, Action: Custom);
423 setOperationAction(Op: ISD::VSELECT, VT, Action: Legal);
424 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT, Action: Custom);
425 }
426 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v2f64, Action: Legal);
427
428 // We can do bitwise operations on v2i64 vectors
429 setOperationAction(Op: ISD::AND, VT: MVT::v2i64, Action: Legal);
430 setOperationAction(Op: ISD::OR, VT: MVT::v2i64, Action: Legal);
431 setOperationAction(Op: ISD::XOR, VT: MVT::v2i64, Action: Legal);
432
433 // It is legal to extload from v4i8 to v4i16 or v4i32.
434 addAllExtLoads(From: MVT::v8i16, To: MVT::v8i8, Action: Legal);
435 addAllExtLoads(From: MVT::v4i32, To: MVT::v4i16, Action: Legal);
436 addAllExtLoads(From: MVT::v4i32, To: MVT::v4i8, Action: Legal);
437
438 // It is legal to sign extend from v4i8/v4i16 to v4i32 or v8i8 to v8i16.
439 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::v4i8, Action: Legal);
440 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::v4i16, Action: Legal);
441 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::v4i32, Action: Legal);
442 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::v8i8, Action: Legal);
443 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::v8i16, Action: Legal);
444
445 // Some truncating stores are legal too.
446 setTruncStoreAction(ValVT: MVT::v4i32, MemVT: MVT::v4i16, Action: Legal);
447 setTruncStoreAction(ValVT: MVT::v4i32, MemVT: MVT::v4i8, Action: Legal);
448 setTruncStoreAction(ValVT: MVT::v8i16, MemVT: MVT::v8i8, Action: Legal);
449
450 // Pre and Post inc on these are legal, given the correct extends
451 for (unsigned im = (unsigned)ISD::PRE_INC;
452 im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) {
453 for (auto VT : {MVT::v8i8, MVT::v4i8, MVT::v4i16}) {
454 setIndexedLoadAction(IdxModes: im, VT, Action: Legal);
455 setIndexedStoreAction(IdxModes: im, VT, Action: Legal);
456 setIndexedMaskedLoadAction(IdxMode: im, VT, Action: Legal);
457 setIndexedMaskedStoreAction(IdxMode: im, VT, Action: Legal);
458 }
459 }
460
461 // Predicate types
462 const MVT pTypes[] = {MVT::v16i1, MVT::v8i1, MVT::v4i1, MVT::v2i1};
463 for (auto VT : pTypes) {
464 addRegisterClass(VT, RC: &ARM::VCCRRegClass);
465 setOperationAction(Op: ISD::BUILD_VECTOR, VT, Action: Custom);
466 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT, Action: Custom);
467 setOperationAction(Op: ISD::EXTRACT_SUBVECTOR, VT, Action: Custom);
468 setOperationAction(Op: ISD::CONCAT_VECTORS, VT, Action: Custom);
469 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT, Action: Custom);
470 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT, Action: Custom);
471 setOperationAction(Op: ISD::SETCC, VT, Action: Custom);
472 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT, Action: Expand);
473 setOperationAction(Op: ISD::LOAD, VT, Action: Custom);
474 setOperationAction(Op: ISD::STORE, VT, Action: Custom);
475 setOperationAction(Op: ISD::TRUNCATE, VT, Action: Custom);
476 setOperationAction(Op: ISD::VSELECT, VT, Action: Expand);
477 setOperationAction(Op: ISD::SELECT, VT, Action: Expand);
478 setOperationAction(Op: ISD::SELECT_CC, VT, Action: Expand);
479
480 if (!HasMVEFP) {
481 setOperationAction(Op: ISD::SINT_TO_FP, VT, Action: Expand);
482 setOperationAction(Op: ISD::UINT_TO_FP, VT, Action: Expand);
483 setOperationAction(Op: ISD::FP_TO_SINT, VT, Action: Expand);
484 setOperationAction(Op: ISD::FP_TO_UINT, VT, Action: Expand);
485 }
486 }
487 setOperationAction(Op: ISD::SETCC, VT: MVT::v2i1, Action: Expand);
488 setOperationAction(Op: ISD::TRUNCATE, VT: MVT::v2i1, Action: Expand);
489 setOperationAction(Op: ISD::AND, VT: MVT::v2i1, Action: Expand);
490 setOperationAction(Op: ISD::OR, VT: MVT::v2i1, Action: Expand);
491 setOperationAction(Op: ISD::XOR, VT: MVT::v2i1, Action: Expand);
492 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v2i1, Action: Expand);
493 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v2i1, Action: Expand);
494 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::v2i1, Action: Expand);
495 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::v2i1, Action: Expand);
496
497 setOperationAction(Op: ISD::SIGN_EXTEND, VT: MVT::v8i32, Action: Custom);
498 setOperationAction(Op: ISD::SIGN_EXTEND, VT: MVT::v16i16, Action: Custom);
499 setOperationAction(Op: ISD::SIGN_EXTEND, VT: MVT::v16i32, Action: Custom);
500 setOperationAction(Op: ISD::ZERO_EXTEND, VT: MVT::v8i32, Action: Custom);
501 setOperationAction(Op: ISD::ZERO_EXTEND, VT: MVT::v16i16, Action: Custom);
502 setOperationAction(Op: ISD::ZERO_EXTEND, VT: MVT::v16i32, Action: Custom);
503 setOperationAction(Op: ISD::TRUNCATE, VT: MVT::v8i32, Action: Custom);
504 setOperationAction(Op: ISD::TRUNCATE, VT: MVT::v16i16, Action: Custom);
505}
506
507const ARMBaseTargetMachine &ARMTargetLowering::getTM() const {
508 return static_cast<const ARMBaseTargetMachine &>(getTargetMachine());
509}
510
511ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM_,
512 const ARMSubtarget &STI)
513 : TargetLowering(TM_, STI), Subtarget(&STI),
514 RegInfo(Subtarget->getRegisterInfo()),
515 Itins(Subtarget->getInstrItineraryData()) {
516 const auto &TM = static_cast<const ARMBaseTargetMachine &>(TM_);
517
518 setBooleanContents(ZeroOrOneBooleanContent);
519 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
520
521 const Triple &TT = TM.getTargetTriple();
522
523 if (Subtarget->isThumb1Only())
524 addRegisterClass(VT: MVT::i32, RC: &ARM::tGPRRegClass);
525 else
526 addRegisterClass(VT: MVT::i32, RC: &ARM::GPRRegClass);
527
528 if (!Subtarget->useSoftFloat() && !Subtarget->isThumb1Only() &&
529 Subtarget->hasFPRegs()) {
530 addRegisterClass(VT: MVT::f32, RC: &ARM::SPRRegClass);
531 addRegisterClass(VT: MVT::f64, RC: &ARM::DPRRegClass);
532
533 if (!Subtarget->hasVFP2Base()) {
534 setAllExpand(MVT::f32);
535 } else {
536 setOperationAction(Op: ISD::FP_TO_SINT_SAT, VT: MVT::i32, Action: Custom);
537 setOperationAction(Op: ISD::FP_TO_UINT_SAT, VT: MVT::i32, Action: Custom);
538
539 for (auto Op : {ISD::STRICT_FADD, ISD::STRICT_FSUB, ISD::STRICT_FMUL,
540 ISD::STRICT_FDIV, ISD::STRICT_FMA, ISD::STRICT_FSQRT})
541 setOperationAction(Op, VT: MVT::f32, Action: Legal);
542 }
543 if (!Subtarget->hasFP64()) {
544 setAllExpand(MVT::f64);
545 } else {
546 for (auto Op : {ISD::STRICT_FADD, ISD::STRICT_FSUB, ISD::STRICT_FMUL,
547 ISD::STRICT_FDIV, ISD::STRICT_FMA, ISD::STRICT_FSQRT})
548 setOperationAction(Op, VT: MVT::f64, Action: Legal);
549
550 setOperationAction(Op: ISD::STRICT_FP_ROUND, VT: MVT::f32, Action: Legal);
551 }
552 }
553
554 if (Subtarget->hasFullFP16()) {
555 for (auto Op : {ISD::STRICT_FADD, ISD::STRICT_FSUB, ISD::STRICT_FMUL,
556 ISD::STRICT_FDIV, ISD::STRICT_FMA, ISD::STRICT_FSQRT})
557 setOperationAction(Op, VT: MVT::f16, Action: Legal);
558
559 addRegisterClass(VT: MVT::f16, RC: &ARM::HPRRegClass);
560 setOperationAction(Op: ISD::BITCAST, VT: MVT::i16, Action: Custom);
561 setOperationAction(Op: ISD::BITCAST, VT: MVT::f16, Action: Custom);
562
563 setOperationAction(Op: ISD::FMINNUM, VT: MVT::f16, Action: Legal);
564 setOperationAction(Op: ISD::FMAXNUM, VT: MVT::f16, Action: Legal);
565 setOperationAction(Op: ISD::STRICT_FMINNUM, VT: MVT::f16, Action: Legal);
566 setOperationAction(Op: ISD::STRICT_FMAXNUM, VT: MVT::f16, Action: Legal);
567 }
568
569 if (Subtarget->hasBF16()) {
570 addRegisterClass(VT: MVT::bf16, RC: &ARM::HPRRegClass);
571 setAllExpand(MVT::bf16);
572 if (!Subtarget->hasFullFP16())
573 setOperationAction(Op: ISD::BITCAST, VT: MVT::bf16, Action: Custom);
574 setOperationAction(Op: ISD::FP_ROUND, VT: MVT::bf16, Action: Custom);
575 setOperationAction(Op: ISD::STRICT_FP_ROUND, VT: MVT::bf16, Action: Custom);
576 } else {
577 setOperationAction(Op: ISD::BF16_TO_FP, VT: MVT::f32, Action: Expand);
578 setOperationAction(Op: ISD::BF16_TO_FP, VT: MVT::f64, Action: Expand);
579 setOperationAction(Op: ISD::FP_TO_BF16, VT: MVT::f32, Action: Custom);
580 setOperationAction(Op: ISD::FP_TO_BF16, VT: MVT::f64, Action: Custom);
581 }
582
583 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
584 for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) {
585 setTruncStoreAction(ValVT: VT, MemVT: InnerVT, Action: Expand);
586 addAllExtLoads(From: VT, To: InnerVT, Action: Expand);
587 }
588
589 setOperationAction(Op: ISD::SMUL_LOHI, VT, Action: Expand);
590 setOperationAction(Op: ISD::UMUL_LOHI, VT, Action: Expand);
591
592 setOperationAction(Op: ISD::BSWAP, VT, Action: Expand);
593 }
594
595 if (!Subtarget->isThumb1Only() && !Subtarget->hasV8_1MMainlineOps())
596 setOperationAction(Op: ISD::SCMP, VT: MVT::i32, Action: Custom);
597
598 if (!Subtarget->hasV8_1MMainlineOps())
599 setOperationAction(Op: ISD::UCMP, VT: MVT::i32, Action: Custom);
600
601 if (!Subtarget->isThumb1Only())
602 setOperationAction(Op: ISD::ABS, VT: MVT::i32, Action: Custom);
603
604 setOperationAction(Op: ISD::ConstantFP, VT: MVT::f32, Action: Custom);
605 setOperationAction(Op: ISD::ConstantFP, VT: MVT::f64, Action: Custom);
606
607 setOperationAction(Op: ISD::READ_REGISTER, VT: MVT::i64, Action: Custom);
608 setOperationAction(Op: ISD::WRITE_REGISTER, VT: MVT::i64, Action: Custom);
609
610 if (Subtarget->hasMVEIntegerOps())
611 addMVEVectorTypes(HasMVEFP: Subtarget->hasMVEFloatOps());
612
613 // Combine low-overhead loop intrinsics so that we can lower i1 types.
614 if (Subtarget->hasLOB()) {
615 setTargetDAGCombine({ISD::BRCOND, ISD::BR_CC});
616 }
617
618 if (Subtarget->hasNEON()) {
619 addDRTypeForNEON(VT: MVT::v2f32);
620 addDRTypeForNEON(VT: MVT::v8i8);
621 addDRTypeForNEON(VT: MVT::v4i16);
622 addDRTypeForNEON(VT: MVT::v2i32);
623 addDRTypeForNEON(VT: MVT::v1i64);
624
625 addQRTypeForNEON(VT: MVT::v4f32);
626 addQRTypeForNEON(VT: MVT::v2f64);
627 addQRTypeForNEON(VT: MVT::v16i8);
628 addQRTypeForNEON(VT: MVT::v8i16);
629 addQRTypeForNEON(VT: MVT::v4i32);
630 addQRTypeForNEON(VT: MVT::v2i64);
631
632 if (Subtarget->hasFullFP16()) {
633 addQRTypeForNEON(VT: MVT::v8f16);
634 addDRTypeForNEON(VT: MVT::v4f16);
635 }
636
637 if (Subtarget->hasBF16()) {
638 addQRTypeForNEON(VT: MVT::v8bf16);
639 addDRTypeForNEON(VT: MVT::v4bf16);
640 }
641 }
642
643 if (Subtarget->hasMVEIntegerOps() || Subtarget->hasNEON()) {
644 // v2f64 is legal so that QR subregs can be extracted as f64 elements, but
645 // none of Neon, MVE or VFP supports any arithmetic operations on it.
646 setOperationAction(Op: ISD::FADD, VT: MVT::v2f64, Action: Expand);
647 setOperationAction(Op: ISD::FSUB, VT: MVT::v2f64, Action: Expand);
648 setOperationAction(Op: ISD::FMUL, VT: MVT::v2f64, Action: Expand);
649 // FIXME: Code duplication: FDIV and FREM are expanded always, see
650 // ARMTargetLowering::addTypeForNEON method for details.
651 setOperationAction(Op: ISD::FDIV, VT: MVT::v2f64, Action: Expand);
652 setOperationAction(Op: ISD::FREM, VT: MVT::v2f64, Action: Expand);
653 // FIXME: Create unittest.
654 // In another words, find a way when "copysign" appears in DAG with vector
655 // operands.
656 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::v2f64, Action: Expand);
657 // FIXME: Code duplication: SETCC has custom operation action, see
658 // ARMTargetLowering::addTypeForNEON method for details.
659 setOperationAction(Op: ISD::SETCC, VT: MVT::v2f64, Action: Expand);
660 // FIXME: Create unittest for FNEG and for FABS.
661 setOperationAction(Op: ISD::FNEG, VT: MVT::v2f64, Action: Expand);
662 setOperationAction(Op: ISD::FABS, VT: MVT::v2f64, Action: Expand);
663 setOperationAction(Op: ISD::FSQRT, VT: MVT::v2f64, Action: Expand);
664 setOperationAction(Op: ISD::FSIN, VT: MVT::v2f64, Action: Expand);
665 setOperationAction(Op: ISD::FCOS, VT: MVT::v2f64, Action: Expand);
666 setOperationAction(Op: ISD::FTAN, VT: MVT::v2f64, Action: Expand);
667 setOperationAction(Op: ISD::FPOW, VT: MVT::v2f64, Action: Expand);
668 setOperationAction(Op: ISD::FLOG, VT: MVT::v2f64, Action: Expand);
669 setOperationAction(Op: ISD::FLOG2, VT: MVT::v2f64, Action: Expand);
670 setOperationAction(Op: ISD::FLOG10, VT: MVT::v2f64, Action: Expand);
671 setOperationAction(Op: ISD::FEXP, VT: MVT::v2f64, Action: Expand);
672 setOperationAction(Op: ISD::FEXP2, VT: MVT::v2f64, Action: Expand);
673 setOperationAction(Op: ISD::FEXP10, VT: MVT::v2f64, Action: Expand);
674 setOperationAction(Op: ISD::FCEIL, VT: MVT::v2f64, Action: Expand);
675 setOperationAction(Op: ISD::FTRUNC, VT: MVT::v2f64, Action: Expand);
676 setOperationAction(Op: ISD::FRINT, VT: MVT::v2f64, Action: Expand);
677 setOperationAction(Op: ISD::FROUNDEVEN, VT: MVT::v2f64, Action: Expand);
678 setOperationAction(Op: ISD::FNEARBYINT, VT: MVT::v2f64, Action: Expand);
679 setOperationAction(Op: ISD::FFLOOR, VT: MVT::v2f64, Action: Expand);
680 setOperationAction(Op: ISD::FMA, VT: MVT::v2f64, Action: Expand);
681 }
682
683 if (Subtarget->hasNEON()) {
684 // The same with v4f32. But keep in mind that vadd, vsub, vmul are natively
685 // supported for v4f32.
686 setOperationAction(Op: ISD::FSQRT, VT: MVT::v4f32, Action: Expand);
687 setOperationAction(Op: ISD::FSIN, VT: MVT::v4f32, Action: Expand);
688 setOperationAction(Op: ISD::FCOS, VT: MVT::v4f32, Action: Expand);
689 setOperationAction(Op: ISD::FTAN, VT: MVT::v4f32, Action: Expand);
690 setOperationAction(Op: ISD::FPOW, VT: MVT::v4f32, Action: Expand);
691 setOperationAction(Op: ISD::FLOG, VT: MVT::v4f32, Action: Expand);
692 setOperationAction(Op: ISD::FLOG2, VT: MVT::v4f32, Action: Expand);
693 setOperationAction(Op: ISD::FLOG10, VT: MVT::v4f32, Action: Expand);
694 setOperationAction(Op: ISD::FEXP, VT: MVT::v4f32, Action: Expand);
695 setOperationAction(Op: ISD::FEXP2, VT: MVT::v4f32, Action: Expand);
696 setOperationAction(Op: ISD::FEXP10, VT: MVT::v4f32, Action: Expand);
697 setOperationAction(Op: ISD::FCEIL, VT: MVT::v4f32, Action: Expand);
698 setOperationAction(Op: ISD::FTRUNC, VT: MVT::v4f32, Action: Expand);
699 setOperationAction(Op: ISD::FRINT, VT: MVT::v4f32, Action: Expand);
700 setOperationAction(Op: ISD::FROUNDEVEN, VT: MVT::v4f32, Action: Expand);
701 setOperationAction(Op: ISD::FNEARBYINT, VT: MVT::v4f32, Action: Expand);
702 setOperationAction(Op: ISD::FFLOOR, VT: MVT::v4f32, Action: Expand);
703
704 // Mark v2f32 intrinsics.
705 setOperationAction(Op: ISD::FSQRT, VT: MVT::v2f32, Action: Expand);
706 setOperationAction(Op: ISD::FSIN, VT: MVT::v2f32, Action: Expand);
707 setOperationAction(Op: ISD::FCOS, VT: MVT::v2f32, Action: Expand);
708 setOperationAction(Op: ISD::FTAN, VT: MVT::v2f32, Action: Expand);
709 setOperationAction(Op: ISD::FPOW, VT: MVT::v2f32, Action: Expand);
710 setOperationAction(Op: ISD::FLOG, VT: MVT::v2f32, Action: Expand);
711 setOperationAction(Op: ISD::FLOG2, VT: MVT::v2f32, Action: Expand);
712 setOperationAction(Op: ISD::FLOG10, VT: MVT::v2f32, Action: Expand);
713 setOperationAction(Op: ISD::FEXP, VT: MVT::v2f32, Action: Expand);
714 setOperationAction(Op: ISD::FEXP2, VT: MVT::v2f32, Action: Expand);
715 setOperationAction(Op: ISD::FEXP10, VT: MVT::v2f32, Action: Expand);
716 setOperationAction(Op: ISD::FCEIL, VT: MVT::v2f32, Action: Expand);
717 setOperationAction(Op: ISD::FTRUNC, VT: MVT::v2f32, Action: Expand);
718 setOperationAction(Op: ISD::FRINT, VT: MVT::v2f32, Action: Expand);
719 setOperationAction(Op: ISD::FROUNDEVEN, VT: MVT::v2f32, Action: Expand);
720 setOperationAction(Op: ISD::FNEARBYINT, VT: MVT::v2f32, Action: Expand);
721 setOperationAction(Op: ISD::FFLOOR, VT: MVT::v2f32, Action: Expand);
722
723 for (ISD::NodeType Op : {ISD::FFLOOR, ISD::FNEARBYINT, ISD::FCEIL,
724 ISD::FRINT, ISD::FTRUNC, ISD::FROUNDEVEN}) {
725 setOperationAction(Op, VT: MVT::v4f16, Action: Expand);
726 setOperationAction(Op, VT: MVT::v8f16, Action: Expand);
727 }
728
729 // Neon does not support some operations on v1i64 and v2i64 types.
730 setOperationAction(Op: ISD::MUL, VT: MVT::v1i64, Action: Expand);
731 // Custom handling for some quad-vector types to detect VMULL.
732 setOperationAction(Op: ISD::MUL, VT: MVT::v8i16, Action: Custom);
733 setOperationAction(Op: ISD::MUL, VT: MVT::v4i32, Action: Custom);
734 setOperationAction(Op: ISD::MUL, VT: MVT::v2i64, Action: Custom);
735 // Custom handling for some vector types to avoid expensive expansions
736 setOperationAction(Op: ISD::SDIV, VT: MVT::v4i16, Action: Custom);
737 setOperationAction(Op: ISD::SDIV, VT: MVT::v8i8, Action: Custom);
738 setOperationAction(Op: ISD::UDIV, VT: MVT::v4i16, Action: Custom);
739 setOperationAction(Op: ISD::UDIV, VT: MVT::v8i8, Action: Custom);
740 // Neon does not have single instruction SINT_TO_FP and UINT_TO_FP with
741 // a destination type that is wider than the source, and nor does
742 // it have a FP_TO_[SU]INT instruction with a narrower destination than
743 // source.
744 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v4i16, Action: Custom);
745 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v8i16, Action: Custom);
746 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v4i16, Action: Custom);
747 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v8i16, Action: Custom);
748 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::v4i16, Action: Custom);
749 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::v8i16, Action: Custom);
750 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::v4i16, Action: Custom);
751 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::v8i16, Action: Custom);
752
753 setOperationAction(Op: ISD::FP_ROUND, VT: MVT::v2f32, Action: Expand);
754 setOperationAction(Op: ISD::FP_EXTEND, VT: MVT::v2f64, Action: Expand);
755
756 // NEON does not have single instruction CTPOP for vectors with element
757 // types wider than 8-bits. However, custom lowering can leverage the
758 // v8i8/v16i8 vcnt instruction.
759 setOperationAction(Op: ISD::CTPOP, VT: MVT::v2i32, Action: Custom);
760 setOperationAction(Op: ISD::CTPOP, VT: MVT::v4i32, Action: Custom);
761 setOperationAction(Op: ISD::CTPOP, VT: MVT::v4i16, Action: Custom);
762 setOperationAction(Op: ISD::CTPOP, VT: MVT::v8i16, Action: Custom);
763 setOperationAction(Op: ISD::CTPOP, VT: MVT::v1i64, Action: Custom);
764 setOperationAction(Op: ISD::CTPOP, VT: MVT::v2i64, Action: Custom);
765
766 setOperationAction(Op: ISD::CTLZ, VT: MVT::v1i64, Action: Expand);
767 setOperationAction(Op: ISD::CTLZ, VT: MVT::v2i64, Action: Expand);
768
769 // NEON does not have single instruction CTTZ for vectors.
770 setOperationAction(Op: ISD::CTTZ, VT: MVT::v8i8, Action: Custom);
771 setOperationAction(Op: ISD::CTTZ, VT: MVT::v4i16, Action: Custom);
772 setOperationAction(Op: ISD::CTTZ, VT: MVT::v2i32, Action: Custom);
773 setOperationAction(Op: ISD::CTTZ, VT: MVT::v1i64, Action: Custom);
774
775 setOperationAction(Op: ISD::CTTZ, VT: MVT::v16i8, Action: Custom);
776 setOperationAction(Op: ISD::CTTZ, VT: MVT::v8i16, Action: Custom);
777 setOperationAction(Op: ISD::CTTZ, VT: MVT::v4i32, Action: Custom);
778 setOperationAction(Op: ISD::CTTZ, VT: MVT::v2i64, Action: Custom);
779
780 setOperationAction(Op: ISD::CTTZ_ZERO_POISON, VT: MVT::v8i8, Action: Custom);
781 setOperationAction(Op: ISD::CTTZ_ZERO_POISON, VT: MVT::v4i16, Action: Custom);
782 setOperationAction(Op: ISD::CTTZ_ZERO_POISON, VT: MVT::v2i32, Action: Custom);
783 setOperationAction(Op: ISD::CTTZ_ZERO_POISON, VT: MVT::v1i64, Action: Custom);
784
785 setOperationAction(Op: ISD::CTTZ_ZERO_POISON, VT: MVT::v16i8, Action: Custom);
786 setOperationAction(Op: ISD::CTTZ_ZERO_POISON, VT: MVT::v8i16, Action: Custom);
787 setOperationAction(Op: ISD::CTTZ_ZERO_POISON, VT: MVT::v4i32, Action: Custom);
788 setOperationAction(Op: ISD::CTTZ_ZERO_POISON, VT: MVT::v2i64, Action: Custom);
789
790 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
791 setOperationAction(Op: ISD::MULHS, VT, Action: Expand);
792 setOperationAction(Op: ISD::MULHU, VT, Action: Expand);
793 }
794
795 // NEON only has FMA instructions as of VFP4.
796 if (!Subtarget->hasVFP4Base()) {
797 setOperationAction(Op: ISD::FMA, VT: MVT::v2f32, Action: Expand);
798 setOperationAction(Op: ISD::FMA, VT: MVT::v4f32, Action: Expand);
799 }
800
801 setTargetDAGCombine({ISD::SHL, ISD::SRL, ISD::SRA, ISD::FP_TO_SINT,
802 ISD::FP_TO_UINT, ISD::FMUL, ISD::LOAD});
803
804 // It is legal to extload from v4i8 to v4i16 or v4i32.
805 for (MVT Ty : {MVT::v8i8, MVT::v4i8, MVT::v2i8, MVT::v4i16, MVT::v2i16,
806 MVT::v2i32}) {
807 for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) {
808 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: Ty, Action: Legal);
809 setLoadExtAction(ExtType: ISD::ZEXTLOAD, ValVT: VT, MemVT: Ty, Action: Legal);
810 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: VT, MemVT: Ty, Action: Legal);
811 }
812 }
813
814 for (auto VT : {MVT::v8i8, MVT::v4i16, MVT::v2i32, MVT::v16i8, MVT::v8i16,
815 MVT::v4i32}) {
816 setOperationAction(Op: ISD::VECREDUCE_SMAX, VT, Action: Custom);
817 setOperationAction(Op: ISD::VECREDUCE_UMAX, VT, Action: Custom);
818 setOperationAction(Op: ISD::VECREDUCE_SMIN, VT, Action: Custom);
819 setOperationAction(Op: ISD::VECREDUCE_UMIN, VT, Action: Custom);
820 }
821 }
822
823 if (Subtarget->hasNEON() || Subtarget->hasMVEIntegerOps()) {
824 setTargetDAGCombine(
825 {ISD::BUILD_VECTOR, ISD::VECTOR_SHUFFLE, ISD::INSERT_SUBVECTOR,
826 ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT,
827 ISD::SIGN_EXTEND_INREG, ISD::STORE, ISD::SIGN_EXTEND, ISD::ZERO_EXTEND,
828 ISD::ANY_EXTEND, ISD::INTRINSIC_WO_CHAIN, ISD::INTRINSIC_W_CHAIN,
829 ISD::INTRINSIC_VOID, ISD::VECREDUCE_ADD, ISD::ADD, ISD::BITCAST});
830 }
831 if (Subtarget->hasMVEIntegerOps()) {
832 setTargetDAGCombine({ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX,
833 ISD::FP_EXTEND, ISD::SELECT, ISD::SELECT_CC,
834 ISD::SETCC});
835 }
836 if (Subtarget->hasMVEFloatOps()) {
837 setTargetDAGCombine(ISD::FADD);
838 }
839
840 if (!Subtarget->hasFP64()) {
841 // When targeting a floating-point unit with only single-precision
842 // operations, f64 is legal for the few double-precision instructions which
843 // are present However, no double-precision operations other than moves,
844 // loads and stores are provided by the hardware.
845 setOperationAction(Op: ISD::FADD, VT: MVT::f64, Action: Expand);
846 setOperationAction(Op: ISD::FSUB, VT: MVT::f64, Action: Expand);
847 setOperationAction(Op: ISD::FMUL, VT: MVT::f64, Action: Expand);
848 setOperationAction(Op: ISD::FMA, VT: MVT::f64, Action: Expand);
849 setOperationAction(Op: ISD::FDIV, VT: MVT::f64, Action: Expand);
850 setOperationAction(Op: ISD::FREM, VT: MVT::f64, Action: LibCall);
851 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::f64, Action: Expand);
852 setOperationAction(Op: ISD::FGETSIGN, VT: MVT::f64, Action: Expand);
853 setOperationAction(Op: ISD::FNEG, VT: MVT::f64, Action: Expand);
854 setOperationAction(Op: ISD::FABS, VT: MVT::f64, Action: Expand);
855 setOperationAction(Op: ISD::FSQRT, VT: MVT::f64, Action: Expand);
856 setOperationAction(Op: ISD::FSIN, VT: MVT::f64, Action: Expand);
857 setOperationAction(Op: ISD::FCOS, VT: MVT::f64, Action: Expand);
858 setOperationAction(Op: ISD::FPOW, VT: MVT::f64, Action: Expand);
859 setOperationAction(Op: ISD::FLOG, VT: MVT::f64, Action: Expand);
860 setOperationAction(Op: ISD::FLOG2, VT: MVT::f64, Action: Expand);
861 setOperationAction(Op: ISD::FLOG10, VT: MVT::f64, Action: Expand);
862 setOperationAction(Op: ISD::FEXP, VT: MVT::f64, Action: Expand);
863 setOperationAction(Op: ISD::FEXP2, VT: MVT::f64, Action: Expand);
864 setOperationAction(Op: ISD::FEXP10, VT: MVT::f64, Action: Expand);
865 setOperationAction(Op: ISD::FCEIL, VT: MVT::f64, Action: Expand);
866 setOperationAction(Op: ISD::FTRUNC, VT: MVT::f64, Action: Expand);
867 setOperationAction(Op: ISD::FRINT, VT: MVT::f64, Action: Expand);
868 setOperationAction(Op: ISD::FROUNDEVEN, VT: MVT::f64, Action: Expand);
869 setOperationAction(Op: ISD::FNEARBYINT, VT: MVT::f64, Action: Expand);
870 setOperationAction(Op: ISD::FFLOOR, VT: MVT::f64, Action: Expand);
871 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i32, Action: Custom);
872 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::i32, Action: Custom);
873 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::i32, Action: Custom);
874 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i32, Action: Custom);
875 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::f64, Action: Custom);
876 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::f64, Action: Custom);
877 setOperationAction(Op: ISD::FP_ROUND, VT: MVT::f32, Action: Custom);
878 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::f64, Action: Custom);
879 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::f64, Action: Custom);
880 setOperationAction(Op: ISD::STRICT_FP_ROUND, VT: MVT::f32, Action: Custom);
881 }
882
883 // STRICT_(U/S)INT_TO_FP specifically use the input MVT to register with
884 // setOperationAction() as opposed to other opcodes that use the output MVT
885 // All inputs should be i32 due to type legalization
886 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::i32, Action: Custom);
887 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::i32, Action: Custom);
888
889 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::i32, Action: Custom);
890 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::i32, Action: Custom);
891
892 if (!Subtarget->hasFP64() || !Subtarget->hasFPARMv8Base()) {
893 setOperationAction(Op: ISD::FP_EXTEND, VT: MVT::f64, Action: Custom);
894 setOperationAction(Op: ISD::STRICT_FP_EXTEND, VT: MVT::f64, Action: Custom);
895 if (Subtarget->hasFullFP16()) {
896 setOperationAction(Op: ISD::FP_ROUND, VT: MVT::f16, Action: Custom);
897 setOperationAction(Op: ISD::STRICT_FP_ROUND, VT: MVT::f16, Action: Custom);
898 }
899 } else {
900 setOperationAction(Op: ISD::STRICT_FP_EXTEND, VT: MVT::f64, Action: Legal);
901 }
902
903 if (!Subtarget->hasFP16()) {
904 setOperationAction(Op: ISD::FP_EXTEND, VT: MVT::f32, Action: Custom);
905 setOperationAction(Op: ISD::STRICT_FP_EXTEND, VT: MVT::f32, Action: Custom);
906 } else {
907 setOperationAction(Op: ISD::STRICT_FP_EXTEND, VT: MVT::f32, Action: Legal);
908 setOperationAction(Op: ISD::STRICT_FP_ROUND, VT: MVT::f16, Action: Legal);
909 }
910
911 computeRegisterProperties(TRI: Subtarget->getRegisterInfo());
912
913 // ARM does not have floating-point extending loads.
914 for (MVT VT : MVT::fp_valuetypes()) {
915 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: MVT::f32, Action: Expand);
916 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: MVT::f16, Action: Expand);
917 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: MVT::bf16, Action: Expand);
918 }
919
920 // ... or truncating stores
921 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f32, Action: Expand);
922 setTruncStoreAction(ValVT: MVT::f32, MemVT: MVT::f16, Action: Expand);
923 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f16, Action: Expand);
924 setTruncStoreAction(ValVT: MVT::f32, MemVT: MVT::bf16, Action: Expand);
925 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::bf16, Action: Expand);
926
927 // ARM does not have i1 sign extending load.
928 for (MVT VT : MVT::integer_valuetypes())
929 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: VT, MemVT: MVT::i1, Action: Promote);
930
931 // ARM supports all 4 flavors of integer indexed load / store.
932 if (!Subtarget->isThumb1Only()) {
933 for (unsigned im = (unsigned)ISD::PRE_INC;
934 im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) {
935 setIndexedLoadAction(IdxModes: im, VT: MVT::i1, Action: Legal);
936 setIndexedLoadAction(IdxModes: im, VT: MVT::i8, Action: Legal);
937 setIndexedLoadAction(IdxModes: im, VT: MVT::i16, Action: Legal);
938 setIndexedLoadAction(IdxModes: im, VT: MVT::i32, Action: Legal);
939 setIndexedStoreAction(IdxModes: im, VT: MVT::i1, Action: Legal);
940 setIndexedStoreAction(IdxModes: im, VT: MVT::i8, Action: Legal);
941 setIndexedStoreAction(IdxModes: im, VT: MVT::i16, Action: Legal);
942 setIndexedStoreAction(IdxModes: im, VT: MVT::i32, Action: Legal);
943 }
944 } else {
945 // Thumb-1 has limited post-inc load/store support - LDM r0!, {r1}.
946 setIndexedLoadAction(IdxModes: ISD::POST_INC, VT: MVT::i32, Action: Legal);
947 setIndexedStoreAction(IdxModes: ISD::POST_INC, VT: MVT::i32, Action: Legal);
948 }
949
950 // Custom loads/stores to possible use __aeabi_uread/write*
951 if (TT.isTargetAEABI() && !Subtarget->allowsUnalignedMem()) {
952 setOperationAction(Op: ISD::STORE, VT: MVT::i32, Action: Custom);
953 setOperationAction(Op: ISD::STORE, VT: MVT::i64, Action: Custom);
954 setOperationAction(Op: ISD::LOAD, VT: MVT::i32, Action: Custom);
955 setOperationAction(Op: ISD::LOAD, VT: MVT::i64, Action: Custom);
956 }
957
958 setOperationAction(Op: ISD::SADDO, VT: MVT::i32, Action: Custom);
959 setOperationAction(Op: ISD::UADDO, VT: MVT::i32, Action: Custom);
960 setOperationAction(Op: ISD::SSUBO, VT: MVT::i32, Action: Custom);
961 setOperationAction(Op: ISD::USUBO, VT: MVT::i32, Action: Custom);
962
963 if (!Subtarget->isThumb1Only()) {
964 setOperationAction(Op: ISD::UMULO, VT: MVT::i32, Action: Custom);
965 setOperationAction(Op: ISD::SMULO, VT: MVT::i32, Action: Custom);
966 }
967
968 setOperationAction(Op: ISD::UADDO_CARRY, VT: MVT::i32, Action: Custom);
969 setOperationAction(Op: ISD::USUBO_CARRY, VT: MVT::i32, Action: Custom);
970 setOperationAction(Op: ISD::SADDO_CARRY, VT: MVT::i32, Action: Custom);
971 setOperationAction(Op: ISD::SSUBO_CARRY, VT: MVT::i32, Action: Custom);
972 if (Subtarget->hasDSP()) {
973 setOperationAction(Op: ISD::SADDSAT, VT: MVT::i8, Action: Custom);
974 setOperationAction(Op: ISD::SSUBSAT, VT: MVT::i8, Action: Custom);
975 setOperationAction(Op: ISD::SADDSAT, VT: MVT::i16, Action: Custom);
976 setOperationAction(Op: ISD::SSUBSAT, VT: MVT::i16, Action: Custom);
977 setOperationAction(Op: ISD::UADDSAT, VT: MVT::i8, Action: Custom);
978 setOperationAction(Op: ISD::USUBSAT, VT: MVT::i8, Action: Custom);
979 setOperationAction(Op: ISD::UADDSAT, VT: MVT::i16, Action: Custom);
980 setOperationAction(Op: ISD::USUBSAT, VT: MVT::i16, Action: Custom);
981 }
982 if (Subtarget->hasBaseDSP()) {
983 setOperationAction(Op: ISD::SADDSAT, VT: MVT::i32, Action: Legal);
984 setOperationAction(Op: ISD::SSUBSAT, VT: MVT::i32, Action: Legal);
985 }
986
987 // i64 operation support.
988 setOperationAction(Op: ISD::MUL, VT: MVT::i64, Action: Expand);
989 setOperationAction(Op: ISD::MULHU, VT: MVT::i32, Action: Expand);
990 if (Subtarget->isThumb1Only()) {
991 setOperationAction(Op: ISD::UMUL_LOHI, VT: MVT::i32, Action: Expand);
992 setOperationAction(Op: ISD::SMUL_LOHI, VT: MVT::i32, Action: Expand);
993 }
994 if (Subtarget->isThumb1Only() || !Subtarget->hasV6Ops()
995 || (Subtarget->isThumb2() && !Subtarget->hasDSP()))
996 setOperationAction(Op: ISD::MULHS, VT: MVT::i32, Action: Expand);
997
998 setOperationAction(Op: ISD::SHL_PARTS, VT: MVT::i32, Action: Custom);
999 setOperationAction(Op: ISD::SRA_PARTS, VT: MVT::i32, Action: Custom);
1000 setOperationAction(Op: ISD::SRL_PARTS, VT: MVT::i32, Action: Custom);
1001 setOperationAction(Op: ISD::SRL, VT: MVT::i64, Action: Custom);
1002 setOperationAction(Op: ISD::SRA, VT: MVT::i64, Action: Custom);
1003 setOperationAction(Op: ISD::INTRINSIC_VOID, VT: MVT::Other, Action: Custom);
1004 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::i64, Action: Custom);
1005 setOperationAction(Op: ISD::LOAD, VT: MVT::i64, Action: Custom);
1006 setOperationAction(Op: ISD::STORE, VT: MVT::i64, Action: Custom);
1007
1008 // MVE lowers 64 bit shifts to lsll and lsrl
1009 // assuming that ISD::SRL and SRA of i64 are already marked custom
1010 if (Subtarget->hasMVEIntegerOps())
1011 setOperationAction(Op: ISD::SHL, VT: MVT::i64, Action: Custom);
1012
1013 // Expand to __aeabi_l{lsl,lsr,asr} calls for Thumb1.
1014 if (Subtarget->isThumb1Only()) {
1015 setOperationAction(Op: ISD::SHL_PARTS, VT: MVT::i32, Action: Expand);
1016 setOperationAction(Op: ISD::SRA_PARTS, VT: MVT::i32, Action: Expand);
1017 setOperationAction(Op: ISD::SRL_PARTS, VT: MVT::i32, Action: Expand);
1018 }
1019
1020 if (!Subtarget->isThumb1Only() && Subtarget->hasV6T2Ops())
1021 setOperationAction(Op: ISD::BITREVERSE, VT: MVT::i32, Action: Legal);
1022
1023 // ARM does not have ROTL.
1024 setOperationAction(Op: ISD::ROTL, VT: MVT::i32, Action: Expand);
1025 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
1026 setOperationAction(Op: ISD::ROTL, VT, Action: Expand);
1027 setOperationAction(Op: ISD::ROTR, VT, Action: Expand);
1028 }
1029 setOperationAction(Op: ISD::CTTZ, VT: MVT::i32, Action: Custom);
1030 // TODO: These two should be set to LibCall, but this currently breaks
1031 // the Linux kernel build. See #101786.
1032 setOperationAction(Op: ISD::CTPOP, VT: MVT::i32, Action: Expand);
1033 setOperationAction(Op: ISD::CTPOP, VT: MVT::i64, Action: Expand);
1034 if (!Subtarget->hasV5TOps() || Subtarget->isThumb1Only()) {
1035 setOperationAction(Op: ISD::CTLZ, VT: MVT::i32, Action: Expand);
1036 setOperationAction(Op: ISD::CTLZ_ZERO_POISON, VT: MVT::i32, Action: LibCall);
1037 }
1038
1039 // @llvm.readcyclecounter requires the Performance Monitors extension.
1040 // Default to the 0 expansion on unsupported platforms.
1041 // FIXME: Technically there are older ARM CPUs that have
1042 // implementation-specific ways of obtaining this information.
1043 if (Subtarget->hasPerfMon())
1044 setOperationAction(Op: ISD::READCYCLECOUNTER, VT: MVT::i64, Action: Custom);
1045
1046 // Only ARMv6 has BSWAP.
1047 if (!Subtarget->hasV6Ops())
1048 setOperationAction(Op: ISD::BSWAP, VT: MVT::i32, Action: Expand);
1049
1050 bool hasDivide = Subtarget->isThumb() ? Subtarget->hasDivideInThumbMode()
1051 : Subtarget->hasDivideInARMMode();
1052 if (!hasDivide) {
1053 // These are expanded into libcalls if the cpu doesn't have HW divider.
1054 setOperationAction(Op: ISD::SDIV, VT: MVT::i32, Action: LibCall);
1055 setOperationAction(Op: ISD::UDIV, VT: MVT::i32, Action: LibCall);
1056 }
1057
1058 if (TT.isOSWindows() && !Subtarget->hasDivideInThumbMode()) {
1059 setOperationAction(Op: ISD::SDIV, VT: MVT::i32, Action: Custom);
1060 setOperationAction(Op: ISD::UDIV, VT: MVT::i32, Action: Custom);
1061
1062 setOperationAction(Op: ISD::SDIV, VT: MVT::i64, Action: Custom);
1063 setOperationAction(Op: ISD::UDIV, VT: MVT::i64, Action: Custom);
1064 }
1065
1066 setOperationAction(Op: ISD::SREM, VT: MVT::i32, Action: Expand);
1067 setOperationAction(Op: ISD::UREM, VT: MVT::i32, Action: Expand);
1068
1069 // Register based DivRem for AEABI (RTABI 4.2)
1070 if (TT.isTargetAEABI() || TT.isAndroid() || TT.isTargetGNUAEABI() ||
1071 TT.isTargetMuslAEABI() || TT.isOSFuchsia() || TT.isOSWindows()) {
1072 setOperationAction(Op: ISD::SREM, VT: MVT::i64, Action: Custom);
1073 setOperationAction(Op: ISD::UREM, VT: MVT::i64, Action: Custom);
1074 HasStandaloneRem = false;
1075
1076 setOperationAction(Op: ISD::SDIVREM, VT: MVT::i32, Action: Custom);
1077 setOperationAction(Op: ISD::UDIVREM, VT: MVT::i32, Action: Custom);
1078 setOperationAction(Op: ISD::SDIVREM, VT: MVT::i64, Action: Custom);
1079 setOperationAction(Op: ISD::UDIVREM, VT: MVT::i64, Action: Custom);
1080 } else {
1081 setOperationAction(Op: ISD::SDIVREM, VT: MVT::i32, Action: Expand);
1082 setOperationAction(Op: ISD::UDIVREM, VT: MVT::i32, Action: Expand);
1083 }
1084
1085 setOperationAction(Op: ISD::GlobalAddress, VT: MVT::i32, Action: Custom);
1086 setOperationAction(Op: ISD::ConstantPool, VT: MVT::i32, Action: Custom);
1087 setOperationAction(Op: ISD::GlobalTLSAddress, VT: MVT::i32, Action: Custom);
1088 setOperationAction(Op: ISD::BlockAddress, VT: MVT::i32, Action: Custom);
1089
1090 setOperationAction(Op: ISD::TRAP, VT: MVT::Other, Action: Legal);
1091 setOperationAction(Op: ISD::DEBUGTRAP, VT: MVT::Other, Action: Legal);
1092
1093 // Use the default implementation.
1094 setOperationAction(Op: ISD::VASTART, VT: MVT::Other, Action: Custom);
1095 setOperationAction(Op: ISD::VAARG, VT: MVT::Other, Action: Expand);
1096 setOperationAction(Op: ISD::VACOPY, VT: MVT::Other, Action: Expand);
1097 setOperationAction(Op: ISD::VAEND, VT: MVT::Other, Action: Expand);
1098 setOperationAction(Op: ISD::STACKSAVE, VT: MVT::Other, Action: Expand);
1099 setOperationAction(Op: ISD::STACKRESTORE, VT: MVT::Other, Action: Expand);
1100
1101 if (TT.isOSWindows())
1102 setOperationAction(Op: ISD::DYNAMIC_STACKALLOC, VT: MVT::i32, Action: Custom);
1103 else
1104 setOperationAction(Op: ISD::DYNAMIC_STACKALLOC, VT: MVT::i32, Action: Expand);
1105
1106 // ARMv6 Thumb1 (except for CPUs that support dmb / dsb) and earlier use
1107 // the default expansion.
1108 InsertFencesForAtomic = false;
1109 if (Subtarget->hasAnyDataBarrier() &&
1110 (!Subtarget->isThumb() || Subtarget->hasV8MBaselineOps())) {
1111 // ATOMIC_FENCE needs custom lowering; the others should have been expanded
1112 // to ldrex/strex loops already.
1113 setOperationAction(Op: ISD::ATOMIC_FENCE, VT: MVT::Other, Action: Custom);
1114 if (!Subtarget->isThumb() || !Subtarget->isMClass())
1115 setOperationAction(Op: ISD::ATOMIC_CMP_SWAP, VT: MVT::i64, Action: Custom);
1116
1117 // On v8, we have particularly efficient implementations of atomic fences
1118 // if they can be combined with nearby atomic loads and stores.
1119 if (!Subtarget->hasAcquireRelease() ||
1120 getTargetMachine().getOptLevel() == CodeGenOptLevel::None) {
1121 // Automatically insert fences (dmb ish) around ATOMIC_SWAP etc.
1122 InsertFencesForAtomic = true;
1123 }
1124 } else {
1125 // If there's anything we can use as a barrier, go through custom lowering
1126 // for ATOMIC_FENCE.
1127 // If target has DMB in thumb, Fences can be inserted.
1128 if (Subtarget->hasDataBarrier())
1129 InsertFencesForAtomic = true;
1130
1131 setOperationAction(Op: ISD::ATOMIC_FENCE, VT: MVT::Other,
1132 Action: Subtarget->hasAnyDataBarrier() ? Custom : Expand);
1133
1134 // Set them all for libcall, which will force libcalls.
1135 setOperationAction(Op: ISD::ATOMIC_CMP_SWAP, VT: MVT::i32, Action: LibCall);
1136 setOperationAction(Op: ISD::ATOMIC_SWAP, VT: MVT::i32, Action: LibCall);
1137 setOperationAction(Op: ISD::ATOMIC_LOAD_ADD, VT: MVT::i32, Action: LibCall);
1138 setOperationAction(Op: ISD::ATOMIC_LOAD_SUB, VT: MVT::i32, Action: LibCall);
1139 setOperationAction(Op: ISD::ATOMIC_LOAD_AND, VT: MVT::i32, Action: LibCall);
1140 setOperationAction(Op: ISD::ATOMIC_LOAD_OR, VT: MVT::i32, Action: LibCall);
1141 setOperationAction(Op: ISD::ATOMIC_LOAD_XOR, VT: MVT::i32, Action: LibCall);
1142 setOperationAction(Op: ISD::ATOMIC_LOAD_NAND, VT: MVT::i32, Action: LibCall);
1143 setOperationAction(Op: ISD::ATOMIC_LOAD_MIN, VT: MVT::i32, Action: LibCall);
1144 setOperationAction(Op: ISD::ATOMIC_LOAD_MAX, VT: MVT::i32, Action: LibCall);
1145 setOperationAction(Op: ISD::ATOMIC_LOAD_UMIN, VT: MVT::i32, Action: LibCall);
1146 setOperationAction(Op: ISD::ATOMIC_LOAD_UMAX, VT: MVT::i32, Action: LibCall);
1147 // Mark ATOMIC_LOAD and ATOMIC_STORE custom so we can handle the
1148 // Unordered/Monotonic case.
1149 if (!InsertFencesForAtomic) {
1150 setOperationAction(Op: ISD::ATOMIC_LOAD, VT: MVT::i32, Action: Custom);
1151 setOperationAction(Op: ISD::ATOMIC_STORE, VT: MVT::i32, Action: Custom);
1152 }
1153 }
1154
1155 // Compute supported atomic widths.
1156 if (TT.isOSLinux() || (!Subtarget->isMClass() && Subtarget->hasV6Ops())) {
1157 // For targets where __sync_* routines are reliably available, we use them
1158 // if necessary.
1159 //
1160 // ARM Linux always supports 64-bit atomics through kernel-assisted atomic
1161 // routines (kernel 3.1 or later). FIXME: Not with compiler-rt?
1162 //
1163 // ARMv6 targets have native instructions in ARM mode. For Thumb mode,
1164 // such targets should provide __sync_* routines, which use the ARM mode
1165 // instructions. (ARMv6 doesn't have dmb, but it has an equivalent
1166 // encoding; see ARMISD::MEMBARRIER_MCR.)
1167 setMaxAtomicSizeInBitsSupported(64);
1168 } else if ((Subtarget->isMClass() && Subtarget->hasV8MBaselineOps()) ||
1169 Subtarget->hasForced32BitAtomics()) {
1170 // Cortex-M (besides Cortex-M0) have 32-bit atomics.
1171 setMaxAtomicSizeInBitsSupported(32);
1172 } else {
1173 // We can't assume anything about other targets; just use libatomic
1174 // routines.
1175 setMaxAtomicSizeInBitsSupported(0);
1176 }
1177
1178 setMaxDivRemBitWidthSupported(64);
1179
1180 setOperationAction(Op: ISD::PREFETCH, VT: MVT::Other, Action: Custom);
1181
1182 // Requires SXTB/SXTH, available on v6 and up in both ARM and Thumb modes.
1183 if (!Subtarget->hasV6Ops()) {
1184 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i16, Action: Expand);
1185 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i8, Action: Expand);
1186 }
1187 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i1, Action: Expand);
1188
1189 if (!Subtarget->useSoftFloat() && Subtarget->hasFPRegs() &&
1190 !Subtarget->isThumb1Only()) {
1191 // Turn f64->i64 into VMOVRRD, i64 -> f64 to VMOVDRR
1192 // iff target supports vfp2.
1193 setOperationAction(Op: ISD::BITCAST, VT: MVT::i64, Action: Custom);
1194 setOperationAction(Op: ISD::GET_ROUNDING, VT: MVT::i32, Action: Custom);
1195 setOperationAction(Op: ISD::SET_ROUNDING, VT: MVT::Other, Action: Custom);
1196 setOperationAction(Op: ISD::GET_FPENV, VT: MVT::i32, Action: Legal);
1197 setOperationAction(Op: ISD::SET_FPENV, VT: MVT::i32, Action: Legal);
1198 setOperationAction(Op: ISD::RESET_FPENV, VT: MVT::Other, Action: Legal);
1199 setOperationAction(Op: ISD::GET_FPMODE, VT: MVT::i32, Action: Legal);
1200 setOperationAction(Op: ISD::SET_FPMODE, VT: MVT::i32, Action: Custom);
1201 setOperationAction(Op: ISD::RESET_FPMODE, VT: MVT::Other, Action: Custom);
1202 }
1203
1204 // We want to custom lower some of our intrinsics.
1205 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::Other, Action: Custom);
1206 setOperationAction(Op: ISD::EH_SJLJ_SETJMP, VT: MVT::i32, Action: Custom);
1207 setOperationAction(Op: ISD::EH_SJLJ_LONGJMP, VT: MVT::Other, Action: Custom);
1208 setOperationAction(Op: ISD::EH_SJLJ_SETUP_DISPATCH, VT: MVT::Other, Action: Custom);
1209
1210 setOperationAction(Op: ISD::SETCC, VT: MVT::i32, Action: Expand);
1211 setOperationAction(Op: ISD::SETCC, VT: MVT::f32, Action: Expand);
1212 setOperationAction(Op: ISD::SETCC, VT: MVT::f64, Action: Expand);
1213 setOperationAction(Op: ISD::SELECT, VT: MVT::i32, Action: Custom);
1214 setOperationAction(Op: ISD::SELECT, VT: MVT::f32, Action: Custom);
1215 setOperationAction(Op: ISD::SELECT, VT: MVT::f64, Action: Custom);
1216 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::i32, Action: Custom);
1217 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::f32, Action: Custom);
1218 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::f64, Action: Custom);
1219 if (Subtarget->hasFullFP16()) {
1220 setOperationAction(Op: ISD::SETCC, VT: MVT::f16, Action: Expand);
1221 setOperationAction(Op: ISD::SELECT, VT: MVT::f16, Action: Custom);
1222 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::f16, Action: Custom);
1223 }
1224
1225 setOperationAction(Op: ISD::SETCCCARRY, VT: MVT::i32, Action: Custom);
1226
1227 setOperationAction(Op: ISD::BRCOND, VT: MVT::Other, Action: Custom);
1228 setOperationAction(Op: ISD::BR_CC, VT: MVT::i32, Action: Custom);
1229 if (Subtarget->hasFullFP16())
1230 setOperationAction(Op: ISD::BR_CC, VT: MVT::f16, Action: Custom);
1231 setOperationAction(Op: ISD::BR_CC, VT: MVT::f32, Action: Custom);
1232 setOperationAction(Op: ISD::BR_CC, VT: MVT::f64, Action: Custom);
1233 setOperationAction(Op: ISD::BR_JT, VT: MVT::Other, Action: Custom);
1234
1235 // We don't support sin/cos/fmod/copysign/pow
1236 setOperationAction(Op: ISD::FSIN, VT: MVT::f64, Action: Expand);
1237 setOperationAction(Op: ISD::FSIN, VT: MVT::f32, Action: Expand);
1238 setOperationAction(Op: ISD::FCOS, VT: MVT::f32, Action: Expand);
1239 setOperationAction(Op: ISD::FCOS, VT: MVT::f64, Action: Expand);
1240 setOperationAction(Op: ISD::FSINCOS, VT: MVT::f64, Action: Expand);
1241 setOperationAction(Op: ISD::FSINCOS, VT: MVT::f32, Action: Expand);
1242 setOperationAction(Op: ISD::FREM, VT: MVT::f64, Action: LibCall);
1243 setOperationAction(Op: ISD::FREM, VT: MVT::f32, Action: LibCall);
1244 if (!Subtarget->useSoftFloat() && Subtarget->hasVFP2Base() &&
1245 !Subtarget->isThumb1Only()) {
1246 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::f64, Action: Custom);
1247 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::f32, Action: Custom);
1248 }
1249 setOperationAction(Op: ISD::FPOW, VT: MVT::f64, Action: Expand);
1250 setOperationAction(Op: ISD::FPOW, VT: MVT::f32, Action: Expand);
1251
1252 if (!Subtarget->hasVFP4Base()) {
1253 setOperationAction(Op: ISD::FMA, VT: MVT::f64, Action: Expand);
1254 setOperationAction(Op: ISD::FMA, VT: MVT::f32, Action: Expand);
1255 }
1256
1257 // Various VFP goodness
1258 if (!Subtarget->useSoftFloat() && !Subtarget->isThumb1Only()) {
1259 // FP-ARMv8 adds f64 <-> f16 conversion. Before that it should be expanded.
1260 if (!Subtarget->hasFPARMv8Base() || !Subtarget->hasFP64()) {
1261 setOperationAction(Op: ISD::FP16_TO_FP, VT: MVT::f64, Action: Expand);
1262 setOperationAction(Op: ISD::FP_TO_FP16, VT: MVT::f64, Action: Expand);
1263 setOperationAction(Op: ISD::STRICT_FP16_TO_FP, VT: MVT::f64, Action: Expand);
1264 setOperationAction(Op: ISD::STRICT_FP_TO_FP16, VT: MVT::f64, Action: Expand);
1265 }
1266
1267 // fp16 is a special v7 extension that adds f16 <-> f32 conversions.
1268 if (!Subtarget->hasFP16()) {
1269 setOperationAction(Op: ISD::FP16_TO_FP, VT: MVT::f32, Action: Expand);
1270 setOperationAction(Op: ISD::FP_TO_FP16, VT: MVT::f32, Action: Expand);
1271 setOperationAction(Op: ISD::STRICT_FP16_TO_FP, VT: MVT::f32, Action: Expand);
1272 setOperationAction(Op: ISD::STRICT_FP_TO_FP16, VT: MVT::f32, Action: Expand);
1273 }
1274
1275 // Strict floating-point comparisons need custom lowering.
1276 setOperationAction(Op: ISD::STRICT_FSETCC, VT: MVT::f16, Action: Custom);
1277 setOperationAction(Op: ISD::STRICT_FSETCCS, VT: MVT::f16, Action: Custom);
1278 setOperationAction(Op: ISD::STRICT_FSETCC, VT: MVT::f32, Action: Custom);
1279 setOperationAction(Op: ISD::STRICT_FSETCCS, VT: MVT::f32, Action: Custom);
1280 setOperationAction(Op: ISD::STRICT_FSETCC, VT: MVT::f64, Action: Custom);
1281 setOperationAction(Op: ISD::STRICT_FSETCCS, VT: MVT::f64, Action: Custom);
1282 }
1283
1284 // FP-ARMv8 implements a lot of rounding-like FP operations.
1285 if (Subtarget->hasFPARMv8Base()) {
1286 for (auto Op :
1287 {ISD::FFLOOR, ISD::FCEIL, ISD::FROUND,
1288 ISD::FTRUNC, ISD::FNEARBYINT, ISD::FRINT,
1289 ISD::FROUNDEVEN, ISD::FMINNUM, ISD::FMAXNUM,
1290 ISD::STRICT_FFLOOR, ISD::STRICT_FCEIL, ISD::STRICT_FROUND,
1291 ISD::STRICT_FTRUNC, ISD::STRICT_FNEARBYINT, ISD::STRICT_FRINT,
1292 ISD::STRICT_FROUNDEVEN, ISD::STRICT_FMINNUM, ISD::STRICT_FMAXNUM}) {
1293 setOperationAction(Op, VT: MVT::f32, Action: Legal);
1294
1295 if (Subtarget->hasFP64())
1296 setOperationAction(Op, VT: MVT::f64, Action: Legal);
1297 }
1298
1299 if (Subtarget->hasNEON()) {
1300 setOperationAction(Op: ISD::FMINNUM, VT: MVT::v2f32, Action: Legal);
1301 setOperationAction(Op: ISD::FMAXNUM, VT: MVT::v2f32, Action: Legal);
1302 setOperationAction(Op: ISD::FMINNUM, VT: MVT::v4f32, Action: Legal);
1303 setOperationAction(Op: ISD::FMAXNUM, VT: MVT::v4f32, Action: Legal);
1304 }
1305 }
1306
1307 // FP16 often need to be promoted to call lib functions
1308 // clang-format off
1309 if (Subtarget->hasFullFP16()) {
1310 setOperationAction(Op: ISD::LRINT, VT: MVT::f16, Action: Expand);
1311 setOperationAction(Op: ISD::LROUND, VT: MVT::f16, Action: Expand);
1312 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::f16, Action: Expand);
1313
1314 for (auto Op : {ISD::FREM, ISD::FPOW, ISD::FPOWI,
1315 ISD::FCOS, ISD::FSIN, ISD::FSINCOS,
1316 ISD::FSINCOSPI, ISD::FMODF, ISD::FACOS,
1317 ISD::FASIN, ISD::FATAN, ISD::FATAN2,
1318 ISD::FCOSH, ISD::FSINH, ISD::FTANH,
1319 ISD::FTAN, ISD::FEXP, ISD::FEXP2,
1320 ISD::FEXP10, ISD::FLOG, ISD::FLOG2,
1321 ISD::FLOG10, ISD::STRICT_FREM, ISD::STRICT_FPOW,
1322 ISD::STRICT_FPOWI, ISD::STRICT_FCOS, ISD::STRICT_FSIN,
1323 ISD::STRICT_FACOS, ISD::STRICT_FASIN, ISD::STRICT_FATAN,
1324 ISD::STRICT_FATAN2, ISD::STRICT_FCOSH, ISD::STRICT_FSINH,
1325 ISD::STRICT_FTANH, ISD::STRICT_FEXP, ISD::STRICT_FEXP2,
1326 ISD::STRICT_FLOG, ISD::STRICT_FLOG2, ISD::STRICT_FLOG10,
1327 ISD::STRICT_FTAN}) {
1328 setOperationAction(Op, VT: MVT::f16, Action: Promote);
1329 }
1330
1331 // Round-to-integer need custom lowering for fp16, as Promote doesn't work
1332 // because the result type is integer.
1333 for (auto Op : {ISD::STRICT_LROUND, ISD::STRICT_LLROUND, ISD::STRICT_LRINT, ISD::STRICT_LLRINT})
1334 setOperationAction(Op, VT: MVT::f16, Action: Custom);
1335
1336 for (auto Op : {ISD::FROUND, ISD::FROUNDEVEN, ISD::FTRUNC,
1337 ISD::FNEARBYINT, ISD::FRINT, ISD::FFLOOR,
1338 ISD::FCEIL, ISD::STRICT_FROUND, ISD::STRICT_FROUNDEVEN,
1339 ISD::STRICT_FTRUNC, ISD::STRICT_FNEARBYINT, ISD::STRICT_FRINT,
1340 ISD::STRICT_FFLOOR, ISD::STRICT_FCEIL}) {
1341 setOperationAction(Op, VT: MVT::f16, Action: Legal);
1342 }
1343 // clang-format on
1344 }
1345
1346 if (Subtarget->hasNEON()) {
1347 // vmin and vmax aren't available in a scalar form, so we can use
1348 // a NEON instruction with an undef lane instead.
1349 setOperationAction(Op: ISD::FMINIMUM, VT: MVT::f32, Action: Legal);
1350 setOperationAction(Op: ISD::FMAXIMUM, VT: MVT::f32, Action: Legal);
1351 setOperationAction(Op: ISD::FMINIMUM, VT: MVT::f16, Action: Legal);
1352 setOperationAction(Op: ISD::FMAXIMUM, VT: MVT::f16, Action: Legal);
1353 setOperationAction(Op: ISD::FMINIMUM, VT: MVT::v2f32, Action: Legal);
1354 setOperationAction(Op: ISD::FMAXIMUM, VT: MVT::v2f32, Action: Legal);
1355 setOperationAction(Op: ISD::FMINIMUM, VT: MVT::v4f32, Action: Legal);
1356 setOperationAction(Op: ISD::FMAXIMUM, VT: MVT::v4f32, Action: Legal);
1357
1358 if (Subtarget->hasV8Ops()) {
1359 for (auto Op : {ISD::FROUND, ISD::STRICT_FROUND, ISD::FROUNDEVEN,
1360 ISD::STRICT_FROUNDEVEN, ISD::FTRUNC, ISD::STRICT_FTRUNC,
1361 ISD::FRINT, ISD::STRICT_FRINT, ISD::FFLOOR,
1362 ISD::STRICT_FFLOOR, ISD::FCEIL, ISD::STRICT_FCEIL}) {
1363 setOperationAction(Op, VT: MVT::v2f32, Action: Legal);
1364 setOperationAction(Op, VT: MVT::v4f32, Action: Legal);
1365 }
1366 }
1367
1368 if (Subtarget->hasFullFP16()) {
1369 setOperationAction(Op: ISD::FMINNUM, VT: MVT::v4f16, Action: Legal);
1370 setOperationAction(Op: ISD::FMAXNUM, VT: MVT::v4f16, Action: Legal);
1371 setOperationAction(Op: ISD::FMINNUM, VT: MVT::v8f16, Action: Legal);
1372 setOperationAction(Op: ISD::FMAXNUM, VT: MVT::v8f16, Action: Legal);
1373
1374 setOperationAction(Op: ISD::FMINIMUM, VT: MVT::v4f16, Action: Legal);
1375 setOperationAction(Op: ISD::FMAXIMUM, VT: MVT::v4f16, Action: Legal);
1376 setOperationAction(Op: ISD::FMINIMUM, VT: MVT::v8f16, Action: Legal);
1377 setOperationAction(Op: ISD::FMAXIMUM, VT: MVT::v8f16, Action: Legal);
1378
1379 for (auto Op : {ISD::FROUND, ISD::STRICT_FROUND, ISD::FROUNDEVEN,
1380 ISD::STRICT_FROUNDEVEN, ISD::FTRUNC, ISD::STRICT_FTRUNC,
1381 ISD::FRINT, ISD::STRICT_FRINT, ISD::FFLOOR,
1382 ISD::STRICT_FFLOOR, ISD::FCEIL, ISD::STRICT_FCEIL}) {
1383 setOperationAction(Op, VT: MVT::v4f16, Action: Legal);
1384 setOperationAction(Op, VT: MVT::v8f16, Action: Legal);
1385 }
1386 }
1387 }
1388
1389 // On MSVC, both 32-bit and 64-bit, ldexpf(f32) is not defined. MinGW has
1390 // it, but it's just a wrapper around ldexp.
1391 if (TT.isOSWindows()) {
1392 for (ISD::NodeType Op : {ISD::FLDEXP, ISD::STRICT_FLDEXP, ISD::FFREXP})
1393 if (isOperationExpand(Op, VT: MVT::f32))
1394 setOperationAction(Op, VT: MVT::f32, Action: Promote);
1395 }
1396
1397 // LegalizeDAG currently can't expand fp16 LDEXP/FREXP on targets where i16
1398 // isn't legal.
1399 for (ISD::NodeType Op : {ISD::FLDEXP, ISD::STRICT_FLDEXP, ISD::FFREXP})
1400 if (isOperationExpand(Op, VT: MVT::f16))
1401 setOperationAction(Op, VT: MVT::f16, Action: Promote);
1402
1403 // We have target-specific dag combine patterns for the following nodes:
1404 // ARMISD::VMOVRRD - No need to call setTargetDAGCombine
1405 setTargetDAGCombine(
1406 {ISD::ADD, ISD::SUB, ISD::MUL, ISD::AND, ISD::OR, ISD::XOR});
1407
1408 if (Subtarget->hasMVEIntegerOps())
1409 setTargetDAGCombine(ISD::VSELECT);
1410
1411 if (Subtarget->hasV6Ops())
1412 setTargetDAGCombine(ISD::SRL);
1413 if (Subtarget->isThumb1Only())
1414 setTargetDAGCombine(ISD::SHL);
1415 // Attempt to lower smin/smax to ssat/usat
1416 if ((!Subtarget->isThumb() && Subtarget->hasV6Ops()) ||
1417 Subtarget->isThumb2()) {
1418 setTargetDAGCombine({ISD::SMIN, ISD::SMAX});
1419 }
1420
1421 setStackPointerRegisterToSaveRestore(ARM::SP);
1422
1423 if (Subtarget->useSoftFloat() || Subtarget->isThumb1Only() ||
1424 !Subtarget->hasVFP2Base() || Subtarget->hasMinSize())
1425 setSchedulingPreference(Sched::RegPressure);
1426 else
1427 setSchedulingPreference(Sched::Hybrid);
1428
1429 //// temporary - rewrite interface to use type
1430 MaxStoresPerMemset = 8;
1431 MaxStoresPerMemsetOptSize = 4;
1432 MaxStoresPerMemcpy = 4; // For @llvm.memcpy -> sequence of stores
1433 MaxStoresPerMemcpyOptSize = 2;
1434 MaxStoresPerMemmove = 4; // For @llvm.memmove -> sequence of stores
1435 MaxStoresPerMemmoveOptSize = 2;
1436
1437 // On ARM arguments smaller than 4 bytes are extended, so all arguments
1438 // are at least 4 bytes aligned.
1439 setMinStackArgumentAlignment(Align(4));
1440
1441 // Prefer likely predicted branches to selects on out-of-order cores.
1442 PredictableSelectIsExpensive = Subtarget->getSchedModel().isOutOfOrder();
1443
1444 setPrefLoopAlignment(Align(1ULL << Subtarget->getPreferBranchLogAlignment()));
1445 setPrefFunctionAlignment(
1446 Align(1ULL << Subtarget->getPreferBranchLogAlignment()));
1447
1448 setMinFunctionAlignment(Subtarget->isThumb() ? Align(2) : Align(4));
1449
1450 IsStrictFPEnabled = true;
1451}
1452
1453bool ARMTargetLowering::useSoftFloat() const {
1454 return Subtarget->useSoftFloat();
1455}
1456
1457bool ARMTargetLowering::preferSelectsOverBooleanArithmetic(EVT VT) const {
1458 return !Subtarget->isThumb1Only() && VT.getSizeInBits() <= 32;
1459}
1460
1461// FIXME: It might make sense to define the representative register class as the
1462// nearest super-register that has a non-null superset. For example, DPR_VFP2 is
1463// a super-register of SPR, and DPR is a superset if DPR_VFP2. Consequently,
1464// SPR's representative would be DPR_VFP2. This should work well if register
1465// pressure tracking were modified such that a register use would increment the
1466// pressure of the register class's representative and all of it's super
1467// classes' representatives transitively. We have not implemented this because
1468// of the difficulty prior to coalescing of modeling operand register classes
1469// due to the common occurrence of cross class copies and subregister insertions
1470// and extractions.
1471std::pair<const TargetRegisterClass *, uint8_t>
1472ARMTargetLowering::findRepresentativeClass(const TargetRegisterInfo *TRI,
1473 MVT VT) const {
1474 const TargetRegisterClass *RRC = nullptr;
1475 uint8_t Cost = 1;
1476 switch (VT.SimpleTy) {
1477 default:
1478 return TargetLowering::findRepresentativeClass(TRI, VT);
1479 // Use DPR as representative register class for all floating point
1480 // and vector types. Since there are 32 SPR registers and 32 DPR registers so
1481 // the cost is 1 for both f32 and f64.
1482 case MVT::f32: case MVT::f64: case MVT::v8i8: case MVT::v4i16:
1483 case MVT::v2i32: case MVT::v1i64: case MVT::v2f32:
1484 RRC = &ARM::DPRRegClass;
1485 // When NEON is used for SP, only half of the register file is available
1486 // because operations that define both SP and DP results will be constrained
1487 // to the VFP2 class (D0-D15). We currently model this constraint prior to
1488 // coalescing by double-counting the SP regs. See the FIXME above.
1489 if (Subtarget->useNEONForSinglePrecisionFP())
1490 Cost = 2;
1491 break;
1492 case MVT::v16i8: case MVT::v8i16: case MVT::v4i32: case MVT::v2i64:
1493 case MVT::v4f32: case MVT::v2f64:
1494 RRC = &ARM::DPRRegClass;
1495 Cost = 2;
1496 break;
1497 case MVT::v4i64:
1498 RRC = &ARM::DPRRegClass;
1499 Cost = 4;
1500 break;
1501 case MVT::v8i64:
1502 RRC = &ARM::DPRRegClass;
1503 Cost = 8;
1504 break;
1505 }
1506 return std::make_pair(x&: RRC, y&: Cost);
1507}
1508
1509EVT ARMTargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &C,
1510 EVT VT) const {
1511 if (!VT.isVector())
1512 return getPointerTy(DL);
1513
1514 // MVE has a predicate register.
1515 if (Subtarget->hasMVEIntegerOps())
1516 return EVT::getVectorVT(Context&: C, VT: MVT::i1, EC: VT.getVectorElementCount());
1517
1518 return VT.changeVectorElementTypeToInteger();
1519}
1520
1521/// getRegClassFor - Return the register class that should be used for the
1522/// specified value type.
1523const TargetRegisterClass *
1524ARMTargetLowering::getRegClassFor(MVT VT, bool isDivergent) const {
1525 (void)isDivergent;
1526 // Map v4i64 to QQ registers but do not make the type legal. Similarly map
1527 // v8i64 to QQQQ registers. v4i64 and v8i64 are only used for REG_SEQUENCE to
1528 // load / store 4 to 8 consecutive NEON D registers, or 2 to 4 consecutive
1529 // MVE Q registers.
1530 if (Subtarget->hasNEON()) {
1531 if (VT == MVT::v4i64)
1532 return &ARM::QQPRRegClass;
1533 if (VT == MVT::v8i64)
1534 return &ARM::QQQQPRRegClass;
1535 }
1536 if (Subtarget->hasMVEIntegerOps()) {
1537 if (VT == MVT::v4i64)
1538 return &ARM::MQQPRRegClass;
1539 if (VT == MVT::v8i64)
1540 return &ARM::MQQQQPRRegClass;
1541 }
1542 return TargetLowering::getRegClassFor(VT);
1543}
1544
1545// memcpy, and other memory intrinsics, typically tries to use LDM/STM if the
1546// source/dest is aligned and the copy size is large enough. We therefore want
1547// to align such objects passed to memory intrinsics.
1548bool ARMTargetLowering::shouldAlignPointerArgs(CallInst *CI, unsigned &MinSize,
1549 Align &PrefAlign) const {
1550 if (!isa<MemIntrinsic>(Val: CI))
1551 return false;
1552 MinSize = 8;
1553 // On ARM11 onwards (excluding M class) 8-byte aligned LDM is typically 1
1554 // cycle faster than 4-byte aligned LDM.
1555 PrefAlign =
1556 (Subtarget->hasV6Ops() && !Subtarget->isMClass() ? Align(8) : Align(4));
1557 return true;
1558}
1559
1560// Create a fast isel object.
1561FastISel *ARMTargetLowering::createFastISel(
1562 FunctionLoweringInfo &funcInfo, const TargetLibraryInfo *libInfo,
1563 const LibcallLoweringInfo *libcallLowering) const {
1564 return ARM::createFastISel(funcInfo, libInfo, libcallLowering);
1565}
1566
1567Sched::Preference ARMTargetLowering::getSchedulingPreference(SDNode *N) const {
1568 unsigned NumVals = N->getNumValues();
1569 if (!NumVals)
1570 return Sched::RegPressure;
1571
1572 for (unsigned i = 0; i != NumVals; ++i) {
1573 EVT VT = N->getValueType(ResNo: i);
1574 if (VT == MVT::Glue || VT == MVT::Other)
1575 continue;
1576 if (VT.isFloatingPoint() || VT.isVector())
1577 return Sched::ILP;
1578 }
1579
1580 if (!N->isMachineOpcode())
1581 return Sched::RegPressure;
1582
1583 // Load are scheduled for latency even if there instruction itinerary
1584 // is not available.
1585 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
1586 const MCInstrDesc &MCID = TII->get(Opcode: N->getMachineOpcode());
1587
1588 if (MCID.getNumDefs() == 0)
1589 return Sched::RegPressure;
1590 if (!Itins->isEmpty() &&
1591 Itins->getOperandCycle(ItinClassIndx: MCID.getSchedClass(), OperandIdx: 0) > 2U)
1592 return Sched::ILP;
1593
1594 return Sched::RegPressure;
1595}
1596
1597//===----------------------------------------------------------------------===//
1598// Lowering Code
1599//===----------------------------------------------------------------------===//
1600
1601static bool isSRL16(const SDValue &Op) {
1602 if (Op.getOpcode() != ISD::SRL)
1603 return false;
1604 if (auto Const = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1)))
1605 return Const->getZExtValue() == 16;
1606 return false;
1607}
1608
1609static bool isSRA16(const SDValue &Op) {
1610 if (Op.getOpcode() != ISD::SRA)
1611 return false;
1612 if (auto Const = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1)))
1613 return Const->getZExtValue() == 16;
1614 return false;
1615}
1616
1617static bool isSHL16(const SDValue &Op) {
1618 if (Op.getOpcode() != ISD::SHL)
1619 return false;
1620 if (auto Const = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1)))
1621 return Const->getZExtValue() == 16;
1622 return false;
1623}
1624
1625// Check for a signed 16-bit value. We special case SRA because it makes it
1626// more simple when also looking for SRAs that aren't sign extending a
1627// smaller value. Without the check, we'd need to take extra care with
1628// checking order for some operations.
1629static bool isS16(const SDValue &Op, SelectionDAG &DAG) {
1630 if (isSRA16(Op))
1631 return isSHL16(Op: Op.getOperand(i: 0));
1632 return DAG.ComputeNumSignBits(Op) == 17;
1633}
1634
1635/// IntCCToARMCC - Convert a DAG integer condition code to an ARM CC
1636static ARMCC::CondCodes IntCCToARMCC(ISD::CondCode CC) {
1637 switch (CC) {
1638 default: llvm_unreachable("Unknown condition code!");
1639 case ISD::SETNE: return ARMCC::NE;
1640 case ISD::SETEQ: return ARMCC::EQ;
1641 case ISD::SETGT: return ARMCC::GT;
1642 case ISD::SETGE: return ARMCC::GE;
1643 case ISD::SETLT: return ARMCC::LT;
1644 case ISD::SETLE: return ARMCC::LE;
1645 case ISD::SETUGT: return ARMCC::HI;
1646 case ISD::SETUGE: return ARMCC::HS;
1647 case ISD::SETULT: return ARMCC::LO;
1648 case ISD::SETULE: return ARMCC::LS;
1649 }
1650}
1651
1652/// FPCCToARMCC - Convert a DAG fp condition code to an ARM CC.
1653static void FPCCToARMCC(ISD::CondCode CC, ARMCC::CondCodes &CondCode,
1654 ARMCC::CondCodes &CondCode2) {
1655 CondCode2 = ARMCC::AL;
1656 switch (CC) {
1657 default: llvm_unreachable("Unknown FP condition!");
1658 case ISD::SETEQ:
1659 case ISD::SETOEQ: CondCode = ARMCC::EQ; break;
1660 case ISD::SETGT:
1661 case ISD::SETOGT: CondCode = ARMCC::GT; break;
1662 case ISD::SETGE:
1663 case ISD::SETOGE: CondCode = ARMCC::GE; break;
1664 case ISD::SETOLT: CondCode = ARMCC::MI; break;
1665 case ISD::SETOLE: CondCode = ARMCC::LS; break;
1666 case ISD::SETONE: CondCode = ARMCC::MI; CondCode2 = ARMCC::GT; break;
1667 case ISD::SETO: CondCode = ARMCC::VC; break;
1668 case ISD::SETUO: CondCode = ARMCC::VS; break;
1669 case ISD::SETUEQ: CondCode = ARMCC::EQ; CondCode2 = ARMCC::VS; break;
1670 case ISD::SETUGT: CondCode = ARMCC::HI; break;
1671 case ISD::SETUGE: CondCode = ARMCC::PL; break;
1672 case ISD::SETLT:
1673 case ISD::SETULT: CondCode = ARMCC::LT; break;
1674 case ISD::SETLE:
1675 case ISD::SETULE: CondCode = ARMCC::LE; break;
1676 case ISD::SETNE:
1677 case ISD::SETUNE: CondCode = ARMCC::NE; break;
1678 }
1679}
1680
1681//===----------------------------------------------------------------------===//
1682// Calling Convention Implementation
1683//===----------------------------------------------------------------------===//
1684
1685/// getEffectiveCallingConv - Get the effective calling convention, taking into
1686/// account presence of floating point hardware and calling convention
1687/// limitations, such as support for variadic functions.
1688CallingConv::ID
1689ARMTargetLowering::getEffectiveCallingConv(CallingConv::ID CC,
1690 bool isVarArg) const {
1691 switch (CC) {
1692 default:
1693 // Unknown CCs are rejected when calling convention lowering is required.
1694 case CallingConv::ARM_AAPCS:
1695 case CallingConv::ARM_APCS:
1696 case CallingConv::GHC:
1697 case CallingConv::CFGuard_Check:
1698 return CC;
1699 case CallingConv::PreserveMost:
1700 return CallingConv::PreserveMost;
1701 case CallingConv::PreserveAll:
1702 return CallingConv::PreserveAll;
1703 case CallingConv::ARM_AAPCS_VFP:
1704 case CallingConv::Swift:
1705 case CallingConv::SwiftTail:
1706 return isVarArg ? CallingConv::ARM_AAPCS : CallingConv::ARM_AAPCS_VFP;
1707 case CallingConv::C:
1708 case CallingConv::Tail:
1709 if (!Subtarget->isAAPCS_ABI())
1710 return CallingConv::ARM_APCS;
1711 else if (Subtarget->isTargetHardFloat() && !isVarArg)
1712 return CallingConv::ARM_AAPCS_VFP;
1713 else
1714 return CallingConv::ARM_AAPCS;
1715 case CallingConv::Fast:
1716 case CallingConv::CXX_FAST_TLS:
1717 if (!Subtarget->isAAPCS_ABI()) {
1718 if (Subtarget->hasFPRegs() && !Subtarget->isThumb1Only() && !isVarArg)
1719 return CallingConv::Fast;
1720 return CallingConv::ARM_APCS;
1721 } else if (Subtarget->hasFPRegs() && !Subtarget->isThumb1Only() &&
1722 !isVarArg)
1723 return CallingConv::ARM_AAPCS_VFP;
1724 else
1725 return CallingConv::ARM_AAPCS;
1726 }
1727}
1728
1729CCAssignFn *ARMTargetLowering::CCAssignFnForCall(CallingConv::ID CC,
1730 bool isVarArg) const {
1731 return CCAssignFnForNode(CC, Return: false, isVarArg);
1732}
1733
1734CCAssignFn *ARMTargetLowering::CCAssignFnForReturn(CallingConv::ID CC,
1735 bool isVarArg) const {
1736 return CCAssignFnForNode(CC, Return: true, isVarArg);
1737}
1738
1739/// CCAssignFnForNode - Selects the correct CCAssignFn for the given
1740/// CallingConvention.
1741CCAssignFn *ARMTargetLowering::CCAssignFnForNode(CallingConv::ID CC,
1742 bool Return,
1743 bool isVarArg) const {
1744 switch (getEffectiveCallingConv(CC, isVarArg)) {
1745 default:
1746 report_fatal_error(reason: "Unsupported calling convention");
1747 case CallingConv::ARM_APCS:
1748 return (Return ? RetCC_ARM_APCS : CC_ARM_APCS);
1749 case CallingConv::ARM_AAPCS:
1750 return (Return ? RetCC_ARM_AAPCS : CC_ARM_AAPCS);
1751 case CallingConv::ARM_AAPCS_VFP:
1752 return (Return ? RetCC_ARM_AAPCS_VFP : CC_ARM_AAPCS_VFP);
1753 case CallingConv::Fast:
1754 return (Return ? RetFastCC_ARM_APCS : FastCC_ARM_APCS);
1755 case CallingConv::GHC:
1756 return (Return ? RetCC_ARM_APCS : CC_ARM_APCS_GHC);
1757 case CallingConv::PreserveMost:
1758 return (Return ? RetCC_ARM_AAPCS : CC_ARM_AAPCS);
1759 case CallingConv::PreserveAll:
1760 return (Return ? RetCC_ARM_AAPCS : CC_ARM_AAPCS);
1761 case CallingConv::CFGuard_Check:
1762 return (Return ? RetCC_ARM_AAPCS : CC_ARM_Win32_CFGuard_Check);
1763 }
1764}
1765
1766SDValue ARMTargetLowering::MoveToHPR(const SDLoc &dl, SelectionDAG &DAG,
1767 MVT LocVT, MVT ValVT, SDValue Val) const {
1768 Val = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::getIntegerVT(BitWidth: LocVT.getSizeInBits()),
1769 Operand: Val);
1770 if (Subtarget->hasFullFP16()) {
1771 Val = DAG.getNode(Opcode: ARMISD::VMOVhr, DL: dl, VT: ValVT, Operand: Val);
1772 } else {
1773 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl,
1774 VT: MVT::getIntegerVT(BitWidth: ValVT.getSizeInBits()), Operand: Val);
1775 Val = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: ValVT, Operand: Val);
1776 }
1777 return Val;
1778}
1779
1780SDValue ARMTargetLowering::MoveFromHPR(const SDLoc &dl, SelectionDAG &DAG,
1781 MVT LocVT, MVT ValVT,
1782 SDValue Val) const {
1783 if (Subtarget->hasFullFP16()) {
1784 Val = DAG.getNode(Opcode: ARMISD::VMOVrh, DL: dl,
1785 VT: MVT::getIntegerVT(BitWidth: LocVT.getSizeInBits()), Operand: Val);
1786 } else {
1787 Val = DAG.getNode(Opcode: ISD::BITCAST, DL: dl,
1788 VT: MVT::getIntegerVT(BitWidth: ValVT.getSizeInBits()), Operand: Val);
1789 Val = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl,
1790 VT: MVT::getIntegerVT(BitWidth: LocVT.getSizeInBits()), Operand: Val);
1791 }
1792 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LocVT, Operand: Val);
1793}
1794
1795/// LowerCallResult - Lower the result values of a call into the
1796/// appropriate copies out of appropriate physical registers.
1797SDValue ARMTargetLowering::LowerCallResult(
1798 SDValue Chain, SDValue InGlue, CallingConv::ID CallConv, bool isVarArg,
1799 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
1800 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals, bool isThisReturn,
1801 SDValue ThisVal, bool isCmseNSCall) const {
1802 // Assign locations to each value returned by this call.
1803 SmallVector<CCValAssign, 16> RVLocs;
1804 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
1805 *DAG.getContext());
1806 CCInfo.AnalyzeCallResult(Ins, Fn: CCAssignFnForReturn(CC: CallConv, isVarArg));
1807
1808 // Copy all of the result registers out of their specified physreg.
1809 for (unsigned i = 0; i != RVLocs.size(); ++i) {
1810 CCValAssign VA = RVLocs[i];
1811
1812 // Pass 'this' value directly from the argument to return value, to avoid
1813 // reg unit interference
1814 if (i == 0 && isThisReturn) {
1815 assert(!VA.needsCustom() && VA.getLocVT() == MVT::i32 &&
1816 "unexpected return calling convention register assignment");
1817 InVals.push_back(Elt: ThisVal);
1818 continue;
1819 }
1820
1821 SDValue Val;
1822 if (VA.needsCustom() &&
1823 (VA.getLocVT() == MVT::f64 || VA.getLocVT() == MVT::v2f64)) {
1824 // Handle f64 or half of a v2f64.
1825 SDValue Lo = DAG.getCopyFromReg(Chain, dl, Reg: VA.getLocReg(), VT: MVT::i32,
1826 Glue: InGlue);
1827 Chain = Lo.getValue(R: 1);
1828 InGlue = Lo.getValue(R: 2);
1829 VA = RVLocs[++i]; // skip ahead to next loc
1830 SDValue Hi = DAG.getCopyFromReg(Chain, dl, Reg: VA.getLocReg(), VT: MVT::i32,
1831 Glue: InGlue);
1832 Chain = Hi.getValue(R: 1);
1833 InGlue = Hi.getValue(R: 2);
1834 if (!Subtarget->isLittle())
1835 std::swap (a&: Lo, b&: Hi);
1836 Val = DAG.getNode(Opcode: ARMISD::VMOVDRR, DL: dl, VT: MVT::f64, N1: Lo, N2: Hi);
1837
1838 if (VA.getLocVT() == MVT::v2f64) {
1839 SDValue Vec = DAG.getNode(Opcode: ISD::UNDEF, DL: dl, VT: MVT::v2f64);
1840 Vec = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: MVT::v2f64, N1: Vec, N2: Val,
1841 N3: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
1842
1843 VA = RVLocs[++i]; // skip ahead to next loc
1844 Lo = DAG.getCopyFromReg(Chain, dl, Reg: VA.getLocReg(), VT: MVT::i32, Glue: InGlue);
1845 Chain = Lo.getValue(R: 1);
1846 InGlue = Lo.getValue(R: 2);
1847 VA = RVLocs[++i]; // skip ahead to next loc
1848 Hi = DAG.getCopyFromReg(Chain, dl, Reg: VA.getLocReg(), VT: MVT::i32, Glue: InGlue);
1849 Chain = Hi.getValue(R: 1);
1850 InGlue = Hi.getValue(R: 2);
1851 if (!Subtarget->isLittle())
1852 std::swap (a&: Lo, b&: Hi);
1853 Val = DAG.getNode(Opcode: ARMISD::VMOVDRR, DL: dl, VT: MVT::f64, N1: Lo, N2: Hi);
1854 Val = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: MVT::v2f64, N1: Vec, N2: Val,
1855 N3: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
1856 }
1857 } else {
1858 Val = DAG.getCopyFromReg(Chain, dl, Reg: VA.getLocReg(), VT: VA.getLocVT(),
1859 Glue: InGlue);
1860 Chain = Val.getValue(R: 1);
1861 InGlue = Val.getValue(R: 2);
1862 }
1863
1864 switch (VA.getLocInfo()) {
1865 default: llvm_unreachable("Unknown loc info!");
1866 case CCValAssign::Full: break;
1867 case CCValAssign::BCvt:
1868 Val = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: VA.getValVT(), Operand: Val);
1869 break;
1870 }
1871
1872 // f16 arguments have their size extended to 4 bytes and passed as if they
1873 // had been copied to the LSBs of a 32-bit register.
1874 // For that, it's passed extended to i32 (soft ABI) or to f32 (hard ABI)
1875 if (VA.needsCustom() &&
1876 (VA.getValVT() == MVT::f16 || VA.getValVT() == MVT::bf16))
1877 Val = MoveToHPR(dl, DAG, LocVT: VA.getLocVT(), ValVT: VA.getValVT(), Val);
1878
1879 // On CMSE Non-secure Calls, call results (returned values) whose bitwidth
1880 // is less than 32 bits must be sign- or zero-extended after the call for
1881 // security reasons. Although the ABI mandates an extension done by the
1882 // callee, the latter cannot be trusted to follow the rules of the ABI.
1883 const ISD::InputArg &Arg = Ins[VA.getValNo()];
1884 if (isCmseNSCall && Arg.ArgVT.isScalarInteger() &&
1885 VA.getLocVT().isScalarInteger() && Arg.ArgVT.bitsLT(VT: MVT::i32))
1886 Val = handleCMSEValue(Value: Val, Arg, DAG, DL: dl);
1887
1888 InVals.push_back(Elt: Val);
1889 }
1890
1891 return Chain;
1892}
1893
1894std::pair<SDValue, MachinePointerInfo> ARMTargetLowering::computeAddrForCallArg(
1895 const SDLoc &dl, SelectionDAG &DAG, const CCValAssign &VA, SDValue StackPtr,
1896 bool IsTailCall, int SPDiff) const {
1897 SDValue DstAddr;
1898 MachinePointerInfo DstInfo;
1899 int32_t Offset = VA.getLocMemOffset();
1900 MachineFunction &MF = DAG.getMachineFunction();
1901
1902 if (IsTailCall) {
1903 Offset += SPDiff;
1904 auto PtrVT = getPointerTy(DL: DAG.getDataLayout());
1905 int Size = VA.getLocVT().getFixedSizeInBits() / 8;
1906 int FI = MF.getFrameInfo().CreateFixedObject(Size, SPOffset: Offset, IsImmutable: true);
1907 DstAddr = DAG.getFrameIndex(FI, VT: PtrVT);
1908 DstInfo =
1909 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI);
1910 } else {
1911 SDValue PtrOff = DAG.getIntPtrConstant(Val: Offset, DL: dl);
1912 DstAddr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: getPointerTy(DL: DAG.getDataLayout()),
1913 N1: StackPtr, N2: PtrOff);
1914 DstInfo =
1915 MachinePointerInfo::getStack(MF&: DAG.getMachineFunction(), Offset);
1916 }
1917
1918 return std::make_pair(x&: DstAddr, y&: DstInfo);
1919}
1920
1921// Returns the type of copying which is required to set up a byval argument to
1922// a tail-called function. This isn't needed for non-tail calls, because they
1923// always need the equivalent of CopyOnce, but tail-calls sometimes need two to
1924// avoid clobbering another argument (CopyViaTemp), and sometimes can be
1925// optimised to zero copies when forwarding an argument from the caller's
1926// caller (NoCopy).
1927ARMTargetLowering::ByValCopyKind ARMTargetLowering::ByValNeedsCopyForTailCall(
1928 SelectionDAG &DAG, SDValue Src, SDValue Dst, ISD::ArgFlagsTy Flags) const {
1929 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
1930 ARMFunctionInfo *AFI = DAG.getMachineFunction().getInfo<ARMFunctionInfo>();
1931
1932 // Globals are always safe to copy from.
1933 if (isa<GlobalAddressSDNode>(Val: Src) || isa<ExternalSymbolSDNode>(Val: Src))
1934 return CopyOnce;
1935
1936 // Can only analyse frame index nodes, conservatively assume we need a
1937 // temporary.
1938 auto *SrcFrameIdxNode = dyn_cast<FrameIndexSDNode>(Val&: Src);
1939 auto *DstFrameIdxNode = dyn_cast<FrameIndexSDNode>(Val&: Dst);
1940 if (!SrcFrameIdxNode || !DstFrameIdxNode)
1941 return CopyViaTemp;
1942
1943 int SrcFI = SrcFrameIdxNode->getIndex();
1944 int DstFI = DstFrameIdxNode->getIndex();
1945 assert(MFI.isFixedObjectIndex(DstFI) &&
1946 "byval passed in non-fixed stack slot");
1947
1948 int64_t SrcOffset = MFI.getObjectOffset(ObjectIdx: SrcFI);
1949 int64_t DstOffset = MFI.getObjectOffset(ObjectIdx: DstFI);
1950
1951 // If the source is in the local frame, then the copy to the argument memory
1952 // is always valid.
1953 bool FixedSrc = MFI.isFixedObjectIndex(ObjectIdx: SrcFI);
1954 if (!FixedSrc ||
1955 (FixedSrc && SrcOffset < -(int64_t)AFI->getArgRegsSaveSize()))
1956 return CopyOnce;
1957
1958 // In the case of byval arguments split between registers and the stack,
1959 // computeAddrForCallArg returns a FrameIndex which corresponds only to the
1960 // stack portion, but the Src SDValue will refer to the full value, including
1961 // the local stack memory that the register portion gets stored into. We only
1962 // need to compare them for equality, so normalise on the full value version.
1963 uint64_t RegSize = Flags.getByValSize() - MFI.getObjectSize(ObjectIdx: DstFI);
1964 DstOffset -= RegSize;
1965
1966 // If the value is already in the correct location, then no copying is
1967 // needed. If not, then we need to copy via a temporary.
1968 if (SrcOffset == DstOffset)
1969 return NoCopy;
1970 else
1971 return CopyViaTemp;
1972}
1973
1974void ARMTargetLowering::PassF64ArgInRegs(const SDLoc &dl, SelectionDAG &DAG,
1975 SDValue Chain, SDValue &Arg,
1976 RegsToPassVector &RegsToPass,
1977 CCValAssign &VA, CCValAssign &NextVA,
1978 SDValue &StackPtr,
1979 SmallVectorImpl<SDValue> &MemOpChains,
1980 bool IsTailCall,
1981 int SPDiff) const {
1982 SDValue fmrrd = DAG.getNode(Opcode: ARMISD::VMOVRRD, DL: dl,
1983 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N: Arg);
1984 unsigned id = Subtarget->isLittle() ? 0 : 1;
1985 RegsToPass.push_back(Elt: std::make_pair(x: VA.getLocReg(), y: fmrrd.getValue(R: id)));
1986
1987 if (NextVA.isRegLoc())
1988 RegsToPass.push_back(Elt: std::make_pair(x: NextVA.getLocReg(), y: fmrrd.getValue(R: 1-id)));
1989 else {
1990 assert(NextVA.isMemLoc());
1991 if (!StackPtr.getNode())
1992 StackPtr = DAG.getCopyFromReg(Chain, dl, Reg: ARM::SP,
1993 VT: getPointerTy(DL: DAG.getDataLayout()));
1994
1995 SDValue DstAddr;
1996 MachinePointerInfo DstInfo;
1997 std::tie(args&: DstAddr, args&: DstInfo) =
1998 computeAddrForCallArg(dl, DAG, VA: NextVA, StackPtr, IsTailCall, SPDiff);
1999 MemOpChains.push_back(
2000 Elt: DAG.getStore(Chain, dl, Val: fmrrd.getValue(R: 1 - id), Ptr: DstAddr, PtrInfo: DstInfo));
2001 }
2002}
2003
2004static bool canGuaranteeTCO(CallingConv::ID CC, bool GuaranteeTailCalls) {
2005 return (CC == CallingConv::Fast && GuaranteeTailCalls) ||
2006 CC == CallingConv::Tail || CC == CallingConv::SwiftTail;
2007}
2008
2009/// LowerCall - Lowering a call into a callseq_start <-
2010/// ARMISD:CALL <- callseq_end chain. Also add input and output parameter
2011/// nodes.
2012SDValue
2013ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
2014 SmallVectorImpl<SDValue> &InVals) const {
2015 SelectionDAG &DAG = CLI.DAG;
2016 SDLoc &dl = CLI.DL;
2017 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
2018 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
2019 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
2020 SDValue Chain = CLI.Chain;
2021 SDValue Callee = CLI.Callee;
2022 bool &isTailCall = CLI.IsTailCall;
2023 CallingConv::ID CallConv = CLI.CallConv;
2024 bool doesNotRet = CLI.DoesNotReturn;
2025 bool isVarArg = CLI.IsVarArg;
2026 const CallBase *CB = CLI.CB;
2027
2028 MachineFunction &MF = DAG.getMachineFunction();
2029 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
2030 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
2031 MachineFunction::CallSiteInfo CSInfo;
2032 bool isStructRet = (Outs.empty()) ? false : Outs[0].Flags.isSRet();
2033 bool isThisReturn = false;
2034 bool isCmseNSCall = false;
2035 bool isSibCall = false;
2036 bool PreferIndirect = false;
2037 bool GuardWithBTI = false;
2038
2039 // Analyze operands of the call, assigning locations to each operand.
2040 SmallVector<CCValAssign, 16> ArgLocs;
2041 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs,
2042 *DAG.getContext());
2043 CCInfo.AnalyzeCallOperands(Outs, Fn: CCAssignFnForCall(CC: CallConv, isVarArg));
2044
2045 // Lower 'returns_twice' calls to a pseudo-instruction.
2046 if (CLI.CB && CLI.CB->getAttributes().hasFnAttr(Kind: Attribute::ReturnsTwice) &&
2047 !Subtarget->noBTIAtReturnTwice())
2048 GuardWithBTI = AFI->branchTargetEnforcement();
2049
2050 // Set type id for call site info.
2051 setTypeIdForCallsiteInfo(CB, MF, CSInfo);
2052
2053 // Determine whether this is a non-secure function call.
2054 if (CLI.CB && CLI.CB->getAttributes().hasFnAttr(Kind: "cmse_nonsecure_call"))
2055 isCmseNSCall = true;
2056
2057 // Disable tail calls if they're not supported.
2058 if (!Subtarget->supportsTailCall())
2059 isTailCall = false;
2060
2061 // For both the non-secure calls and the returns from a CMSE entry function,
2062 // the function needs to do some extra work after the call, or before the
2063 // return, respectively, thus it cannot end with a tail call
2064 if (isCmseNSCall || AFI->isCmseNSEntryFunction())
2065 isTailCall = false;
2066
2067 if (isa<GlobalAddressSDNode>(Val: Callee)) {
2068 // If we're optimizing for minimum size and the function is called three or
2069 // more times in this block, we can improve codesize by calling indirectly
2070 // as BLXr has a 16-bit encoding.
2071 auto *GV = cast<GlobalAddressSDNode>(Val&: Callee)->getGlobal();
2072 if (CLI.CB) {
2073 auto *BB = CLI.CB->getParent();
2074 PreferIndirect = Subtarget->isThumb() && Subtarget->hasMinSize() &&
2075 count_if(Range: GV->users(), P: [&BB](const User *U) {
2076 return isa<Instruction>(Val: U) &&
2077 cast<Instruction>(Val: U)->getParent() == BB;
2078 }) > 2;
2079 }
2080 }
2081 if (isTailCall) {
2082 // Check if it's really possible to do a tail call.
2083 isTailCall =
2084 IsEligibleForTailCallOptimization(CLI, CCInfo, ArgLocs, isIndirect: PreferIndirect);
2085
2086 if (isTailCall && !getTargetMachine().Options.GuaranteedTailCallOpt &&
2087 CallConv != CallingConv::Tail && CallConv != CallingConv::SwiftTail)
2088 isSibCall = true;
2089
2090 // We don't support GuaranteedTailCallOpt for ARM, only automatically
2091 // detected sibcalls.
2092 if (isTailCall)
2093 ++NumTailCalls;
2094 }
2095
2096 if (!isTailCall && CLI.CB && CLI.CB->isMustTailCall())
2097 report_fatal_error(reason: "failed to perform tail call elimination on a call "
2098 "site marked musttail");
2099
2100 // Get a count of how many bytes are to be pushed on the stack.
2101 unsigned NumBytes = CCInfo.getStackSize();
2102
2103 // SPDiff is the byte offset of the call's argument area from the callee's.
2104 // Stores to callee stack arguments will be placed in FixedStackSlots offset
2105 // by this amount for a tail call. In a sibling call it must be 0 because the
2106 // caller will deallocate the entire stack and the callee still expects its
2107 // arguments to begin at SP+0. Completely unused for non-tail calls.
2108 int SPDiff = 0;
2109
2110 if (isTailCall && !isSibCall) {
2111 auto FuncInfo = MF.getInfo<ARMFunctionInfo>();
2112 unsigned NumReusableBytes = FuncInfo->getArgumentStackSize();
2113
2114 // Since callee will pop argument stack as a tail call, we must keep the
2115 // popped size 16-byte aligned.
2116 MaybeAlign StackAlign = DAG.getDataLayout().getStackAlignment();
2117 assert(StackAlign && "data layout string is missing stack alignment");
2118 NumBytes = alignTo(Size: NumBytes, A: *StackAlign);
2119
2120 // SPDiff will be negative if this tail call requires more space than we
2121 // would automatically have in our incoming argument space. Positive if we
2122 // can actually shrink the stack.
2123 SPDiff = NumReusableBytes - NumBytes;
2124
2125 // If this call requires more stack than we have available from
2126 // LowerFormalArguments, tell FrameLowering to reserve space for it.
2127 if (SPDiff < 0 && AFI->getArgRegsSaveSize() < (unsigned)-SPDiff)
2128 AFI->setArgRegsSaveSize(-SPDiff);
2129 }
2130
2131 if (isSibCall) {
2132 // For sibling tail calls, memory operands are available in our caller's stack.
2133 NumBytes = 0;
2134 } else {
2135 // Adjust the stack pointer for the new arguments...
2136 // These operations are automatically eliminated by the prolog/epilog pass
2137 Chain = DAG.getCALLSEQ_START(Chain, InSize: isTailCall ? 0 : NumBytes, OutSize: 0, DL: dl);
2138 }
2139
2140 SDValue StackPtr =
2141 DAG.getCopyFromReg(Chain, dl, Reg: ARM::SP, VT: getPointerTy(DL: DAG.getDataLayout()));
2142
2143 RegsToPassVector RegsToPass;
2144 SmallVector<SDValue, 8> MemOpChains;
2145
2146 // If we are doing a tail-call, any byval arguments will be written to stack
2147 // space which was used for incoming arguments. If any the values being used
2148 // are incoming byval arguments to this function, then they might be
2149 // overwritten by the stores of the outgoing arguments. To avoid this, we
2150 // need to make a temporary copy of them in local stack space, then copy back
2151 // to the argument area.
2152 DenseMap<unsigned, SDValue> ByValTemporaries;
2153 SDValue ByValTempChain;
2154 if (isTailCall) {
2155 SmallVector<SDValue, 8> ByValCopyChains;
2156 for (const CCValAssign &VA : ArgLocs) {
2157 unsigned ArgIdx = VA.getValNo();
2158 SDValue Src = OutVals[ArgIdx];
2159 ISD::ArgFlagsTy Flags = Outs[ArgIdx].Flags;
2160
2161 if (!Flags.isByVal())
2162 continue;
2163
2164 SDValue Dst;
2165 MachinePointerInfo DstInfo;
2166 std::tie(args&: Dst, args&: DstInfo) =
2167 computeAddrForCallArg(dl, DAG, VA, StackPtr: SDValue(), IsTailCall: true, SPDiff);
2168 ByValCopyKind Copy = ByValNeedsCopyForTailCall(DAG, Src, Dst, Flags);
2169
2170 if (Copy == NoCopy) {
2171 // If the argument is already at the correct offset on the stack
2172 // (because we are forwarding a byval argument from our caller), we
2173 // don't need any copying.
2174 continue;
2175 } else if (Copy == CopyOnce) {
2176 // If the argument is in our local stack frame, no other argument
2177 // preparation can clobber it, so we can copy it to the final location
2178 // later.
2179 ByValTemporaries[ArgIdx] = Src;
2180 } else {
2181 assert(Copy == CopyViaTemp && "unexpected enum value");
2182 // If we might be copying this argument from the outgoing argument
2183 // stack area, we need to copy via a temporary in the local stack
2184 // frame.
2185 int TempFrameIdx = MFI.CreateStackObject(
2186 Size: Flags.getByValSize(), Alignment: Flags.getNonZeroByValAlign(), isSpillSlot: false);
2187 SDValue Temp =
2188 DAG.getFrameIndex(FI: TempFrameIdx, VT: getPointerTy(DL: DAG.getDataLayout()));
2189
2190 SDValue SizeNode = DAG.getConstant(Val: Flags.getByValSize(), DL: dl, VT: MVT::i32);
2191 SDValue AlignNode =
2192 DAG.getConstant(Val: Flags.getNonZeroByValAlign().value(), DL: dl, VT: MVT::i32);
2193
2194 SDVTList VTs = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
2195 SDValue Ops[] = {Chain, Temp, Src, SizeNode, AlignNode};
2196 ByValCopyChains.push_back(
2197 Elt: DAG.getNode(Opcode: ARMISD::COPY_STRUCT_BYVAL, DL: dl, VTList: VTs, Ops));
2198 ByValTemporaries[ArgIdx] = Temp;
2199 }
2200 }
2201 if (!ByValCopyChains.empty())
2202 ByValTempChain =
2203 DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: ByValCopyChains);
2204 }
2205
2206 // During a tail call, stores to the argument area must happen after all of
2207 // the function's incoming arguments have been loaded because they may alias.
2208 // This is done by folding in a TokenFactor from LowerFormalArguments, but
2209 // there's no point in doing so repeatedly so this tracks whether that's
2210 // happened yet.
2211 bool AfterFormalArgLoads = false;
2212
2213 // Walk the register/memloc assignments, inserting copies/loads. In the case
2214 // of tail call optimization, arguments are handled later.
2215 for (unsigned i = 0, realArgIdx = 0, e = ArgLocs.size();
2216 i != e;
2217 ++i, ++realArgIdx) {
2218 CCValAssign &VA = ArgLocs[i];
2219 SDValue Arg = OutVals[realArgIdx];
2220 ISD::ArgFlagsTy Flags = Outs[realArgIdx].Flags;
2221 bool isByVal = Flags.isByVal();
2222
2223 // Promote the value if needed.
2224 switch (VA.getLocInfo()) {
2225 default: llvm_unreachable("Unknown loc info!");
2226 case CCValAssign::Full: break;
2227 case CCValAssign::SExt:
2228 Arg = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: Arg);
2229 break;
2230 case CCValAssign::ZExt:
2231 Arg = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: Arg);
2232 break;
2233 case CCValAssign::AExt:
2234 Arg = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: Arg);
2235 break;
2236 case CCValAssign::BCvt:
2237 Arg = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: VA.getLocVT(), Operand: Arg);
2238 break;
2239 }
2240
2241 if (isTailCall && VA.isMemLoc() && !AfterFormalArgLoads) {
2242 Chain = DAG.getStackArgumentTokenFactor(Chain);
2243 if (ByValTempChain) {
2244 // In case of large byval copies, re-using the stackframe for tail-calls
2245 // can lead to overwriting incoming arguments on the stack. Force
2246 // loading these stack arguments before the copy to avoid that.
2247 SmallVector<SDValue, 8> IncomingLoad;
2248 for (unsigned I = 0; I < OutVals.size(); ++I) {
2249 if (Outs[I].Flags.isByVal())
2250 continue;
2251
2252 SDValue OutVal = OutVals[I];
2253 LoadSDNode *OutLN = dyn_cast_or_null<LoadSDNode>(Val&: OutVal);
2254 if (!OutLN)
2255 continue;
2256
2257 FrameIndexSDNode *FIN =
2258 dyn_cast_or_null<FrameIndexSDNode>(Val: OutLN->getBasePtr());
2259 if (!FIN)
2260 continue;
2261
2262 if (!MFI.isFixedObjectIndex(ObjectIdx: FIN->getIndex()))
2263 continue;
2264
2265 for (const CCValAssign &VA : ArgLocs) {
2266 if (VA.isMemLoc())
2267 IncomingLoad.push_back(Elt: OutVal.getValue(R: 1));
2268 }
2269 }
2270
2271 // Update the chain to force loads for potentially clobbered argument
2272 // loads to happen before the byval copy.
2273 if (!IncomingLoad.empty()) {
2274 IncomingLoad.push_back(Elt: Chain);
2275 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: IncomingLoad);
2276 }
2277
2278 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Chain,
2279 N2: ByValTempChain);
2280 }
2281 AfterFormalArgLoads = true;
2282 }
2283
2284 // f16 arguments have their size extended to 4 bytes and passed as if they
2285 // had been copied to the LSBs of a 32-bit register.
2286 // For that, it's passed extended to i32 (soft ABI) or to f32 (hard ABI)
2287 if (VA.needsCustom() &&
2288 (VA.getValVT() == MVT::f16 || VA.getValVT() == MVT::bf16)) {
2289 Arg = MoveFromHPR(dl, DAG, LocVT: VA.getLocVT(), ValVT: VA.getValVT(), Val: Arg);
2290 } else {
2291 // f16 arguments could have been extended prior to argument lowering.
2292 // Mask them arguments if this is a CMSE nonsecure call.
2293 auto ArgVT = Outs[realArgIdx].ArgVT;
2294 if (isCmseNSCall && (ArgVT == MVT::f16)) {
2295 auto LocBits = VA.getLocVT().getSizeInBits();
2296 auto MaskValue = APInt::getLowBitsSet(numBits: LocBits, loBitsSet: ArgVT.getSizeInBits());
2297 SDValue Mask =
2298 DAG.getConstant(Val: MaskValue, DL: dl, VT: MVT::getIntegerVT(BitWidth: LocBits));
2299 Arg = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::getIntegerVT(BitWidth: LocBits), Operand: Arg);
2300 Arg = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::getIntegerVT(BitWidth: LocBits), N1: Arg, N2: Mask);
2301 Arg = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: VA.getLocVT(), Operand: Arg);
2302 }
2303 }
2304
2305 // f64 and v2f64 might be passed in i32 pairs and must be split into pieces
2306 if (VA.needsCustom() && VA.getLocVT() == MVT::v2f64) {
2307 SDValue Op0 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: MVT::f64, N1: Arg,
2308 N2: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
2309 SDValue Op1 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: MVT::f64, N1: Arg,
2310 N2: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
2311
2312 PassF64ArgInRegs(dl, DAG, Chain, Arg&: Op0, RegsToPass, VA, NextVA&: ArgLocs[++i],
2313 StackPtr, MemOpChains, IsTailCall: isTailCall, SPDiff);
2314
2315 VA = ArgLocs[++i]; // skip ahead to next loc
2316 if (VA.isRegLoc()) {
2317 PassF64ArgInRegs(dl, DAG, Chain, Arg&: Op1, RegsToPass, VA, NextVA&: ArgLocs[++i],
2318 StackPtr, MemOpChains, IsTailCall: isTailCall, SPDiff);
2319 } else {
2320 assert(VA.isMemLoc());
2321 SDValue DstAddr;
2322 MachinePointerInfo DstInfo;
2323 std::tie(args&: DstAddr, args&: DstInfo) =
2324 computeAddrForCallArg(dl, DAG, VA, StackPtr, IsTailCall: isTailCall, SPDiff);
2325 MemOpChains.push_back(Elt: DAG.getStore(Chain, dl, Val: Op1, Ptr: DstAddr, PtrInfo: DstInfo));
2326 }
2327 } else if (VA.needsCustom() && VA.getLocVT() == MVT::f64) {
2328 PassF64ArgInRegs(dl, DAG, Chain, Arg, RegsToPass, VA, NextVA&: ArgLocs[++i],
2329 StackPtr, MemOpChains, IsTailCall: isTailCall, SPDiff);
2330 } else if (VA.isRegLoc()) {
2331 if (realArgIdx == 0 && Flags.isReturned() && !Flags.isSwiftSelf() &&
2332 Outs[0].VT == MVT::i32) {
2333 assert(VA.getLocVT() == MVT::i32 &&
2334 "unexpected calling convention register assignment");
2335 assert(!Ins.empty() && Ins[0].VT == MVT::i32 &&
2336 "unexpected use of 'returned'");
2337 isThisReturn = true;
2338 }
2339 const TargetOptions &Options = DAG.getTarget().Options;
2340 if (Options.EmitCallSiteInfo)
2341 CSInfo.ArgRegPairs.emplace_back(Args: VA.getLocReg(), Args&: i);
2342 RegsToPass.push_back(Elt: std::make_pair(x: VA.getLocReg(), y&: Arg));
2343 } else if (isByVal) {
2344 assert(VA.isMemLoc());
2345 unsigned offset = 0;
2346
2347 // True if this byval aggregate will be split between registers
2348 // and memory.
2349 unsigned ByValArgsCount = CCInfo.getInRegsParamsCount();
2350 unsigned CurByValIdx = CCInfo.getInRegsParamsProcessed();
2351
2352 SDValue ByValSrc;
2353 bool NeedsStackCopy;
2354 if (auto It = ByValTemporaries.find(Val: realArgIdx);
2355 It != ByValTemporaries.end()) {
2356 ByValSrc = It->second;
2357 NeedsStackCopy = true;
2358 } else {
2359 ByValSrc = Arg;
2360 NeedsStackCopy = !isTailCall;
2361 }
2362
2363 // If part of the argument is in registers, load them.
2364 if (CurByValIdx < ByValArgsCount) {
2365 unsigned RegBegin, RegEnd;
2366 CCInfo.getInRegsParamInfo(InRegsParamRecordIndex: CurByValIdx, BeginReg&: RegBegin, EndReg&: RegEnd);
2367
2368 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
2369 unsigned int i, j;
2370 for (i = 0, j = RegBegin; j < RegEnd; i++, j++) {
2371 SDValue Const = DAG.getConstant(Val: 4*i, DL: dl, VT: MVT::i32);
2372 SDValue AddArg = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: ByValSrc, N2: Const);
2373 SDValue Load =
2374 DAG.getLoad(VT: PtrVT, dl, Chain, Ptr: AddArg, PtrInfo: MachinePointerInfo(),
2375 Alignment: DAG.InferPtrAlign(Ptr: AddArg));
2376 MemOpChains.push_back(Elt: Load.getValue(R: 1));
2377 RegsToPass.push_back(Elt: std::make_pair(x&: j, y&: Load));
2378 }
2379
2380 // If parameter size outsides register area, "offset" value
2381 // helps us to calculate stack slot for remained part properly.
2382 offset = RegEnd - RegBegin;
2383
2384 CCInfo.nextInRegsParam();
2385 }
2386
2387 // If the memory part of the argument isn't already in the correct place
2388 // (which can happen with tail calls), copy it into the argument area.
2389 if (NeedsStackCopy && Flags.getByValSize() > 4 * offset) {
2390 auto PtrVT = getPointerTy(DL: DAG.getDataLayout());
2391 SDValue Dst;
2392 MachinePointerInfo DstInfo;
2393 std::tie(args&: Dst, args&: DstInfo) =
2394 computeAddrForCallArg(dl, DAG, VA, StackPtr, IsTailCall: isTailCall, SPDiff);
2395 SDValue SrcOffset = DAG.getIntPtrConstant(Val: 4*offset, DL: dl);
2396 SDValue Src = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: ByValSrc, N2: SrcOffset);
2397 SDValue SizeNode = DAG.getConstant(Val: Flags.getByValSize() - 4*offset, DL: dl,
2398 VT: MVT::i32);
2399 SDValue AlignNode =
2400 DAG.getConstant(Val: Flags.getNonZeroByValAlign().value(), DL: dl, VT: MVT::i32);
2401
2402 SDVTList VTs = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
2403 SDValue Ops[] = { Chain, Dst, Src, SizeNode, AlignNode};
2404 MemOpChains.push_back(Elt: DAG.getNode(Opcode: ARMISD::COPY_STRUCT_BYVAL, DL: dl, VTList: VTs,
2405 Ops));
2406 }
2407 } else {
2408 assert(VA.isMemLoc());
2409 SDValue DstAddr;
2410 MachinePointerInfo DstInfo;
2411 std::tie(args&: DstAddr, args&: DstInfo) =
2412 computeAddrForCallArg(dl, DAG, VA, StackPtr, IsTailCall: isTailCall, SPDiff);
2413
2414 SDValue Store = DAG.getStore(Chain, dl, Val: Arg, Ptr: DstAddr, PtrInfo: DstInfo);
2415 MemOpChains.push_back(Elt: Store);
2416 }
2417 }
2418
2419 if (!MemOpChains.empty())
2420 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: MemOpChains);
2421
2422 // Build a sequence of copy-to-reg nodes chained together with token chain
2423 // and flag operands which copy the outgoing args into the appropriate regs.
2424 SDValue InGlue;
2425 for (const auto &[Reg, N] : RegsToPass) {
2426 Chain = DAG.getCopyToReg(Chain, dl, Reg, N, Glue: InGlue);
2427 InGlue = Chain.getValue(R: 1);
2428 }
2429
2430 // If the callee is a GlobalAddress/ExternalSymbol node (quite common, every
2431 // direct call is) turn it into a TargetGlobalAddress/TargetExternalSymbol
2432 // node so that legalize doesn't hack it.
2433 bool isDirect = false;
2434
2435 const TargetMachine &TM = getTargetMachine();
2436 const Triple &TT = TM.getTargetTriple();
2437 const GlobalValue *GVal = nullptr;
2438 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Val&: Callee))
2439 GVal = G->getGlobal();
2440 bool isStub = !TM.shouldAssumeDSOLocal(GV: GVal) && TT.isOSBinFormatMachO();
2441
2442 bool isARMFunc = !Subtarget->isThumb() || (isStub && !Subtarget->isMClass());
2443 bool isLocalARMFunc = false;
2444 auto PtrVt = getPointerTy(DL: DAG.getDataLayout());
2445
2446 if (Subtarget->genLongCalls()) {
2447 bool isPIC = isPositionIndependent() && !TT.isOSWindows();
2448 if (isPIC && Subtarget->genExecuteOnly())
2449 reportFatalUsageError(reason: "long-calls with execute-only and "
2450 "position-independent code is not supported");
2451 if (Subtarget->isROPI())
2452 reportFatalUsageError(reason: "long-calls with ROPI is not currently supported");
2453
2454 // Handle a global address or an external symbol. If it's not one of
2455 // those, the target's already in a register, so we don't need to do
2456 // anything extra.
2457 if (isa<GlobalAddressSDNode>(Val: Callee)) {
2458 if (Subtarget->genExecuteOnly()) {
2459 // Execute-only forbids constant pools in .text, so use movw/movt.
2460 // fPIC is not supported with execute-only.
2461 if (Subtarget->useMovt())
2462 ++NumMovwMovt;
2463 Callee = DAG.getNode(Opcode: ARMISD::Wrapper, DL: dl, VT: PtrVt,
2464 Operand: DAG.getTargetGlobalAddress(GV: GVal, DL: dl, VT: PtrVt));
2465 } else if (isPIC) {
2466 // PIC without execute-only: use GOT-based addressing.
2467 // DSO-local symbols use a plain PC-relative WrapperPIC;
2468 // non-DSO-local symbols additionally load the address from the GOT.
2469 SDValue G = DAG.getTargetGlobalAddress(
2470 GV: GVal, DL: dl, VT: PtrVt, offset: 0, TargetFlags: GVal->isDSOLocal() ? 0 : ARMII::MO_GOT);
2471 Callee = DAG.getNode(Opcode: ARMISD::WrapperPIC, DL: dl, VT: PtrVt, Operand: G);
2472 if (!GVal->isDSOLocal())
2473 Callee =
2474 DAG.getLoad(VT: PtrVt, dl, Chain: DAG.getEntryNode(), Ptr: Callee,
2475 PtrInfo: MachinePointerInfo::getGOT(MF&: DAG.getMachineFunction()));
2476 } else {
2477 // Neither execute-only nor PIC: load the address from a constant pool.
2478 unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
2479 ARMConstantPoolValue *CPV = ARMConstantPoolConstant::Create(
2480 C: GVal, ID: ARMPCLabelIndex, Kind: ARMCP::CPValue, PCAdj: 0);
2481
2482 // Get the address of the callee into a register
2483 SDValue Addr = DAG.getTargetConstantPool(C: CPV, VT: PtrVt, Align: Align(4));
2484 Addr = DAG.getNode(Opcode: ARMISD::Wrapper, DL: dl, VT: MVT::i32, Operand: Addr);
2485 Callee = DAG.getLoad(
2486 VT: PtrVt, dl, Chain: DAG.getEntryNode(), Ptr: Addr,
2487 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
2488 }
2489 } else if (ExternalSymbolSDNode *S=dyn_cast<ExternalSymbolSDNode>(Val&: Callee)) {
2490 const char *Sym = S->getSymbol();
2491
2492 if (Subtarget->genExecuteOnly()) {
2493 // Execute-only forbids constant pools in .text, so use movw/movt.
2494 // fPIC is not supported with execute-only.
2495 if (Subtarget->useMovt())
2496 ++NumMovwMovt;
2497 Callee = DAG.getNode(Opcode: ARMISD::Wrapper, DL: dl, VT: PtrVt,
2498 Operand: DAG.getTargetExternalSymbol(Sym, VT: PtrVt, TargetFlags: 0));
2499 } else if (isPIC) {
2500 // PIC without execute-only: load the symbol's address from the GOT via
2501 // a GOT_PREL constant pool entry consumed by a PICLDR.
2502 unsigned PCAdj = Subtarget->isThumb() ? 4 : 8;
2503 unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
2504 ARMConstantPoolValue *CPV = ARMConstantPoolSymbol::Create(
2505 C&: *DAG.getContext(), s: Sym, ID: ARMPCLabelIndex, PCAdj, Modifier: ARMCP::GOT_PREL,
2506 /*AddCurrentAddress=*/true);
2507 SDValue CPAddr = DAG.getTargetConstantPool(C: CPV, VT: PtrVt, Align: Align(4));
2508 CPAddr = DAG.getNode(Opcode: ARMISD::Wrapper, DL: dl, VT: MVT::i32, Operand: CPAddr);
2509 SDValue GOTOffset = DAG.getLoad(
2510 VT: PtrVt, dl, Chain: DAG.getEntryNode(), Ptr: CPAddr,
2511 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
2512 SDValue PICLabel = DAG.getConstant(Val: ARMPCLabelIndex, DL: dl, VT: MVT::i32);
2513 Callee = DAG.getNode(Opcode: ARMISD::PIC_ADD, DL: dl, VT: PtrVt, N1: GOTOffset, N2: PICLabel);
2514 Callee =
2515 DAG.getLoad(VT: PtrVt, dl, Chain: DAG.getEntryNode(), Ptr: Callee,
2516 PtrInfo: MachinePointerInfo::getGOT(MF&: DAG.getMachineFunction()));
2517 } else {
2518 // Neither execute-only nor PIC: load the address from a constant pool.
2519 unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
2520 ARMConstantPoolValue *CPV = ARMConstantPoolSymbol::Create(
2521 C&: *DAG.getContext(), s: Sym, ID: ARMPCLabelIndex, PCAdj: 0);
2522
2523 // Get the address of the callee into a register
2524 SDValue Addr = DAG.getTargetConstantPool(C: CPV, VT: PtrVt, Align: Align(4));
2525 Addr = DAG.getNode(Opcode: ARMISD::Wrapper, DL: dl, VT: MVT::i32, Operand: Addr);
2526 Callee = DAG.getLoad(
2527 VT: PtrVt, dl, Chain: DAG.getEntryNode(), Ptr: Addr,
2528 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
2529 }
2530 }
2531 } else if (isa<GlobalAddressSDNode>(Val: Callee)) {
2532 if (!PreferIndirect) {
2533 isDirect = true;
2534 bool isDef = GVal->isStrongDefinitionForLinker();
2535
2536 // ARM call to a local ARM function is predicable.
2537 isLocalARMFunc = !Subtarget->isThumb() && (isDef || !ARMInterworking);
2538 // tBX takes a register source operand.
2539 if (isStub && Subtarget->isThumb1Only() && !Subtarget->hasV5TOps()) {
2540 assert(TT.isOSBinFormatMachO() && "WrapperPIC use on non-MachO?");
2541 Callee = DAG.getNode(
2542 Opcode: ARMISD::WrapperPIC, DL: dl, VT: PtrVt,
2543 Operand: DAG.getTargetGlobalAddress(GV: GVal, DL: dl, VT: PtrVt, offset: 0, TargetFlags: ARMII::MO_NONLAZY));
2544 Callee = DAG.getLoad(
2545 VT: PtrVt, dl, Chain: DAG.getEntryNode(), Ptr: Callee,
2546 PtrInfo: MachinePointerInfo::getGOT(MF&: DAG.getMachineFunction()), Alignment: MaybeAlign(),
2547 MMOFlags: MachineMemOperand::MODereferenceable |
2548 MachineMemOperand::MOInvariant);
2549 } else if (Subtarget->isTargetCOFF()) {
2550 assert(Subtarget->isTargetWindows() &&
2551 "Windows is the only supported COFF target");
2552 unsigned TargetFlags = ARMII::MO_NO_FLAG;
2553 if (GVal->hasDLLImportStorageClass())
2554 TargetFlags = ARMII::MO_DLLIMPORT;
2555 else if (!TM.shouldAssumeDSOLocal(GV: GVal))
2556 TargetFlags = ARMII::MO_COFFSTUB;
2557 Callee = DAG.getTargetGlobalAddress(GV: GVal, DL: dl, VT: PtrVt, /*offset=*/0,
2558 TargetFlags);
2559 if (TargetFlags & (ARMII::MO_DLLIMPORT | ARMII::MO_COFFSTUB))
2560 Callee =
2561 DAG.getLoad(VT: PtrVt, dl, Chain: DAG.getEntryNode(),
2562 Ptr: DAG.getNode(Opcode: ARMISD::Wrapper, DL: dl, VT: PtrVt, Operand: Callee),
2563 PtrInfo: MachinePointerInfo::getGOT(MF&: DAG.getMachineFunction()));
2564 } else {
2565 Callee = DAG.getTargetGlobalAddress(GV: GVal, DL: dl, VT: PtrVt, offset: 0, TargetFlags: 0);
2566 }
2567 }
2568 } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Val&: Callee)) {
2569 isDirect = true;
2570 // tBX takes a register source operand.
2571 const char *Sym = S->getSymbol();
2572 if (isARMFunc && Subtarget->isThumb1Only() && !Subtarget->hasV5TOps()) {
2573 unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
2574 ARMConstantPoolValue *CPV =
2575 ARMConstantPoolSymbol::Create(C&: *DAG.getContext(), s: Sym,
2576 ID: ARMPCLabelIndex, PCAdj: 4);
2577 SDValue CPAddr = DAG.getTargetConstantPool(C: CPV, VT: PtrVt, Align: Align(4));
2578 CPAddr = DAG.getNode(Opcode: ARMISD::Wrapper, DL: dl, VT: MVT::i32, Operand: CPAddr);
2579 Callee = DAG.getLoad(
2580 VT: PtrVt, dl, Chain: DAG.getEntryNode(), Ptr: CPAddr,
2581 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
2582 SDValue PICLabel = DAG.getConstant(Val: ARMPCLabelIndex, DL: dl, VT: MVT::i32);
2583 Callee = DAG.getNode(Opcode: ARMISD::PIC_ADD, DL: dl, VT: PtrVt, N1: Callee, N2: PICLabel);
2584 } else {
2585 Callee = DAG.getTargetExternalSymbol(Sym, VT: PtrVt, TargetFlags: 0);
2586 }
2587 }
2588
2589 if (isCmseNSCall) {
2590 assert(!isARMFunc && !isDirect &&
2591 "Cannot handle call to ARM function or direct call");
2592 if (NumBytes > 0) {
2593 DAG.getContext()->diagnose(
2594 DI: DiagnosticInfoUnsupported(DAG.getMachineFunction().getFunction(),
2595 "call to non-secure function would require "
2596 "passing arguments on stack",
2597 dl.getDebugLoc()));
2598 }
2599 if (isStructRet) {
2600 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
2601 DAG.getMachineFunction().getFunction(),
2602 "call to non-secure function would return value through pointer",
2603 dl.getDebugLoc()));
2604 }
2605 }
2606
2607 // FIXME: handle tail calls differently.
2608 unsigned CallOpc;
2609 if (Subtarget->isThumb()) {
2610 if (GuardWithBTI)
2611 CallOpc = ARMISD::t2CALL_BTI;
2612 else if (isCmseNSCall)
2613 CallOpc = ARMISD::tSECALL;
2614 else if ((!isDirect || isARMFunc) && !Subtarget->hasV5TOps())
2615 CallOpc = ARMISD::CALL_NOLINK;
2616 else
2617 CallOpc = ARMISD::CALL;
2618 } else {
2619 if (!isDirect && !Subtarget->hasV5TOps())
2620 CallOpc = ARMISD::CALL_NOLINK;
2621 else if (doesNotRet && isDirect && Subtarget->hasRetAddrStack() &&
2622 // Emit regular call when code size is the priority
2623 !Subtarget->hasMinSize())
2624 // "mov lr, pc; b _foo" to avoid confusing the RSP
2625 CallOpc = ARMISD::CALL_NOLINK;
2626 else
2627 CallOpc = isLocalARMFunc ? ARMISD::CALL_PRED : ARMISD::CALL;
2628 }
2629
2630 // We don't usually want to end the call-sequence here because we would tidy
2631 // the frame up *after* the call, however in the ABI-changing tail-call case
2632 // we've carefully laid out the parameters so that when sp is reset they'll be
2633 // in the correct location.
2634 if (isTailCall && !isSibCall) {
2635 Chain = DAG.getCALLSEQ_END(Chain, Size1: 0, Size2: 0, Glue: InGlue, DL: dl);
2636 InGlue = Chain.getValue(R: 1);
2637 }
2638
2639 std::vector<SDValue> Ops;
2640 Ops.push_back(x: Chain);
2641 Ops.push_back(x: Callee);
2642
2643 if (isTailCall) {
2644 Ops.push_back(x: DAG.getSignedTargetConstant(Val: SPDiff, DL: dl, VT: MVT::i32));
2645 }
2646
2647 // Add argument registers to the end of the list so that they are known live
2648 // into the call.
2649 for (const auto &[Reg, N] : RegsToPass)
2650 Ops.push_back(x: DAG.getRegister(Reg, VT: N.getValueType()));
2651
2652 // Add a register mask operand representing the call-preserved registers.
2653 const uint32_t *Mask;
2654 const ARMBaseRegisterInfo *ARI = Subtarget->getRegisterInfo();
2655 if (isThisReturn) {
2656 // For 'this' returns, use the R0-preserving mask if applicable
2657 Mask = ARI->getThisReturnPreservedMask(MF, CallConv);
2658 if (!Mask) {
2659 // Set isThisReturn to false if the calling convention is not one that
2660 // allows 'returned' to be modeled in this way, so LowerCallResult does
2661 // not try to pass 'this' straight through
2662 isThisReturn = false;
2663 Mask = ARI->getCallPreservedMask(MF, CallConv);
2664 }
2665 } else
2666 Mask = ARI->getCallPreservedMask(MF, CallConv);
2667
2668 assert(Mask && "Missing call preserved mask for calling convention");
2669 Ops.push_back(x: DAG.getRegisterMask(RegMask: Mask));
2670
2671 if (InGlue.getNode())
2672 Ops.push_back(x: InGlue);
2673
2674 if (isTailCall) {
2675 MF.getFrameInfo().setHasTailCall();
2676 SDValue Ret = DAG.getNode(Opcode: ARMISD::TC_RETURN, DL: dl, VT: MVT::Other, Ops);
2677 if (CLI.CFIType)
2678 Ret.getNode()->setCFIType(CLI.CFIType->getZExtValue());
2679 DAG.addNoMergeSiteInfo(Node: Ret.getNode(), NoMerge: CLI.NoMerge);
2680 DAG.addCallSiteInfo(Node: Ret.getNode(), CallInfo: std::move(CSInfo));
2681 return Ret;
2682 }
2683
2684 // Returns a chain and a flag for retval copy to use.
2685 Chain = DAG.getNode(Opcode: CallOpc, DL: dl, ResultTys: {MVT::Other, MVT::Glue}, Ops);
2686 if (CLI.CFIType)
2687 Chain.getNode()->setCFIType(CLI.CFIType->getZExtValue());
2688 DAG.addNoMergeSiteInfo(Node: Chain.getNode(), NoMerge: CLI.NoMerge);
2689 InGlue = Chain.getValue(R: 1);
2690 DAG.addCallSiteInfo(Node: Chain.getNode(), CallInfo: std::move(CSInfo));
2691
2692 // If we're guaranteeing tail-calls will be honoured, the callee must
2693 // pop its own argument stack on return. But this call is *not* a tail call so
2694 // we need to undo that after it returns to restore the status-quo.
2695 bool TailCallOpt = getTargetMachine().Options.GuaranteedTailCallOpt;
2696 uint64_t CalleePopBytes =
2697 canGuaranteeTCO(CC: CallConv, GuaranteeTailCalls: TailCallOpt) ? alignTo(Value: NumBytes, Align: 16) : -1U;
2698
2699 Chain = DAG.getCALLSEQ_END(Chain, Size1: NumBytes, Size2: CalleePopBytes, Glue: InGlue, DL: dl);
2700 if (!Ins.empty())
2701 InGlue = Chain.getValue(R: 1);
2702
2703 // Handle result values, copying them out of physregs into vregs that we
2704 // return.
2705 return LowerCallResult(Chain, InGlue, CallConv, isVarArg, Ins, dl, DAG,
2706 InVals, isThisReturn,
2707 ThisVal: isThisReturn ? OutVals[0] : SDValue(), isCmseNSCall);
2708}
2709
2710/// HandleByVal - Every parameter *after* a byval parameter is passed
2711/// on the stack. Remember the next parameter register to allocate,
2712/// and then confiscate the rest of the parameter registers to insure
2713/// this.
2714void ARMTargetLowering::HandleByVal(CCState *State, unsigned &Size,
2715 Align Alignment) const {
2716 // Byval (as with any stack) slots are always at least 4 byte aligned.
2717 Alignment = std::max(a: Alignment, b: Align(4));
2718
2719 MCRegister Reg = State->AllocateReg(Regs: GPRArgRegs);
2720 if (!Reg)
2721 return;
2722
2723 unsigned AlignInRegs = Alignment.value() / 4;
2724 unsigned Waste = (ARM::R4 - Reg) % AlignInRegs;
2725 for (unsigned i = 0; i < Waste; ++i)
2726 Reg = State->AllocateReg(Regs: GPRArgRegs);
2727
2728 if (!Reg)
2729 return;
2730
2731 unsigned Excess = 4 * (ARM::R4 - Reg);
2732
2733 // Special case when NSAA != SP and parameter size greater than size of
2734 // all remained GPR regs. In that case we can't split parameter, we must
2735 // send it to stack. We also must set NCRN to R4, so waste all
2736 // remained registers.
2737 const unsigned NSAAOffset = State->getStackSize();
2738 if (NSAAOffset != 0 && Size > Excess) {
2739 while (State->AllocateReg(Regs: GPRArgRegs))
2740 ;
2741 return;
2742 }
2743
2744 // First register for byval parameter is the first register that wasn't
2745 // allocated before this method call, so it would be "reg".
2746 // If parameter is small enough to be saved in range [reg, r4), then
2747 // the end (first after last) register would be reg + param-size-in-regs,
2748 // else parameter would be splitted between registers and stack,
2749 // end register would be r4 in this case.
2750 unsigned ByValRegBegin = Reg;
2751 unsigned ByValRegEnd = std::min<unsigned>(a: Reg + Size / 4, b: ARM::R4);
2752 State->addInRegsParamInfo(RegBegin: ByValRegBegin, RegEnd: ByValRegEnd);
2753 // Note, first register is allocated in the beginning of function already,
2754 // allocate remained amount of registers we need.
2755 for (unsigned i = Reg + 1; i != ByValRegEnd; ++i)
2756 State->AllocateReg(Regs: GPRArgRegs);
2757 // A byval parameter that is split between registers and memory needs its
2758 // size truncated here.
2759 // In the case where the entire structure fits in registers, we set the
2760 // size in memory to zero.
2761 Size = std::max<int>(a: Size - Excess, b: 0);
2762}
2763
2764/// IsEligibleForTailCallOptimization - Check whether the call is eligible
2765/// for tail call optimization. Targets which want to do tail call
2766/// optimization should implement this function. Note that this function also
2767/// processes musttail calls, so when this function returns false on a valid
2768/// musttail call, a fatal backend error occurs.
2769bool ARMTargetLowering::IsEligibleForTailCallOptimization(
2770 TargetLowering::CallLoweringInfo &CLI, CCState &CCInfo,
2771 SmallVectorImpl<CCValAssign> &ArgLocs, const bool isIndirect) const {
2772 CallingConv::ID CalleeCC = CLI.CallConv;
2773 SDValue Callee = CLI.Callee;
2774 bool isVarArg = CLI.IsVarArg;
2775 const SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
2776 const SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
2777 const SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
2778 const SelectionDAG &DAG = CLI.DAG;
2779 MachineFunction &MF = DAG.getMachineFunction();
2780 const Function &CallerF = MF.getFunction();
2781 CallingConv::ID CallerCC = CallerF.getCallingConv();
2782
2783 assert(Subtarget->supportsTailCall());
2784
2785 // Indirect tail-calls require a register to hold the target address. That
2786 // register must be:
2787 // * Allocatable (i.e. r0-r7 if the target is Thumb1).
2788 // * Not callee-saved, so must be one of r0-r3 or r12.
2789 // * Not used to hold an argument to the tail-called function, which might be
2790 // in r0-r3.
2791 // * Not used to hold the return address authentication code, which is in r12
2792 // if enabled.
2793 // Sometimes, no register matches all of these conditions, so we can't do a
2794 // tail-call.
2795 if (!isa<GlobalAddressSDNode>(Val: Callee.getNode()) || isIndirect) {
2796 SmallSet<MCPhysReg, 5> AddressRegisters = {ARM::R0, ARM::R1, ARM::R2,
2797 ARM::R3};
2798 if (!(Subtarget->isThumb1Only() ||
2799 MF.getInfo<ARMFunctionInfo>()->shouldSignReturnAddress(SpillsLR: true)))
2800 AddressRegisters.insert(V: ARM::R12);
2801 for (const CCValAssign &AL : ArgLocs)
2802 if (AL.isRegLoc())
2803 AddressRegisters.erase(V: AL.getLocReg());
2804 if (AddressRegisters.empty()) {
2805 LLVM_DEBUG(dbgs() << "false (no reg to hold function pointer)\n");
2806 return false;
2807 }
2808 }
2809
2810 // Look for obvious safe cases to perform tail call optimization that do not
2811 // require ABI changes. This is what gcc calls sibcall.
2812
2813 // Exception-handling functions need a special set of instructions to indicate
2814 // a return to the hardware. Tail-calling another function would probably
2815 // break this.
2816 if (CallerF.hasFnAttribute(Kind: "interrupt")) {
2817 LLVM_DEBUG(dbgs() << "false (interrupt attribute)\n");
2818 return false;
2819 }
2820
2821 if (canGuaranteeTCO(CC: CalleeCC,
2822 GuaranteeTailCalls: getTargetMachine().Options.GuaranteedTailCallOpt)) {
2823 LLVM_DEBUG(dbgs() << (CalleeCC == CallerCC ? "true" : "false")
2824 << " (guaranteed tail-call CC)\n");
2825 return CalleeCC == CallerCC;
2826 }
2827
2828 // Also avoid sibcall optimization if either caller or callee uses struct
2829 // return semantics.
2830 bool isCalleeStructRet = Outs.empty() ? false : Outs[0].Flags.isSRet();
2831 bool isCallerStructRet = MF.getFunction().hasStructRetAttr();
2832 if (isCalleeStructRet != isCallerStructRet) {
2833 LLVM_DEBUG(dbgs() << "false (struct-ret)\n");
2834 return false;
2835 }
2836
2837 // Externally-defined functions with weak linkage should not be
2838 // tail-called on ARM when the OS does not support dynamic
2839 // pre-emption of symbols, as the AAELF spec requires normal calls
2840 // to undefined weak functions to be replaced with a NOP or jump to the
2841 // next instruction. The behaviour of branch instructions in this
2842 // situation (as used for tail calls) is implementation-defined, so we
2843 // cannot rely on the linker replacing the tail call with a return.
2844 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Val&: Callee)) {
2845 const GlobalValue *GV = G->getGlobal();
2846 const Triple &TT = getTargetMachine().getTargetTriple();
2847 if (GV->hasExternalWeakLinkage() &&
2848 (!TT.isOSWindows() || TT.isOSBinFormatELF() ||
2849 TT.isOSBinFormatMachO())) {
2850 LLVM_DEBUG(dbgs() << "false (external weak linkage)\n");
2851 return false;
2852 }
2853 }
2854
2855 // Check that the call results are passed in the same way.
2856 LLVMContext &C = *DAG.getContext();
2857 if (!CCState::resultsCompatible(
2858 CalleeCC: getEffectiveCallingConv(CC: CalleeCC, isVarArg),
2859 CallerCC: getEffectiveCallingConv(CC: CallerCC, isVarArg: CallerF.isVarArg()), MF, C, Ins,
2860 CalleeFn: CCAssignFnForReturn(CC: CalleeCC, isVarArg),
2861 CallerFn: CCAssignFnForReturn(CC: CallerCC, isVarArg: CallerF.isVarArg()))) {
2862 LLVM_DEBUG(dbgs() << "false (incompatible results)\n");
2863 return false;
2864 }
2865 // The callee has to preserve all registers the caller needs to preserve.
2866 const ARMBaseRegisterInfo *TRI = Subtarget->getRegisterInfo();
2867 const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
2868 if (CalleeCC != CallerCC) {
2869 const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
2870 if (!TRI->regmaskSubsetEqual(mask0: CallerPreserved, mask1: CalleePreserved)) {
2871 LLVM_DEBUG(dbgs() << "false (not all registers preserved)\n");
2872 return false;
2873 }
2874 }
2875
2876 // If Caller's vararg argument has been split between registers and stack, do
2877 // not perform tail call, since part of the argument is in caller's local
2878 // frame.
2879 const ARMFunctionInfo *AFI_Caller = MF.getInfo<ARMFunctionInfo>();
2880 if (CLI.IsVarArg && AFI_Caller->getArgRegsSaveSize()) {
2881 LLVM_DEBUG(dbgs() << "false (arg reg save area)\n");
2882 return false;
2883 }
2884
2885 // If the callee takes no arguments then go on to check the results of the
2886 // call.
2887 const MachineRegisterInfo &MRI = MF.getRegInfo();
2888 if (!parametersInCSRMatch(MRI, CallerPreservedMask: CallerPreserved, ArgLocs, OutVals)) {
2889 LLVM_DEBUG(dbgs() << "false (parameters in CSRs do not match)\n");
2890 return false;
2891 }
2892
2893 // If the stack arguments for this call do not fit into our own save area then
2894 // the call cannot be made tail.
2895 if (CCInfo.getStackSize() > AFI_Caller->getArgumentStackSize())
2896 return false;
2897
2898 LLVM_DEBUG(dbgs() << "true\n");
2899 return true;
2900}
2901
2902bool
2903ARMTargetLowering::CanLowerReturn(CallingConv::ID CallConv,
2904 MachineFunction &MF, bool isVarArg,
2905 const SmallVectorImpl<ISD::OutputArg> &Outs,
2906 LLVMContext &Context, const Type *RetTy) const {
2907 SmallVector<CCValAssign, 16> RVLocs;
2908 CCState CCInfo(CallConv, isVarArg, MF, RVLocs, Context);
2909 return CCInfo.CheckReturn(Outs, Fn: CCAssignFnForReturn(CC: CallConv, isVarArg));
2910}
2911
2912static SDValue LowerInterruptReturn(SmallVectorImpl<SDValue> &RetOps,
2913 const SDLoc &DL, SelectionDAG &DAG) {
2914 const MachineFunction &MF = DAG.getMachineFunction();
2915 const Function &F = MF.getFunction();
2916
2917 StringRef IntKind = F.getFnAttribute(Kind: "interrupt").getValueAsString();
2918
2919 // See ARM ARM v7 B1.8.3. On exception entry LR is set to a possibly offset
2920 // version of the "preferred return address". These offsets affect the return
2921 // instruction if this is a return from PL1 without hypervisor extensions.
2922 // IRQ/FIQ: +4 "subs pc, lr, #4"
2923 // SWI: 0 "subs pc, lr, #0"
2924 // ABORT: +4 "subs pc, lr, #4"
2925 // UNDEF: +4/+2 "subs pc, lr, #0"
2926 // UNDEF varies depending on where the exception came from ARM or Thumb
2927 // mode. Alongside GCC, we throw our hands up in disgust and pretend it's 0.
2928
2929 int64_t LROffset;
2930 if (IntKind == "" || IntKind == "IRQ" || IntKind == "FIQ" ||
2931 IntKind == "ABORT")
2932 LROffset = 4;
2933 else if (IntKind == "SWI" || IntKind == "UNDEF")
2934 LROffset = 0;
2935 else
2936 report_fatal_error(reason: "Unsupported interrupt attribute. If present, value "
2937 "must be one of: IRQ, FIQ, SWI, ABORT or UNDEF");
2938
2939 RetOps.insert(I: RetOps.begin() + 1,
2940 Elt: DAG.getConstant(Val: LROffset, DL, VT: MVT::i32, isTarget: false));
2941
2942 return DAG.getNode(Opcode: ARMISD::INTRET_GLUE, DL, VT: MVT::Other, Ops: RetOps);
2943}
2944
2945SDValue
2946ARMTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
2947 bool isVarArg,
2948 const SmallVectorImpl<ISD::OutputArg> &Outs,
2949 const SmallVectorImpl<SDValue> &OutVals,
2950 const SDLoc &dl, SelectionDAG &DAG) const {
2951 // CCValAssign - represent the assignment of the return value to a location.
2952 SmallVector<CCValAssign, 16> RVLocs;
2953
2954 // CCState - Info about the registers and stack slots.
2955 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
2956 *DAG.getContext());
2957
2958 // Analyze outgoing return values.
2959 CCInfo.AnalyzeReturn(Outs, Fn: CCAssignFnForReturn(CC: CallConv, isVarArg));
2960
2961 SDValue Glue;
2962 SmallVector<SDValue, 4> RetOps;
2963 RetOps.push_back(Elt: Chain); // Operand #0 = Chain (updated below)
2964 bool isLittleEndian = Subtarget->isLittle();
2965
2966 MachineFunction &MF = DAG.getMachineFunction();
2967 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
2968 AFI->setReturnRegsCount(RVLocs.size());
2969
2970 // Report error if cmse entry function returns structure through first ptr arg.
2971 if (AFI->isCmseNSEntryFunction() && MF.getFunction().hasStructRetAttr()) {
2972 // Note: using an empty SDLoc(), as the first line of the function is a
2973 // better place to report than the last line.
2974 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
2975 DAG.getMachineFunction().getFunction(),
2976 "secure entry function would return value through pointer",
2977 SDLoc().getDebugLoc()));
2978 }
2979
2980 // Copy the result values into the output registers.
2981 for (unsigned i = 0, realRVLocIdx = 0;
2982 i != RVLocs.size();
2983 ++i, ++realRVLocIdx) {
2984 CCValAssign &VA = RVLocs[i];
2985 assert(VA.isRegLoc() && "Can only return in registers!");
2986
2987 SDValue Arg = OutVals[realRVLocIdx];
2988 bool ReturnF16 = false;
2989
2990 if (Subtarget->hasFullFP16() && Subtarget->isTargetHardFloat()) {
2991 // Half-precision return values can be returned like this:
2992 //
2993 // t11 f16 = fadd ...
2994 // t12: i16 = bitcast t11
2995 // t13: i32 = zero_extend t12
2996 // t14: f32 = bitcast t13 <~~~~~~~ Arg
2997 //
2998 // to avoid code generation for bitcasts, we simply set Arg to the node
2999 // that produces the f16 value, t11 in this case.
3000 //
3001 if (Arg.getValueType() == MVT::f32 && Arg.getOpcode() == ISD::BITCAST) {
3002 SDValue ZE = Arg.getOperand(i: 0);
3003 if (ZE.getOpcode() == ISD::ZERO_EXTEND && ZE.getValueType() == MVT::i32) {
3004 SDValue BC = ZE.getOperand(i: 0);
3005 if (BC.getOpcode() == ISD::BITCAST && BC.getValueType() == MVT::i16) {
3006 Arg = BC.getOperand(i: 0);
3007 ReturnF16 = true;
3008 }
3009 }
3010 }
3011 }
3012
3013 switch (VA.getLocInfo()) {
3014 default: llvm_unreachable("Unknown loc info!");
3015 case CCValAssign::Full: break;
3016 case CCValAssign::BCvt:
3017 if (!ReturnF16)
3018 Arg = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: VA.getLocVT(), Operand: Arg);
3019 break;
3020 }
3021
3022 // Mask f16 arguments if this is a CMSE nonsecure entry.
3023 auto RetVT = Outs[realRVLocIdx].ArgVT;
3024 if (AFI->isCmseNSEntryFunction() && (RetVT == MVT::f16)) {
3025 if (VA.needsCustom() && VA.getValVT() == MVT::f16) {
3026 Arg = MoveFromHPR(dl, DAG, LocVT: VA.getLocVT(), ValVT: VA.getValVT(), Val: Arg);
3027 } else {
3028 auto LocBits = VA.getLocVT().getSizeInBits();
3029 auto MaskValue = APInt::getLowBitsSet(numBits: LocBits, loBitsSet: RetVT.getSizeInBits());
3030 SDValue Mask =
3031 DAG.getConstant(Val: MaskValue, DL: dl, VT: MVT::getIntegerVT(BitWidth: LocBits));
3032 Arg = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::getIntegerVT(BitWidth: LocBits), Operand: Arg);
3033 Arg = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::getIntegerVT(BitWidth: LocBits), N1: Arg, N2: Mask);
3034 Arg = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: VA.getLocVT(), Operand: Arg);
3035 }
3036 }
3037
3038 if (VA.needsCustom() &&
3039 (VA.getLocVT() == MVT::v2f64 || VA.getLocVT() == MVT::f64)) {
3040 if (VA.getLocVT() == MVT::v2f64) {
3041 // Extract the first half and return it in two registers.
3042 SDValue Half = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: MVT::f64, N1: Arg,
3043 N2: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
3044 SDValue HalfGPRs = DAG.getNode(Opcode: ARMISD::VMOVRRD, DL: dl,
3045 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N: Half);
3046
3047 Chain =
3048 DAG.getCopyToReg(Chain, dl, Reg: VA.getLocReg(),
3049 N: HalfGPRs.getValue(R: isLittleEndian ? 0 : 1), Glue);
3050 Glue = Chain.getValue(R: 1);
3051 RetOps.push_back(Elt: DAG.getRegister(Reg: VA.getLocReg(), VT: VA.getLocVT()));
3052 VA = RVLocs[++i]; // skip ahead to next loc
3053 Chain =
3054 DAG.getCopyToReg(Chain, dl, Reg: VA.getLocReg(),
3055 N: HalfGPRs.getValue(R: isLittleEndian ? 1 : 0), Glue);
3056 Glue = Chain.getValue(R: 1);
3057 RetOps.push_back(Elt: DAG.getRegister(Reg: VA.getLocReg(), VT: VA.getLocVT()));
3058 VA = RVLocs[++i]; // skip ahead to next loc
3059
3060 // Extract the 2nd half and fall through to handle it as an f64 value.
3061 Arg = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: MVT::f64, N1: Arg,
3062 N2: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
3063 }
3064 // Legalize ret f64 -> ret 2 x i32. We always have fmrrd if f64 is
3065 // available.
3066 SDValue fmrrd = DAG.getNode(Opcode: ARMISD::VMOVRRD, DL: dl,
3067 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N: Arg);
3068 Chain = DAG.getCopyToReg(Chain, dl, Reg: VA.getLocReg(),
3069 N: fmrrd.getValue(R: isLittleEndian ? 0 : 1), Glue);
3070 Glue = Chain.getValue(R: 1);
3071 RetOps.push_back(Elt: DAG.getRegister(Reg: VA.getLocReg(), VT: VA.getLocVT()));
3072 VA = RVLocs[++i]; // skip ahead to next loc
3073 Chain = DAG.getCopyToReg(Chain, dl, Reg: VA.getLocReg(),
3074 N: fmrrd.getValue(R: isLittleEndian ? 1 : 0), Glue);
3075 } else
3076 Chain = DAG.getCopyToReg(Chain, dl, Reg: VA.getLocReg(), N: Arg, Glue);
3077
3078 // Guarantee that all emitted copies are
3079 // stuck together, avoiding something bad.
3080 Glue = Chain.getValue(R: 1);
3081 RetOps.push_back(Elt: DAG.getRegister(
3082 Reg: VA.getLocReg(), VT: ReturnF16 ? Arg.getValueType() : VA.getLocVT()));
3083 }
3084 const ARMBaseRegisterInfo *TRI = Subtarget->getRegisterInfo();
3085 const MCPhysReg *I =
3086 TRI->getCalleeSavedRegsViaCopy(MF: &DAG.getMachineFunction());
3087 if (I) {
3088 for (; *I; ++I) {
3089 if (ARM::GPRRegClass.contains(Reg: *I))
3090 RetOps.push_back(Elt: DAG.getRegister(Reg: *I, VT: MVT::i32));
3091 else if (ARM::DPRRegClass.contains(Reg: *I))
3092 RetOps.push_back(Elt: DAG.getRegister(Reg: *I, VT: MVT::getFloatingPointVT(BitWidth: 64)));
3093 else
3094 llvm_unreachable("Unexpected register class in CSRsViaCopy!");
3095 }
3096 }
3097
3098 // Update chain and glue.
3099 RetOps[0] = Chain;
3100 if (Glue.getNode())
3101 RetOps.push_back(Elt: Glue);
3102
3103 // CPUs which aren't M-class use a special sequence to return from
3104 // exceptions (roughly, any instruction setting pc and cpsr simultaneously,
3105 // though we use "subs pc, lr, #N").
3106 //
3107 // M-class CPUs actually use a normal return sequence with a special
3108 // (hardware-provided) value in LR, so the normal code path works.
3109 if (DAG.getMachineFunction().getFunction().hasFnAttribute(Kind: "interrupt") &&
3110 !Subtarget->isMClass()) {
3111 if (Subtarget->isThumb1Only())
3112 report_fatal_error(reason: "interrupt attribute is not supported in Thumb1");
3113 return LowerInterruptReturn(RetOps, DL: dl, DAG);
3114 }
3115
3116 unsigned RetNode =
3117 AFI->isCmseNSEntryFunction() ? ARMISD::SERET_GLUE : ARMISD::RET_GLUE;
3118 return DAG.getNode(Opcode: RetNode, DL: dl, VT: MVT::Other, Ops: RetOps);
3119}
3120
3121bool ARMTargetLowering::isUsedByReturnOnly(SDNode *N, SDValue &Chain) const {
3122 if (N->getNumValues() != 1)
3123 return false;
3124 if (!N->hasNUsesOfValue(NUses: 1, Value: 0))
3125 return false;
3126
3127 SDValue TCChain = Chain;
3128 SDNode *Copy = *N->user_begin();
3129 if (Copy->getOpcode() == ISD::CopyToReg) {
3130 // If the copy has a glue operand, we conservatively assume it isn't safe to
3131 // perform a tail call.
3132 if (Copy->getOperand(Num: Copy->getNumOperands()-1).getValueType() == MVT::Glue)
3133 return false;
3134 TCChain = Copy->getOperand(Num: 0);
3135 } else if (Copy->getOpcode() == ARMISD::VMOVRRD) {
3136 SDNode *VMov = Copy;
3137 // f64 returned in a pair of GPRs.
3138 SmallPtrSet<SDNode*, 2> Copies;
3139 for (SDNode *U : VMov->users()) {
3140 if (U->getOpcode() != ISD::CopyToReg)
3141 return false;
3142 Copies.insert(Ptr: U);
3143 }
3144 if (Copies.size() > 2)
3145 return false;
3146
3147 for (SDNode *U : VMov->users()) {
3148 SDValue UseChain = U->getOperand(Num: 0);
3149 if (Copies.count(Ptr: UseChain.getNode()))
3150 // Second CopyToReg
3151 Copy = U;
3152 else {
3153 // We are at the top of this chain.
3154 // If the copy has a glue operand, we conservatively assume it
3155 // isn't safe to perform a tail call.
3156 if (U->getOperand(Num: U->getNumOperands() - 1).getValueType() == MVT::Glue)
3157 return false;
3158 // First CopyToReg
3159 TCChain = UseChain;
3160 }
3161 }
3162 } else if (Copy->getOpcode() == ISD::BITCAST) {
3163 // f32 returned in a single GPR.
3164 if (!Copy->hasOneUse())
3165 return false;
3166 Copy = *Copy->user_begin();
3167 if (Copy->getOpcode() != ISD::CopyToReg || !Copy->hasNUsesOfValue(NUses: 1, Value: 0))
3168 return false;
3169 // If the copy has a glue operand, we conservatively assume it isn't safe to
3170 // perform a tail call.
3171 if (Copy->getOperand(Num: Copy->getNumOperands()-1).getValueType() == MVT::Glue)
3172 return false;
3173 TCChain = Copy->getOperand(Num: 0);
3174 } else {
3175 return false;
3176 }
3177
3178 bool HasRet = false;
3179 for (const SDNode *U : Copy->users()) {
3180 if (U->getOpcode() != ARMISD::RET_GLUE &&
3181 U->getOpcode() != ARMISD::INTRET_GLUE)
3182 return false;
3183 HasRet = true;
3184 }
3185
3186 if (!HasRet)
3187 return false;
3188
3189 Chain = TCChain;
3190 return true;
3191}
3192
3193bool ARMTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
3194 if (!Subtarget->supportsTailCall())
3195 return false;
3196
3197 if (!CI->isTailCall())
3198 return false;
3199
3200 return true;
3201}
3202
3203// Trying to write a 64 bit value so need to split into two 32 bit values first,
3204// and pass the lower and high parts through.
3205static SDValue LowerWRITE_REGISTER(SDValue Op, SelectionDAG &DAG) {
3206 SDLoc DL(Op);
3207 SDValue WriteValue = Op->getOperand(Num: 2);
3208
3209 // This function is only supposed to be called for i64 type argument.
3210 assert(WriteValue.getValueType() == MVT::i64
3211 && "LowerWRITE_REGISTER called for non-i64 type argument.");
3212
3213 SDValue Lo, Hi;
3214 std::tie(args&: Lo, args&: Hi) = DAG.SplitScalar(N: WriteValue, DL, LoVT: MVT::i32, HiVT: MVT::i32);
3215 SDValue Ops[] = { Op->getOperand(Num: 0), Op->getOperand(Num: 1), Lo, Hi };
3216 return DAG.getNode(Opcode: ISD::WRITE_REGISTER, DL, VT: MVT::Other, Ops);
3217}
3218
3219// ConstantPool, JumpTable, GlobalAddress, and ExternalSymbol are lowered as
3220// their target counterpart wrapped in the ARMISD::Wrapper node. Suppose N is
3221// one of the above mentioned nodes. It has to be wrapped because otherwise
3222// Select(N) returns N. So the raw TargetGlobalAddress nodes, etc. can only
3223// be used to form addressing mode. These wrapped nodes will be selected
3224// into MOVi.
3225SDValue ARMTargetLowering::LowerConstantPool(SDValue Op,
3226 SelectionDAG &DAG) const {
3227 EVT PtrVT = Op.getValueType();
3228 // FIXME there is no actual debug info here
3229 SDLoc dl(Op);
3230 ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Val&: Op);
3231 SDValue Res;
3232
3233 // When generating execute-only code Constant Pools must be promoted to the
3234 // global data section. It's a bit ugly that we can't share them across basic
3235 // blocks, but this way we guarantee that execute-only behaves correct with
3236 // position-independent addressing modes.
3237 if (Subtarget->genExecuteOnly()) {
3238 auto AFI = DAG.getMachineFunction().getInfo<ARMFunctionInfo>();
3239 auto *T = CP->getType();
3240 auto C = const_cast<Constant*>(CP->getConstVal());
3241 auto M = DAG.getMachineFunction().getFunction().getParent();
3242 auto GV = new GlobalVariable(
3243 *M, T, /*isConstant=*/true, GlobalVariable::InternalLinkage, C,
3244 Twine(DAG.getDataLayout().getInternalSymbolPrefix()) + "CP" +
3245 Twine(DAG.getMachineFunction().getFunctionNumber()) + "_" +
3246 Twine(AFI->createPICLabelUId()));
3247 SDValue GA = DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT);
3248 return LowerGlobalAddress(Op: GA, DAG);
3249 }
3250
3251 // The 16-bit ADR instruction can only encode offsets that are multiples of 4,
3252 // so we need to align to at least 4 bytes when we don't have 32-bit ADR.
3253 Align CPAlign = CP->getAlign();
3254 if (Subtarget->isThumb1Only())
3255 CPAlign = std::max(a: CPAlign, b: Align(4));
3256 if (CP->isMachineConstantPoolEntry())
3257 Res =
3258 DAG.getTargetConstantPool(C: CP->getMachineCPVal(), VT: PtrVT, Align: CPAlign);
3259 else
3260 Res = DAG.getTargetConstantPool(C: CP->getConstVal(), VT: PtrVT, Align: CPAlign);
3261 return DAG.getNode(Opcode: ARMISD::Wrapper, DL: dl, VT: MVT::i32, Operand: Res);
3262}
3263
3264unsigned ARMTargetLowering::getJumpTableEncoding() const {
3265 // If we don't have a 32-bit pc-relative branch instruction then the jump
3266 // table consists of block addresses. Usually this is inline, but for
3267 // execute-only it must be placed out-of-line.
3268 if (Subtarget->genExecuteOnly() && !Subtarget->hasV8MBaselineOps())
3269 return MachineJumpTableInfo::EK_BlockAddress;
3270 return MachineJumpTableInfo::EK_Inline;
3271}
3272
3273SDValue ARMTargetLowering::LowerBlockAddress(SDValue Op,
3274 SelectionDAG &DAG) const {
3275 MachineFunction &MF = DAG.getMachineFunction();
3276 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
3277 unsigned ARMPCLabelIndex = 0;
3278 SDLoc DL(Op);
3279 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
3280 const BlockAddress *BA = cast<BlockAddressSDNode>(Val&: Op)->getBlockAddress();
3281 SDValue CPAddr;
3282 bool IsPositionIndependent = isPositionIndependent() || Subtarget->isROPI();
3283 if (!IsPositionIndependent) {
3284 CPAddr = DAG.getTargetConstantPool(C: BA, VT: PtrVT, Align: Align(4));
3285 } else {
3286 unsigned PCAdj = Subtarget->isThumb() ? 4 : 8;
3287 ARMPCLabelIndex = AFI->createPICLabelUId();
3288 ARMConstantPoolValue *CPV =
3289 ARMConstantPoolConstant::Create(C: BA, ID: ARMPCLabelIndex,
3290 Kind: ARMCP::CPBlockAddress, PCAdj);
3291 CPAddr = DAG.getTargetConstantPool(C: CPV, VT: PtrVT, Align: Align(4));
3292 }
3293 CPAddr = DAG.getNode(Opcode: ARMISD::Wrapper, DL, VT: PtrVT, Operand: CPAddr);
3294 SDValue Result = DAG.getLoad(
3295 VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: CPAddr,
3296 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
3297 if (!IsPositionIndependent)
3298 return Result;
3299 SDValue PICLabel = DAG.getConstant(Val: ARMPCLabelIndex, DL, VT: MVT::i32);
3300 return DAG.getNode(Opcode: ARMISD::PIC_ADD, DL, VT: PtrVT, N1: Result, N2: PICLabel);
3301}
3302
3303/// Convert a TLS address reference into the correct sequence of loads
3304/// and calls to compute the variable's address for Darwin, and return an
3305/// SDValue containing the final node.
3306
3307/// Darwin only has one TLS scheme which must be capable of dealing with the
3308/// fully general situation, in the worst case. This means:
3309/// + "extern __thread" declaration.
3310/// + Defined in a possibly unknown dynamic library.
3311///
3312/// The general system is that each __thread variable has a [3 x i32] descriptor
3313/// which contains information used by the runtime to calculate the address. The
3314/// only part of this the compiler needs to know about is the first word, which
3315/// contains a function pointer that must be called with the address of the
3316/// entire descriptor in "r0".
3317///
3318/// Since this descriptor may be in a different unit, in general access must
3319/// proceed along the usual ARM rules. A common sequence to produce is:
3320///
3321/// movw rT1, :lower16:_var$non_lazy_ptr
3322/// movt rT1, :upper16:_var$non_lazy_ptr
3323/// ldr r0, [rT1]
3324/// ldr rT2, [r0]
3325/// blx rT2
3326/// [...address now in r0...]
3327SDValue
3328ARMTargetLowering::LowerGlobalTLSAddressDarwin(SDValue Op,
3329 SelectionDAG &DAG) const {
3330 assert(getTargetMachine().getTargetTriple().isOSDarwin() &&
3331 "This function expects a Darwin target");
3332 SDLoc DL(Op);
3333
3334 // First step is to get the address of the actua global symbol. This is where
3335 // the TLS descriptor lives.
3336 SDValue DescAddr = LowerGlobalAddressDarwin(Op, DAG);
3337
3338 // The first entry in the descriptor is a function pointer that we must call
3339 // to obtain the address of the variable.
3340 SDValue Chain = DAG.getEntryNode();
3341 SDValue FuncTLVGet = DAG.getLoad(
3342 VT: MVT::i32, dl: DL, Chain, Ptr: DescAddr,
3343 PtrInfo: MachinePointerInfo::getGOT(MF&: DAG.getMachineFunction()), Alignment: Align(4),
3344 MMOFlags: MachineMemOperand::MONonTemporal | MachineMemOperand::MODereferenceable |
3345 MachineMemOperand::MOInvariant);
3346 Chain = FuncTLVGet.getValue(R: 1);
3347
3348 MachineFunction &F = DAG.getMachineFunction();
3349 MachineFrameInfo &MFI = F.getFrameInfo();
3350 MFI.setAdjustsStack(true);
3351
3352 // TLS calls preserve all registers except those that absolutely must be
3353 // trashed: R0 (it takes an argument), LR (it's a call) and CPSR (let's not be
3354 // silly).
3355 auto TRI =
3356 getTargetMachine().getSubtargetImpl(F.getFunction())->getRegisterInfo();
3357 auto ARI = static_cast<const ARMRegisterInfo *>(TRI);
3358 const uint32_t *Mask = ARI->getTLSCallPreservedMask(MF: DAG.getMachineFunction());
3359
3360 // Finally, we can make the call. This is just a degenerate version of a
3361 // normal AArch64 call node: r0 takes the address of the descriptor, and
3362 // returns the address of the variable in this thread.
3363 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: ARM::R0, N: DescAddr, Glue: SDValue());
3364 Chain =
3365 DAG.getNode(Opcode: ARMISD::CALL, DL, VTList: DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue),
3366 N1: Chain, N2: FuncTLVGet, N3: DAG.getRegister(Reg: ARM::R0, VT: MVT::i32),
3367 N4: DAG.getRegisterMask(RegMask: Mask), N5: Chain.getValue(R: 1));
3368 return DAG.getCopyFromReg(Chain, dl: DL, Reg: ARM::R0, VT: MVT::i32, Glue: Chain.getValue(R: 1));
3369}
3370
3371SDValue
3372ARMTargetLowering::LowerGlobalTLSAddressWindows(SDValue Op,
3373 SelectionDAG &DAG) const {
3374 assert(getTargetMachine().getTargetTriple().isOSWindows() &&
3375 "Windows specific TLS lowering");
3376
3377 SDValue Chain = DAG.getEntryNode();
3378 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
3379 SDLoc DL(Op);
3380
3381 // Load the current TEB (thread environment block)
3382 SDValue Ops[] = {Chain,
3383 DAG.getTargetConstant(Val: Intrinsic::arm_mrc, DL, VT: MVT::i32),
3384 DAG.getTargetConstant(Val: 15, DL, VT: MVT::i32),
3385 DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32),
3386 DAG.getTargetConstant(Val: 13, DL, VT: MVT::i32),
3387 DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32),
3388 DAG.getTargetConstant(Val: 2, DL, VT: MVT::i32)};
3389 SDValue CurrentTEB = DAG.getNode(Opcode: ISD::INTRINSIC_W_CHAIN, DL,
3390 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other), Ops);
3391
3392 SDValue TEB = CurrentTEB.getValue(R: 0);
3393 Chain = CurrentTEB.getValue(R: 1);
3394
3395 // Load the ThreadLocalStoragePointer from the TEB
3396 // A pointer to the TLS array is located at offset 0x2c from the TEB.
3397 SDValue TLSArray =
3398 DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: TEB, N2: DAG.getIntPtrConstant(Val: 0x2c, DL));
3399 TLSArray = DAG.getLoad(VT: PtrVT, dl: DL, Chain, Ptr: TLSArray, PtrInfo: MachinePointerInfo());
3400
3401 // The pointer to the thread's TLS data area is at the TLS Index scaled by 4
3402 // offset into the TLSArray.
3403
3404 // Load the TLS index from the C runtime
3405 SDValue TLSIndex =
3406 DAG.getTargetExternalSymbol(Sym: "_tls_index", VT: PtrVT, TargetFlags: ARMII::MO_NO_FLAG);
3407 TLSIndex = DAG.getNode(Opcode: ARMISD::Wrapper, DL, VT: PtrVT, Operand: TLSIndex);
3408 TLSIndex = DAG.getLoad(VT: PtrVT, dl: DL, Chain, Ptr: TLSIndex, PtrInfo: MachinePointerInfo());
3409
3410 SDValue Slot = DAG.getNode(Opcode: ISD::SHL, DL, VT: PtrVT, N1: TLSIndex,
3411 N2: DAG.getConstant(Val: 2, DL, VT: MVT::i32));
3412 SDValue TLS = DAG.getLoad(VT: PtrVT, dl: DL, Chain,
3413 Ptr: DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: TLSArray, N2: Slot),
3414 PtrInfo: MachinePointerInfo());
3415
3416 // Get the offset of the start of the .tls section (section base)
3417 const auto *GA = cast<GlobalAddressSDNode>(Val&: Op);
3418 auto *CPV = ARMConstantPoolConstant::Create(GV: GA->getGlobal(), Modifier: ARMCP::SECREL);
3419 SDValue Offset = DAG.getLoad(
3420 VT: PtrVT, dl: DL, Chain,
3421 Ptr: DAG.getNode(Opcode: ARMISD::Wrapper, DL, VT: MVT::i32,
3422 Operand: DAG.getTargetConstantPool(C: CPV, VT: PtrVT, Align: Align(4))),
3423 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
3424
3425 return DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: TLS, N2: Offset);
3426}
3427
3428// Lower ISD::GlobalTLSAddress using the "general dynamic" model
3429SDValue
3430ARMTargetLowering::LowerToTLSGeneralDynamicModel(GlobalAddressSDNode *GA,
3431 SelectionDAG &DAG) const {
3432 SDLoc dl(GA);
3433 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
3434 unsigned char PCAdj = Subtarget->isThumb() ? 4 : 8;
3435 MachineFunction &MF = DAG.getMachineFunction();
3436 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
3437 unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
3438 ARMConstantPoolValue *CPV =
3439 ARMConstantPoolConstant::Create(C: GA->getGlobal(), ID: ARMPCLabelIndex,
3440 Kind: ARMCP::CPValue, PCAdj, Modifier: ARMCP::TLSGD, AddCurrentAddress: true);
3441 SDValue Argument = DAG.getTargetConstantPool(C: CPV, VT: PtrVT, Align: Align(4));
3442 Argument = DAG.getNode(Opcode: ARMISD::Wrapper, DL: dl, VT: MVT::i32, Operand: Argument);
3443 Argument = DAG.getLoad(
3444 VT: PtrVT, dl, Chain: DAG.getEntryNode(), Ptr: Argument,
3445 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
3446 SDValue Chain = Argument.getValue(R: 1);
3447
3448 SDValue PICLabel = DAG.getConstant(Val: ARMPCLabelIndex, DL: dl, VT: MVT::i32);
3449 Argument = DAG.getNode(Opcode: ARMISD::PIC_ADD, DL: dl, VT: PtrVT, N1: Argument, N2: PICLabel);
3450
3451 // call __tls_get_addr.
3452 ArgListTy Args;
3453 Args.emplace_back(args&: Argument, args: Type::getInt32Ty(C&: *DAG.getContext()));
3454
3455 // FIXME: is there useful debug info available here?
3456 TargetLowering::CallLoweringInfo CLI(DAG);
3457 CLI.setDebugLoc(dl).setChain(Chain).setLibCallee(
3458 CC: CallingConv::C, ResultType: Type::getInt32Ty(C&: *DAG.getContext()),
3459 Target: DAG.getExternalSymbol(Sym: "__tls_get_addr", VT: PtrVT), ArgsList: std::move(Args));
3460
3461 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
3462 return CallResult.first;
3463}
3464
3465// Lower ISD::GlobalTLSAddress using the "initial exec" or
3466// "local exec" model.
3467SDValue
3468ARMTargetLowering::LowerToTLSExecModels(GlobalAddressSDNode *GA,
3469 SelectionDAG &DAG,
3470 TLSModel::Model model) const {
3471 const GlobalValue *GV = GA->getGlobal();
3472 SDLoc dl(GA);
3473 SDValue Offset;
3474 SDValue Chain = DAG.getEntryNode();
3475 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
3476 // Get the Thread Pointer
3477 SDValue ThreadPointer = DAG.getNode(Opcode: ARMISD::THREAD_POINTER, DL: dl, VT: PtrVT);
3478
3479 if (model == TLSModel::InitialExec) {
3480 MachineFunction &MF = DAG.getMachineFunction();
3481 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
3482 unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
3483 // Initial exec model.
3484 unsigned char PCAdj = Subtarget->isThumb() ? 4 : 8;
3485 ARMConstantPoolValue *CPV =
3486 ARMConstantPoolConstant::Create(C: GA->getGlobal(), ID: ARMPCLabelIndex,
3487 Kind: ARMCP::CPValue, PCAdj, Modifier: ARMCP::GOTTPOFF,
3488 AddCurrentAddress: true);
3489 Offset = DAG.getTargetConstantPool(C: CPV, VT: PtrVT, Align: Align(4));
3490 Offset = DAG.getNode(Opcode: ARMISD::Wrapper, DL: dl, VT: MVT::i32, Operand: Offset);
3491 Offset = DAG.getLoad(
3492 VT: PtrVT, dl, Chain, Ptr: Offset,
3493 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
3494 Chain = Offset.getValue(R: 1);
3495
3496 SDValue PICLabel = DAG.getConstant(Val: ARMPCLabelIndex, DL: dl, VT: MVT::i32);
3497 Offset = DAG.getNode(Opcode: ARMISD::PIC_ADD, DL: dl, VT: PtrVT, N1: Offset, N2: PICLabel);
3498
3499 Offset = DAG.getLoad(
3500 VT: PtrVT, dl, Chain, Ptr: Offset,
3501 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
3502 } else {
3503 // local exec model
3504 assert(model == TLSModel::LocalExec);
3505 ARMConstantPoolValue *CPV =
3506 ARMConstantPoolConstant::Create(GV, Modifier: ARMCP::TPOFF);
3507 Offset = DAG.getTargetConstantPool(C: CPV, VT: PtrVT, Align: Align(4));
3508 Offset = DAG.getNode(Opcode: ARMISD::Wrapper, DL: dl, VT: MVT::i32, Operand: Offset);
3509 Offset = DAG.getLoad(
3510 VT: PtrVT, dl, Chain, Ptr: Offset,
3511 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
3512 }
3513
3514 // The address of the thread local variable is the add of the thread
3515 // pointer with the offset of the variable.
3516 return DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: ThreadPointer, N2: Offset);
3517}
3518
3519SDValue
3520ARMTargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const {
3521 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Val&: Op);
3522 if (DAG.getTarget().useEmulatedTLS())
3523 return LowerToTLSEmulatedModel(GA, DAG);
3524
3525 const Triple &TT = getTargetMachine().getTargetTriple();
3526 if (TT.isOSDarwin())
3527 return LowerGlobalTLSAddressDarwin(Op, DAG);
3528
3529 if (TT.isOSWindows())
3530 return LowerGlobalTLSAddressWindows(Op, DAG);
3531
3532 // TODO: implement the "local dynamic" model
3533 assert(TT.isOSBinFormatELF() && "Only ELF implemented here");
3534 TLSModel::Model model = getTargetMachine().getTLSModel(GV: GA->getGlobal());
3535
3536 switch (model) {
3537 case TLSModel::GeneralDynamic:
3538 case TLSModel::LocalDynamic:
3539 return LowerToTLSGeneralDynamicModel(GA, DAG);
3540 case TLSModel::InitialExec:
3541 case TLSModel::LocalExec:
3542 return LowerToTLSExecModels(GA, DAG, model);
3543 }
3544 llvm_unreachable("bogus TLS model");
3545}
3546
3547/// Return true if all users of V are within function F, looking through
3548/// ConstantExprs.
3549static bool allUsersAreInFunction(const Value *V, const Function *F) {
3550 SmallVector<const User*,4> Worklist(V->users());
3551 while (!Worklist.empty()) {
3552 auto *U = Worklist.pop_back_val();
3553 if (isa<ConstantExpr>(Val: U)) {
3554 append_range(C&: Worklist, R: U->users());
3555 continue;
3556 }
3557
3558 auto *I = dyn_cast<Instruction>(Val: U);
3559 if (!I || I->getParent()->getParent() != F)
3560 return false;
3561 }
3562 return true;
3563}
3564
3565static SDValue promoteToConstantPool(const ARMTargetLowering *TLI,
3566 const GlobalValue *GV, SelectionDAG &DAG,
3567 EVT PtrVT, const SDLoc &dl) {
3568 // If we're creating a pool entry for a constant global with unnamed address,
3569 // and the global is small enough, we can emit it inline into the constant pool
3570 // to save ourselves an indirection.
3571 //
3572 // This is a win if the constant is only used in one function (so it doesn't
3573 // need to be duplicated) or duplicating the constant wouldn't increase code
3574 // size (implying the constant is no larger than 4 bytes).
3575 const Function &F = DAG.getMachineFunction().getFunction();
3576
3577 // We rely on this decision to inline being idempotent and unrelated to the
3578 // use-site. We know that if we inline a variable at one use site, we'll
3579 // inline it elsewhere too (and reuse the constant pool entry). Fast-isel
3580 // doesn't know about this optimization, so bail out if it's enabled else
3581 // we could decide to inline here (and thus never emit the GV) but require
3582 // the GV from fast-isel generated code.
3583 if (!EnableConstpoolPromotion ||
3584 DAG.getMachineFunction().getTarget().Options.EnableFastISel)
3585 return SDValue();
3586
3587 auto *GVar = dyn_cast<GlobalVariable>(Val: GV);
3588 if (!GVar || !GVar->hasInitializer() ||
3589 !GVar->isConstant() || !GVar->hasGlobalUnnamedAddr() ||
3590 !GVar->hasLocalLinkage())
3591 return SDValue();
3592
3593 // If we inline a value that contains relocations, we move the relocations
3594 // from .data to .text. This is not allowed in position-independent code.
3595 auto *Init = GVar->getInitializer();
3596 if ((TLI->isPositionIndependent() || TLI->getSubtarget()->isROPI()) &&
3597 Init->needsDynamicRelocation())
3598 return SDValue();
3599
3600 // The constant islands pass can only really deal with alignment requests
3601 // <= 4 bytes and cannot pad constants itself. Therefore we cannot promote
3602 // any type wanting greater alignment requirements than 4 bytes. We also
3603 // can only promote constants that are multiples of 4 bytes in size or
3604 // are paddable to a multiple of 4. Currently we only try and pad constants
3605 // that are strings for simplicity.
3606 auto *CDAInit = dyn_cast<ConstantDataArray>(Val: Init);
3607 unsigned Size = DAG.getDataLayout().getTypeAllocSize(Ty: Init->getType());
3608 Align PrefAlign = DAG.getDataLayout().getPreferredAlign(GV: GVar);
3609 unsigned RequiredPadding = 4 - (Size % 4);
3610 bool PaddingPossible =
3611 RequiredPadding == 4 || (CDAInit && CDAInit->isString());
3612 if (!PaddingPossible || PrefAlign > 4 || Size > ConstpoolPromotionMaxSize ||
3613 Size == 0)
3614 return SDValue();
3615
3616 unsigned PaddedSize = Size + ((RequiredPadding == 4) ? 0 : RequiredPadding);
3617 MachineFunction &MF = DAG.getMachineFunction();
3618 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
3619
3620 // We can't bloat the constant pool too much, else the ConstantIslands pass
3621 // may fail to converge. If we haven't promoted this global yet (it may have
3622 // multiple uses), and promoting it would increase the constant pool size (Sz
3623 // > 4), ensure we have space to do so up to MaxTotal.
3624 if (!AFI->getGlobalsPromotedToConstantPool().count(Ptr: GVar) && Size > 4)
3625 if (AFI->getPromotedConstpoolIncrease() + PaddedSize - 4 >=
3626 ConstpoolPromotionMaxTotal)
3627 return SDValue();
3628
3629 // This is only valid if all users are in a single function; we can't clone
3630 // the constant in general. The LLVM IR unnamed_addr allows merging
3631 // constants, but not cloning them.
3632 //
3633 // We could potentially allow cloning if we could prove all uses of the
3634 // constant in the current function don't care about the address, like
3635 // printf format strings. But that isn't implemented for now.
3636 if (!allUsersAreInFunction(V: GVar, F: &F))
3637 return SDValue();
3638
3639 // We're going to inline this global. Pad it out if needed.
3640 if (RequiredPadding != 4) {
3641 StringRef S = CDAInit->getAsString();
3642
3643 SmallVector<uint8_t,16> V(S.size());
3644 std::copy(first: S.bytes_begin(), last: S.bytes_end(), result: V.begin());
3645 while (RequiredPadding--)
3646 V.push_back(Elt: 0);
3647 Init = ConstantDataArray::get(Context&: *DAG.getContext(), Elts&: V);
3648 }
3649
3650 auto CPVal = ARMConstantPoolConstant::Create(GV: GVar, Initializer: Init);
3651 SDValue CPAddr = DAG.getTargetConstantPool(C: CPVal, VT: PtrVT, Align: Align(4));
3652 if (!AFI->getGlobalsPromotedToConstantPool().count(Ptr: GVar)) {
3653 AFI->markGlobalAsPromotedToConstantPool(GV: GVar);
3654 AFI->setPromotedConstpoolIncrease(AFI->getPromotedConstpoolIncrease() +
3655 PaddedSize - 4);
3656 }
3657 ++NumConstpoolPromoted;
3658 return DAG.getNode(Opcode: ARMISD::Wrapper, DL: dl, VT: MVT::i32, Operand: CPAddr);
3659}
3660
3661bool ARMTargetLowering::isReadOnly(const GlobalValue *GV) const {
3662 if (const GlobalAlias *GA = dyn_cast<GlobalAlias>(Val: GV))
3663 if (!(GV = GA->getAliaseeObject()))
3664 return false;
3665 if (const auto *V = dyn_cast<GlobalVariable>(Val: GV))
3666 return V->isConstant();
3667 return isa<Function>(Val: GV);
3668}
3669
3670SDValue ARMTargetLowering::LowerGlobalAddress(SDValue Op,
3671 SelectionDAG &DAG) const {
3672 switch (Subtarget->getTargetTriple().getObjectFormat()) {
3673 default: llvm_unreachable("unknown object format");
3674 case Triple::COFF:
3675 return LowerGlobalAddressWindows(Op, DAG);
3676 case Triple::ELF:
3677 return LowerGlobalAddressELF(Op, DAG);
3678 case Triple::MachO:
3679 return LowerGlobalAddressDarwin(Op, DAG);
3680 }
3681}
3682
3683SDValue ARMTargetLowering::LowerGlobalAddressELF(SDValue Op,
3684 SelectionDAG &DAG) const {
3685 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
3686 SDLoc dl(Op);
3687 const GlobalValue *GV = cast<GlobalAddressSDNode>(Val&: Op)->getGlobal();
3688 bool IsRO = isReadOnly(GV);
3689
3690 // promoteToConstantPool only if not generating XO text section
3691 if (GV->isDSOLocal() && !Subtarget->genExecuteOnly())
3692 if (SDValue V = promoteToConstantPool(TLI: this, GV, DAG, PtrVT, dl))
3693 return V;
3694
3695 if (isPositionIndependent()) {
3696 SDValue G = DAG.getTargetGlobalAddress(
3697 GV, DL: dl, VT: PtrVT, offset: 0, TargetFlags: GV->isDSOLocal() ? 0 : ARMII::MO_GOT);
3698 SDValue Result = DAG.getNode(Opcode: ARMISD::WrapperPIC, DL: dl, VT: PtrVT, Operand: G);
3699 if (!GV->isDSOLocal())
3700 Result =
3701 DAG.getLoad(VT: PtrVT, dl, Chain: DAG.getEntryNode(), Ptr: Result,
3702 PtrInfo: MachinePointerInfo::getGOT(MF&: DAG.getMachineFunction()));
3703 return Result;
3704 } else if (Subtarget->isROPI() && IsRO) {
3705 // PC-relative.
3706 SDValue G = DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT);
3707 SDValue Result = DAG.getNode(Opcode: ARMISD::WrapperPIC, DL: dl, VT: PtrVT, Operand: G);
3708 return Result;
3709 } else if (Subtarget->isRWPI() && !IsRO) {
3710 // SB-relative.
3711 SDValue RelAddr;
3712 if (Subtarget->useMovt()) {
3713 ++NumMovwMovt;
3714 SDValue G = DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: 0, TargetFlags: ARMII::MO_SBREL);
3715 RelAddr = DAG.getNode(Opcode: ARMISD::Wrapper, DL: dl, VT: PtrVT, Operand: G);
3716 } else { // use literal pool for address constant
3717 ARMConstantPoolValue *CPV =
3718 ARMConstantPoolConstant::Create(GV, Modifier: ARMCP::SBREL);
3719 SDValue CPAddr = DAG.getTargetConstantPool(C: CPV, VT: PtrVT, Align: Align(4));
3720 CPAddr = DAG.getNode(Opcode: ARMISD::Wrapper, DL: dl, VT: MVT::i32, Operand: CPAddr);
3721 RelAddr = DAG.getLoad(
3722 VT: PtrVT, dl, Chain: DAG.getEntryNode(), Ptr: CPAddr,
3723 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
3724 }
3725 SDValue SB = DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl, Reg: ARM::R9, VT: PtrVT);
3726 SDValue Result = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: SB, N2: RelAddr);
3727 return Result;
3728 }
3729
3730 // If we have T2 ops, we can materialize the address directly via movt/movw
3731 // pair. This is always cheaper. If need to generate Execute Only code, and we
3732 // only have Thumb1 available, we can't use a constant pool and are forced to
3733 // use immediate relocations.
3734 if (Subtarget->useMovt() || Subtarget->genExecuteOnly()) {
3735 if (Subtarget->useMovt())
3736 ++NumMovwMovt;
3737 // FIXME: Once remat is capable of dealing with instructions with register
3738 // operands, expand this into two nodes.
3739 return DAG.getNode(Opcode: ARMISD::Wrapper, DL: dl, VT: PtrVT,
3740 Operand: DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT));
3741 } else {
3742 SDValue CPAddr = DAG.getTargetConstantPool(C: GV, VT: PtrVT, Align: Align(4));
3743 CPAddr = DAG.getNode(Opcode: ARMISD::Wrapper, DL: dl, VT: MVT::i32, Operand: CPAddr);
3744 return DAG.getLoad(
3745 VT: PtrVT, dl, Chain: DAG.getEntryNode(), Ptr: CPAddr,
3746 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
3747 }
3748}
3749
3750SDValue ARMTargetLowering::LowerGlobalAddressDarwin(SDValue Op,
3751 SelectionDAG &DAG) const {
3752 assert(!Subtarget->isROPI() && !Subtarget->isRWPI() &&
3753 "ROPI/RWPI not currently supported for Darwin");
3754 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
3755 SDLoc dl(Op);
3756 const GlobalValue *GV = cast<GlobalAddressSDNode>(Val&: Op)->getGlobal();
3757
3758 if (Subtarget->useMovt())
3759 ++NumMovwMovt;
3760
3761 // FIXME: Once remat is capable of dealing with instructions with register
3762 // operands, expand this into multiple nodes
3763 unsigned Wrapper =
3764 isPositionIndependent() ? ARMISD::WrapperPIC : ARMISD::Wrapper;
3765
3766 SDValue G = DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: 0, TargetFlags: ARMII::MO_NONLAZY);
3767 SDValue Result = DAG.getNode(Opcode: Wrapper, DL: dl, VT: PtrVT, Operand: G);
3768
3769 if (Subtarget->isGVIndirectSymbol(GV))
3770 Result = DAG.getLoad(VT: PtrVT, dl, Chain: DAG.getEntryNode(), Ptr: Result,
3771 PtrInfo: MachinePointerInfo::getGOT(MF&: DAG.getMachineFunction()));
3772 return Result;
3773}
3774
3775SDValue ARMTargetLowering::LowerGlobalAddressWindows(SDValue Op,
3776 SelectionDAG &DAG) const {
3777 assert(getTargetMachine().getTargetTriple().isOSWindows() &&
3778 "non-Windows COFF is not supported");
3779 assert(Subtarget->useMovt() &&
3780 "Windows on ARM expects to use movw/movt");
3781 assert(!Subtarget->isROPI() && !Subtarget->isRWPI() &&
3782 "ROPI/RWPI not currently supported for Windows");
3783
3784 const TargetMachine &TM = getTargetMachine();
3785 const GlobalValue *GV = cast<GlobalAddressSDNode>(Val&: Op)->getGlobal();
3786 ARMII::TOF TargetFlags = ARMII::MO_NO_FLAG;
3787 if (GV->hasDLLImportStorageClass())
3788 TargetFlags = ARMII::MO_DLLIMPORT;
3789 else if (!TM.shouldAssumeDSOLocal(GV))
3790 TargetFlags = ARMII::MO_COFFSTUB;
3791 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
3792 SDValue Result;
3793 SDLoc DL(Op);
3794
3795 ++NumMovwMovt;
3796
3797 // FIXME: Once remat is capable of dealing with instructions with register
3798 // operands, expand this into two nodes.
3799 Result = DAG.getNode(Opcode: ARMISD::Wrapper, DL, VT: PtrVT,
3800 Operand: DAG.getTargetGlobalAddress(GV, DL, VT: PtrVT, /*offset=*/0,
3801 TargetFlags));
3802 if (TargetFlags & (ARMII::MO_DLLIMPORT | ARMII::MO_COFFSTUB))
3803 Result = DAG.getLoad(VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: Result,
3804 PtrInfo: MachinePointerInfo::getGOT(MF&: DAG.getMachineFunction()));
3805 return Result;
3806}
3807
3808SDValue
3809ARMTargetLowering::LowerEH_SJLJ_SETJMP(SDValue Op, SelectionDAG &DAG) const {
3810 SDLoc dl(Op);
3811 SDValue Val = DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32);
3812 return DAG.getNode(Opcode: ARMISD::EH_SJLJ_SETJMP, DL: dl,
3813 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other), N1: Op.getOperand(i: 0),
3814 N2: Op.getOperand(i: 1), N3: Val);
3815}
3816
3817SDValue
3818ARMTargetLowering::LowerEH_SJLJ_LONGJMP(SDValue Op, SelectionDAG &DAG) const {
3819 SDLoc dl(Op);
3820 return DAG.getNode(Opcode: ARMISD::EH_SJLJ_LONGJMP, DL: dl, VT: MVT::Other, N1: Op.getOperand(i: 0),
3821 N2: Op.getOperand(i: 1), N3: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
3822}
3823
3824SDValue ARMTargetLowering::LowerEH_SJLJ_SETUP_DISPATCH(SDValue Op,
3825 SelectionDAG &DAG) const {
3826 SDLoc dl(Op);
3827 return DAG.getNode(Opcode: ARMISD::EH_SJLJ_SETUP_DISPATCH, DL: dl, VT: MVT::Other,
3828 Operand: Op.getOperand(i: 0));
3829}
3830
3831SDValue ARMTargetLowering::LowerINTRINSIC_VOID(
3832 SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget) const {
3833 unsigned IntNo =
3834 Op.getConstantOperandVal(i: Op.getOperand(i: 0).getValueType() == MVT::Other);
3835 switch (IntNo) {
3836 default:
3837 return SDValue(); // Don't custom lower most intrinsics.
3838 case Intrinsic::arm_gnu_eabi_mcount: {
3839 MachineFunction &MF = DAG.getMachineFunction();
3840 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
3841 SDLoc dl(Op);
3842 SDValue Chain = Op.getOperand(i: 0);
3843 // call "\01__gnu_mcount_nc"
3844 const ARMBaseRegisterInfo *ARI = Subtarget->getRegisterInfo();
3845 const uint32_t *Mask =
3846 ARI->getCallPreservedMask(MF: DAG.getMachineFunction(), CallingConv::C);
3847 assert(Mask && "Missing call preserved mask for calling convention");
3848 // Mark LR an implicit live-in.
3849 Register Reg = MF.addLiveIn(PReg: ARM::LR, RC: getRegClassFor(VT: MVT::i32));
3850 SDValue ReturnAddress =
3851 DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl, Reg, VT: PtrVT);
3852 constexpr EVT ResultTys[] = {MVT::Other, MVT::Glue};
3853 SDValue Callee =
3854 DAG.getTargetExternalSymbol(Sym: "\01__gnu_mcount_nc", VT: PtrVT, TargetFlags: 0);
3855 SDValue RegisterMask = DAG.getRegisterMask(RegMask: Mask);
3856 if (Subtarget->isThumb())
3857 return SDValue(
3858 DAG.getMachineNode(
3859 Opcode: ARM::tBL_PUSHLR, dl, ResultTys,
3860 Ops: {ReturnAddress, DAG.getTargetConstant(Val: ARMCC::AL, DL: dl, VT: PtrVT),
3861 DAG.getRegister(Reg: 0, VT: PtrVT), Callee, RegisterMask, Chain}),
3862 0);
3863 return SDValue(
3864 DAG.getMachineNode(Opcode: ARM::BL_PUSHLR, dl, ResultTys,
3865 Ops: {ReturnAddress, Callee, RegisterMask, Chain}),
3866 0);
3867 }
3868 }
3869}
3870
3871SDValue
3872ARMTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG,
3873 const ARMSubtarget *Subtarget) const {
3874 unsigned IntNo = Op.getConstantOperandVal(i: 0);
3875 SDLoc dl(Op);
3876 switch (IntNo) {
3877 default: return SDValue(); // Don't custom lower most intrinsics.
3878 case Intrinsic::localaddress: {
3879 const MachineFunction &MF = DAG.getMachineFunction();
3880 const auto *RegInfo = Subtarget->getRegisterInfo();
3881 unsigned Reg = RegInfo->getLocalAddressRegister(MF);
3882 return DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl, Reg,
3883 VT: Op.getSimpleValueType());
3884 }
3885 case Intrinsic::eh_recoverfp: {
3886 SDValue FnOp = Op.getOperand(i: 1);
3887 GlobalAddressSDNode *GSD = dyn_cast<GlobalAddressSDNode>(Val&: FnOp);
3888 auto *Fn = dyn_cast_or_null<Function>(Val: GSD ? GSD->getGlobal() : nullptr);
3889 if (!Fn)
3890 report_fatal_error(
3891 reason: "llvm.eh.recoverfp must take a function as the first argument");
3892 const auto *RegInfo = Subtarget->getRegisterInfo();
3893 Register BaseReg = RegInfo->getBaseRegister();
3894 MachineFunction &MF = DAG.getMachineFunction();
3895 MachineBasicBlock &MBB = *MF.begin();
3896 if (!MBB.isLiveIn(Reg: BaseReg))
3897 MBB.addLiveIn(PhysReg: BaseReg);
3898 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
3899 return DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl, Reg: BaseReg, VT: PtrVT);
3900 }
3901 case Intrinsic::thread_pointer: {
3902 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
3903 return DAG.getNode(Opcode: ARMISD::THREAD_POINTER, DL: dl, VT: PtrVT);
3904 }
3905 case Intrinsic::arm_cls: {
3906 // Note: arm_cls and arm_cls64 intrinsics are expanded directly here
3907 // in LowerINTRINSIC_WO_CHAIN since there's no native scalar CLS
3908 // instruction.
3909 const SDValue &Operand = Op.getOperand(i: 1);
3910 const EVT VTy = Op.getValueType();
3911 return DAG.getNode(Opcode: ISD::CTLS, DL: dl, VT: VTy, Operand);
3912 }
3913 case Intrinsic::arm_cls64: {
3914 // arm_cls64 returns i32 but takes i64 input.
3915 // Use ISD::CTLS for i64 and truncate the result.
3916 SDValue CTLS64 = DAG.getNode(Opcode: ISD::CTLS, DL: dl, VT: MVT::i64, Operand: Op.getOperand(i: 1));
3917 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i32, Operand: CTLS64);
3918 }
3919 case Intrinsic::arm_neon_vcls:
3920 case Intrinsic::arm_mve_vcls: {
3921 // Lower vector CLS intrinsics to ISD::CTLS.
3922 // Vector CTLS is Legal when NEON/MVE is available (set elsewhere).
3923 const EVT VTy = Op.getValueType();
3924 return DAG.getNode(Opcode: ISD::CTLS, DL: dl, VT: VTy, Operand: Op.getOperand(i: 1));
3925 }
3926 case Intrinsic::eh_sjlj_lsda: {
3927 MachineFunction &MF = DAG.getMachineFunction();
3928 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
3929 unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
3930 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
3931 SDValue CPAddr;
3932 bool IsPositionIndependent = isPositionIndependent();
3933 unsigned PCAdj = IsPositionIndependent ? (Subtarget->isThumb() ? 4 : 8) : 0;
3934 ARMConstantPoolValue *CPV =
3935 ARMConstantPoolConstant::Create(C: &MF.getFunction(), ID: ARMPCLabelIndex,
3936 Kind: ARMCP::CPLSDA, PCAdj);
3937 CPAddr = DAG.getTargetConstantPool(C: CPV, VT: PtrVT, Align: Align(4));
3938 CPAddr = DAG.getNode(Opcode: ARMISD::Wrapper, DL: dl, VT: MVT::i32, Operand: CPAddr);
3939 SDValue Result = DAG.getLoad(
3940 VT: PtrVT, dl, Chain: DAG.getEntryNode(), Ptr: CPAddr,
3941 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
3942
3943 if (IsPositionIndependent) {
3944 SDValue PICLabel = DAG.getConstant(Val: ARMPCLabelIndex, DL: dl, VT: MVT::i32);
3945 Result = DAG.getNode(Opcode: ARMISD::PIC_ADD, DL: dl, VT: PtrVT, N1: Result, N2: PICLabel);
3946 }
3947 return Result;
3948 }
3949 case Intrinsic::arm_neon_vabs:
3950 return DAG.getNode(Opcode: ISD::ABS, DL: SDLoc(Op), VT: Op.getValueType(),
3951 Operand: Op.getOperand(i: 1));
3952 case Intrinsic::arm_neon_vabds:
3953 if (Op.getValueType().isInteger())
3954 return DAG.getNode(Opcode: ISD::ABDS, DL: SDLoc(Op), VT: Op.getValueType(),
3955 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
3956 return SDValue();
3957 case Intrinsic::arm_neon_vabdu:
3958 return DAG.getNode(Opcode: ISD::ABDU, DL: SDLoc(Op), VT: Op.getValueType(),
3959 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
3960 case Intrinsic::arm_neon_vmulls:
3961 case Intrinsic::arm_neon_vmullu: {
3962 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmulls)
3963 ? ARMISD::VMULLs : ARMISD::VMULLu;
3964 return DAG.getNode(Opcode: NewOpc, DL: SDLoc(Op), VT: Op.getValueType(),
3965 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
3966 }
3967 case Intrinsic::arm_neon_vminnm:
3968 case Intrinsic::arm_neon_vmaxnm: {
3969 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vminnm)
3970 ? ISD::FMINNUM : ISD::FMAXNUM;
3971 return DAG.getNode(Opcode: NewOpc, DL: SDLoc(Op), VT: Op.getValueType(),
3972 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
3973 }
3974 case Intrinsic::arm_neon_vminu:
3975 case Intrinsic::arm_neon_vmaxu: {
3976 if (Op.getValueType().isFloatingPoint())
3977 return SDValue();
3978 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vminu)
3979 ? ISD::UMIN : ISD::UMAX;
3980 return DAG.getNode(Opcode: NewOpc, DL: SDLoc(Op), VT: Op.getValueType(),
3981 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
3982 }
3983 case Intrinsic::arm_neon_vmins:
3984 case Intrinsic::arm_neon_vmaxs: {
3985 // v{min,max}s is overloaded between signed integers and floats.
3986 if (!Op.getValueType().isFloatingPoint()) {
3987 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmins)
3988 ? ISD::SMIN : ISD::SMAX;
3989 return DAG.getNode(Opcode: NewOpc, DL: SDLoc(Op), VT: Op.getValueType(),
3990 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
3991 }
3992 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmins)
3993 ? ISD::FMINIMUM : ISD::FMAXIMUM;
3994 return DAG.getNode(Opcode: NewOpc, DL: SDLoc(Op), VT: Op.getValueType(),
3995 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
3996 }
3997 case Intrinsic::arm_neon_vtbl1:
3998 return DAG.getNode(Opcode: ARMISD::VTBL1, DL: SDLoc(Op), VT: Op.getValueType(),
3999 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
4000 case Intrinsic::arm_neon_vtbl2:
4001 return DAG.getNode(Opcode: ARMISD::VTBL2, DL: SDLoc(Op), VT: Op.getValueType(),
4002 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
4003 case Intrinsic::arm_mve_pred_i2v:
4004 case Intrinsic::arm_mve_pred_v2i:
4005 return DAG.getNode(Opcode: ARMISD::PREDICATE_CAST, DL: SDLoc(Op), VT: Op.getValueType(),
4006 Operand: Op.getOperand(i: 1));
4007 case Intrinsic::arm_mve_vreinterpretq:
4008 return DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: SDLoc(Op), VT: Op.getValueType(),
4009 Operand: Op.getOperand(i: 1));
4010 case Intrinsic::arm_mve_lsll:
4011 return DAG.getNode(Opcode: ARMISD::LSLL, DL: SDLoc(Op), VTList: Op->getVTList(),
4012 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
4013 case Intrinsic::arm_mve_asrl:
4014 return DAG.getNode(Opcode: ARMISD::ASRL, DL: SDLoc(Op), VTList: Op->getVTList(),
4015 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
4016 case Intrinsic::arm_mve_vsli:
4017 return DAG.getNode(Opcode: ARMISD::VSLIIMM, DL: SDLoc(Op), VTList: Op->getVTList(),
4018 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
4019 case Intrinsic::arm_mve_vsri:
4020 return DAG.getNode(Opcode: ARMISD::VSRIIMM, DL: SDLoc(Op), VTList: Op->getVTList(),
4021 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
4022 }
4023}
4024
4025static SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG,
4026 const ARMSubtarget *Subtarget) {
4027 SDLoc dl(Op);
4028 auto SSID = static_cast<SyncScope::ID>(Op.getConstantOperandVal(i: 2));
4029 if (SSID == SyncScope::SingleThread)
4030 return Op;
4031
4032 if (!Subtarget->hasDataBarrier()) {
4033 // Some ARMv6 cpus can support data barriers with an mcr instruction.
4034 // Thumb1 and pre-v6 ARM mode use a libcall instead and should never get
4035 // here.
4036 assert(Subtarget->hasV6Ops() && !Subtarget->isThumb() &&
4037 "Unexpected ISD::ATOMIC_FENCE encountered. Should be libcall!");
4038 return DAG.getNode(Opcode: ARMISD::MEMBARRIER_MCR, DL: dl, VT: MVT::Other, N1: Op.getOperand(i: 0),
4039 N2: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
4040 }
4041
4042 AtomicOrdering Ord =
4043 static_cast<AtomicOrdering>(Op.getConstantOperandVal(i: 1));
4044 ARM_MB::MemBOpt Domain = ARM_MB::ISH;
4045 if (Subtarget->isMClass()) {
4046 // Only a full system barrier exists in the M-class architectures.
4047 Domain = ARM_MB::SY;
4048 } else if (Subtarget->preferISHSTBarriers() &&
4049 Ord == AtomicOrdering::Release) {
4050 // Swift happens to implement ISHST barriers in a way that's compatible with
4051 // Release semantics but weaker than ISH so we'd be fools not to use
4052 // it. Beware: other processors probably don't!
4053 Domain = ARM_MB::ISHST;
4054 }
4055
4056 return DAG.getNode(Opcode: ISD::INTRINSIC_VOID, DL: dl, VT: MVT::Other, N1: Op.getOperand(i: 0),
4057 N2: DAG.getConstant(Val: Intrinsic::arm_dmb, DL: dl, VT: MVT::i32),
4058 N3: DAG.getConstant(Val: Domain, DL: dl, VT: MVT::i32));
4059}
4060
4061static SDValue LowerPREFETCH(SDValue Op, SelectionDAG &DAG,
4062 const ARMSubtarget *Subtarget) {
4063 // ARM pre v5TE and Thumb1 does not have preload instructions.
4064 if (!(Subtarget->isThumb2() ||
4065 (!Subtarget->isThumb1Only() && Subtarget->hasV5TEOps())))
4066 // Just preserve the chain.
4067 return Op.getOperand(i: 0);
4068
4069 SDLoc dl(Op);
4070 unsigned isRead = ~Op.getConstantOperandVal(i: 2) & 1;
4071 if (!isRead &&
4072 (!Subtarget->hasV7Ops() || !Subtarget->hasMPExtension()))
4073 // ARMv7 with MP extension has PLDW.
4074 return Op.getOperand(i: 0);
4075
4076 unsigned isData = Op.getConstantOperandVal(i: 4);
4077 if (Subtarget->isThumb()) {
4078 // Invert the bits.
4079 isRead = ~isRead & 1;
4080 isData = ~isData & 1;
4081 }
4082
4083 return DAG.getNode(Opcode: ARMISD::PRELOAD, DL: dl, VT: MVT::Other, N1: Op.getOperand(i: 0),
4084 N2: Op.getOperand(i: 1), N3: DAG.getConstant(Val: isRead, DL: dl, VT: MVT::i32),
4085 N4: DAG.getConstant(Val: isData, DL: dl, VT: MVT::i32));
4086}
4087
4088static SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) {
4089 MachineFunction &MF = DAG.getMachineFunction();
4090 ARMFunctionInfo *FuncInfo = MF.getInfo<ARMFunctionInfo>();
4091
4092 // vastart just stores the address of the VarArgsFrameIndex slot into the
4093 // memory location argument.
4094 SDLoc dl(Op);
4095 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DL: DAG.getDataLayout());
4096 SDValue FR = DAG.getFrameIndex(FI: FuncInfo->getVarArgsFrameIndex(), VT: PtrVT);
4097 const Value *SV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 2))->getValue();
4098 return DAG.getStore(Chain: Op.getOperand(i: 0), dl, Val: FR, Ptr: Op.getOperand(i: 1),
4099 PtrInfo: MachinePointerInfo(SV));
4100}
4101
4102SDValue ARMTargetLowering::GetF64FormalArgument(CCValAssign &VA,
4103 CCValAssign &NextVA,
4104 SDValue &Root,
4105 SelectionDAG &DAG,
4106 const SDLoc &dl) const {
4107 MachineFunction &MF = DAG.getMachineFunction();
4108 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
4109
4110 const TargetRegisterClass *RC;
4111 if (AFI->isThumb1OnlyFunction())
4112 RC = &ARM::tGPRRegClass;
4113 else
4114 RC = &ARM::GPRRegClass;
4115
4116 // Transform the arguments stored in physical registers into virtual ones.
4117 Register Reg = MF.addLiveIn(PReg: VA.getLocReg(), RC);
4118 SDValue ArgValue = DAG.getCopyFromReg(Chain: Root, dl, Reg, VT: MVT::i32);
4119
4120 SDValue ArgValue2;
4121 if (NextVA.isMemLoc()) {
4122 MachineFrameInfo &MFI = MF.getFrameInfo();
4123 int FI = MFI.CreateFixedObject(Size: 4, SPOffset: NextVA.getLocMemOffset(), IsImmutable: true);
4124
4125 // Create load node to retrieve arguments from the stack.
4126 SDValue FIN = DAG.getFrameIndex(FI, VT: getPointerTy(DL: DAG.getDataLayout()));
4127 ArgValue2 = DAG.getLoad(
4128 VT: MVT::i32, dl, Chain: Root, Ptr: FIN,
4129 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI));
4130 } else {
4131 Reg = MF.addLiveIn(PReg: NextVA.getLocReg(), RC);
4132 ArgValue2 = DAG.getCopyFromReg(Chain: Root, dl, Reg, VT: MVT::i32);
4133 }
4134 if (!Subtarget->isLittle())
4135 std::swap (a&: ArgValue, b&: ArgValue2);
4136 return DAG.getNode(Opcode: ARMISD::VMOVDRR, DL: dl, VT: MVT::f64, N1: ArgValue, N2: ArgValue2);
4137}
4138
4139// The remaining GPRs hold either the beginning of variable-argument
4140// data, or the beginning of an aggregate passed by value (usually
4141// byval). Either way, we allocate stack slots adjacent to the data
4142// provided by our caller, and store the unallocated registers there.
4143// If this is a variadic function, the va_list pointer will begin with
4144// these values; otherwise, this reassembles a (byval) structure that
4145// was split between registers and memory.
4146// Return: The frame index registers were stored into.
4147int ARMTargetLowering::StoreByValRegs(CCState &CCInfo, SelectionDAG &DAG,
4148 const SDLoc &dl, SDValue &Chain,
4149 const Value *OrigArg,
4150 unsigned InRegsParamRecordIdx,
4151 int ArgOffset, unsigned ArgSize) const {
4152 // Currently, two use-cases possible:
4153 // Case #1. Non-var-args function, and we meet first byval parameter.
4154 // Setup first unallocated register as first byval register;
4155 // eat all remained registers
4156 // (these two actions are performed by HandleByVal method).
4157 // Then, here, we initialize stack frame with
4158 // "store-reg" instructions.
4159 // Case #2. Var-args function, that doesn't contain byval parameters.
4160 // The same: eat all remained unallocated registers,
4161 // initialize stack frame.
4162
4163 MachineFunction &MF = DAG.getMachineFunction();
4164 MachineFrameInfo &MFI = MF.getFrameInfo();
4165 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
4166 unsigned RBegin, REnd;
4167 if (InRegsParamRecordIdx < CCInfo.getInRegsParamsCount()) {
4168 CCInfo.getInRegsParamInfo(InRegsParamRecordIndex: InRegsParamRecordIdx, BeginReg&: RBegin, EndReg&: REnd);
4169 } else {
4170 unsigned RBeginIdx = CCInfo.getFirstUnallocated(Regs: GPRArgRegs);
4171 RBegin = RBeginIdx == 4 ? (unsigned)ARM::R4 : GPRArgRegs[RBeginIdx];
4172 REnd = ARM::R4;
4173 }
4174
4175 if (REnd != RBegin)
4176 ArgOffset = -4 * (ARM::R4 - RBegin);
4177
4178 auto PtrVT = getPointerTy(DL: DAG.getDataLayout());
4179 int FrameIndex = MFI.CreateFixedObject(Size: ArgSize, SPOffset: ArgOffset, IsImmutable: false);
4180 SDValue FIN = DAG.getFrameIndex(FI: FrameIndex, VT: PtrVT);
4181
4182 SmallVector<SDValue, 4> MemOps;
4183 const TargetRegisterClass *RC =
4184 AFI->isThumb1OnlyFunction() ? &ARM::tGPRRegClass : &ARM::GPRRegClass;
4185
4186 for (unsigned Reg = RBegin, i = 0; Reg < REnd; ++Reg, ++i) {
4187 Register VReg = MF.addLiveIn(PReg: Reg, RC);
4188 SDValue Val = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: MVT::i32);
4189 SDValue Store = DAG.getStore(Chain: Val.getValue(R: 1), dl, Val, Ptr: FIN,
4190 PtrInfo: MachinePointerInfo(OrigArg, 4 * i));
4191 MemOps.push_back(Elt: Store);
4192 FIN = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: FIN, N2: DAG.getConstant(Val: 4, DL: dl, VT: PtrVT));
4193 }
4194
4195 if (!MemOps.empty())
4196 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: MemOps);
4197 return FrameIndex;
4198}
4199
4200// Setup stack frame, the va_list pointer will start from.
4201void ARMTargetLowering::VarArgStyleRegisters(CCState &CCInfo, SelectionDAG &DAG,
4202 const SDLoc &dl, SDValue &Chain,
4203 unsigned ArgOffset,
4204 unsigned TotalArgRegsSaveSize,
4205 bool ForceMutable) const {
4206 MachineFunction &MF = DAG.getMachineFunction();
4207 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
4208
4209 // Try to store any remaining integer argument regs
4210 // to their spots on the stack so that they may be loaded by dereferencing
4211 // the result of va_next.
4212 // If there is no regs to be stored, just point address after last
4213 // argument passed via stack.
4214 int FrameIndex = StoreByValRegs(
4215 CCInfo, DAG, dl, Chain, OrigArg: nullptr, InRegsParamRecordIdx: CCInfo.getInRegsParamsCount(),
4216 ArgOffset: CCInfo.getStackSize(), ArgSize: std::max(a: 4U, b: TotalArgRegsSaveSize));
4217 AFI->setVarArgsFrameIndex(FrameIndex);
4218}
4219
4220bool ARMTargetLowering::splitValueIntoRegisterParts(
4221 SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
4222 unsigned NumParts, MVT PartVT, std::optional<CallingConv::ID> CC) const {
4223 EVT ValueVT = Val.getValueType();
4224 if ((ValueVT == MVT::f16 || ValueVT == MVT::bf16) && PartVT == MVT::f32) {
4225 unsigned ValueBits = ValueVT.getSizeInBits();
4226 unsigned PartBits = PartVT.getSizeInBits();
4227 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::getIntegerVT(BitWidth: ValueBits), Operand: Val);
4228 Val = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::getIntegerVT(BitWidth: PartBits), Operand: Val);
4229 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: PartVT, Operand: Val);
4230 Parts[0] = Val;
4231 return true;
4232 }
4233 return false;
4234}
4235
4236SDValue ARMTargetLowering::joinRegisterPartsIntoValue(
4237 SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts,
4238 MVT PartVT, EVT ValueVT, std::optional<CallingConv::ID> CC) const {
4239 if ((ValueVT == MVT::f16 || ValueVT == MVT::bf16) && PartVT == MVT::f32) {
4240 unsigned ValueBits = ValueVT.getSizeInBits();
4241 unsigned PartBits = PartVT.getSizeInBits();
4242 SDValue Val = Parts[0];
4243
4244 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::getIntegerVT(BitWidth: PartBits), Operand: Val);
4245 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::getIntegerVT(BitWidth: ValueBits), Operand: Val);
4246 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ValueVT, Operand: Val);
4247 return Val;
4248 }
4249 return SDValue();
4250}
4251
4252SDValue ARMTargetLowering::LowerFormalArguments(
4253 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
4254 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
4255 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
4256 MachineFunction &MF = DAG.getMachineFunction();
4257 MachineFrameInfo &MFI = MF.getFrameInfo();
4258
4259 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
4260
4261 // Assign locations to all of the incoming arguments.
4262 SmallVector<CCValAssign, 16> ArgLocs;
4263 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs,
4264 *DAG.getContext());
4265 CCInfo.AnalyzeFormalArguments(Ins, Fn: CCAssignFnForCall(CC: CallConv, isVarArg));
4266
4267 Function::const_arg_iterator CurOrigArg = MF.getFunction().arg_begin();
4268 unsigned CurArgIdx = 0;
4269
4270 // Initially ArgRegsSaveSize is zero.
4271 // Then we increase this value each time we meet byval parameter.
4272 // We also increase this value in case of varargs function.
4273 AFI->setArgRegsSaveSize(0);
4274
4275 // Calculate the amount of stack space that we need to allocate to store
4276 // byval and variadic arguments that are passed in registers.
4277 // We need to know this before we allocate the first byval or variadic
4278 // argument, as they will be allocated a stack slot below the CFA (Canonical
4279 // Frame Address, the stack pointer at entry to the function).
4280 unsigned ArgRegBegin = ARM::R4;
4281 for (const CCValAssign &VA : ArgLocs) {
4282 if (CCInfo.getInRegsParamsProcessed() >= CCInfo.getInRegsParamsCount())
4283 break;
4284
4285 unsigned Index = VA.getValNo();
4286 ISD::ArgFlagsTy Flags = Ins[Index].Flags;
4287 if (!Flags.isByVal())
4288 continue;
4289
4290 assert(VA.isMemLoc() && "unexpected byval pointer in reg");
4291 unsigned RBegin, REnd;
4292 CCInfo.getInRegsParamInfo(InRegsParamRecordIndex: CCInfo.getInRegsParamsProcessed(), BeginReg&: RBegin, EndReg&: REnd);
4293 ArgRegBegin = std::min(a: ArgRegBegin, b: RBegin);
4294
4295 CCInfo.nextInRegsParam();
4296 }
4297 CCInfo.rewindByValRegsInfo();
4298
4299 int lastInsIndex = -1;
4300 if (isVarArg && MFI.hasVAStart()) {
4301 unsigned RegIdx = CCInfo.getFirstUnallocated(Regs: GPRArgRegs);
4302 if (RegIdx != std::size(GPRArgRegs))
4303 ArgRegBegin = std::min(a: ArgRegBegin, b: (unsigned)GPRArgRegs[RegIdx]);
4304 }
4305
4306 unsigned TotalArgRegsSaveSize = 4 * (ARM::R4 - ArgRegBegin);
4307 AFI->setArgRegsSaveSize(TotalArgRegsSaveSize);
4308 auto PtrVT = getPointerTy(DL: DAG.getDataLayout());
4309
4310 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
4311 CCValAssign &VA = ArgLocs[i];
4312 if (Ins[VA.getValNo()].isOrigArg()) {
4313 std::advance(i&: CurOrigArg,
4314 n: Ins[VA.getValNo()].getOrigArgIndex() - CurArgIdx);
4315 CurArgIdx = Ins[VA.getValNo()].getOrigArgIndex();
4316 }
4317 // Arguments stored in registers.
4318 if (VA.isRegLoc()) {
4319 EVT RegVT = VA.getLocVT();
4320 SDValue ArgValue;
4321
4322 if (VA.needsCustom() && VA.getLocVT() == MVT::v2f64) {
4323 // f64 and vector types are split up into multiple registers or
4324 // combinations of registers and stack slots.
4325 SDValue ArgValue1 =
4326 GetF64FormalArgument(VA, NextVA&: ArgLocs[++i], Root&: Chain, DAG, dl);
4327 VA = ArgLocs[++i]; // skip ahead to next loc
4328 SDValue ArgValue2;
4329 if (VA.isMemLoc()) {
4330 int FI = MFI.CreateFixedObject(Size: 8, SPOffset: VA.getLocMemOffset(), IsImmutable: true);
4331 SDValue FIN = DAG.getFrameIndex(FI, VT: PtrVT);
4332 ArgValue2 = DAG.getLoad(
4333 VT: MVT::f64, dl, Chain, Ptr: FIN,
4334 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI));
4335 } else {
4336 ArgValue2 = GetF64FormalArgument(VA, NextVA&: ArgLocs[++i], Root&: Chain, DAG, dl);
4337 }
4338 ArgValue = DAG.getNode(Opcode: ISD::UNDEF, DL: dl, VT: MVT::v2f64);
4339 ArgValue = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: MVT::v2f64, N1: ArgValue,
4340 N2: ArgValue1, N3: DAG.getIntPtrConstant(Val: 0, DL: dl));
4341 ArgValue = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: MVT::v2f64, N1: ArgValue,
4342 N2: ArgValue2, N3: DAG.getIntPtrConstant(Val: 1, DL: dl));
4343 } else if (VA.needsCustom() && VA.getLocVT() == MVT::f64) {
4344 ArgValue = GetF64FormalArgument(VA, NextVA&: ArgLocs[++i], Root&: Chain, DAG, dl);
4345 } else {
4346 const TargetRegisterClass *RC;
4347
4348 if (RegVT == MVT::f16 || RegVT == MVT::bf16)
4349 RC = &ARM::HPRRegClass;
4350 else if (RegVT == MVT::f32)
4351 RC = &ARM::SPRRegClass;
4352 else if (RegVT == MVT::f64 || RegVT == MVT::v4f16 ||
4353 RegVT == MVT::v4bf16)
4354 RC = &ARM::DPRRegClass;
4355 else if (RegVT == MVT::v2f64 || RegVT == MVT::v8f16 ||
4356 RegVT == MVT::v8bf16)
4357 RC = &ARM::QPRRegClass;
4358 else if (RegVT == MVT::i32)
4359 RC = AFI->isThumb1OnlyFunction() ? &ARM::tGPRRegClass
4360 : &ARM::GPRRegClass;
4361 else
4362 llvm_unreachable("RegVT not supported by FORMAL_ARGUMENTS Lowering");
4363
4364 // Transform the arguments in physical registers into virtual ones.
4365 Register Reg = MF.addLiveIn(PReg: VA.getLocReg(), RC);
4366 ArgValue = DAG.getCopyFromReg(Chain, dl, Reg, VT: RegVT);
4367
4368 // If this value is passed in r0 and has the returned attribute (e.g.
4369 // C++ 'structors), record this fact for later use.
4370 if (VA.getLocReg() == ARM::R0 && Ins[VA.getValNo()].Flags.isReturned()) {
4371 AFI->setPreservesR0();
4372 }
4373 }
4374
4375 // If this is an 8 or 16-bit value, it is really passed promoted
4376 // to 32 bits. Insert an assert[sz]ext to capture this, then
4377 // truncate to the right size.
4378 switch (VA.getLocInfo()) {
4379 default: llvm_unreachable("Unknown loc info!");
4380 case CCValAssign::Full: break;
4381 case CCValAssign::BCvt:
4382 ArgValue = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: VA.getValVT(), Operand: ArgValue);
4383 break;
4384 }
4385
4386 // f16 arguments have their size extended to 4 bytes and passed as if they
4387 // had been copied to the LSBs of a 32-bit register.
4388 // For that, it's passed extended to i32 (soft ABI) or to f32 (hard ABI)
4389 if (VA.needsCustom() &&
4390 (VA.getValVT() == MVT::f16 || VA.getValVT() == MVT::bf16))
4391 ArgValue = MoveToHPR(dl, DAG, LocVT: VA.getLocVT(), ValVT: VA.getValVT(), Val: ArgValue);
4392
4393 // On CMSE Entry Functions, formal integer arguments whose bitwidth is
4394 // less than 32 bits must be sign- or zero-extended in the callee for
4395 // security reasons. Although the ABI mandates an extension done by the
4396 // caller, the latter cannot be trusted to follow the rules of the ABI.
4397 const ISD::InputArg &Arg = Ins[VA.getValNo()];
4398 if (AFI->isCmseNSEntryFunction() && Arg.ArgVT.isScalarInteger() &&
4399 RegVT.isScalarInteger() && Arg.ArgVT.bitsLT(VT: MVT::i32))
4400 ArgValue = handleCMSEValue(Value: ArgValue, Arg, DAG, DL: dl);
4401
4402 InVals.push_back(Elt: ArgValue);
4403 } else { // VA.isRegLoc()
4404 // Only arguments passed on the stack should make it here.
4405 assert(VA.isMemLoc());
4406 assert(VA.getValVT() != MVT::i64 && "i64 should already be lowered");
4407
4408 int index = VA.getValNo();
4409
4410 // Some Ins[] entries become multiple ArgLoc[] entries.
4411 // Process them only once.
4412 if (index != lastInsIndex)
4413 {
4414 ISD::ArgFlagsTy Flags = Ins[index].Flags;
4415 // FIXME: For now, all byval parameter objects are marked mutable.
4416 // This can be changed with more analysis.
4417 // In case of tail call optimization mark all arguments mutable.
4418 // Since they could be overwritten by lowering of arguments in case of
4419 // a tail call.
4420 if (Flags.isByVal()) {
4421 assert(Ins[index].isOrigArg() &&
4422 "Byval arguments cannot be implicit");
4423 unsigned CurByValIndex = CCInfo.getInRegsParamsProcessed();
4424
4425 int FrameIndex = StoreByValRegs(
4426 CCInfo, DAG, dl, Chain, OrigArg: &*CurOrigArg, InRegsParamRecordIdx: CurByValIndex,
4427 ArgOffset: VA.getLocMemOffset(), ArgSize: Flags.getByValSize());
4428 InVals.push_back(Elt: DAG.getFrameIndex(FI: FrameIndex, VT: PtrVT));
4429 CCInfo.nextInRegsParam();
4430 } else if (VA.needsCustom() && (VA.getValVT() == MVT::f16 ||
4431 VA.getValVT() == MVT::bf16)) {
4432 // f16 and bf16 values are passed in the least-significant half of
4433 // a 4 byte stack slot. This is done as-if the extension was done
4434 // in a 32-bit register, so the actual bytes used for the value
4435 // differ between little and big endian.
4436 assert(VA.getLocVT().getSizeInBits() == 32);
4437 unsigned FIOffset = VA.getLocMemOffset();
4438 int FI = MFI.CreateFixedObject(Size: VA.getLocVT().getSizeInBits() / 8,
4439 SPOffset: FIOffset, IsImmutable: true);
4440
4441 SDValue Addr = DAG.getFrameIndex(FI, VT: PtrVT);
4442 if (DAG.getDataLayout().isBigEndian())
4443 Addr = DAG.getObjectPtrOffset(SL: dl, Ptr: Addr, Offset: TypeSize::getFixed(ExactSize: 2));
4444
4445 InVals.push_back(Elt: DAG.getLoad(VT: VA.getValVT(), dl, Chain, Ptr: Addr,
4446 PtrInfo: MachinePointerInfo::getFixedStack(
4447 MF&: DAG.getMachineFunction(), FI)));
4448
4449 } else {
4450 unsigned FIOffset = VA.getLocMemOffset();
4451 int FI = MFI.CreateFixedObject(Size: VA.getLocVT().getSizeInBits()/8,
4452 SPOffset: FIOffset, IsImmutable: true);
4453
4454 // Create load nodes to retrieve arguments from the stack.
4455 SDValue FIN = DAG.getFrameIndex(FI, VT: PtrVT);
4456 InVals.push_back(Elt: DAG.getLoad(VT: VA.getValVT(), dl, Chain, Ptr: FIN,
4457 PtrInfo: MachinePointerInfo::getFixedStack(
4458 MF&: DAG.getMachineFunction(), FI)));
4459 }
4460 lastInsIndex = index;
4461 }
4462 }
4463 }
4464
4465 // varargs
4466 if (isVarArg && MFI.hasVAStart()) {
4467 VarArgStyleRegisters(CCInfo, DAG, dl, Chain, ArgOffset: CCInfo.getStackSize(),
4468 TotalArgRegsSaveSize);
4469 if (AFI->isCmseNSEntryFunction()) {
4470 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
4471 DAG.getMachineFunction().getFunction(),
4472 "secure entry function must not be variadic", dl.getDebugLoc()));
4473 }
4474 }
4475
4476 unsigned StackArgSize = CCInfo.getStackSize();
4477 bool TailCallOpt = MF.getTarget().Options.GuaranteedTailCallOpt;
4478 if (canGuaranteeTCO(CC: CallConv, GuaranteeTailCalls: TailCallOpt)) {
4479 // The only way to guarantee a tail call is if the callee restores its
4480 // argument area, but it must also keep the stack aligned when doing so.
4481 MaybeAlign StackAlign = DAG.getDataLayout().getStackAlignment();
4482 assert(StackAlign && "data layout string is missing stack alignment");
4483 StackArgSize = alignTo(Size: StackArgSize, A: *StackAlign);
4484
4485 AFI->setArgumentStackToRestore(StackArgSize);
4486 }
4487 AFI->setArgumentStackSize(StackArgSize);
4488
4489 if (CCInfo.getStackSize() > 0 && AFI->isCmseNSEntryFunction()) {
4490 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
4491 DAG.getMachineFunction().getFunction(),
4492 "secure entry function requires arguments on stack", dl.getDebugLoc()));
4493 }
4494
4495 return Chain;
4496}
4497
4498/// isFloatingPointZero - Return true if this is +0.0.
4499static bool isFloatingPointZero(SDValue Op) {
4500 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Val&: Op))
4501 return CFP->getValueAPF().isPosZero();
4502 else if (ISD::isEXTLoad(N: Op.getNode()) || ISD::isNON_EXTLoad(N: Op.getNode())) {
4503 // Maybe this has already been legalized into the constant pool?
4504 if (Op.getOperand(i: 1).getOpcode() == ARMISD::Wrapper) {
4505 SDValue WrapperOp = Op.getOperand(i: 1).getOperand(i: 0);
4506 if (ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(Val&: WrapperOp))
4507 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(Val: CP->getConstVal()))
4508 return CFP->getValueAPF().isPosZero();
4509 }
4510 } else if (Op->getOpcode() == ISD::BITCAST &&
4511 Op->getValueType(ResNo: 0) == MVT::f64) {
4512 // Handle (ISD::BITCAST (ARMISD::VMOVIMM (ISD::TargetConstant 0)) MVT::f64)
4513 // created by LowerConstantFP().
4514 SDValue BitcastOp = Op->getOperand(Num: 0);
4515 if (BitcastOp->getOpcode() == ARMISD::VMOVIMM &&
4516 isNullConstant(V: BitcastOp->getOperand(Num: 0)))
4517 return true;
4518 }
4519 return false;
4520}
4521
4522static bool isSafeSignedCMN(SDValue Op, SelectionDAG &DAG) {
4523 // 0 - INT_MIN sign wraps, so no signed wrap means cmn is safe.
4524 if (Op->getFlags().hasNoSignedWrap())
4525 return true;
4526
4527 // We can still figure out if the second operand is safe to use
4528 // in a CMN instruction by checking if it is known to be not the minimum
4529 // signed value. If it is not, then we can safely use CMN.
4530 // Note: We can eventually remove this check and simply rely on
4531 // Op->getFlags().hasNoSignedWrap() once SelectionDAG/ISelLowering
4532 // consistently sets them appropriately when making said nodes.
4533
4534 KnownBits KnownSrc = DAG.computeKnownBits(Op: Op.getOperand(i: 1));
4535 return !KnownSrc.getSignedMinValue().isMinSignedValue();
4536}
4537
4538static bool isCMN(SDValue Op, ISD::CondCode CC, SelectionDAG &DAG) {
4539 return Op.getOpcode() == ISD::SUB && isNullConstant(V: Op.getOperand(i: 0)) &&
4540 (isIntEqualitySetCC(Code: CC) ||
4541 (isUnsignedIntSetCC(Code: CC) && DAG.isKnownNeverZero(Op: Op.getOperand(i: 1))) ||
4542 (isSignedIntSetCC(Code: CC) && isSafeSignedCMN(Op, DAG)));
4543}
4544
4545/// Returns how profitable it is to fold a comparison's operand's shift and/or
4546/// extension operations into the comparison instruction's second operand
4547/// (so_reg_imm / so_reg_reg for ARM, t2_so_reg for Thumb-2).
4548static unsigned getCmpOperandFoldingProfit(SDValue Op, const ARMSubtarget &ST) {
4549 // Thumb-1 CMP does not support shifted second operands.
4550 if (ST.isThumb1Only() || !Op.hasOneUse())
4551 return 0;
4552
4553 unsigned Opc = Op.getOpcode();
4554 if (Opc == ISD::SHL || Opc == ISD::SRL || Opc == ISD::SRA) {
4555 if (auto *ShiftAmt = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1)))
4556 return ShiftAmt->getZExtValue() <= 31 ? 1 : 0;
4557 // Register-controlled shift: only ARM-mode CMP/CMN (so_reg_reg) supports
4558 // this; Thumb-2 t2_so_reg requires an immediate shift amount.
4559 return ST.isThumb() ? 0 : 1;
4560 }
4561
4562 if (Opc == ISD::ROTR) {
4563 // Rotr constants will be normalized via mod 32, or & 31,
4564 // so we do not have to bounds check.
4565 if (isa<ConstantSDNode>(Val: Op.getOperand(i: 1)))
4566 return 1;
4567 return ST.isThumb() ? 0 : 1;
4568 }
4569
4570 return 0;
4571}
4572
4573/// Returns appropriate ARM CMP (cmp) and corresponding condition code for
4574/// the given operands.
4575SDValue ARMTargetLowering::getARMCmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
4576 SDValue &ARMcc, SelectionDAG &DAG,
4577 const SDLoc &dl) const {
4578 if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(Val: RHS.getNode())) {
4579 unsigned C = RHSC->getZExtValue();
4580 if (!isLegalICmpImmediate(Imm: (int32_t)C)) {
4581 // Constant does not fit, try adjusting it by one.
4582 switch (CC) {
4583 default: break;
4584 case ISD::SETLT:
4585 case ISD::SETGE:
4586 if (C != 0x80000000 && isLegalICmpImmediate(Imm: C-1)) {
4587 CC = (CC == ISD::SETLT) ? ISD::SETLE : ISD::SETGT;
4588 RHS = DAG.getConstant(Val: C - 1, DL: dl, VT: MVT::i32);
4589 }
4590 break;
4591 case ISD::SETULT:
4592 case ISD::SETUGE:
4593 if (C != 0 && isLegalICmpImmediate(Imm: C-1)) {
4594 CC = (CC == ISD::SETULT) ? ISD::SETULE : ISD::SETUGT;
4595 RHS = DAG.getConstant(Val: C - 1, DL: dl, VT: MVT::i32);
4596 }
4597 break;
4598 case ISD::SETLE:
4599 case ISD::SETGT:
4600 if (C != 0x7fffffff && isLegalICmpImmediate(Imm: C+1)) {
4601 CC = (CC == ISD::SETLE) ? ISD::SETLT : ISD::SETGE;
4602 RHS = DAG.getConstant(Val: C + 1, DL: dl, VT: MVT::i32);
4603 }
4604 break;
4605 case ISD::SETULE:
4606 case ISD::SETUGT:
4607 if (C != 0xffffffff && isLegalICmpImmediate(Imm: C+1)) {
4608 CC = (CC == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE;
4609 RHS = DAG.getConstant(Val: C + 1, DL: dl, VT: MVT::i32);
4610 }
4611 break;
4612 }
4613 }
4614 }
4615
4616 // Thumb1 has very limited immediate modes, so turning an "and" into a
4617 // shift can save multiple instructions.
4618 //
4619 // If we have (x & C1), and C1 is an appropriate mask, we can transform it
4620 // into "((x << n) >> n)". But that isn't necessarily profitable on its
4621 // own. If it's the operand to an unsigned comparison with an immediate,
4622 // we can eliminate one of the shifts: we transform
4623 // "((x << n) >> n) == C2" to "(x << n) == (C2 << n)".
4624 //
4625 // We avoid transforming cases which aren't profitable due to encoding
4626 // details:
4627 //
4628 // 1. C2 fits into the immediate field of a cmp, and the transformed version
4629 // would not; in that case, we're essentially trading one immediate load for
4630 // another.
4631 // 2. C1 is 255 or 65535, so we can use uxtb or uxth.
4632 // 3. C2 is zero; we have other code for this special case.
4633 //
4634 // FIXME: Figure out profitability for Thumb2; we usually can't save an
4635 // instruction, since the AND is always one instruction anyway, but we could
4636 // use narrow instructions in some cases.
4637 if (Subtarget->isThumb1Only() && LHS->getOpcode() == ISD::AND &&
4638 LHS->hasOneUse() && isa<ConstantSDNode>(Val: LHS.getOperand(i: 1)) &&
4639 LHS.getValueType() == MVT::i32 && isa<ConstantSDNode>(Val: RHS) &&
4640 !isSignedIntSetCC(Code: CC)) {
4641 unsigned Mask = LHS.getConstantOperandVal(i: 1);
4642 auto *RHSC = cast<ConstantSDNode>(Val: RHS.getNode());
4643 uint64_t RHSV = RHSC->getZExtValue();
4644 if (isMask_32(Value: Mask) && (RHSV & ~Mask) == 0 && Mask != 255 && Mask != 65535) {
4645 unsigned ShiftBits = llvm::countl_zero(Val: Mask);
4646 if (RHSV && (RHSV > 255 || (RHSV << ShiftBits) <= 255)) {
4647 SDValue ShiftAmt = DAG.getConstant(Val: ShiftBits, DL: dl, VT: MVT::i32);
4648 LHS = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: MVT::i32, N1: LHS.getOperand(i: 0), N2: ShiftAmt);
4649 RHS = DAG.getConstant(Val: RHSV << ShiftBits, DL: dl, VT: MVT::i32);
4650 }
4651 }
4652 }
4653
4654 // The specific comparison "(x<<c) > 0x80000000U" can be optimized to a
4655 // single "lsls x, c+1". The shift sets the "C" and "Z" flags the same
4656 // way a cmp would.
4657 // FIXME: Add support for ARM/Thumb2; this would need isel patterns, and
4658 // some tweaks to the heuristics for the previous and->shift transform.
4659 // FIXME: Optimize cases where the LHS isn't a shift.
4660 if (Subtarget->isThumb1Only() && LHS->getOpcode() == ISD::SHL &&
4661 isa<ConstantSDNode>(Val: RHS) && RHS->getAsZExtVal() == 0x80000000U &&
4662 CC == ISD::SETUGT && isa<ConstantSDNode>(Val: LHS.getOperand(i: 1)) &&
4663 LHS.getConstantOperandVal(i: 1) < 31) {
4664 unsigned ShiftAmt = LHS.getConstantOperandVal(i: 1) + 1;
4665 SDValue Shift =
4666 DAG.getNode(Opcode: ARMISD::LSLS, DL: dl, VTList: DAG.getVTList(VT1: MVT::i32, VT2: FlagsVT),
4667 N1: LHS.getOperand(i: 0), N2: DAG.getConstant(Val: ShiftAmt, DL: dl, VT: MVT::i32));
4668 ARMcc = DAG.getConstant(Val: ARMCC::HI, DL: dl, VT: MVT::i32);
4669 return Shift.getValue(R: 1);
4670 }
4671
4672 ARMCC::CondCodes CondCode = IntCCToARMCC(CC);
4673
4674 unsigned CompareType;
4675 switch (CondCode) {
4676 default:
4677 CompareType = ARMISD::CMP;
4678 break;
4679 case ARMCC::EQ:
4680 case ARMCC::NE:
4681 // Uses only Z Flag
4682 CompareType = ARMISD::CMPZ;
4683 break;
4684 }
4685
4686 // TODO: Remove CMPZ check once we generalize and remove the CMPZ enum from
4687 // the codebase.
4688
4689 // TODO: When we have a solution to the vselect predicate not allowing pl/mi
4690 // all the time, allow those cases to be cmn too no matter what.
4691 if (CompareType != ARMISD::CMPZ && isCMN(Op: RHS, CC, DAG)) {
4692 CompareType = ARMISD::CMN;
4693 RHS = RHS.getOperand(i: 1);
4694 } else if (CompareType != ARMISD::CMPZ && isCMN(Op: LHS, CC, DAG)) {
4695 CompareType = ARMISD::CMN;
4696 LHS = LHS.getOperand(i: 1);
4697 CondCode = IntCCToARMCC(CC: ISD::getSetCCSwappedOperands(Operation: CC));
4698 }
4699
4700 // Prefer folding shifts / CMN into the cmp/cmn second operand (so_reg /
4701 // t2_so_reg). When both sides compete, pick the higher
4702 // getCmpOperandFoldingProfit. Only when RHS is not a legal icmp
4703 // immediate: otherwise keep the canonical (reg, imm) form.
4704 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val: RHS.getNode());
4705 if (!C || !isLegalICmpImmediate(Imm: C->getSExtValue())) {
4706 if (getCmpOperandFoldingProfit(Op: LHS, ST: *Subtarget) >
4707 getCmpOperandFoldingProfit(Op: RHS, ST: *Subtarget)) {
4708 std::swap(a&: LHS, b&: RHS);
4709 if (CompareType == ARMISD::CMP)
4710 CondCode = ARMCC::getSwappedCondition(CC: CondCode);
4711 }
4712 }
4713
4714 // If the RHS is a constant zero then the V (overflow) flag will never be
4715 // set. This can allow us to simplify GE to PL or LT to MI, which can be
4716 // simpler for other passes (like the peephole optimiser) to deal with.
4717 if (isNullConstant(V: RHS)) {
4718 switch (CondCode) {
4719 default:
4720 break;
4721 case ARMCC::GE:
4722 CondCode = ARMCC::PL;
4723 break;
4724 case ARMCC::LT:
4725 CondCode = ARMCC::MI;
4726 break;
4727 }
4728 }
4729
4730 ARMcc = DAG.getConstant(Val: CondCode, DL: dl, VT: MVT::i32);
4731 return DAG.getNode(Opcode: CompareType, DL: dl, VT: FlagsVT, N1: LHS, N2: RHS);
4732}
4733
4734/// Returns a appropriate VFP CMP (fcmp{s|d}+fmstat) for the given operands.
4735SDValue ARMTargetLowering::getVFPCmp(SDValue LHS, SDValue RHS,
4736 SelectionDAG &DAG, const SDLoc &dl,
4737 bool Signaling) const {
4738 assert(Subtarget->hasFP64() || RHS.getValueType() != MVT::f64);
4739 SDValue Flags;
4740 if (!isFloatingPointZero(Op: RHS))
4741 Flags = DAG.getNode(Opcode: Signaling ? ARMISD::CMPFPE : ARMISD::CMPFP, DL: dl, VT: FlagsVT,
4742 N1: LHS, N2: RHS);
4743 else
4744 Flags = DAG.getNode(Opcode: Signaling ? ARMISD::CMPFPEw0 : ARMISD::CMPFPw0, DL: dl,
4745 VT: FlagsVT, Operand: LHS);
4746 return DAG.getNode(Opcode: ARMISD::FMSTAT, DL: dl, VT: FlagsVT, Operand: Flags);
4747}
4748
4749// This function returns three things: the arithmetic computation itself
4750// (Value), a comparison (OverflowCmp), and a condition code (ARMcc). The
4751// comparison and the condition code define the case in which the arithmetic
4752// computation *does not* overflow.
4753std::pair<SDValue, SDValue>
4754ARMTargetLowering::getARMXALUOOp(SDValue Op, SelectionDAG &DAG,
4755 SDValue &ARMcc) const {
4756 assert(Op.getValueType() == MVT::i32 && "Unsupported value type");
4757
4758 SDValue Value, OverflowCmp;
4759 SDValue LHS = Op.getOperand(i: 0);
4760 SDValue RHS = Op.getOperand(i: 1);
4761 SDLoc dl(Op);
4762
4763 // FIXME: We are currently always generating CMPs because we don't support
4764 // generating CMN through the backend. This is not as good as the natural
4765 // CMP case because it causes a register dependency and cannot be folded
4766 // later.
4767
4768 switch (Op.getOpcode()) {
4769 default:
4770 llvm_unreachable("Unknown overflow instruction!");
4771 case ISD::SADDO:
4772 ARMcc = DAG.getConstant(Val: ARMCC::VC, DL: dl, VT: MVT::i32);
4773 Value = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: Op.getValueType(), N1: LHS, N2: RHS);
4774 OverflowCmp = DAG.getNode(Opcode: ARMISD::CMP, DL: dl, VT: FlagsVT, N1: Value, N2: LHS);
4775 break;
4776 case ISD::UADDO:
4777 ARMcc = DAG.getConstant(Val: ARMCC::HS, DL: dl, VT: MVT::i32);
4778 // We use ADDC here to correspond to its use in LowerALUO.
4779 // We do not use it in the USUBO case as Value may not be used.
4780 Value = DAG.getNode(Opcode: ARMISD::ADDC, DL: dl,
4781 VTList: DAG.getVTList(VT1: Op.getValueType(), VT2: MVT::i32), N1: LHS, N2: RHS)
4782 .getValue(R: 0);
4783 OverflowCmp = DAG.getNode(Opcode: ARMISD::CMP, DL: dl, VT: FlagsVT, N1: Value, N2: LHS);
4784 break;
4785 case ISD::SSUBO:
4786 ARMcc = DAG.getConstant(Val: ARMCC::VC, DL: dl, VT: MVT::i32);
4787 Value = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: Op.getValueType(), N1: LHS, N2: RHS);
4788 OverflowCmp = DAG.getNode(Opcode: ARMISD::CMP, DL: dl, VT: FlagsVT, N1: LHS, N2: RHS);
4789 break;
4790 case ISD::USUBO:
4791 ARMcc = DAG.getConstant(Val: ARMCC::HS, DL: dl, VT: MVT::i32);
4792 Value = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: Op.getValueType(), N1: LHS, N2: RHS);
4793 OverflowCmp = DAG.getNode(Opcode: ARMISD::CMP, DL: dl, VT: FlagsVT, N1: LHS, N2: RHS);
4794 break;
4795 case ISD::UMULO:
4796 // We generate a UMUL_LOHI and then check if the high word is 0.
4797 ARMcc = DAG.getConstant(Val: ARMCC::EQ, DL: dl, VT: MVT::i32);
4798 Value = DAG.getNode(Opcode: ISD::UMUL_LOHI, DL: dl,
4799 VTList: DAG.getVTList(VT1: Op.getValueType(), VT2: Op.getValueType()),
4800 N1: LHS, N2: RHS);
4801 OverflowCmp = DAG.getNode(Opcode: ARMISD::CMPZ, DL: dl, VT: FlagsVT, N1: Value.getValue(R: 1),
4802 N2: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
4803 Value = Value.getValue(R: 0); // We only want the low 32 bits for the result.
4804 break;
4805 case ISD::SMULO:
4806 // We generate a SMUL_LOHI and then check if all the bits of the high word
4807 // are the same as the sign bit of the low word.
4808 ARMcc = DAG.getConstant(Val: ARMCC::EQ, DL: dl, VT: MVT::i32);
4809 Value = DAG.getNode(Opcode: ISD::SMUL_LOHI, DL: dl,
4810 VTList: DAG.getVTList(VT1: Op.getValueType(), VT2: Op.getValueType()),
4811 N1: LHS, N2: RHS);
4812 OverflowCmp = DAG.getNode(Opcode: ARMISD::CMPZ, DL: dl, VT: FlagsVT, N1: Value.getValue(R: 1),
4813 N2: DAG.getNode(Opcode: ISD::SRA, DL: dl, VT: Op.getValueType(),
4814 N1: Value.getValue(R: 0),
4815 N2: DAG.getConstant(Val: 31, DL: dl, VT: MVT::i32)));
4816 Value = Value.getValue(R: 0); // We only want the low 32 bits for the result.
4817 break;
4818 } // switch (...)
4819
4820 return std::make_pair(x&: Value, y&: OverflowCmp);
4821}
4822
4823static SDValue valueToCarryFlag(SDValue Value, SelectionDAG &DAG, bool Invert) {
4824 SDLoc DL(Value);
4825 EVT VT = Value.getValueType();
4826
4827 if (Invert)
4828 Value = DAG.getNode(Opcode: ISD::SUB, DL, VT: MVT::i32,
4829 N1: DAG.getConstant(Val: 1, DL, VT: MVT::i32), N2: Value);
4830
4831 SDValue Cmp = DAG.getNode(Opcode: ARMISD::SUBC, DL, VTList: DAG.getVTList(VT1: VT, VT2: MVT::i32),
4832 N1: Value, N2: DAG.getConstant(Val: 1, DL, VT));
4833 return Cmp.getValue(R: 1);
4834}
4835
4836static SDValue carryFlagToValue(SDValue Flags, EVT VT, SelectionDAG &DAG,
4837 bool Invert) {
4838 SDLoc DL(Flags);
4839
4840 if (Invert) {
4841 // Convert flags to boolean with ADDE 0,0,Carry then compute 1 - bool.
4842 SDValue BoolCarry = DAG.getNode(
4843 Opcode: ARMISD::ADDE, DL, VTList: DAG.getVTList(VT1: VT, VT2: MVT::i32),
4844 N1: DAG.getConstant(Val: 0, DL, VT), N2: DAG.getConstant(Val: 0, DL, VT), N3: Flags);
4845 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: DAG.getConstant(Val: 1, DL, VT), N2: BoolCarry);
4846 }
4847
4848 // Now convert the carry flag into a boolean carry. We do this
4849 // using ARMISD::ADDE 0, 0, Carry
4850 return DAG.getNode(Opcode: ARMISD::ADDE, DL, VTList: DAG.getVTList(VT1: VT, VT2: MVT::i32),
4851 N1: DAG.getConstant(Val: 0, DL, VT), N2: DAG.getConstant(Val: 0, DL, VT),
4852 N3: Flags);
4853}
4854
4855// Value is 1 if 'V' bit is 1, else 0
4856static SDValue overflowFlagToValue(SDValue Flags, EVT VT, SelectionDAG &DAG) {
4857 SDLoc DL(Flags);
4858 SDValue Zero = DAG.getConstant(Val: 0, DL, VT);
4859 SDValue One = DAG.getConstant(Val: 1, DL, VT);
4860 SDValue ARMcc = DAG.getConstant(Val: ARMCC::VS, DL, VT: MVT::i32);
4861 return DAG.getNode(Opcode: ARMISD::CMOV, DL, VT, N1: Zero, N2: One, N3: ARMcc, N4: Flags);
4862}
4863
4864SDValue ARMTargetLowering::LowerALUO(SDValue Op, SelectionDAG &DAG) const {
4865 // Let legalize expand this if it isn't a legal type yet.
4866 if (!isTypeLegal(VT: Op.getValueType()))
4867 return SDValue();
4868
4869 SDValue LHS = Op.getOperand(i: 0);
4870 SDValue RHS = Op.getOperand(i: 1);
4871 SDLoc dl(Op);
4872
4873 EVT VT = Op.getValueType();
4874 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: MVT::i32);
4875 SDValue Value;
4876 SDValue Overflow;
4877 switch (Op.getOpcode()) {
4878 case ISD::UADDO:
4879 Value = DAG.getNode(Opcode: ARMISD::ADDC, DL: dl, VTList: VTs, N1: LHS, N2: RHS);
4880 // Convert the carry flag into a boolean value.
4881 Overflow = carryFlagToValue(Flags: Value.getValue(R: 1), VT, DAG, Invert: false);
4882 break;
4883 case ISD::USUBO:
4884 Value = DAG.getNode(Opcode: ARMISD::SUBC, DL: dl, VTList: VTs, N1: LHS, N2: RHS);
4885 // Convert the carry flag into a boolean value.
4886 Overflow = carryFlagToValue(Flags: Value.getValue(R: 1), VT, DAG, Invert: true);
4887 break;
4888 default: {
4889 // Handle other operations with getARMXALUOOp
4890 SDValue OverflowCmp, ARMcc;
4891 std::tie(args&: Value, args&: OverflowCmp) = getARMXALUOOp(Op, DAG, ARMcc);
4892 // We use 0 and 1 as false and true values.
4893 // ARMcc represents the "no overflow" condition (e.g., VC for signed ops).
4894 // CMOV operand order is (FalseVal, TrueVal), so we put 1 in FalseVal
4895 // position to get Overflow=1 when the "no overflow" condition is false.
4896 Overflow =
4897 DAG.getNode(Opcode: ARMISD::CMOV, DL: dl, VT: MVT::i32,
4898 N1: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32), // FalseVal: overflow
4899 N2: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32), // TrueVal: no overflow
4900 N3: ARMcc, N4: OverflowCmp);
4901 break;
4902 }
4903 }
4904
4905 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: dl, VTList: VTs, N1: Value, N2: Overflow);
4906}
4907
4908static SDValue LowerADDSUBSAT(SDValue Op, SelectionDAG &DAG,
4909 const ARMSubtarget *Subtarget) {
4910 EVT VT = Op.getValueType();
4911 if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP() || Subtarget->isThumb1Only())
4912 return SDValue();
4913 if (!VT.isSimple())
4914 return SDValue();
4915
4916 unsigned NewOpcode;
4917 switch (VT.getSimpleVT().SimpleTy) {
4918 default:
4919 return SDValue();
4920 case MVT::i8:
4921 switch (Op->getOpcode()) {
4922 case ISD::UADDSAT:
4923 NewOpcode = ARMISD::UQADD8b;
4924 break;
4925 case ISD::SADDSAT:
4926 NewOpcode = ARMISD::QADD8b;
4927 break;
4928 case ISD::USUBSAT:
4929 NewOpcode = ARMISD::UQSUB8b;
4930 break;
4931 case ISD::SSUBSAT:
4932 NewOpcode = ARMISD::QSUB8b;
4933 break;
4934 }
4935 break;
4936 case MVT::i16:
4937 switch (Op->getOpcode()) {
4938 case ISD::UADDSAT:
4939 NewOpcode = ARMISD::UQADD16b;
4940 break;
4941 case ISD::SADDSAT:
4942 NewOpcode = ARMISD::QADD16b;
4943 break;
4944 case ISD::USUBSAT:
4945 NewOpcode = ARMISD::UQSUB16b;
4946 break;
4947 case ISD::SSUBSAT:
4948 NewOpcode = ARMISD::QSUB16b;
4949 break;
4950 }
4951 break;
4952 }
4953
4954 SDLoc dl(Op);
4955 SDValue Add =
4956 DAG.getNode(Opcode: NewOpcode, DL: dl, VT: MVT::i32,
4957 N1: DAG.getSExtOrTrunc(Op: Op->getOperand(Num: 0), DL: dl, VT: MVT::i32),
4958 N2: DAG.getSExtOrTrunc(Op: Op->getOperand(Num: 1), DL: dl, VT: MVT::i32));
4959 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: Add);
4960}
4961
4962SDValue ARMTargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
4963 SDValue Cond = Op.getOperand(i: 0);
4964 SDValue SelectTrue = Op.getOperand(i: 1);
4965 SDValue SelectFalse = Op.getOperand(i: 2);
4966 SDLoc dl(Op);
4967 unsigned Opc = Cond.getOpcode();
4968
4969 if (Cond.getResNo() == 1 &&
4970 (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO ||
4971 Opc == ISD::USUBO)) {
4972 if (!isTypeLegal(VT: Cond->getValueType(ResNo: 0)))
4973 return SDValue();
4974
4975 SDValue Value, OverflowCmp;
4976 SDValue ARMcc;
4977 std::tie(args&: Value, args&: OverflowCmp) = getARMXALUOOp(Op: Cond, DAG, ARMcc);
4978 EVT VT = Op.getValueType();
4979
4980 return getCMOV(dl, VT, FalseVal: SelectTrue, TrueVal: SelectFalse, ARMcc, Flags: OverflowCmp, DAG);
4981 }
4982
4983 // Convert:
4984 //
4985 // (select (cmov 1, 0, cond), t, f) -> (cmov t, f, cond)
4986 // (select (cmov 0, 1, cond), t, f) -> (cmov f, t, cond)
4987 //
4988 if (Cond.getOpcode() == ARMISD::CMOV && Cond.hasOneUse()) {
4989 const ConstantSDNode *CMOVTrue =
4990 dyn_cast<ConstantSDNode>(Val: Cond.getOperand(i: 0));
4991 const ConstantSDNode *CMOVFalse =
4992 dyn_cast<ConstantSDNode>(Val: Cond.getOperand(i: 1));
4993
4994 if (CMOVTrue && CMOVFalse) {
4995 unsigned CMOVTrueVal = CMOVTrue->getZExtValue();
4996 unsigned CMOVFalseVal = CMOVFalse->getZExtValue();
4997
4998 SDValue True;
4999 SDValue False;
5000 if (CMOVTrueVal == 1 && CMOVFalseVal == 0) {
5001 True = SelectTrue;
5002 False = SelectFalse;
5003 } else if (CMOVTrueVal == 0 && CMOVFalseVal == 1) {
5004 True = SelectFalse;
5005 False = SelectTrue;
5006 }
5007
5008 if (True.getNode() && False.getNode())
5009 return getCMOV(dl, VT: Op.getValueType(), FalseVal: True, TrueVal: False, ARMcc: Cond.getOperand(i: 2),
5010 Flags: Cond.getOperand(i: 3), DAG);
5011 }
5012 }
5013
5014 return DAG.getSelectCC(DL: dl, LHS: Cond,
5015 RHS: DAG.getConstant(Val: 0, DL: dl, VT: Cond.getValueType()),
5016 True: SelectTrue, False: SelectFalse, Cond: ISD::SETNE);
5017}
5018
5019static void checkVSELConstraints(ISD::CondCode CC, ARMCC::CondCodes &CondCode,
5020 bool &swpCmpOps, bool &swpVselOps) {
5021 // Start by selecting the GE condition code for opcodes that return true for
5022 // 'equality'
5023 if (CC == ISD::SETUGE || CC == ISD::SETOGE || CC == ISD::SETOLE ||
5024 CC == ISD::SETULE || CC == ISD::SETGE || CC == ISD::SETLE)
5025 CondCode = ARMCC::GE;
5026
5027 // and GT for opcodes that return false for 'equality'.
5028 else if (CC == ISD::SETUGT || CC == ISD::SETOGT || CC == ISD::SETOLT ||
5029 CC == ISD::SETULT || CC == ISD::SETGT || CC == ISD::SETLT)
5030 CondCode = ARMCC::GT;
5031
5032 // Since we are constrained to GE/GT, if the opcode contains 'less', we need
5033 // to swap the compare operands.
5034 if (CC == ISD::SETOLE || CC == ISD::SETULE || CC == ISD::SETOLT ||
5035 CC == ISD::SETULT || CC == ISD::SETLE || CC == ISD::SETLT)
5036 swpCmpOps = true;
5037
5038 // Both GT and GE are ordered comparisons, and return false for 'unordered'.
5039 // If we have an unordered opcode, we need to swap the operands to the VSEL
5040 // instruction (effectively negating the condition).
5041 //
5042 // This also has the effect of swapping which one of 'less' or 'greater'
5043 // returns true, so we also swap the compare operands. It also switches
5044 // whether we return true for 'equality', so we compensate by picking the
5045 // opposite condition code to our original choice.
5046 if (CC == ISD::SETULE || CC == ISD::SETULT || CC == ISD::SETUGE ||
5047 CC == ISD::SETUGT) {
5048 swpCmpOps = !swpCmpOps;
5049 swpVselOps = !swpVselOps;
5050 CondCode = CondCode == ARMCC::GT ? ARMCC::GE : ARMCC::GT;
5051 }
5052
5053 // 'ordered' is 'anything but unordered', so use the VS condition code and
5054 // swap the VSEL operands.
5055 if (CC == ISD::SETO) {
5056 CondCode = ARMCC::VS;
5057 swpVselOps = true;
5058 }
5059
5060 // 'unordered or not equal' is 'anything but equal', so use the EQ condition
5061 // code and swap the VSEL operands. Also do this if we don't care about the
5062 // unordered case.
5063 if (CC == ISD::SETUNE || CC == ISD::SETNE) {
5064 CondCode = ARMCC::EQ;
5065 swpVselOps = true;
5066 }
5067}
5068
5069SDValue ARMTargetLowering::getCMOV(const SDLoc &dl, EVT VT, SDValue FalseVal,
5070 SDValue TrueVal, SDValue ARMcc,
5071 SDValue Flags, SelectionDAG &DAG) const {
5072 if (!Subtarget->hasFP64() && VT == MVT::f64) {
5073 FalseVal = DAG.getNode(Opcode: ARMISD::VMOVRRD, DL: dl,
5074 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N: FalseVal);
5075 TrueVal = DAG.getNode(Opcode: ARMISD::VMOVRRD, DL: dl,
5076 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N: TrueVal);
5077
5078 SDValue TrueLow = TrueVal.getValue(R: 0);
5079 SDValue TrueHigh = TrueVal.getValue(R: 1);
5080 SDValue FalseLow = FalseVal.getValue(R: 0);
5081 SDValue FalseHigh = FalseVal.getValue(R: 1);
5082
5083 SDValue Low = DAG.getNode(Opcode: ARMISD::CMOV, DL: dl, VT: MVT::i32, N1: FalseLow, N2: TrueLow,
5084 N3: ARMcc, N4: Flags);
5085 SDValue High = DAG.getNode(Opcode: ARMISD::CMOV, DL: dl, VT: MVT::i32, N1: FalseHigh, N2: TrueHigh,
5086 N3: ARMcc, N4: Flags);
5087
5088 return DAG.getNode(Opcode: ARMISD::VMOVDRR, DL: dl, VT: MVT::f64, N1: Low, N2: High);
5089 }
5090 return DAG.getNode(Opcode: ARMISD::CMOV, DL: dl, VT, N1: FalseVal, N2: TrueVal, N3: ARMcc, N4: Flags);
5091}
5092
5093static bool isGTorGE(ISD::CondCode CC) {
5094 return CC == ISD::SETGT || CC == ISD::SETGE;
5095}
5096
5097static bool isLTorLE(ISD::CondCode CC) {
5098 return CC == ISD::SETLT || CC == ISD::SETLE;
5099}
5100
5101// See if a conditional (LHS CC RHS ? TrueVal : FalseVal) is lower-saturating.
5102// All of these conditions (and their <= and >= counterparts) will do:
5103// x < k ? k : x
5104// x > k ? x : k
5105// k < x ? x : k
5106// k > x ? k : x
5107static bool isLowerSaturate(const SDValue LHS, const SDValue RHS,
5108 const SDValue TrueVal, const SDValue FalseVal,
5109 const ISD::CondCode CC, const SDValue K) {
5110 return (isGTorGE(CC) &&
5111 ((K == LHS && K == TrueVal) || (K == RHS && K == FalseVal))) ||
5112 (isLTorLE(CC) &&
5113 ((K == RHS && K == TrueVal) || (K == LHS && K == FalseVal)));
5114}
5115
5116// Check if two chained conditionals could be converted into SSAT or USAT.
5117//
5118// SSAT can replace a set of two conditional selectors that bound a number to an
5119// interval of type [k, ~k] when k + 1 is a power of 2. Here are some examples:
5120//
5121// x < -k ? -k : (x > k ? k : x)
5122// x < -k ? -k : (x < k ? x : k)
5123// x > -k ? (x > k ? k : x) : -k
5124// x < k ? (x < -k ? -k : x) : k
5125// etc.
5126//
5127// LLVM canonicalizes these to either a min(max()) or a max(min())
5128// pattern. This function tries to match one of these and will return a SSAT
5129// node if successful.
5130//
5131// USAT works similarly to SSAT but bounds on the interval [0, k] where k + 1
5132// is a power of 2.
5133static SDValue LowerSaturatingConditional(SDValue Op, SelectionDAG &DAG) {
5134 EVT VT = Op.getValueType();
5135 SDValue V1 = Op.getOperand(i: 0);
5136 SDValue K1 = Op.getOperand(i: 1);
5137 SDValue TrueVal1 = Op.getOperand(i: 2);
5138 SDValue FalseVal1 = Op.getOperand(i: 3);
5139 ISD::CondCode CC1 = cast<CondCodeSDNode>(Val: Op.getOperand(i: 4))->get();
5140
5141 const SDValue Op2 = isa<ConstantSDNode>(Val: TrueVal1) ? FalseVal1 : TrueVal1;
5142 if (Op2.getOpcode() != ISD::SELECT_CC)
5143 return SDValue();
5144
5145 SDValue V2 = Op2.getOperand(i: 0);
5146 SDValue K2 = Op2.getOperand(i: 1);
5147 SDValue TrueVal2 = Op2.getOperand(i: 2);
5148 SDValue FalseVal2 = Op2.getOperand(i: 3);
5149 ISD::CondCode CC2 = cast<CondCodeSDNode>(Val: Op2.getOperand(i: 4))->get();
5150
5151 SDValue V1Tmp = V1;
5152 SDValue V2Tmp = V2;
5153
5154 // Check that the registers and the constants match a max(min()) or min(max())
5155 // pattern
5156 if (V1Tmp != TrueVal1 || V2Tmp != TrueVal2 || K1 != FalseVal1 ||
5157 K2 != FalseVal2 ||
5158 !((isGTorGE(CC: CC1) && isLTorLE(CC: CC2)) || (isLTorLE(CC: CC1) && isGTorGE(CC: CC2))))
5159 return SDValue();
5160
5161 // Check that the constant in the lower-bound check is
5162 // the opposite of the constant in the upper-bound check
5163 // in 1's complement.
5164 if (!isa<ConstantSDNode>(Val: K1) || !isa<ConstantSDNode>(Val: K2))
5165 return SDValue();
5166
5167 int64_t Val1 = cast<ConstantSDNode>(Val&: K1)->getSExtValue();
5168 int64_t Val2 = cast<ConstantSDNode>(Val&: K2)->getSExtValue();
5169 int64_t PosVal = std::max(a: Val1, b: Val2);
5170 int64_t NegVal = std::min(a: Val1, b: Val2);
5171
5172 if (!((Val1 > Val2 && isLTorLE(CC: CC1)) || (Val1 < Val2 && isLTorLE(CC: CC2))) ||
5173 !isPowerOf2_64(Value: PosVal + 1))
5174 return SDValue();
5175
5176 // Handle the difference between USAT (unsigned) and SSAT (signed)
5177 // saturation
5178 // At this point, PosVal is guaranteed to be positive
5179 uint64_t K = PosVal;
5180 SDLoc dl(Op);
5181 if (Val1 == ~Val2)
5182 return DAG.getNode(Opcode: ARMISD::SSAT, DL: dl, VT, N1: V2Tmp,
5183 N2: DAG.getConstant(Val: llvm::countr_one(Value: K), DL: dl, VT));
5184 if (NegVal == 0)
5185 return DAG.getNode(Opcode: ARMISD::USAT, DL: dl, VT, N1: V2Tmp,
5186 N2: DAG.getConstant(Val: llvm::countr_one(Value: K), DL: dl, VT));
5187
5188 return SDValue();
5189}
5190
5191// Check if a condition of the type x < k ? k : x can be converted into a
5192// bit operation instead of conditional moves.
5193// Currently this is allowed given:
5194// - The conditions and values match up
5195// - k is 0 or -1 (all ones)
5196// This function will not check the last condition, thats up to the caller
5197// It returns true if the transformation can be made, and in such case
5198// returns x in V, and k in SatK.
5199static bool isLowerSaturatingConditional(const SDValue &Op, SDValue &V,
5200 SDValue &SatK)
5201{
5202 SDValue LHS = Op.getOperand(i: 0);
5203 SDValue RHS = Op.getOperand(i: 1);
5204 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 4))->get();
5205 SDValue TrueVal = Op.getOperand(i: 2);
5206 SDValue FalseVal = Op.getOperand(i: 3);
5207
5208 SDValue *K = isa<ConstantSDNode>(Val: LHS) ? &LHS : isa<ConstantSDNode>(Val: RHS)
5209 ? &RHS
5210 : nullptr;
5211
5212 // No constant operation in comparison, early out
5213 if (!K)
5214 return false;
5215
5216 SDValue KTmp = isa<ConstantSDNode>(Val: TrueVal) ? TrueVal : FalseVal;
5217 V = (KTmp == TrueVal) ? FalseVal : TrueVal;
5218 SDValue VTmp = (K && *K == LHS) ? RHS : LHS;
5219
5220 // If the constant on left and right side, or variable on left and right,
5221 // does not match, early out
5222 if (*K != KTmp || V != VTmp)
5223 return false;
5224
5225 if (isLowerSaturate(LHS, RHS, TrueVal, FalseVal, CC, K: *K)) {
5226 SatK = *K;
5227 return true;
5228 }
5229
5230 return false;
5231}
5232
5233bool ARMTargetLowering::isUnsupportedFloatingType(EVT VT) const {
5234 if (VT == MVT::f32)
5235 return !Subtarget->hasVFP2Base();
5236 if (VT == MVT::f64)
5237 return !Subtarget->hasFP64();
5238 if (VT == MVT::f16)
5239 return !Subtarget->hasFullFP16();
5240 return false;
5241}
5242
5243static SDValue matchCSET(unsigned &Opcode, bool &InvertCond, SDValue TrueVal,
5244 SDValue FalseVal, const ARMSubtarget *Subtarget) {
5245 ConstantSDNode *CFVal = dyn_cast<ConstantSDNode>(Val&: FalseVal);
5246 ConstantSDNode *CTVal = dyn_cast<ConstantSDNode>(Val&: TrueVal);
5247 if (!CFVal || !CTVal || !Subtarget->hasV8_1MMainlineOps())
5248 return SDValue();
5249
5250 unsigned TVal = CTVal->getZExtValue();
5251 unsigned FVal = CFVal->getZExtValue();
5252
5253 Opcode = 0;
5254 InvertCond = false;
5255 if (TVal == ~FVal) {
5256 Opcode = ARMISD::CSINV;
5257 } else if (TVal == ~FVal + 1) {
5258 Opcode = ARMISD::CSNEG;
5259 } else if (TVal + 1 == FVal) {
5260 Opcode = ARMISD::CSINC;
5261 } else if (TVal == FVal + 1) {
5262 Opcode = ARMISD::CSINC;
5263 std::swap(a&: TrueVal, b&: FalseVal);
5264 std::swap(a&: TVal, b&: FVal);
5265 InvertCond = !InvertCond;
5266 } else {
5267 return SDValue();
5268 }
5269
5270 // If one of the constants is cheaper than another, materialise the
5271 // cheaper one and let the csel generate the other.
5272 if (Opcode != ARMISD::CSINC &&
5273 HasLowerConstantMaterializationCost(Val1: FVal, Val2: TVal, Subtarget)) {
5274 std::swap(a&: TrueVal, b&: FalseVal);
5275 std::swap(a&: TVal, b&: FVal);
5276 InvertCond = !InvertCond;
5277 }
5278
5279 // Attempt to use ZR checking TVal is 0, possibly inverting the condition
5280 // to get there. CSINC not is invertable like the other two (~(~a) == a,
5281 // -(-a) == a, but (a+1)+1 != a).
5282 if (FVal == 0 && Opcode != ARMISD::CSINC) {
5283 std::swap(a&: TrueVal, b&: FalseVal);
5284 std::swap(a&: TVal, b&: FVal);
5285 InvertCond = !InvertCond;
5286 }
5287
5288 return TrueVal;
5289}
5290
5291SDValue ARMTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
5292 EVT VT = Op.getValueType();
5293 SDLoc dl(Op);
5294
5295 // Try to convert two saturating conditional selects into a single SSAT
5296 if ((!Subtarget->isThumb() && Subtarget->hasV6Ops()) || Subtarget->isThumb2())
5297 if (SDValue SatValue = LowerSaturatingConditional(Op, DAG))
5298 return SatValue;
5299
5300 // Try to convert expressions of the form x < k ? k : x (and similar forms)
5301 // into more efficient bit operations, which is possible when k is 0 or -1
5302 // On ARM and Thumb-2 which have flexible operand 2 this will result in
5303 // single instructions. On Thumb the shift and the bit operation will be two
5304 // instructions.
5305 // Only allow this transformation on full-width (32-bit) operations
5306 SDValue LowerSatConstant;
5307 SDValue SatValue;
5308 if (VT == MVT::i32 &&
5309 isLowerSaturatingConditional(Op, V&: SatValue, SatK&: LowerSatConstant)) {
5310 SDValue ShiftV = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: SatValue,
5311 N2: DAG.getConstant(Val: 31, DL: dl, VT));
5312 if (isNullConstant(V: LowerSatConstant)) {
5313 SDValue NotShiftV = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: ShiftV,
5314 N2: DAG.getAllOnesConstant(DL: dl, VT));
5315 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: SatValue, N2: NotShiftV);
5316 } else if (isAllOnesConstant(V: LowerSatConstant))
5317 return DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: SatValue, N2: ShiftV);
5318 }
5319
5320 SDValue LHS = Op.getOperand(i: 0);
5321 SDValue RHS = Op.getOperand(i: 1);
5322 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 4))->get();
5323 SDValue TrueVal = Op.getOperand(i: 2);
5324 SDValue FalseVal = Op.getOperand(i: 3);
5325 ConstantSDNode *CFVal = dyn_cast<ConstantSDNode>(Val&: FalseVal);
5326 ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(Val&: RHS);
5327 if (Op.getValueType().isInteger()) {
5328
5329 // Check for SMAX(lhs, 0) and SMIN(lhs, 0) patterns.
5330 // (SELECT_CC setgt, lhs, 0, lhs, 0) -> (BIC lhs, (SRA lhs, typesize-1))
5331 // (SELECT_CC setlt, lhs, 0, lhs, 0) -> (AND lhs, (SRA lhs, typesize-1))
5332 // Both require less instructions than compare and conditional select.
5333 if ((CC == ISD::SETGT || CC == ISD::SETLT) && LHS == TrueVal && RHSC &&
5334 RHSC->isZero() && CFVal && CFVal->isZero() &&
5335 LHS.getValueType() == RHS.getValueType()) {
5336 EVT VT = LHS.getValueType();
5337 SDValue Shift =
5338 DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: LHS,
5339 N2: DAG.getConstant(Val: VT.getSizeInBits() - 1, DL: dl, VT));
5340
5341 if (CC == ISD::SETGT)
5342 Shift = DAG.getNOT(DL: dl, Val: Shift, VT);
5343
5344 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: LHS, N2: Shift);
5345 }
5346
5347 // (SELECT_CC setlt, x, 0, 1, 0) -> SRL(x, bw-1)
5348 if (CC == ISD::SETLT && isNullConstant(V: RHS) && isOneConstant(V: TrueVal) &&
5349 isNullConstant(V: FalseVal) && LHS.getValueType() == VT)
5350 return DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: LHS,
5351 N2: DAG.getConstant(Val: VT.getSizeInBits() - 1, DL: dl, VT));
5352 }
5353
5354 if (LHS.getValueType() == MVT::i32) {
5355 unsigned Opcode;
5356 bool InvertCond;
5357 if (SDValue Op =
5358 matchCSET(Opcode, InvertCond, TrueVal, FalseVal, Subtarget)) {
5359 if (InvertCond)
5360 CC = ISD::getSetCCInverse(Operation: CC, Type: LHS.getValueType());
5361
5362 SDValue ARMcc;
5363 SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMcc, DAG, dl);
5364 EVT VT = Op.getValueType();
5365 return DAG.getNode(Opcode, DL: dl, VT, N1: Op, N2: Op, N3: ARMcc, N4: Cmp);
5366 }
5367 }
5368
5369 if (isUnsupportedFloatingType(VT: LHS.getValueType())) {
5370 softenSetCCOperands(DAG, VT: LHS.getValueType(), NewLHS&: LHS, NewRHS&: RHS, CCCode&: CC, DL: dl, OldLHS: LHS, OldRHS: RHS);
5371
5372 // If softenSetCCOperands only returned one value, we should compare it to
5373 // zero.
5374 if (!RHS.getNode()) {
5375 RHS = DAG.getConstant(Val: 0, DL: dl, VT: LHS.getValueType());
5376 CC = ISD::SETNE;
5377 }
5378 }
5379
5380 if (LHS.getValueType() == MVT::i32) {
5381 // Try to generate VSEL on ARMv8.
5382 // The VSEL instruction can't use all the usual ARM condition
5383 // codes: it only has two bits to select the condition code, so it's
5384 // constrained to use only GE, GT, VS and EQ.
5385 //
5386 // To implement all the various ISD::SETXXX opcodes, we sometimes need to
5387 // swap the operands of the previous compare instruction (effectively
5388 // inverting the compare condition, swapping 'less' and 'greater') and
5389 // sometimes need to swap the operands to the VSEL (which inverts the
5390 // condition in the sense of firing whenever the previous condition didn't)
5391 if (Subtarget->hasFPARMv8Base() && (TrueVal.getValueType() == MVT::f16 ||
5392 TrueVal.getValueType() == MVT::f32 ||
5393 TrueVal.getValueType() == MVT::f64)) {
5394 ARMCC::CondCodes CondCode = IntCCToARMCC(CC);
5395 if (CondCode == ARMCC::LT || CondCode == ARMCC::LE ||
5396 CondCode == ARMCC::VC || CondCode == ARMCC::NE) {
5397 CC = ISD::getSetCCInverse(Operation: CC, Type: LHS.getValueType());
5398 std::swap(a&: TrueVal, b&: FalseVal);
5399 }
5400 }
5401
5402 SDValue ARMcc;
5403 SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMcc, DAG, dl);
5404 // Choose GE over PL, which vsel does now support
5405 if (ARMcc->getAsZExtVal() == ARMCC::PL)
5406 ARMcc = DAG.getConstant(Val: ARMCC::GE, DL: dl, VT: MVT::i32);
5407 return getCMOV(dl, VT, FalseVal, TrueVal, ARMcc, Flags: Cmp, DAG);
5408 }
5409
5410 ARMCC::CondCodes CondCode, CondCode2;
5411 FPCCToARMCC(CC, CondCode, CondCode2);
5412
5413 // Normalize the fp compare. If RHS is zero we prefer to keep it there so we
5414 // match CMPFPw0 instead of CMPFP, though we don't do this for f16 because we
5415 // must use VSEL (limited condition codes), due to not having conditional f16
5416 // moves.
5417 if (Subtarget->hasFPARMv8Base() &&
5418 !(isFloatingPointZero(Op: RHS) && TrueVal.getValueType() != MVT::f16) &&
5419 (TrueVal.getValueType() == MVT::f16 ||
5420 TrueVal.getValueType() == MVT::f32 ||
5421 TrueVal.getValueType() == MVT::f64)) {
5422 bool swpCmpOps = false;
5423 bool swpVselOps = false;
5424 checkVSELConstraints(CC, CondCode, swpCmpOps, swpVselOps);
5425
5426 if (CondCode == ARMCC::GT || CondCode == ARMCC::GE ||
5427 CondCode == ARMCC::VS || CondCode == ARMCC::EQ) {
5428 if (swpCmpOps)
5429 std::swap(a&: LHS, b&: RHS);
5430 if (swpVselOps)
5431 std::swap(a&: TrueVal, b&: FalseVal);
5432 }
5433 }
5434
5435 SDValue ARMcc = DAG.getConstant(Val: CondCode, DL: dl, VT: MVT::i32);
5436 SDValue Cmp = getVFPCmp(LHS, RHS, DAG, dl);
5437 SDValue Result = getCMOV(dl, VT, FalseVal, TrueVal, ARMcc, Flags: Cmp, DAG);
5438 if (CondCode2 != ARMCC::AL) {
5439 SDValue ARMcc2 = DAG.getConstant(Val: CondCode2, DL: dl, VT: MVT::i32);
5440 Result = getCMOV(dl, VT, FalseVal: Result, TrueVal, ARMcc: ARMcc2, Flags: Cmp, DAG);
5441 }
5442 return Result;
5443}
5444
5445/// canChangeToInt - Given the fp compare operand, return true if it is suitable
5446/// to morph to an integer compare sequence.
5447static bool canChangeToInt(SDValue Op, bool &SeenZero,
5448 const ARMSubtarget *Subtarget) {
5449 SDNode *N = Op.getNode();
5450 if (!N->hasOneUse())
5451 // Otherwise it requires moving the value from fp to integer registers.
5452 return false;
5453 if (!N->getNumValues())
5454 return false;
5455 EVT VT = Op.getValueType();
5456 if (VT != MVT::f32 && !Subtarget->isFPBrccSlow())
5457 // f32 case is generally profitable. f64 case only makes sense when vcmpe +
5458 // vmrs are very slow, e.g. cortex-a8.
5459 return false;
5460
5461 if (isFloatingPointZero(Op)) {
5462 SeenZero = true;
5463 return true;
5464 }
5465 return ISD::isNormalLoad(N);
5466}
5467
5468static SDValue bitcastf32Toi32(SDValue Op, SelectionDAG &DAG) {
5469 if (isFloatingPointZero(Op))
5470 return DAG.getConstant(Val: 0, DL: SDLoc(Op), VT: MVT::i32);
5471
5472 if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(Val&: Op))
5473 return DAG.getLoad(VT: MVT::i32, dl: SDLoc(Op), Chain: Ld->getChain(), Ptr: Ld->getBasePtr(),
5474 PtrInfo: Ld->getPointerInfo(), Alignment: Ld->getAlign(),
5475 MMOFlags: Ld->getMemOperand()->getFlags());
5476
5477 llvm_unreachable("Unknown VFP cmp argument!");
5478}
5479
5480static void expandf64Toi32(SDValue Op, SelectionDAG &DAG,
5481 SDValue &RetVal1, SDValue &RetVal2) {
5482 SDLoc dl(Op);
5483
5484 if (isFloatingPointZero(Op)) {
5485 RetVal1 = DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32);
5486 RetVal2 = DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32);
5487 return;
5488 }
5489
5490 if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(Val&: Op)) {
5491 SDValue Ptr = Ld->getBasePtr();
5492 RetVal1 =
5493 DAG.getLoad(VT: MVT::i32, dl, Chain: Ld->getChain(), Ptr, PtrInfo: Ld->getPointerInfo(),
5494 Alignment: Ld->getAlign(), MMOFlags: Ld->getMemOperand()->getFlags());
5495
5496 EVT PtrType = Ptr.getValueType();
5497 SDValue NewPtr = DAG.getNode(Opcode: ISD::ADD, DL: dl,
5498 VT: PtrType, N1: Ptr, N2: DAG.getConstant(Val: 4, DL: dl, VT: PtrType));
5499 RetVal2 = DAG.getLoad(VT: MVT::i32, dl, Chain: Ld->getChain(), Ptr: NewPtr,
5500 PtrInfo: Ld->getPointerInfo().getWithOffset(O: 4),
5501 Alignment: commonAlignment(A: Ld->getAlign(), Offset: 4),
5502 MMOFlags: Ld->getMemOperand()->getFlags());
5503 return;
5504 }
5505
5506 llvm_unreachable("Unknown VFP cmp argument!");
5507}
5508
5509/// OptimizeVFPBrcond - With nnan and without daz, it's legal to optimize some
5510/// f32 and even f64 comparisons to integer ones.
5511SDValue
5512ARMTargetLowering::OptimizeVFPBrcond(SDValue Op, SelectionDAG &DAG) const {
5513 SDValue Chain = Op.getOperand(i: 0);
5514 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 1))->get();
5515 SDValue LHS = Op.getOperand(i: 2);
5516 SDValue RHS = Op.getOperand(i: 3);
5517 SDValue Dest = Op.getOperand(i: 4);
5518 SDLoc dl(Op);
5519
5520 bool LHSSeenZero = false;
5521 bool LHSOk = canChangeToInt(Op: LHS, SeenZero&: LHSSeenZero, Subtarget);
5522 bool RHSSeenZero = false;
5523 bool RHSOk = canChangeToInt(Op: RHS, SeenZero&: RHSSeenZero, Subtarget);
5524 if (LHSOk && RHSOk && (LHSSeenZero || RHSSeenZero)) {
5525 // If unsafe fp math optimization is enabled and there are no other uses of
5526 // the CMP operands, and the condition code is EQ or NE, we can optimize it
5527 // to an integer comparison.
5528 if (CC == ISD::SETOEQ)
5529 CC = ISD::SETEQ;
5530 else if (CC == ISD::SETUNE)
5531 CC = ISD::SETNE;
5532
5533 SDValue Mask = DAG.getConstant(Val: 0x7fffffff, DL: dl, VT: MVT::i32);
5534 SDValue ARMcc;
5535 if (LHS.getValueType() == MVT::f32) {
5536 LHS = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32,
5537 N1: bitcastf32Toi32(Op: LHS, DAG), N2: Mask);
5538 RHS = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32,
5539 N1: bitcastf32Toi32(Op: RHS, DAG), N2: Mask);
5540 SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMcc, DAG, dl);
5541 return DAG.getNode(Opcode: ARMISD::BRCOND, DL: dl, VT: MVT::Other, N1: Chain, N2: Dest, N3: ARMcc,
5542 N4: Cmp);
5543 }
5544
5545 SDValue LHS1, LHS2;
5546 SDValue RHS1, RHS2;
5547 expandf64Toi32(Op: LHS, DAG, RetVal1&: LHS1, RetVal2&: LHS2);
5548 expandf64Toi32(Op: RHS, DAG, RetVal1&: RHS1, RetVal2&: RHS2);
5549 LHS2 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32, N1: LHS2, N2: Mask);
5550 RHS2 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32, N1: RHS2, N2: Mask);
5551 ARMCC::CondCodes CondCode = IntCCToARMCC(CC);
5552 ARMcc = DAG.getConstant(Val: CondCode, DL: dl, VT: MVT::i32);
5553 SDValue Ops[] = { Chain, ARMcc, LHS1, LHS2, RHS1, RHS2, Dest };
5554 return DAG.getNode(Opcode: ARMISD::BCC_i64, DL: dl, VT: MVT::Other, Ops);
5555 }
5556
5557 return SDValue();
5558}
5559
5560// Generate CMP + CMOV for integer abs.
5561SDValue ARMTargetLowering::LowerABS(SDValue Op, SelectionDAG &DAG) const {
5562 SDLoc DL(Op);
5563
5564 SDValue Neg = DAG.getNegative(Val: Op.getOperand(i: 0), DL, VT: MVT::i32);
5565
5566 // Generate CMP & CMOV.
5567 SDValue Cmp = DAG.getNode(Opcode: ARMISD::CMP, DL, VT: FlagsVT, N1: Op.getOperand(i: 0),
5568 N2: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
5569 return DAG.getNode(Opcode: ARMISD::CMOV, DL, VT: MVT::i32, N1: Op.getOperand(i: 0), N2: Neg,
5570 N3: DAG.getConstant(Val: ARMCC::MI, DL, VT: MVT::i32), N4: Cmp);
5571}
5572
5573static SDValue getInvertedARMCondCode(SDValue ARMcc, SelectionDAG &DAG) {
5574 ARMCC::CondCodes CondCode =
5575 (ARMCC::CondCodes)cast<ConstantSDNode>(Val&: ARMcc)->getZExtValue();
5576 CondCode = ARMCC::getOppositeCondition(CC: CondCode);
5577 return DAG.getConstant(Val: CondCode, DL: SDLoc(ARMcc), VT: MVT::i32);
5578}
5579
5580SDValue ARMTargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
5581 SDValue Chain = Op.getOperand(i: 0);
5582 SDValue Cond = Op.getOperand(i: 1);
5583 SDValue Dest = Op.getOperand(i: 2);
5584 SDLoc dl(Op);
5585
5586 // Optimize {s|u}{add|sub|mul}.with.overflow feeding into a branch
5587 // instruction.
5588 unsigned Opc = Cond.getOpcode();
5589 bool OptimizeMul = (Opc == ISD::SMULO || Opc == ISD::UMULO) &&
5590 !Subtarget->isThumb1Only();
5591 if (Cond.getResNo() == 1 &&
5592 (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO ||
5593 Opc == ISD::USUBO || OptimizeMul)) {
5594 // Only lower legal XALUO ops.
5595 if (!isTypeLegal(VT: Cond->getValueType(ResNo: 0)))
5596 return SDValue();
5597
5598 // The actual operation with overflow check.
5599 SDValue Value, OverflowCmp;
5600 SDValue ARMcc;
5601 std::tie(args&: Value, args&: OverflowCmp) = getARMXALUOOp(Op: Cond, DAG, ARMcc);
5602
5603 // Reverse the condition code.
5604 ARMcc = getInvertedARMCondCode(ARMcc, DAG);
5605
5606 return DAG.getNode(Opcode: ARMISD::BRCOND, DL: dl, VT: MVT::Other, N1: Chain, N2: Dest, N3: ARMcc,
5607 N4: OverflowCmp);
5608 }
5609
5610 return SDValue();
5611}
5612
5613SDValue ARMTargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
5614 SDValue Chain = Op.getOperand(i: 0);
5615 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 1))->get();
5616 SDValue LHS = Op.getOperand(i: 2);
5617 SDValue RHS = Op.getOperand(i: 3);
5618 SDValue Dest = Op.getOperand(i: 4);
5619 SDLoc dl(Op);
5620
5621 if (isUnsupportedFloatingType(VT: LHS.getValueType())) {
5622 softenSetCCOperands(DAG, VT: LHS.getValueType(), NewLHS&: LHS, NewRHS&: RHS, CCCode&: CC, DL: dl, OldLHS: LHS, OldRHS: RHS);
5623
5624 // If softenSetCCOperands only returned one value, we should compare it to
5625 // zero.
5626 if (!RHS.getNode()) {
5627 RHS = DAG.getConstant(Val: 0, DL: dl, VT: LHS.getValueType());
5628 CC = ISD::SETNE;
5629 }
5630 }
5631
5632 // Optimize {s|u}{add|sub|mul}.with.overflow feeding into a branch
5633 // instruction.
5634 unsigned Opc = LHS.getOpcode();
5635 bool OptimizeMul = (Opc == ISD::SMULO || Opc == ISD::UMULO) &&
5636 !Subtarget->isThumb1Only();
5637 if (LHS.getResNo() == 1 && (isOneConstant(V: RHS) || isNullConstant(V: RHS)) &&
5638 (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO ||
5639 Opc == ISD::USUBO || OptimizeMul) &&
5640 (CC == ISD::SETEQ || CC == ISD::SETNE)) {
5641 // Only lower legal XALUO ops.
5642 if (!isTypeLegal(VT: LHS->getValueType(ResNo: 0)))
5643 return SDValue();
5644
5645 // The actual operation with overflow check.
5646 SDValue Value, OverflowCmp;
5647 SDValue ARMcc;
5648 std::tie(args&: Value, args&: OverflowCmp) = getARMXALUOOp(Op: LHS.getValue(R: 0), DAG, ARMcc);
5649
5650 if ((CC == ISD::SETNE) != isOneConstant(V: RHS)) {
5651 // Reverse the condition code.
5652 ARMcc = getInvertedARMCondCode(ARMcc, DAG);
5653 }
5654
5655 return DAG.getNode(Opcode: ARMISD::BRCOND, DL: dl, VT: MVT::Other, N1: Chain, N2: Dest, N3: ARMcc,
5656 N4: OverflowCmp);
5657 }
5658
5659 if (LHS.getValueType() == MVT::i32) {
5660 SDValue ARMcc;
5661 SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMcc, DAG, dl);
5662 return DAG.getNode(Opcode: ARMISD::BRCOND, DL: dl, VT: MVT::Other, N1: Chain, N2: Dest, N3: ARMcc, N4: Cmp);
5663 }
5664
5665 SDNodeFlags Flags = Op->getFlags();
5666 if (Flags.hasNoNaNs() &&
5667 DAG.getDenormalMode(VT: MVT::f32) == DenormalMode::getIEEE() &&
5668 DAG.getDenormalMode(VT: MVT::f64) == DenormalMode::getIEEE() &&
5669 (CC == ISD::SETEQ || CC == ISD::SETOEQ || CC == ISD::SETNE ||
5670 CC == ISD::SETUNE)) {
5671 if (SDValue Result = OptimizeVFPBrcond(Op, DAG))
5672 return Result;
5673 }
5674
5675 ARMCC::CondCodes CondCode, CondCode2;
5676 FPCCToARMCC(CC, CondCode, CondCode2);
5677
5678 SDValue ARMcc = DAG.getConstant(Val: CondCode, DL: dl, VT: MVT::i32);
5679 SDValue Cmp = getVFPCmp(LHS, RHS, DAG, dl);
5680 SDValue Ops[] = {Chain, Dest, ARMcc, Cmp};
5681 SDValue Res = DAG.getNode(Opcode: ARMISD::BRCOND, DL: dl, VT: MVT::Other, Ops);
5682 if (CondCode2 != ARMCC::AL) {
5683 ARMcc = DAG.getConstant(Val: CondCode2, DL: dl, VT: MVT::i32);
5684 SDValue Ops[] = {Res, Dest, ARMcc, Cmp};
5685 Res = DAG.getNode(Opcode: ARMISD::BRCOND, DL: dl, VT: MVT::Other, Ops);
5686 }
5687 return Res;
5688}
5689
5690SDValue ARMTargetLowering::LowerBR_JT(SDValue Op, SelectionDAG &DAG) const {
5691 SDValue Chain = Op.getOperand(i: 0);
5692 SDValue Table = Op.getOperand(i: 1);
5693 SDValue Index = Op.getOperand(i: 2);
5694 SDLoc dl(Op);
5695
5696 EVT PTy = getPointerTy(DL: DAG.getDataLayout());
5697 JumpTableSDNode *JT = cast<JumpTableSDNode>(Val&: Table);
5698 SDValue JTI = DAG.getTargetJumpTable(JTI: JT->getIndex(), VT: PTy);
5699 Table = DAG.getNode(Opcode: ARMISD::WrapperJT, DL: dl, VT: MVT::i32, Operand: JTI);
5700 Index = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: PTy, N1: Index, N2: DAG.getConstant(Val: 4, DL: dl, VT: PTy));
5701 SDValue Addr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PTy, N1: Table, N2: Index);
5702 if (Subtarget->isThumb2() || (Subtarget->hasV8MBaselineOps() && Subtarget->isThumb())) {
5703 // Thumb2 and ARMv8-M use a two-level jump. That is, it jumps into the jump table
5704 // which does another jump to the destination. This also makes it easier
5705 // to translate it to TBB / TBH later (Thumb2 only).
5706 // FIXME: This might not work if the function is extremely large.
5707 return DAG.getNode(Opcode: ARMISD::BR2_JT, DL: dl, VT: MVT::Other, N1: Chain,
5708 N2: Addr, N3: Op.getOperand(i: 2), N4: JTI);
5709 }
5710 if (isPositionIndependent() || Subtarget->isROPI()) {
5711 Addr =
5712 DAG.getLoad(VT: (EVT)MVT::i32, dl, Chain, Ptr: Addr,
5713 PtrInfo: MachinePointerInfo::getJumpTable(MF&: DAG.getMachineFunction()));
5714 Chain = Addr.getValue(R: 1);
5715 Addr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PTy, N1: Table, N2: Addr);
5716 return DAG.getNode(Opcode: ARMISD::BR_JT, DL: dl, VT: MVT::Other, N1: Chain, N2: Addr, N3: JTI);
5717 } else {
5718 Addr =
5719 DAG.getLoad(VT: PTy, dl, Chain, Ptr: Addr,
5720 PtrInfo: MachinePointerInfo::getJumpTable(MF&: DAG.getMachineFunction()));
5721 Chain = Addr.getValue(R: 1);
5722 return DAG.getNode(Opcode: ARMISD::BR_JT, DL: dl, VT: MVT::Other, N1: Chain, N2: Addr, N3: JTI);
5723 }
5724}
5725
5726static SDValue LowerVectorFP_TO_INT(SDValue Op, SelectionDAG &DAG) {
5727 EVT VT = Op.getValueType();
5728 SDLoc dl(Op);
5729
5730 if (Op.getValueType().getVectorElementType() == MVT::i32) {
5731 if (Op.getOperand(i: 0).getValueType().getVectorElementType() == MVT::f32)
5732 return Op;
5733 return DAG.UnrollVectorOp(N: Op.getNode());
5734 }
5735
5736 const bool HasFullFP16 = DAG.getSubtarget<ARMSubtarget>().hasFullFP16();
5737
5738 EVT NewTy;
5739 const EVT OpTy = Op.getOperand(i: 0).getValueType();
5740 if (OpTy == MVT::v4f32)
5741 NewTy = MVT::v4i32;
5742 else if (OpTy == MVT::v4f16 && HasFullFP16)
5743 NewTy = MVT::v4i16;
5744 else if (OpTy == MVT::v8f16 && HasFullFP16)
5745 NewTy = MVT::v8i16;
5746 else
5747 llvm_unreachable("Invalid type for custom lowering!");
5748
5749 if (VT != MVT::v4i16 && VT != MVT::v8i16)
5750 return DAG.UnrollVectorOp(N: Op.getNode());
5751
5752 Op = DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT: NewTy, Operand: Op.getOperand(i: 0));
5753 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: Op);
5754}
5755
5756SDValue ARMTargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const {
5757 EVT VT = Op.getValueType();
5758 if (VT.isVector())
5759 return LowerVectorFP_TO_INT(Op, DAG);
5760
5761 bool IsStrict = Op->isStrictFPOpcode();
5762 SDValue SrcVal = Op.getOperand(i: IsStrict ? 1 : 0);
5763
5764 if (isUnsupportedFloatingType(VT: SrcVal.getValueType())) {
5765 RTLIB::Libcall LC;
5766 if (Op.getOpcode() == ISD::FP_TO_SINT ||
5767 Op.getOpcode() == ISD::STRICT_FP_TO_SINT)
5768 LC = RTLIB::getFPTOSINT(OpVT: SrcVal.getValueType(),
5769 RetVT: Op.getValueType());
5770 else
5771 LC = RTLIB::getFPTOUINT(OpVT: SrcVal.getValueType(),
5772 RetVT: Op.getValueType());
5773 SDLoc Loc(Op);
5774 MakeLibCallOptions CallOptions;
5775 SDValue Chain = IsStrict ? Op.getOperand(i: 0) : SDValue();
5776 SDValue Result;
5777 std::tie(args&: Result, args&: Chain) = makeLibCall(DAG, LC, RetVT: Op.getValueType(), Ops: SrcVal,
5778 CallOptions, dl: Loc, Chain);
5779 return IsStrict ? DAG.getMergeValues(Ops: {Result, Chain}, dl: Loc) : Result;
5780 }
5781
5782 return Op;
5783}
5784
5785static SDValue LowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG,
5786 const ARMSubtarget *Subtarget) {
5787 EVT VT = Op.getValueType();
5788 EVT ToVT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
5789 EVT FromVT = Op.getOperand(i: 0).getValueType();
5790
5791 if (VT == MVT::i32 && ToVT == MVT::i32 && FromVT == MVT::f32)
5792 return Op;
5793 if (VT == MVT::i32 && ToVT == MVT::i32 && FromVT == MVT::f64 &&
5794 Subtarget->hasFP64())
5795 return Op;
5796 if (VT == MVT::i32 && ToVT == MVT::i32 && FromVT == MVT::f16 &&
5797 Subtarget->hasFullFP16())
5798 return Op;
5799 if (VT == MVT::v4i32 && ToVT == MVT::i32 && FromVT == MVT::v4f32 &&
5800 Subtarget->hasMVEFloatOps())
5801 return Op;
5802 if (VT == MVT::v8i16 && ToVT == MVT::i16 && FromVT == MVT::v8f16 &&
5803 Subtarget->hasMVEFloatOps())
5804 return Op;
5805
5806 if (FromVT != MVT::v4f32 && FromVT != MVT::v8f16)
5807 return SDValue();
5808
5809 SDLoc DL(Op);
5810 bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT_SAT;
5811 unsigned BW = ToVT.getScalarSizeInBits() - IsSigned;
5812 SDValue CVT = DAG.getNode(Opcode: Op.getOpcode(), DL, VT, N1: Op.getOperand(i: 0),
5813 N2: DAG.getValueType(VT.getScalarType()));
5814 SDValue Max = DAG.getNode(Opcode: IsSigned ? ISD::SMIN : ISD::UMIN, DL, VT, N1: CVT,
5815 N2: DAG.getConstant(Val: (1 << BW) - 1, DL, VT));
5816 if (IsSigned)
5817 Max = DAG.getNode(Opcode: ISD::SMAX, DL, VT, N1: Max,
5818 N2: DAG.getSignedConstant(Val: -(1 << BW), DL, VT));
5819 return Max;
5820}
5821
5822static SDValue LowerVectorINT_TO_FP(SDValue Op, SelectionDAG &DAG) {
5823 EVT VT = Op.getValueType();
5824 SDLoc dl(Op);
5825
5826 if (Op.getOperand(i: 0).getValueType().getVectorElementType() == MVT::i32) {
5827 if (VT.getVectorElementType() == MVT::f32)
5828 return Op;
5829 return DAG.UnrollVectorOp(N: Op.getNode());
5830 }
5831
5832 assert((Op.getOperand(0).getValueType() == MVT::v4i16 ||
5833 Op.getOperand(0).getValueType() == MVT::v8i16) &&
5834 "Invalid type for custom lowering!");
5835
5836 const bool HasFullFP16 = DAG.getSubtarget<ARMSubtarget>().hasFullFP16();
5837
5838 EVT DestVecType;
5839 if (VT == MVT::v4f32)
5840 DestVecType = MVT::v4i32;
5841 else if (VT == MVT::v4f16 && HasFullFP16)
5842 DestVecType = MVT::v4i16;
5843 else if (VT == MVT::v8f16 && HasFullFP16)
5844 DestVecType = MVT::v8i16;
5845 else
5846 return DAG.UnrollVectorOp(N: Op.getNode());
5847
5848 unsigned CastOpc;
5849 unsigned Opc;
5850 switch (Op.getOpcode()) {
5851 default: llvm_unreachable("Invalid opcode!");
5852 case ISD::SINT_TO_FP:
5853 CastOpc = ISD::SIGN_EXTEND;
5854 Opc = ISD::SINT_TO_FP;
5855 break;
5856 case ISD::UINT_TO_FP:
5857 CastOpc = ISD::ZERO_EXTEND;
5858 Opc = ISD::UINT_TO_FP;
5859 break;
5860 }
5861
5862 Op = DAG.getNode(Opcode: CastOpc, DL: dl, VT: DestVecType, Operand: Op.getOperand(i: 0));
5863 return DAG.getNode(Opcode: Opc, DL: dl, VT, Operand: Op);
5864}
5865
5866SDValue ARMTargetLowering::LowerINT_TO_FP(SDValue Op, SelectionDAG &DAG) const {
5867 EVT VT = Op.getValueType();
5868 if (VT.isVector())
5869 return LowerVectorINT_TO_FP(Op, DAG);
5870
5871 bool IsStrict = Op->isStrictFPOpcode();
5872 SDValue SrcVal = Op.getOperand(i: IsStrict ? 1 : 0);
5873
5874 if (isUnsupportedFloatingType(VT)) {
5875 RTLIB::Libcall LC;
5876 if (Op.getOpcode() == ISD::SINT_TO_FP ||
5877 Op.getOpcode() == ISD::STRICT_SINT_TO_FP)
5878 LC = RTLIB::getSINTTOFP(OpVT: SrcVal.getValueType(), RetVT: Op.getValueType());
5879 else
5880 LC = RTLIB::getUINTTOFP(OpVT: SrcVal.getValueType(), RetVT: Op.getValueType());
5881 SDLoc Loc(Op);
5882 MakeLibCallOptions CallOptions;
5883 SDValue Chain = IsStrict ? Op.getOperand(i: 0) : SDValue();
5884 SDValue Result;
5885 std::tie(args&: Result, args&: Chain) = makeLibCall(DAG, LC, RetVT: Op.getValueType(), Ops: SrcVal,
5886 CallOptions, dl: Loc, Chain);
5887 return IsStrict ? DAG.getMergeValues(Ops: {Result, Chain}, dl: Loc) : Result;
5888 }
5889
5890 return Op;
5891}
5892
5893SDValue ARMTargetLowering::LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const {
5894 // Implement fcopysign with a fabs and a conditional fneg.
5895 SDValue Tmp0 = Op.getOperand(i: 0);
5896 SDValue Tmp1 = Op.getOperand(i: 1);
5897 SDLoc dl(Op);
5898 EVT VT = Op.getValueType();
5899 EVT SrcVT = Tmp1.getValueType();
5900 bool InGPR = Tmp0.getOpcode() == ISD::BITCAST ||
5901 Tmp0.getOpcode() == ARMISD::VMOVDRR;
5902 bool UseNEON = !InGPR && Subtarget->hasNEON();
5903
5904 if (UseNEON) {
5905 // Use VBSL to copy the sign bit.
5906 unsigned EncodedVal = ARM_AM::createVMOVModImm(OpCmode: 0x6, Val: 0x80);
5907 SDValue Mask = DAG.getNode(Opcode: ARMISD::VMOVIMM, DL: dl, VT: MVT::v2i32,
5908 Operand: DAG.getTargetConstant(Val: EncodedVal, DL: dl, VT: MVT::i32));
5909 EVT OpVT = (VT == MVT::f32) ? MVT::v2i32 : MVT::v1i64;
5910 if (VT == MVT::f64)
5911 Mask = DAG.getNode(Opcode: ARMISD::VSHLIMM, DL: dl, VT: OpVT,
5912 N1: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: OpVT, Operand: Mask),
5913 N2: DAG.getConstant(Val: 32, DL: dl, VT: MVT::i32));
5914 else /*if (VT == MVT::f32)*/
5915 Tmp0 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: MVT::v2f32, Operand: Tmp0);
5916 if (SrcVT == MVT::f32) {
5917 Tmp1 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: MVT::v2f32, Operand: Tmp1);
5918 if (VT == MVT::f64)
5919 Tmp1 = DAG.getNode(Opcode: ARMISD::VSHLIMM, DL: dl, VT: OpVT,
5920 N1: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: OpVT, Operand: Tmp1),
5921 N2: DAG.getConstant(Val: 32, DL: dl, VT: MVT::i32));
5922 } else if (VT == MVT::f32)
5923 Tmp1 = DAG.getNode(Opcode: ARMISD::VSHRuIMM, DL: dl, VT: MVT::v1i64,
5924 N1: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v1i64, Operand: Tmp1),
5925 N2: DAG.getConstant(Val: 32, DL: dl, VT: MVT::i32));
5926 Tmp0 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: OpVT, Operand: Tmp0);
5927 Tmp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: OpVT, Operand: Tmp1);
5928
5929 SDValue AllOnes = DAG.getTargetConstant(Val: ARM_AM::createVMOVModImm(OpCmode: 0xe, Val: 0xff),
5930 DL: dl, VT: MVT::i32);
5931 AllOnes = DAG.getNode(Opcode: ARMISD::VMOVIMM, DL: dl, VT: MVT::v8i8, Operand: AllOnes);
5932 SDValue MaskNot = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: OpVT, N1: Mask,
5933 N2: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: OpVT, Operand: AllOnes));
5934
5935 SDValue Res = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: OpVT,
5936 N1: DAG.getNode(Opcode: ISD::AND, DL: dl, VT: OpVT, N1: Tmp1, N2: Mask),
5937 N2: DAG.getNode(Opcode: ISD::AND, DL: dl, VT: OpVT, N1: Tmp0, N2: MaskNot));
5938 if (VT == MVT::f32) {
5939 Res = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v2f32, Operand: Res);
5940 Res = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: MVT::f32, N1: Res,
5941 N2: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
5942 } else {
5943 Res = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::f64, Operand: Res);
5944 }
5945
5946 return Res;
5947 }
5948
5949 // Bitcast operand 1 to i32.
5950 if (SrcVT == MVT::f64)
5951 Tmp1 = DAG.getNode(Opcode: ARMISD::VMOVRRD, DL: dl, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
5952 N: Tmp1).getValue(R: 1);
5953 Tmp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: Tmp1);
5954
5955 // Or in the signbit with integer operations.
5956 SDValue Mask1 = DAG.getConstant(Val: 0x80000000, DL: dl, VT: MVT::i32);
5957 SDValue Mask2 = DAG.getConstant(Val: 0x7fffffff, DL: dl, VT: MVT::i32);
5958 Tmp1 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32, N1: Tmp1, N2: Mask1);
5959 if (VT == MVT::f32) {
5960 Tmp0 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32,
5961 N1: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: Tmp0), N2: Mask2);
5962 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::f32,
5963 Operand: DAG.getNode(Opcode: ISD::OR, DL: dl, VT: MVT::i32, N1: Tmp0, N2: Tmp1));
5964 }
5965
5966 // f64: Or the high part with signbit and then combine two parts.
5967 Tmp0 = DAG.getNode(Opcode: ARMISD::VMOVRRD, DL: dl, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
5968 N: Tmp0);
5969 SDValue Lo = Tmp0.getValue(R: 0);
5970 SDValue Hi = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32, N1: Tmp0.getValue(R: 1), N2: Mask2);
5971 Hi = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: MVT::i32, N1: Hi, N2: Tmp1);
5972 return DAG.getNode(Opcode: ARMISD::VMOVDRR, DL: dl, VT: MVT::f64, N1: Lo, N2: Hi);
5973}
5974
5975SDValue ARMTargetLowering::LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const{
5976 MachineFunction &MF = DAG.getMachineFunction();
5977 MachineFrameInfo &MFI = MF.getFrameInfo();
5978 MFI.setReturnAddressIsTaken(true);
5979
5980 EVT VT = Op.getValueType();
5981 SDLoc dl(Op);
5982 unsigned Depth = Op.getConstantOperandVal(i: 0);
5983 if (Depth) {
5984 SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
5985 SDValue Offset = DAG.getConstant(Val: 4, DL: dl, VT: MVT::i32);
5986 return DAG.getLoad(VT, dl, Chain: DAG.getEntryNode(),
5987 Ptr: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: FrameAddr, N2: Offset),
5988 PtrInfo: MachinePointerInfo());
5989 }
5990
5991 // Return LR, which contains the return address. Mark it an implicit live-in.
5992 Register Reg = MF.addLiveIn(PReg: ARM::LR, RC: getRegClassFor(VT: MVT::i32));
5993 return DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl, Reg, VT);
5994}
5995
5996SDValue ARMTargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const {
5997 const ARMBaseRegisterInfo &ARI =
5998 *static_cast<const ARMBaseRegisterInfo*>(RegInfo);
5999 MachineFunction &MF = DAG.getMachineFunction();
6000 MachineFrameInfo &MFI = MF.getFrameInfo();
6001 MFI.setFrameAddressIsTaken(true);
6002
6003 EVT VT = Op.getValueType();
6004 SDLoc dl(Op); // FIXME probably not meaningful
6005 unsigned Depth = Op.getConstantOperandVal(i: 0);
6006 Register FrameReg = ARI.getFrameRegister(MF);
6007 SDValue FrameAddr = DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl, Reg: FrameReg, VT);
6008 while (Depth--)
6009 FrameAddr = DAG.getLoad(VT, dl, Chain: DAG.getEntryNode(), Ptr: FrameAddr,
6010 PtrInfo: MachinePointerInfo());
6011 return FrameAddr;
6012}
6013
6014// FIXME? Maybe this could be a TableGen attribute on some registers and
6015// this table could be generated automatically from RegInfo.
6016Register ARMTargetLowering::getRegisterByName(const char* RegName, LLT VT,
6017 const MachineFunction &MF) const {
6018 return StringSwitch<Register>(RegName)
6019 .Case(S: "sp", Value: ARM::SP)
6020 .Default(Value: Register());
6021}
6022
6023// Result is 64 bit value so split into two 32 bit values and return as a
6024// pair of values.
6025static void ExpandREAD_REGISTER(SDNode *N, SmallVectorImpl<SDValue> &Results,
6026 SelectionDAG &DAG) {
6027 SDLoc DL(N);
6028
6029 // This function is only supposed to be called for i64 type destination.
6030 assert(N->getValueType(0) == MVT::i64
6031 && "ExpandREAD_REGISTER called for non-i64 type result.");
6032
6033 SDValue Read = DAG.getNode(Opcode: ISD::READ_REGISTER, DL,
6034 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32, VT3: MVT::Other),
6035 N1: N->getOperand(Num: 0),
6036 N2: N->getOperand(Num: 1));
6037
6038 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: MVT::i64, N1: Read.getValue(R: 0),
6039 N2: Read.getValue(R: 1)));
6040 Results.push_back(Elt: Read.getValue(R: 2)); // Chain
6041}
6042
6043/// \p BC is a bitcast that is about to be turned into a VMOVDRR.
6044/// When \p DstVT, the destination type of \p BC, is on the vector
6045/// register bank and the source of bitcast, \p Op, operates on the same bank,
6046/// it might be possible to combine them, such that everything stays on the
6047/// vector register bank.
6048/// \p return The node that would replace \p BT, if the combine
6049/// is possible.
6050static SDValue CombineVMOVDRRCandidateWithVecOp(const SDNode *BC,
6051 SelectionDAG &DAG) {
6052 SDValue Op = BC->getOperand(Num: 0);
6053 EVT DstVT = BC->getValueType(ResNo: 0);
6054
6055 // The only vector instruction that can produce a scalar (remember,
6056 // since the bitcast was about to be turned into VMOVDRR, the source
6057 // type is i64) from a vector is EXTRACT_VECTOR_ELT.
6058 // Moreover, we can do this combine only if there is one use.
6059 // Finally, if the destination type is not a vector, there is not
6060 // much point on forcing everything on the vector bank.
6061 if (!DstVT.isVector() || Op.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
6062 !Op.hasOneUse())
6063 return SDValue();
6064
6065 // If the index is not constant, we will introduce an additional
6066 // multiply that will stick.
6067 // Give up in that case.
6068 ConstantSDNode *Index = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1));
6069 if (!Index)
6070 return SDValue();
6071 unsigned DstNumElt = DstVT.getVectorNumElements();
6072
6073 // Compute the new index.
6074 const APInt &APIntIndex = Index->getAPIntValue();
6075 APInt NewIndex(APIntIndex.getBitWidth(), DstNumElt);
6076 NewIndex *= APIntIndex;
6077 // Check if the new constant index fits into i32.
6078 if (NewIndex.getBitWidth() > 32)
6079 return SDValue();
6080
6081 // vMTy bitcast(i64 extractelt vNi64 src, i32 index) ->
6082 // vMTy extractsubvector vNxMTy (bitcast vNi64 src), i32 index*M)
6083 SDLoc dl(Op);
6084 SDValue ExtractSrc = Op.getOperand(i: 0);
6085 EVT VecVT = EVT::getVectorVT(
6086 Context&: *DAG.getContext(), VT: DstVT.getScalarType(),
6087 NumElements: ExtractSrc.getValueType().getVectorNumElements() * DstNumElt);
6088 SDValue BitCast = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: VecVT, Operand: ExtractSrc);
6089 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: DstVT, N1: BitCast,
6090 N2: DAG.getConstant(Val: NewIndex.getZExtValue(), DL: dl, VT: MVT::i32));
6091}
6092
6093/// ExpandBITCAST - If the target supports VFP, this function is called to
6094/// expand a bit convert where either the source or destination type is i64 to
6095/// use a VMOVDRR or VMOVRRD node. This should not be done when the non-i64
6096/// operand type is illegal (e.g., v2f32 for a target that doesn't support
6097/// vectors), since the legalizer won't know what to do with that.
6098SDValue ARMTargetLowering::ExpandBITCAST(SDNode *N, SelectionDAG &DAG,
6099 const ARMSubtarget *Subtarget) const {
6100 SDLoc dl(N);
6101 SDValue Op = N->getOperand(Num: 0);
6102
6103 // This function is only supposed to be called for i16 and i64 types, either
6104 // as the source or destination of the bit convert.
6105 EVT SrcVT = Op.getValueType();
6106 EVT DstVT = N->getValueType(ResNo: 0);
6107
6108 if ((SrcVT == MVT::i16 || SrcVT == MVT::i32) &&
6109 (DstVT == MVT::f16 || DstVT == MVT::bf16))
6110 return MoveToHPR(dl: SDLoc(N), DAG, LocVT: MVT::i32, ValVT: DstVT.getSimpleVT(),
6111 Val: DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SDLoc(N), VT: MVT::i32, Operand: Op));
6112
6113 if ((DstVT == MVT::i16 || DstVT == MVT::i32) &&
6114 (SrcVT == MVT::f16 || SrcVT == MVT::bf16)) {
6115 if (Subtarget->hasFullFP16() && !Subtarget->hasBF16())
6116 Op = DAG.getBitcast(VT: MVT::f16, V: Op);
6117 return DAG.getNode(
6118 Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: DstVT,
6119 Operand: MoveFromHPR(dl: SDLoc(N), DAG, LocVT: MVT::i32, ValVT: SrcVT.getSimpleVT(), Val: Op));
6120 }
6121
6122 if (!(SrcVT == MVT::i64 || DstVT == MVT::i64))
6123 return SDValue();
6124
6125 // Turn i64->f64 into VMOVDRR.
6126 if (SrcVT == MVT::i64 && isTypeLegal(VT: DstVT)) {
6127 // Do not force values to GPRs (this is what VMOVDRR does for the inputs)
6128 // if we can combine the bitcast with its source.
6129 if (SDValue Val = CombineVMOVDRRCandidateWithVecOp(BC: N, DAG))
6130 return Val;
6131 SDValue Lo, Hi;
6132 std::tie(args&: Lo, args&: Hi) = DAG.SplitScalar(N: Op, DL: dl, LoVT: MVT::i32, HiVT: MVT::i32);
6133 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: DstVT,
6134 Operand: DAG.getNode(Opcode: ARMISD::VMOVDRR, DL: dl, VT: MVT::f64, N1: Lo, N2: Hi));
6135 }
6136
6137 // Turn f64->i64 into VMOVRRD.
6138 if (DstVT == MVT::i64 && isTypeLegal(VT: SrcVT)) {
6139 SDValue Cvt;
6140 if (DAG.getDataLayout().isBigEndian() && SrcVT.isVector() &&
6141 SrcVT.getVectorNumElements() > 1)
6142 Cvt = DAG.getNode(Opcode: ARMISD::VMOVRRD, DL: dl,
6143 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
6144 N: DAG.getNode(Opcode: ARMISD::VREV64, DL: dl, VT: SrcVT, Operand: Op));
6145 else
6146 Cvt = DAG.getNode(Opcode: ARMISD::VMOVRRD, DL: dl,
6147 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N: Op);
6148 // Merge the pieces into a single i64 value.
6149 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: MVT::i64, N1: Cvt, N2: Cvt.getValue(R: 1));
6150 }
6151
6152 return SDValue();
6153}
6154
6155/// getZeroVector - Returns a vector of specified type with all zero elements.
6156/// Zero vectors are used to represent vector negation and in those cases
6157/// will be implemented with the NEON VNEG instruction. However, VNEG does
6158/// not support i64 elements, so sometimes the zero vectors will need to be
6159/// explicitly constructed. Regardless, use a canonical VMOV to create the
6160/// zero vector.
6161static SDValue getZeroVector(EVT VT, SelectionDAG &DAG, const SDLoc &dl) {
6162 assert(VT.isVector() && "Expected a vector type");
6163 // The canonical modified immediate encoding of a zero vector is....0!
6164 SDValue EncodedVal = DAG.getTargetConstant(Val: 0, DL: dl, VT: MVT::i32);
6165 EVT VmovVT = VT.is128BitVector() ? MVT::v4i32 : MVT::v2i32;
6166 SDValue Vmov = DAG.getNode(Opcode: ARMISD::VMOVIMM, DL: dl, VT: VmovVT, Operand: EncodedVal);
6167 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: Vmov);
6168}
6169
6170/// LowerShiftRightParts - Lower SRA_PARTS, which returns two
6171/// i32 values and take a 2 x i32 value to shift plus a shift amount.
6172SDValue ARMTargetLowering::LowerShiftRightParts(SDValue Op,
6173 SelectionDAG &DAG) const {
6174 assert(Op.getNumOperands() == 3 && "Not a double-shift!");
6175 EVT VT = Op.getValueType();
6176 unsigned VTBits = VT.getSizeInBits();
6177 SDLoc dl(Op);
6178 SDValue ShOpLo = Op.getOperand(i: 0);
6179 SDValue ShOpHi = Op.getOperand(i: 1);
6180 SDValue ShAmt = Op.getOperand(i: 2);
6181 SDValue ARMcc;
6182 unsigned Opc = (Op.getOpcode() == ISD::SRA_PARTS) ? ISD::SRA : ISD::SRL;
6183
6184 assert(Op.getOpcode() == ISD::SRA_PARTS || Op.getOpcode() == ISD::SRL_PARTS);
6185
6186 SDValue RevShAmt = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32,
6187 N1: DAG.getConstant(Val: VTBits, DL: dl, VT: MVT::i32), N2: ShAmt);
6188 SDValue Tmp1 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: ShOpLo, N2: ShAmt);
6189 SDValue ExtraShAmt = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32, N1: ShAmt,
6190 N2: DAG.getConstant(Val: VTBits, DL: dl, VT: MVT::i32));
6191 SDValue Tmp2 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: ShOpHi, N2: RevShAmt);
6192 SDValue LoSmallShift = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp1, N2: Tmp2);
6193 SDValue LoBigShift = DAG.getNode(Opcode: Opc, DL: dl, VT, N1: ShOpHi, N2: ExtraShAmt);
6194 SDValue CmpLo = getARMCmp(LHS: ExtraShAmt, RHS: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32),
6195 CC: ISD::SETGE, ARMcc, DAG, dl);
6196 SDValue Lo =
6197 DAG.getNode(Opcode: ARMISD::CMOV, DL: dl, VT, N1: LoSmallShift, N2: LoBigShift, N3: ARMcc, N4: CmpLo);
6198
6199 SDValue HiSmallShift = DAG.getNode(Opcode: Opc, DL: dl, VT, N1: ShOpHi, N2: ShAmt);
6200 SDValue HiBigShift = Opc == ISD::SRA
6201 ? DAG.getNode(Opcode: Opc, DL: dl, VT, N1: ShOpHi,
6202 N2: DAG.getConstant(Val: VTBits - 1, DL: dl, VT))
6203 : DAG.getConstant(Val: 0, DL: dl, VT);
6204 SDValue CmpHi = getARMCmp(LHS: ExtraShAmt, RHS: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32),
6205 CC: ISD::SETGE, ARMcc, DAG, dl);
6206 SDValue Hi =
6207 DAG.getNode(Opcode: ARMISD::CMOV, DL: dl, VT, N1: HiSmallShift, N2: HiBigShift, N3: ARMcc, N4: CmpHi);
6208
6209 SDValue Ops[2] = { Lo, Hi };
6210 return DAG.getMergeValues(Ops, dl);
6211}
6212
6213/// LowerShiftLeftParts - Lower SHL_PARTS, which returns two
6214/// i32 values and take a 2 x i32 value to shift plus a shift amount.
6215SDValue ARMTargetLowering::LowerShiftLeftParts(SDValue Op,
6216 SelectionDAG &DAG) const {
6217 assert(Op.getNumOperands() == 3 && "Not a double-shift!");
6218 EVT VT = Op.getValueType();
6219 unsigned VTBits = VT.getSizeInBits();
6220 SDLoc dl(Op);
6221 SDValue ShOpLo = Op.getOperand(i: 0);
6222 SDValue ShOpHi = Op.getOperand(i: 1);
6223 SDValue ShAmt = Op.getOperand(i: 2);
6224 SDValue ARMcc;
6225
6226 assert(Op.getOpcode() == ISD::SHL_PARTS);
6227 SDValue RevShAmt = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32,
6228 N1: DAG.getConstant(Val: VTBits, DL: dl, VT: MVT::i32), N2: ShAmt);
6229 SDValue Tmp1 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: ShOpLo, N2: RevShAmt);
6230 SDValue Tmp2 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: ShOpHi, N2: ShAmt);
6231 SDValue HiSmallShift = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp1, N2: Tmp2);
6232
6233 SDValue ExtraShAmt = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32, N1: ShAmt,
6234 N2: DAG.getConstant(Val: VTBits, DL: dl, VT: MVT::i32));
6235 SDValue HiBigShift = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: ShOpLo, N2: ExtraShAmt);
6236 SDValue CmpHi = getARMCmp(LHS: ExtraShAmt, RHS: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32),
6237 CC: ISD::SETGE, ARMcc, DAG, dl);
6238 SDValue Hi =
6239 DAG.getNode(Opcode: ARMISD::CMOV, DL: dl, VT, N1: HiSmallShift, N2: HiBigShift, N3: ARMcc, N4: CmpHi);
6240
6241 SDValue CmpLo = getARMCmp(LHS: ExtraShAmt, RHS: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32),
6242 CC: ISD::SETGE, ARMcc, DAG, dl);
6243 SDValue LoSmallShift = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: ShOpLo, N2: ShAmt);
6244 SDValue Lo = DAG.getNode(Opcode: ARMISD::CMOV, DL: dl, VT, N1: LoSmallShift,
6245 N2: DAG.getConstant(Val: 0, DL: dl, VT), N3: ARMcc, N4: CmpLo);
6246
6247 SDValue Ops[2] = { Lo, Hi };
6248 return DAG.getMergeValues(Ops, dl);
6249}
6250
6251SDValue ARMTargetLowering::LowerGET_ROUNDING(SDValue Op,
6252 SelectionDAG &DAG) const {
6253 // The rounding mode is in bits 23:22 of the FPSCR.
6254 // The ARM rounding mode value to FLT_ROUNDS mapping is 0->1, 1->2, 2->3, 3->0
6255 // The formula we use to implement this is (((FPSCR + 1 << 22) >> 22) & 3)
6256 // so that the shift + and get folded into a bitfield extract.
6257 SDLoc dl(Op);
6258 SDValue Chain = Op.getOperand(i: 0);
6259 SDValue Ops[] = {Chain,
6260 DAG.getConstant(Val: Intrinsic::arm_get_fpscr, DL: dl, VT: MVT::i32)};
6261
6262 SDValue FPSCR =
6263 DAG.getNode(Opcode: ISD::INTRINSIC_W_CHAIN, DL: dl, ResultTys: {MVT::i32, MVT::Other}, Ops);
6264 Chain = FPSCR.getValue(R: 1);
6265 SDValue FltRounds = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::i32, N1: FPSCR,
6266 N2: DAG.getConstant(Val: 1U << 22, DL: dl, VT: MVT::i32));
6267 SDValue RMODE = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i32, N1: FltRounds,
6268 N2: DAG.getConstant(Val: 22, DL: dl, VT: MVT::i32));
6269 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32, N1: RMODE,
6270 N2: DAG.getConstant(Val: 3, DL: dl, VT: MVT::i32));
6271 return DAG.getMergeValues(Ops: {And, Chain}, dl);
6272}
6273
6274SDValue ARMTargetLowering::LowerSET_ROUNDING(SDValue Op,
6275 SelectionDAG &DAG) const {
6276 SDLoc DL(Op);
6277 SDValue Chain = Op->getOperand(Num: 0);
6278 SDValue RMValue = Op->getOperand(Num: 1);
6279
6280 // The rounding mode is in bits 23:22 of the FPSCR.
6281 // The llvm.set.rounding argument value to ARM rounding mode value mapping
6282 // is 0->3, 1->0, 2->1, 3->2. The formula we use to implement this is
6283 // ((arg - 1) & 3) << 22).
6284 //
6285 // It is expected that the argument of llvm.set.rounding is within the
6286 // segment [0, 3], so NearestTiesToAway (4) is not handled here. It is
6287 // responsibility of the code generated llvm.set.rounding to ensure this
6288 // condition.
6289
6290 // Calculate new value of FPSCR[23:22].
6291 RMValue = DAG.getNode(Opcode: ISD::SUB, DL, VT: MVT::i32, N1: RMValue,
6292 N2: DAG.getConstant(Val: 1, DL, VT: MVT::i32));
6293 RMValue = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: RMValue,
6294 N2: DAG.getConstant(Val: 0x3, DL, VT: MVT::i32));
6295 RMValue = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i32, N1: RMValue,
6296 N2: DAG.getConstant(Val: ARM::RoundingBitsPos, DL, VT: MVT::i32));
6297
6298 // Get current value of FPSCR.
6299 SDValue Ops[] = {Chain,
6300 DAG.getConstant(Val: Intrinsic::arm_get_fpscr, DL, VT: MVT::i32)};
6301 SDValue FPSCR =
6302 DAG.getNode(Opcode: ISD::INTRINSIC_W_CHAIN, DL, ResultTys: {MVT::i32, MVT::Other}, Ops);
6303 Chain = FPSCR.getValue(R: 1);
6304 FPSCR = FPSCR.getValue(R: 0);
6305
6306 // Put new rounding mode into FPSCR[23:22].
6307 const unsigned RMMask = ~(ARM::Rounding::rmMask << ARM::RoundingBitsPos);
6308 FPSCR = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: FPSCR,
6309 N2: DAG.getConstant(Val: RMMask, DL, VT: MVT::i32));
6310 FPSCR = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: FPSCR, N2: RMValue);
6311 SDValue Ops2[] = {
6312 Chain, DAG.getConstant(Val: Intrinsic::arm_set_fpscr, DL, VT: MVT::i32), FPSCR};
6313 return DAG.getNode(Opcode: ISD::INTRINSIC_VOID, DL, VT: MVT::Other, Ops: Ops2);
6314}
6315
6316SDValue ARMTargetLowering::LowerSET_FPMODE(SDValue Op,
6317 SelectionDAG &DAG) const {
6318 SDLoc DL(Op);
6319 SDValue Chain = Op->getOperand(Num: 0);
6320 SDValue Mode = Op->getOperand(Num: 1);
6321
6322 // Generate nodes to build:
6323 // FPSCR = (FPSCR & FPStatusBits) | (Mode & ~FPStatusBits)
6324 SDValue Ops[] = {Chain,
6325 DAG.getConstant(Val: Intrinsic::arm_get_fpscr, DL, VT: MVT::i32)};
6326 SDValue FPSCR =
6327 DAG.getNode(Opcode: ISD::INTRINSIC_W_CHAIN, DL, ResultTys: {MVT::i32, MVT::Other}, Ops);
6328 Chain = FPSCR.getValue(R: 1);
6329 FPSCR = FPSCR.getValue(R: 0);
6330
6331 SDValue FPSCRMasked =
6332 DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: FPSCR,
6333 N2: DAG.getConstant(Val: ARM::FPStatusBits, DL, VT: MVT::i32));
6334 SDValue InputMasked =
6335 DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: Mode,
6336 N2: DAG.getConstant(Val: ~ARM::FPStatusBits, DL, VT: MVT::i32));
6337 FPSCR = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: FPSCRMasked, N2: InputMasked);
6338
6339 SDValue Ops2[] = {
6340 Chain, DAG.getConstant(Val: Intrinsic::arm_set_fpscr, DL, VT: MVT::i32), FPSCR};
6341 return DAG.getNode(Opcode: ISD::INTRINSIC_VOID, DL, VT: MVT::Other, Ops: Ops2);
6342}
6343
6344SDValue ARMTargetLowering::LowerRESET_FPMODE(SDValue Op,
6345 SelectionDAG &DAG) const {
6346 SDLoc DL(Op);
6347 SDValue Chain = Op->getOperand(Num: 0);
6348
6349 // To get the default FP mode all control bits are cleared:
6350 // FPSCR = FPSCR & (FPStatusBits | FPReservedBits)
6351 SDValue Ops[] = {Chain,
6352 DAG.getConstant(Val: Intrinsic::arm_get_fpscr, DL, VT: MVT::i32)};
6353 SDValue FPSCR =
6354 DAG.getNode(Opcode: ISD::INTRINSIC_W_CHAIN, DL, ResultTys: {MVT::i32, MVT::Other}, Ops);
6355 Chain = FPSCR.getValue(R: 1);
6356 FPSCR = FPSCR.getValue(R: 0);
6357
6358 SDValue FPSCRMasked = DAG.getNode(
6359 Opcode: ISD::AND, DL, VT: MVT::i32, N1: FPSCR,
6360 N2: DAG.getConstant(Val: ARM::FPStatusBits | ARM::FPReservedBits, DL, VT: MVT::i32));
6361 SDValue Ops2[] = {Chain,
6362 DAG.getConstant(Val: Intrinsic::arm_set_fpscr, DL, VT: MVT::i32),
6363 FPSCRMasked};
6364 return DAG.getNode(Opcode: ISD::INTRINSIC_VOID, DL, VT: MVT::Other, Ops: Ops2);
6365}
6366
6367static SDValue LowerCTTZ(SDNode *N, SelectionDAG &DAG,
6368 const ARMSubtarget *ST) {
6369 SDLoc dl(N);
6370 EVT VT = N->getValueType(ResNo: 0);
6371 if (VT.isVector() && ST->hasNEON()) {
6372
6373 // Compute the least significant set bit: LSB = X & -X
6374 SDValue X = N->getOperand(Num: 0);
6375 SDValue NX = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: getZeroVector(VT, DAG, dl), N2: X);
6376 SDValue LSB = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: X, N2: NX);
6377
6378 EVT ElemTy = VT.getVectorElementType();
6379
6380 if (ElemTy == MVT::i8) {
6381 // Compute with: cttz(x) = ctpop(lsb - 1)
6382 SDValue One = DAG.getNode(Opcode: ARMISD::VMOVIMM, DL: dl, VT,
6383 Operand: DAG.getTargetConstant(Val: 1, DL: dl, VT: ElemTy));
6384 SDValue Bits = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: LSB, N2: One);
6385 return DAG.getNode(Opcode: ISD::CTPOP, DL: dl, VT, Operand: Bits);
6386 }
6387
6388 if ((ElemTy == MVT::i16 || ElemTy == MVT::i32) &&
6389 (N->getOpcode() == ISD::CTTZ_ZERO_POISON)) {
6390 // Compute with: cttz(x) = (width - 1) - ctlz(lsb), if x != 0
6391 unsigned NumBits = ElemTy.getSizeInBits();
6392 SDValue WidthMinus1 =
6393 DAG.getNode(Opcode: ARMISD::VMOVIMM, DL: dl, VT,
6394 Operand: DAG.getTargetConstant(Val: NumBits - 1, DL: dl, VT: ElemTy));
6395 SDValue CTLZ = DAG.getNode(Opcode: ISD::CTLZ, DL: dl, VT, Operand: LSB);
6396 return DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: WidthMinus1, N2: CTLZ);
6397 }
6398
6399 // Compute with: cttz(x) = ctpop(lsb - 1)
6400
6401 // Compute LSB - 1.
6402 SDValue Bits;
6403 if (ElemTy == MVT::i64) {
6404 // Load constant 0xffff'ffff'ffff'ffff to register.
6405 SDValue FF = DAG.getNode(Opcode: ARMISD::VMOVIMM, DL: dl, VT,
6406 Operand: DAG.getTargetConstant(Val: 0x1eff, DL: dl, VT: MVT::i32));
6407 Bits = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: LSB, N2: FF);
6408 } else {
6409 SDValue One = DAG.getNode(Opcode: ARMISD::VMOVIMM, DL: dl, VT,
6410 Operand: DAG.getTargetConstant(Val: 1, DL: dl, VT: ElemTy));
6411 Bits = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: LSB, N2: One);
6412 }
6413 return DAG.getNode(Opcode: ISD::CTPOP, DL: dl, VT, Operand: Bits);
6414 }
6415
6416 if (!ST->hasV6T2Ops())
6417 return SDValue();
6418
6419 SDValue rbit = DAG.getNode(Opcode: ISD::BITREVERSE, DL: dl, VT, Operand: N->getOperand(Num: 0));
6420 return DAG.getNode(Opcode: ISD::CTLZ, DL: dl, VT, Operand: rbit);
6421}
6422
6423static SDValue LowerCTPOP(SDNode *N, SelectionDAG &DAG,
6424 const ARMSubtarget *ST) {
6425 EVT VT = N->getValueType(ResNo: 0);
6426 SDLoc DL(N);
6427
6428 assert(ST->hasNEON() && "Custom ctpop lowering requires NEON.");
6429 assert((VT == MVT::v1i64 || VT == MVT::v2i64 || VT == MVT::v2i32 ||
6430 VT == MVT::v4i32 || VT == MVT::v4i16 || VT == MVT::v8i16) &&
6431 "Unexpected type for custom ctpop lowering");
6432
6433 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6434 EVT VT8Bit = VT.is64BitVector() ? MVT::v8i8 : MVT::v16i8;
6435 SDValue Res = DAG.getBitcast(VT: VT8Bit, V: N->getOperand(Num: 0));
6436 Res = DAG.getNode(Opcode: ISD::CTPOP, DL, VT: VT8Bit, Operand: Res);
6437
6438 // Widen v8i8/v16i8 CTPOP result to VT by repeatedly widening pairwise adds.
6439 unsigned EltSize = 8;
6440 unsigned NumElts = VT.is64BitVector() ? 8 : 16;
6441 while (EltSize != VT.getScalarSizeInBits()) {
6442 SmallVector<SDValue, 8> Ops;
6443 Ops.push_back(Elt: DAG.getConstant(Val: Intrinsic::arm_neon_vpaddlu, DL,
6444 VT: TLI.getPointerTy(DL: DAG.getDataLayout())));
6445 Ops.push_back(Elt: Res);
6446
6447 EltSize *= 2;
6448 NumElts /= 2;
6449 MVT WidenVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: EltSize), NumElements: NumElts);
6450 Res = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: WidenVT, Ops);
6451 }
6452
6453 return Res;
6454}
6455
6456/// Getvshiftimm - Check if this is a valid build_vector for the immediate
6457/// operand of a vector shift operation, where all the elements of the
6458/// build_vector must have the same constant integer value.
6459static bool getVShiftImm(SDValue Op, unsigned ElementBits, int64_t &Cnt) {
6460 // Ignore bit_converts.
6461 while (Op.getOpcode() == ISD::BITCAST)
6462 Op = Op.getOperand(i: 0);
6463 BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(Val: Op.getNode());
6464 APInt SplatBits, SplatUndef;
6465 unsigned SplatBitSize;
6466 bool HasAnyUndefs;
6467 if (!BVN ||
6468 !BVN->isConstantSplat(SplatValue&: SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs,
6469 MinSplatBits: ElementBits) ||
6470 SplatBitSize > ElementBits)
6471 return false;
6472 Cnt = SplatBits.getSExtValue();
6473 return true;
6474}
6475
6476/// isVShiftLImm - Check if this is a valid build_vector for the immediate
6477/// operand of a vector shift left operation. That value must be in the range:
6478/// 0 <= Value < ElementBits for a left shift; or
6479/// 0 <= Value <= ElementBits for a long left shift.
6480static bool isVShiftLImm(SDValue Op, EVT VT, bool isLong, int64_t &Cnt) {
6481 assert(VT.isVector() && "vector shift count is not a vector type");
6482 int64_t ElementBits = VT.getScalarSizeInBits();
6483 if (!getVShiftImm(Op, ElementBits, Cnt))
6484 return false;
6485 return (Cnt >= 0 && (isLong ? Cnt - 1 : Cnt) < ElementBits);
6486}
6487
6488/// isVShiftRImm - Check if this is a valid build_vector for the immediate
6489/// operand of a vector shift right operation. For a shift opcode, the value
6490/// is positive, but for an intrinsic the value count must be negative. The
6491/// absolute value must be in the range:
6492/// 1 <= |Value| <= ElementBits for a right shift; or
6493/// 1 <= |Value| <= ElementBits/2 for a narrow right shift.
6494static bool isVShiftRImm(SDValue Op, EVT VT, bool isNarrow, bool isIntrinsic,
6495 int64_t &Cnt) {
6496 assert(VT.isVector() && "vector shift count is not a vector type");
6497 int64_t ElementBits = VT.getScalarSizeInBits();
6498 if (!getVShiftImm(Op, ElementBits, Cnt))
6499 return false;
6500 if (!isIntrinsic)
6501 return (Cnt >= 1 && Cnt <= (isNarrow ? ElementBits / 2 : ElementBits));
6502 if (Cnt >= -(isNarrow ? ElementBits / 2 : ElementBits) && Cnt <= -1) {
6503 Cnt = -Cnt;
6504 return true;
6505 }
6506 return false;
6507}
6508
6509static SDValue LowerShift(SDNode *N, SelectionDAG &DAG,
6510 const ARMSubtarget *ST) {
6511 EVT VT = N->getValueType(ResNo: 0);
6512 SDLoc dl(N);
6513 int64_t Cnt;
6514
6515 if (!VT.isVector())
6516 return SDValue();
6517
6518 // We essentially have two forms here. Shift by an immediate and shift by a
6519 // vector register (there are also shift by a gpr, but that is just handled
6520 // with a tablegen pattern). We cannot easily match shift by an immediate in
6521 // tablegen so we do that here and generate a VSHLIMM/VSHRsIMM/VSHRuIMM.
6522 // For shifting by a vector, we don't have VSHR, only VSHL (which can be
6523 // signed or unsigned, and a negative shift indicates a shift right).
6524 if (N->getOpcode() == ISD::SHL) {
6525 if (isVShiftLImm(Op: N->getOperand(Num: 1), VT, isLong: false, Cnt))
6526 return DAG.getNode(Opcode: ARMISD::VSHLIMM, DL: dl, VT, N1: N->getOperand(Num: 0),
6527 N2: DAG.getConstant(Val: Cnt, DL: dl, VT: MVT::i32));
6528 return DAG.getNode(Opcode: ARMISD::VSHLu, DL: dl, VT, N1: N->getOperand(Num: 0),
6529 N2: N->getOperand(Num: 1));
6530 }
6531
6532 assert((N->getOpcode() == ISD::SRA || N->getOpcode() == ISD::SRL) &&
6533 "unexpected vector shift opcode");
6534
6535 if (isVShiftRImm(Op: N->getOperand(Num: 1), VT, isNarrow: false, isIntrinsic: false, Cnt)) {
6536 unsigned VShiftOpc =
6537 (N->getOpcode() == ISD::SRA ? ARMISD::VSHRsIMM : ARMISD::VSHRuIMM);
6538 return DAG.getNode(Opcode: VShiftOpc, DL: dl, VT, N1: N->getOperand(Num: 0),
6539 N2: DAG.getConstant(Val: Cnt, DL: dl, VT: MVT::i32));
6540 }
6541
6542 // Other right shifts we don't have operations for (we use a shift left by a
6543 // negative number).
6544 EVT ShiftVT = N->getOperand(Num: 1).getValueType();
6545 SDValue NegatedCount = DAG.getNode(
6546 Opcode: ISD::SUB, DL: dl, VT: ShiftVT, N1: getZeroVector(VT: ShiftVT, DAG, dl), N2: N->getOperand(Num: 1));
6547 unsigned VShiftOpc =
6548 (N->getOpcode() == ISD::SRA ? ARMISD::VSHLs : ARMISD::VSHLu);
6549 return DAG.getNode(Opcode: VShiftOpc, DL: dl, VT, N1: N->getOperand(Num: 0), N2: NegatedCount);
6550}
6551
6552static SDValue Expand64BitShift(SDNode *N, SelectionDAG &DAG,
6553 const ARMSubtarget *ST) {
6554 EVT VT = N->getValueType(ResNo: 0);
6555 SDLoc dl(N);
6556
6557 // We can get here for a node like i32 = ISD::SHL i32, i64
6558 if (VT != MVT::i64)
6559 return SDValue();
6560
6561 assert((N->getOpcode() == ISD::SRL || N->getOpcode() == ISD::SRA ||
6562 N->getOpcode() == ISD::SHL) &&
6563 "Unknown shift to lower!");
6564
6565 unsigned ShOpc = N->getOpcode();
6566 if (ST->hasMVEIntegerOps()) {
6567 SDValue ShAmt = N->getOperand(Num: 1);
6568 unsigned ShPartsOpc = ARMISD::LSLL;
6569 ConstantSDNode *Con = dyn_cast<ConstantSDNode>(Val&: ShAmt);
6570
6571 // If the shift amount is greater than 32 or has a greater bitwidth than 64
6572 // then do the default optimisation
6573 if ((!Con && ShAmt->getValueType(ResNo: 0).getSizeInBits() > 64) ||
6574 (Con && (Con->getAPIntValue() == 0 || Con->getAPIntValue().uge(RHS: 32))))
6575 return SDValue();
6576
6577 // Extract the lower 32 bits of the shift amount if it's not an i32
6578 if (ShAmt->getValueType(ResNo: 0) != MVT::i32)
6579 ShAmt = DAG.getZExtOrTrunc(Op: ShAmt, DL: dl, VT: MVT::i32);
6580
6581 if (ShOpc == ISD::SRL) {
6582 if (!Con)
6583 // There is no t2LSRLr instruction so negate and perform an lsll if the
6584 // shift amount is in a register, emulating a right shift.
6585 ShAmt = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32,
6586 N1: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32), N2: ShAmt);
6587 else
6588 // Else generate an lsrl on the immediate shift amount
6589 ShPartsOpc = ARMISD::LSRL;
6590 } else if (ShOpc == ISD::SRA)
6591 ShPartsOpc = ARMISD::ASRL;
6592
6593 // Split Lower/Upper 32 bits of the destination/source
6594 SDValue Lo, Hi;
6595 std::tie(args&: Lo, args&: Hi) =
6596 DAG.SplitScalar(N: N->getOperand(Num: 0), DL: dl, LoVT: MVT::i32, HiVT: MVT::i32);
6597 // Generate the shift operation as computed above
6598 Lo = DAG.getNode(Opcode: ShPartsOpc, DL: dl, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N1: Lo, N2: Hi,
6599 N3: ShAmt);
6600 // The upper 32 bits come from the second return value of lsll
6601 Hi = SDValue(Lo.getNode(), 1);
6602 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: MVT::i64, N1: Lo, N2: Hi);
6603 }
6604
6605 // We only lower SRA, SRL of 1 here, all others use generic lowering.
6606 if (!isOneConstant(V: N->getOperand(Num: 1)) || N->getOpcode() == ISD::SHL)
6607 return SDValue();
6608
6609 // If we are in thumb mode, we don't have RRX.
6610 if (ST->isThumb1Only())
6611 return SDValue();
6612
6613 // Okay, we have a 64-bit SRA or SRL of 1. Lower this to an RRX expr.
6614 SDValue Lo, Hi;
6615 std::tie(args&: Lo, args&: Hi) = DAG.SplitScalar(N: N->getOperand(Num: 0), DL: dl, LoVT: MVT::i32, HiVT: MVT::i32);
6616
6617 // First, build a LSRS1/ASRS1 op, which shifts the top part by one and
6618 // captures the shifted out bit into a carry flag.
6619 unsigned Opc = N->getOpcode() == ISD::SRL ? ARMISD::LSRS1 : ARMISD::ASRS1;
6620 Hi = DAG.getNode(Opcode: Opc, DL: dl, VTList: DAG.getVTList(VT1: MVT::i32, VT2: FlagsVT), N: Hi);
6621
6622 // The low part is an ARMISD::RRX operand, which shifts the carry in.
6623 Lo = DAG.getNode(Opcode: ARMISD::RRX, DL: dl, VT: MVT::i32, N1: Lo, N2: Hi.getValue(R: 1));
6624
6625 // Merge the pieces into a single i64 value.
6626 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: MVT::i64, N1: Lo, N2: Hi);
6627}
6628
6629static SDValue LowerVSETCC(SDValue Op, SelectionDAG &DAG,
6630 const ARMSubtarget *ST) {
6631 bool Invert = false;
6632 bool Swap = false;
6633 unsigned Opc = ARMCC::AL;
6634
6635 SDValue Op0 = Op.getOperand(i: 0);
6636 SDValue Op1 = Op.getOperand(i: 1);
6637 SDValue CC = Op.getOperand(i: 2);
6638 EVT VT = Op.getValueType();
6639 ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(Val&: CC)->get();
6640 SDLoc dl(Op);
6641
6642 EVT CmpVT;
6643 if (ST->hasNEON())
6644 CmpVT = Op0.getValueType().changeVectorElementTypeToInteger();
6645 else {
6646 assert(ST->hasMVEIntegerOps() &&
6647 "No hardware support for integer vector comparison!");
6648
6649 if (Op.getValueType().getVectorElementType() != MVT::i1)
6650 return SDValue();
6651
6652 // Make sure we expand floating point setcc to scalar if we do not have
6653 // mve.fp, so that we can handle them from there.
6654 if (Op0.getValueType().isFloatingPoint() && !ST->hasMVEFloatOps())
6655 return SDValue();
6656
6657 CmpVT = VT;
6658 }
6659
6660 if (Op0.getValueType().getVectorElementType() == MVT::i64 &&
6661 (SetCCOpcode == ISD::SETEQ || SetCCOpcode == ISD::SETNE)) {
6662 // Special-case integer 64-bit equality comparisons. They aren't legal,
6663 // but they can be lowered with a few vector instructions.
6664 unsigned CmpElements = CmpVT.getVectorNumElements() * 2;
6665 EVT SplitVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32, NumElements: CmpElements);
6666 SDValue CastOp0 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: SplitVT, Operand: Op0);
6667 SDValue CastOp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: SplitVT, Operand: Op1);
6668 SDValue Cmp = DAG.getNode(Opcode: ISD::SETCC, DL: dl, VT: SplitVT, N1: CastOp0, N2: CastOp1,
6669 N3: DAG.getCondCode(Cond: ISD::SETEQ));
6670 SDValue Reversed = DAG.getNode(Opcode: ARMISD::VREV64, DL: dl, VT: SplitVT, Operand: Cmp);
6671 SDValue Merged = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SplitVT, N1: Cmp, N2: Reversed);
6672 Merged = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: CmpVT, Operand: Merged);
6673 if (SetCCOpcode == ISD::SETNE)
6674 Merged = DAG.getNOT(DL: dl, Val: Merged, VT: CmpVT);
6675 Merged = DAG.getSExtOrTrunc(Op: Merged, DL: dl, VT);
6676 return Merged;
6677 }
6678
6679 if (CmpVT.getVectorElementType() == MVT::i64)
6680 // 64-bit comparisons are not legal in general.
6681 return SDValue();
6682
6683 if (Op1.getValueType().isFloatingPoint()) {
6684 switch (SetCCOpcode) {
6685 default: llvm_unreachable("Illegal FP comparison");
6686 case ISD::SETUNE:
6687 case ISD::SETNE:
6688 if (ST->hasMVEFloatOps()) {
6689 Opc = ARMCC::NE; break;
6690 } else {
6691 Invert = true; [[fallthrough]];
6692 }
6693 case ISD::SETOEQ:
6694 case ISD::SETEQ: Opc = ARMCC::EQ; break;
6695 case ISD::SETOLT:
6696 case ISD::SETLT: Swap = true; [[fallthrough]];
6697 case ISD::SETOGT:
6698 case ISD::SETGT: Opc = ARMCC::GT; break;
6699 case ISD::SETOLE:
6700 case ISD::SETLE: Swap = true; [[fallthrough]];
6701 case ISD::SETOGE:
6702 case ISD::SETGE: Opc = ARMCC::GE; break;
6703 case ISD::SETUGE: Swap = true; [[fallthrough]];
6704 case ISD::SETULE: Invert = true; Opc = ARMCC::GT; break;
6705 case ISD::SETUGT: Swap = true; [[fallthrough]];
6706 case ISD::SETULT: Invert = true; Opc = ARMCC::GE; break;
6707 case ISD::SETUEQ: Invert = true; [[fallthrough]];
6708 case ISD::SETONE: {
6709 // Expand this to (OLT | OGT).
6710 SDValue TmpOp0 = DAG.getNode(Opcode: ARMISD::VCMP, DL: dl, VT: CmpVT, N1: Op1, N2: Op0,
6711 N3: DAG.getConstant(Val: ARMCC::GT, DL: dl, VT: MVT::i32));
6712 SDValue TmpOp1 = DAG.getNode(Opcode: ARMISD::VCMP, DL: dl, VT: CmpVT, N1: Op0, N2: Op1,
6713 N3: DAG.getConstant(Val: ARMCC::GT, DL: dl, VT: MVT::i32));
6714 SDValue Result = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: CmpVT, N1: TmpOp0, N2: TmpOp1);
6715 if (Invert)
6716 Result = DAG.getNOT(DL: dl, Val: Result, VT);
6717 return Result;
6718 }
6719 case ISD::SETUO: Invert = true; [[fallthrough]];
6720 case ISD::SETO: {
6721 // Expand this to (OLT | OGE).
6722 SDValue TmpOp0 = DAG.getNode(Opcode: ARMISD::VCMP, DL: dl, VT: CmpVT, N1: Op1, N2: Op0,
6723 N3: DAG.getConstant(Val: ARMCC::GT, DL: dl, VT: MVT::i32));
6724 SDValue TmpOp1 = DAG.getNode(Opcode: ARMISD::VCMP, DL: dl, VT: CmpVT, N1: Op0, N2: Op1,
6725 N3: DAG.getConstant(Val: ARMCC::GE, DL: dl, VT: MVT::i32));
6726 SDValue Result = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: CmpVT, N1: TmpOp0, N2: TmpOp1);
6727 if (Invert)
6728 Result = DAG.getNOT(DL: dl, Val: Result, VT);
6729 return Result;
6730 }
6731 }
6732 } else {
6733 // Integer comparisons.
6734 switch (SetCCOpcode) {
6735 default: llvm_unreachable("Illegal integer comparison");
6736 case ISD::SETNE:
6737 if (ST->hasMVEIntegerOps()) {
6738 Opc = ARMCC::NE; break;
6739 } else {
6740 Invert = true; [[fallthrough]];
6741 }
6742 case ISD::SETEQ: Opc = ARMCC::EQ; break;
6743 case ISD::SETLT: Swap = true; [[fallthrough]];
6744 case ISD::SETGT: Opc = ARMCC::GT; break;
6745 case ISD::SETLE: Swap = true; [[fallthrough]];
6746 case ISD::SETGE: Opc = ARMCC::GE; break;
6747 case ISD::SETULT: Swap = true; [[fallthrough]];
6748 case ISD::SETUGT: Opc = ARMCC::HI; break;
6749 case ISD::SETULE: Swap = true; [[fallthrough]];
6750 case ISD::SETUGE: Opc = ARMCC::HS; break;
6751 }
6752
6753 // Detect VTST (Vector Test Bits) = icmp ne (and (op0, op1), zero).
6754 if (ST->hasNEON() && Opc == ARMCC::EQ) {
6755 SDValue AndOp;
6756 if (ISD::isBuildVectorAllZeros(N: Op1.getNode()))
6757 AndOp = Op0;
6758 else if (ISD::isBuildVectorAllZeros(N: Op0.getNode()))
6759 AndOp = Op1;
6760
6761 // Ignore bitconvert.
6762 if (AndOp.getNode() && AndOp.getOpcode() == ISD::BITCAST)
6763 AndOp = AndOp.getOperand(i: 0);
6764
6765 if (AndOp.getNode() && AndOp.getOpcode() == ISD::AND) {
6766 Op0 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: CmpVT, Operand: AndOp.getOperand(i: 0));
6767 Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: CmpVT, Operand: AndOp.getOperand(i: 1));
6768 SDValue Result = DAG.getNode(Opcode: ARMISD::VTST, DL: dl, VT: CmpVT, N1: Op0, N2: Op1);
6769 if (!Invert)
6770 Result = DAG.getNOT(DL: dl, Val: Result, VT);
6771 return Result;
6772 }
6773 }
6774 }
6775
6776 if (Swap)
6777 std::swap(a&: Op0, b&: Op1);
6778
6779 // If one of the operands is a constant vector zero, attempt to fold the
6780 // comparison to a specialized compare-against-zero form.
6781 if (ISD::isBuildVectorAllZeros(N: Op0.getNode()) &&
6782 (Opc == ARMCC::GE || Opc == ARMCC::GT || Opc == ARMCC::EQ ||
6783 Opc == ARMCC::NE)) {
6784 if (Opc == ARMCC::GE)
6785 Opc = ARMCC::LE;
6786 else if (Opc == ARMCC::GT)
6787 Opc = ARMCC::LT;
6788 std::swap(a&: Op0, b&: Op1);
6789 }
6790
6791 SDValue Result;
6792 if (ISD::isBuildVectorAllZeros(N: Op1.getNode()) &&
6793 (Opc == ARMCC::GE || Opc == ARMCC::GT || Opc == ARMCC::LE ||
6794 Opc == ARMCC::LT || Opc == ARMCC::NE || Opc == ARMCC::EQ))
6795 Result = DAG.getNode(Opcode: ARMISD::VCMPZ, DL: dl, VT: CmpVT, N1: Op0,
6796 N2: DAG.getConstant(Val: Opc, DL: dl, VT: MVT::i32));
6797 else
6798 Result = DAG.getNode(Opcode: ARMISD::VCMP, DL: dl, VT: CmpVT, N1: Op0, N2: Op1,
6799 N3: DAG.getConstant(Val: Opc, DL: dl, VT: MVT::i32));
6800
6801 Result = DAG.getSExtOrTrunc(Op: Result, DL: dl, VT);
6802
6803 if (Invert)
6804 Result = DAG.getNOT(DL: dl, Val: Result, VT);
6805
6806 return Result;
6807}
6808
6809static SDValue LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG) {
6810 SDValue LHS = Op.getOperand(i: 0);
6811 SDValue RHS = Op.getOperand(i: 1);
6812
6813 assert(LHS.getSimpleValueType().isInteger() && "SETCCCARRY is integer only.");
6814
6815 SDValue Carry = Op.getOperand(i: 2);
6816 SDValue Cond = Op.getOperand(i: 3);
6817 SDLoc DL(Op);
6818
6819 // ARMISD::SUBE expects a carry not a borrow like ISD::USUBO_CARRY so we
6820 // have to invert the carry first.
6821 SDValue InvCarry = valueToCarryFlag(Value: Carry, DAG, Invert: true);
6822
6823 SDVTList VTs = DAG.getVTList(VT1: LHS.getValueType(), VT2: MVT::i32);
6824 SDValue Cmp = DAG.getNode(Opcode: ARMISD::SUBE, DL, VTList: VTs, N1: LHS, N2: RHS, N3: InvCarry);
6825
6826 SDValue FVal = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
6827 SDValue TVal = DAG.getConstant(Val: 1, DL, VT: MVT::i32);
6828 SDValue ARMcc = DAG.getConstant(
6829 Val: IntCCToARMCC(CC: cast<CondCodeSDNode>(Val&: Cond)->get()), DL, VT: MVT::i32);
6830 return DAG.getNode(Opcode: ARMISD::CMOV, DL, VT: Op.getValueType(), N1: FVal, N2: TVal, N3: ARMcc,
6831 N4: Cmp.getValue(R: 1));
6832}
6833
6834/// isVMOVModifiedImm - Check if the specified splat value corresponds to a
6835/// valid vector constant for a NEON or MVE instruction with a "modified
6836/// immediate" operand (e.g., VMOV). If so, return the encoded value.
6837static SDValue isVMOVModifiedImm(uint64_t SplatBits, uint64_t SplatUndef,
6838 unsigned SplatBitSize, SelectionDAG &DAG,
6839 const SDLoc &dl, EVT &VT, EVT VectorVT,
6840 VMOVModImmType type) {
6841 unsigned OpCmode, Imm;
6842 bool is128Bits = VectorVT.is128BitVector();
6843
6844 // SplatBitSize is set to the smallest size that splats the vector, so a
6845 // zero vector will always have SplatBitSize == 8. However, NEON modified
6846 // immediate instructions others than VMOV do not support the 8-bit encoding
6847 // of a zero vector, and the default encoding of zero is supposed to be the
6848 // 32-bit version.
6849 if (SplatBits == 0)
6850 SplatBitSize = 32;
6851
6852 switch (SplatBitSize) {
6853 case 8:
6854 if (type != VMOVModImm)
6855 return SDValue();
6856 // Any 1-byte value is OK. Op=0, Cmode=1110.
6857 assert((SplatBits & ~0xff) == 0 && "one byte splat value is too big");
6858 OpCmode = 0xe;
6859 Imm = SplatBits;
6860 VT = is128Bits ? MVT::v16i8 : MVT::v8i8;
6861 break;
6862
6863 case 16:
6864 // NEON's 16-bit VMOV supports splat values where only one byte is nonzero.
6865 VT = is128Bits ? MVT::v8i16 : MVT::v4i16;
6866 if ((SplatBits & ~0xff) == 0) {
6867 // Value = 0x00nn: Op=x, Cmode=100x.
6868 OpCmode = 0x8;
6869 Imm = SplatBits;
6870 break;
6871 }
6872 if ((SplatBits & ~0xff00) == 0) {
6873 // Value = 0xnn00: Op=x, Cmode=101x.
6874 OpCmode = 0xa;
6875 Imm = SplatBits >> 8;
6876 break;
6877 }
6878 return SDValue();
6879
6880 case 32:
6881 // NEON's 32-bit VMOV supports splat values where:
6882 // * only one byte is nonzero, or
6883 // * the least significant byte is 0xff and the second byte is nonzero, or
6884 // * the least significant 2 bytes are 0xff and the third is nonzero.
6885 VT = is128Bits ? MVT::v4i32 : MVT::v2i32;
6886 if ((SplatBits & ~0xff) == 0) {
6887 // Value = 0x000000nn: Op=x, Cmode=000x.
6888 OpCmode = 0;
6889 Imm = SplatBits;
6890 break;
6891 }
6892 if ((SplatBits & ~0xff00) == 0) {
6893 // Value = 0x0000nn00: Op=x, Cmode=001x.
6894 OpCmode = 0x2;
6895 Imm = SplatBits >> 8;
6896 break;
6897 }
6898 if ((SplatBits & ~0xff0000) == 0) {
6899 // Value = 0x00nn0000: Op=x, Cmode=010x.
6900 OpCmode = 0x4;
6901 Imm = SplatBits >> 16;
6902 break;
6903 }
6904 if ((SplatBits & ~0xff000000) == 0) {
6905 // Value = 0xnn000000: Op=x, Cmode=011x.
6906 OpCmode = 0x6;
6907 Imm = SplatBits >> 24;
6908 break;
6909 }
6910
6911 // cmode == 0b1100 and cmode == 0b1101 are not supported for VORR or VBIC
6912 if (type == OtherModImm) return SDValue();
6913
6914 if ((SplatBits & ~0xffff) == 0 &&
6915 ((SplatBits | SplatUndef) & 0xff) == 0xff) {
6916 // Value = 0x0000nnff: Op=x, Cmode=1100.
6917 OpCmode = 0xc;
6918 Imm = SplatBits >> 8;
6919 break;
6920 }
6921
6922 // cmode == 0b1101 is not supported for MVE VMVN
6923 if (type == MVEVMVNModImm)
6924 return SDValue();
6925
6926 if ((SplatBits & ~0xffffff) == 0 &&
6927 ((SplatBits | SplatUndef) & 0xffff) == 0xffff) {
6928 // Value = 0x00nnffff: Op=x, Cmode=1101.
6929 OpCmode = 0xd;
6930 Imm = SplatBits >> 16;
6931 break;
6932 }
6933
6934 // Note: there are a few 32-bit splat values (specifically: 00ffff00,
6935 // ff000000, ff0000ff, and ffff00ff) that are valid for VMOV.I64 but not
6936 // VMOV.I32. A (very) minor optimization would be to replicate the value
6937 // and fall through here to test for a valid 64-bit splat. But, then the
6938 // caller would also need to check and handle the change in size.
6939 return SDValue();
6940
6941 case 64: {
6942 if (type != VMOVModImm)
6943 return SDValue();
6944 // NEON has a 64-bit VMOV splat where each byte is either 0 or 0xff.
6945 uint64_t BitMask = 0xff;
6946 unsigned ImmMask = 1;
6947 Imm = 0;
6948 for (int ByteNum = 0; ByteNum < 8; ++ByteNum) {
6949 if (((SplatBits | SplatUndef) & BitMask) == BitMask) {
6950 Imm |= ImmMask;
6951 } else if ((SplatBits & BitMask) != 0) {
6952 return SDValue();
6953 }
6954 BitMask <<= 8;
6955 ImmMask <<= 1;
6956 }
6957
6958 // Op=1, Cmode=1110.
6959 OpCmode = 0x1e;
6960 VT = is128Bits ? MVT::v2i64 : MVT::v1i64;
6961 break;
6962 }
6963
6964 default:
6965 llvm_unreachable("unexpected size for isVMOVModifiedImm");
6966 }
6967
6968 unsigned EncodedVal = ARM_AM::createVMOVModImm(OpCmode, Val: Imm);
6969 return DAG.getTargetConstant(Val: EncodedVal, DL: dl, VT: MVT::i32);
6970}
6971
6972SDValue ARMTargetLowering::LowerConstantFP(SDValue Op, SelectionDAG &DAG,
6973 const ARMSubtarget *ST) const {
6974 EVT VT = Op.getValueType();
6975 bool IsDouble = (VT == MVT::f64);
6976 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Val&: Op);
6977 const APFloat &FPVal = CFP->getValueAPF();
6978
6979 // Prevent floating-point constants from using literal loads
6980 // when execute-only is enabled.
6981 if (ST->genExecuteOnly()) {
6982 // We shouldn't trigger this for v6m execute-only
6983 assert((!ST->isThumb1Only() || ST->hasV8MBaselineOps()) &&
6984 "Unexpected architecture");
6985
6986 // If we can represent the constant as an immediate, don't lower it
6987 if (isFPImmLegal(Imm: FPVal, VT))
6988 return Op;
6989 // Otherwise, construct as integer, and move to float register
6990 APInt INTVal = FPVal.bitcastToAPInt();
6991 SDLoc DL(CFP);
6992 switch (VT.getSimpleVT().SimpleTy) {
6993 default:
6994 llvm_unreachable("Unknown floating point type!");
6995 break;
6996 case MVT::f64: {
6997 SDValue Lo = DAG.getConstant(Val: INTVal.trunc(width: 32), DL, VT: MVT::i32);
6998 SDValue Hi = DAG.getConstant(Val: INTVal.lshr(shiftAmt: 32).trunc(width: 32), DL, VT: MVT::i32);
6999 return DAG.getNode(Opcode: ARMISD::VMOVDRR, DL, VT: MVT::f64, N1: Lo, N2: Hi);
7000 }
7001 case MVT::f32:
7002 return DAG.getNode(Opcode: ARMISD::VMOVSR, DL, VT,
7003 Operand: DAG.getConstant(Val: INTVal, DL, VT: MVT::i32));
7004 }
7005 }
7006
7007 if (!ST->hasVFP3Base())
7008 return SDValue();
7009
7010 // Use the default (constant pool) lowering for double constants when we have
7011 // an SP-only FPU
7012 if (IsDouble && !Subtarget->hasFP64())
7013 return SDValue();
7014
7015 // Try splatting with a VMOV.f32...
7016 int ImmVal = IsDouble ? ARM_AM::getFP64Imm(FPImm: FPVal) : ARM_AM::getFP32Imm(FPImm: FPVal);
7017
7018 if (ImmVal != -1) {
7019 if (IsDouble || !ST->useNEONForSinglePrecisionFP()) {
7020 // We have code in place to select a valid ConstantFP already, no need to
7021 // do any mangling.
7022 return Op;
7023 }
7024
7025 // It's a float and we are trying to use NEON operations where
7026 // possible. Lower it to a splat followed by an extract.
7027 SDLoc DL(Op);
7028 SDValue NewVal = DAG.getTargetConstant(Val: ImmVal, DL, VT: MVT::i32);
7029 SDValue VecConstant = DAG.getNode(Opcode: ARMISD::VMOVFPIMM, DL, VT: MVT::v2f32,
7030 Operand: NewVal);
7031 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::f32, N1: VecConstant,
7032 N2: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
7033 }
7034
7035 // The rest of our options are NEON only, make sure that's allowed before
7036 // proceeding..
7037 if (!ST->hasNEON() || (!IsDouble && !ST->useNEONForSinglePrecisionFP()))
7038 return SDValue();
7039
7040 EVT VMovVT;
7041 uint64_t iVal = FPVal.bitcastToAPInt().getZExtValue();
7042
7043 // It wouldn't really be worth bothering for doubles except for one very
7044 // important value, which does happen to match: 0.0. So make sure we don't do
7045 // anything stupid.
7046 if (IsDouble && (iVal & 0xffffffff) != (iVal >> 32))
7047 return SDValue();
7048
7049 // Try a VMOV.i32 (FIXME: i8, i16, or i64 could work too).
7050 SDValue NewVal = isVMOVModifiedImm(SplatBits: iVal & 0xffffffffU, SplatUndef: 0, SplatBitSize: 32, DAG, dl: SDLoc(Op),
7051 VT&: VMovVT, VectorVT: VT, type: VMOVModImm);
7052 if (NewVal != SDValue()) {
7053 SDLoc DL(Op);
7054 SDValue VecConstant = DAG.getNode(Opcode: ARMISD::VMOVIMM, DL, VT: VMovVT,
7055 Operand: NewVal);
7056 if (IsDouble)
7057 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f64, Operand: VecConstant);
7058
7059 // It's a float: cast and extract a vector element.
7060 SDValue VecFConstant = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v2f32,
7061 Operand: VecConstant);
7062 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::f32, N1: VecFConstant,
7063 N2: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
7064 }
7065
7066 // Finally, try a VMVN.i32
7067 NewVal = isVMOVModifiedImm(SplatBits: ~iVal & 0xffffffffU, SplatUndef: 0, SplatBitSize: 32, DAG, dl: SDLoc(Op), VT&: VMovVT,
7068 VectorVT: VT, type: VMVNModImm);
7069 if (NewVal != SDValue()) {
7070 SDLoc DL(Op);
7071 SDValue VecConstant = DAG.getNode(Opcode: ARMISD::VMVNIMM, DL, VT: VMovVT, Operand: NewVal);
7072
7073 if (IsDouble)
7074 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f64, Operand: VecConstant);
7075
7076 // It's a float: cast and extract a vector element.
7077 SDValue VecFConstant = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v2f32,
7078 Operand: VecConstant);
7079 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::f32, N1: VecFConstant,
7080 N2: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
7081 }
7082
7083 return SDValue();
7084}
7085
7086// check if an VEXT instruction can handle the shuffle mask when the
7087// vector sources of the shuffle are the same.
7088static bool isSingletonVEXTMask(ArrayRef<int> M, EVT VT, unsigned &Imm) {
7089 unsigned NumElts = VT.getVectorNumElements();
7090
7091 // Assume that the first shuffle index is not UNDEF. Fail if it is.
7092 if (M[0] < 0)
7093 return false;
7094
7095 Imm = M[0];
7096
7097 // If this is a VEXT shuffle, the immediate value is the index of the first
7098 // element. The other shuffle indices must be the successive elements after
7099 // the first one.
7100 unsigned ExpectedElt = Imm;
7101 for (unsigned i = 1; i < NumElts; ++i) {
7102 // Increment the expected index. If it wraps around, just follow it
7103 // back to index zero and keep going.
7104 ++ExpectedElt;
7105 if (ExpectedElt == NumElts)
7106 ExpectedElt = 0;
7107
7108 if (M[i] < 0) continue; // ignore UNDEF indices
7109 if (ExpectedElt != static_cast<unsigned>(M[i]))
7110 return false;
7111 }
7112
7113 return true;
7114}
7115
7116static bool isVEXTMask(ArrayRef<int> M, EVT VT,
7117 bool &ReverseVEXT, unsigned &Imm) {
7118 unsigned NumElts = VT.getVectorNumElements();
7119 ReverseVEXT = false;
7120
7121 // Assume that the first shuffle index is not UNDEF. Fail if it is.
7122 if (M[0] < 0)
7123 return false;
7124
7125 Imm = M[0];
7126
7127 // If this is a VEXT shuffle, the immediate value is the index of the first
7128 // element. The other shuffle indices must be the successive elements after
7129 // the first one.
7130 unsigned ExpectedElt = Imm;
7131 for (unsigned i = 1; i < NumElts; ++i) {
7132 // Increment the expected index. If it wraps around, it may still be
7133 // a VEXT but the source vectors must be swapped.
7134 ExpectedElt += 1;
7135 if (ExpectedElt == NumElts * 2) {
7136 ExpectedElt = 0;
7137 ReverseVEXT = true;
7138 }
7139
7140 if (M[i] < 0) continue; // ignore UNDEF indices
7141 if (ExpectedElt != static_cast<unsigned>(M[i]))
7142 return false;
7143 }
7144
7145 // Adjust the index value if the source operands will be swapped.
7146 if (ReverseVEXT)
7147 Imm -= NumElts;
7148
7149 return true;
7150}
7151
7152static bool isVTBLMask(ArrayRef<int> M, EVT VT) {
7153 // We can handle <8 x i8> vector shuffles. If the index in the mask is out of
7154 // range, then 0 is placed into the resulting vector. So pretty much any mask
7155 // of 8 elements can work here.
7156 return VT == MVT::v8i8 && M.size() == 8;
7157}
7158
7159static unsigned SelectPairHalf(unsigned Elements, ArrayRef<int> Mask,
7160 unsigned Index) {
7161 if (Mask.size() == Elements * 2)
7162 return Index / Elements;
7163 return Mask[Index] == 0 ? 0 : 1;
7164}
7165
7166// Checks whether the shuffle mask represents a vector transpose (VTRN) by
7167// checking that pairs of elements in the shuffle mask represent the same index
7168// in each vector, incrementing the expected index by 2 at each step.
7169// e.g. For v1,v2 of type v4i32 a valid shuffle mask is: [0, 4, 2, 6]
7170// v1={a,b,c,d} => x=shufflevector v1, v2 shufflemask => x={a,e,c,g}
7171// v2={e,f,g,h}
7172// WhichResult gives the offset for each element in the mask based on which
7173// of the two results it belongs to.
7174//
7175// The transpose can be represented either as:
7176// result1 = shufflevector v1, v2, result1_shuffle_mask
7177// result2 = shufflevector v1, v2, result2_shuffle_mask
7178// where v1/v2 and the shuffle masks have the same number of elements
7179// (here WhichResult (see below) indicates which result is being checked)
7180//
7181// or as:
7182// results = shufflevector v1, v2, shuffle_mask
7183// where both results are returned in one vector and the shuffle mask has twice
7184// as many elements as v1/v2 (here WhichResult will always be 0 if true) here we
7185// want to check the low half and high half of the shuffle mask as if it were
7186// the other case
7187static bool isVTRNMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
7188 unsigned EltSz = VT.getScalarSizeInBits();
7189 if (EltSz == 64)
7190 return false;
7191
7192 unsigned NumElts = VT.getVectorNumElements();
7193 if ((M.size() != NumElts && M.size() != NumElts * 2) || NumElts % 2 != 0)
7194 return false;
7195
7196 // If the mask is twice as long as the input vector then we need to check the
7197 // upper and lower parts of the mask with a matching value for WhichResult
7198 // FIXME: A mask with only even values will be rejected in case the first
7199 // element is undefined, e.g. [-1, 4, 2, 6] will be rejected, because only
7200 // M[0] is used to determine WhichResult
7201 for (unsigned i = 0; i < M.size(); i += NumElts) {
7202 WhichResult = SelectPairHalf(Elements: NumElts, Mask: M, Index: i);
7203 for (unsigned j = 0; j < NumElts; j += 2) {
7204 if ((M[i+j] >= 0 && (unsigned) M[i+j] != j + WhichResult) ||
7205 (M[i+j+1] >= 0 && (unsigned) M[i+j+1] != j + NumElts + WhichResult))
7206 return false;
7207 }
7208 }
7209
7210 if (M.size() == NumElts*2)
7211 WhichResult = 0;
7212
7213 return true;
7214}
7215
7216/// isVTRN_v_undef_Mask - Special case of isVTRNMask for canonical form of
7217/// "vector_shuffle v, v", i.e., "vector_shuffle v, undef".
7218/// Mask is e.g., <0, 0, 2, 2> instead of <0, 4, 2, 6>.
7219static bool isVTRN_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult){
7220 unsigned EltSz = VT.getScalarSizeInBits();
7221 if (EltSz == 64)
7222 return false;
7223
7224 unsigned NumElts = VT.getVectorNumElements();
7225 if ((M.size() != NumElts && M.size() != NumElts * 2) || NumElts % 2 != 0)
7226 return false;
7227
7228 for (unsigned i = 0; i < M.size(); i += NumElts) {
7229 WhichResult = SelectPairHalf(Elements: NumElts, Mask: M, Index: i);
7230 for (unsigned j = 0; j < NumElts; j += 2) {
7231 if ((M[i+j] >= 0 && (unsigned) M[i+j] != j + WhichResult) ||
7232 (M[i+j+1] >= 0 && (unsigned) M[i+j+1] != j + WhichResult))
7233 return false;
7234 }
7235 }
7236
7237 if (M.size() == NumElts*2)
7238 WhichResult = 0;
7239
7240 return true;
7241}
7242
7243// Checks whether the shuffle mask represents a vector unzip (VUZP) by checking
7244// that the mask elements are either all even and in steps of size 2 or all odd
7245// and in steps of size 2.
7246// e.g. For v1,v2 of type v4i32 a valid shuffle mask is: [0, 2, 4, 6]
7247// v1={a,b,c,d} => x=shufflevector v1, v2 shufflemask => x={a,c,e,g}
7248// v2={e,f,g,h}
7249// Requires similar checks to that of isVTRNMask with
7250// respect the how results are returned.
7251static bool isVUZPMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
7252 unsigned EltSz = VT.getScalarSizeInBits();
7253 if (EltSz == 64)
7254 return false;
7255
7256 unsigned NumElts = VT.getVectorNumElements();
7257 if (M.size() != NumElts && M.size() != NumElts*2)
7258 return false;
7259
7260 for (unsigned i = 0; i < M.size(); i += NumElts) {
7261 WhichResult = SelectPairHalf(Elements: NumElts, Mask: M, Index: i);
7262 for (unsigned j = 0; j < NumElts; ++j) {
7263 if (M[i+j] >= 0 && (unsigned) M[i+j] != 2 * j + WhichResult)
7264 return false;
7265 }
7266 }
7267
7268 if (M.size() == NumElts*2)
7269 WhichResult = 0;
7270
7271 // VUZP.32 for 64-bit vectors is a pseudo-instruction alias for VTRN.32.
7272 if (VT.is64BitVector() && EltSz == 32)
7273 return false;
7274
7275 return true;
7276}
7277
7278/// isVUZP_v_undef_Mask - Special case of isVUZPMask for canonical form of
7279/// "vector_shuffle v, v", i.e., "vector_shuffle v, undef".
7280/// Mask is e.g., <0, 2, 0, 2> instead of <0, 2, 4, 6>,
7281static bool isVUZP_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult){
7282 unsigned EltSz = VT.getScalarSizeInBits();
7283 if (EltSz == 64)
7284 return false;
7285
7286 unsigned NumElts = VT.getVectorNumElements();
7287 if (M.size() != NumElts && M.size() != NumElts*2)
7288 return false;
7289
7290 unsigned Half = NumElts / 2;
7291 for (unsigned i = 0; i < M.size(); i += NumElts) {
7292 WhichResult = SelectPairHalf(Elements: NumElts, Mask: M, Index: i);
7293 for (unsigned j = 0; j < NumElts; j += Half) {
7294 unsigned Idx = WhichResult;
7295 for (unsigned k = 0; k < Half; ++k) {
7296 int MIdx = M[i + j + k];
7297 if (MIdx >= 0 && (unsigned) MIdx != Idx)
7298 return false;
7299 Idx += 2;
7300 }
7301 }
7302 }
7303
7304 if (M.size() == NumElts*2)
7305 WhichResult = 0;
7306
7307 // VUZP.32 for 64-bit vectors is a pseudo-instruction alias for VTRN.32.
7308 if (VT.is64BitVector() && EltSz == 32)
7309 return false;
7310
7311 return true;
7312}
7313
7314// Checks whether the shuffle mask represents a vector zip (VZIP) by checking
7315// that pairs of elements of the shufflemask represent the same index in each
7316// vector incrementing sequentially through the vectors.
7317// e.g. For v1,v2 of type v4i32 a valid shuffle mask is: [0, 4, 1, 5]
7318// v1={a,b,c,d} => x=shufflevector v1, v2 shufflemask => x={a,e,b,f}
7319// v2={e,f,g,h}
7320// Requires similar checks to that of isVTRNMask with respect the how results
7321// are returned.
7322static bool isVZIPMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
7323 unsigned EltSz = VT.getScalarSizeInBits();
7324 if (EltSz == 64)
7325 return false;
7326
7327 unsigned NumElts = VT.getVectorNumElements();
7328 if ((M.size() != NumElts && M.size() != NumElts * 2) || NumElts % 2 != 0)
7329 return false;
7330
7331 for (unsigned i = 0; i < M.size(); i += NumElts) {
7332 WhichResult = SelectPairHalf(Elements: NumElts, Mask: M, Index: i);
7333 unsigned Idx = WhichResult * NumElts / 2;
7334 for (unsigned j = 0; j < NumElts; j += 2) {
7335 if ((M[i+j] >= 0 && (unsigned) M[i+j] != Idx) ||
7336 (M[i+j+1] >= 0 && (unsigned) M[i+j+1] != Idx + NumElts))
7337 return false;
7338 Idx += 1;
7339 }
7340 }
7341
7342 if (M.size() == NumElts*2)
7343 WhichResult = 0;
7344
7345 // VZIP.32 for 64-bit vectors is a pseudo-instruction alias for VTRN.32.
7346 if (VT.is64BitVector() && EltSz == 32)
7347 return false;
7348
7349 return true;
7350}
7351
7352/// isVZIP_v_undef_Mask - Special case of isVZIPMask for canonical form of
7353/// "vector_shuffle v, v", i.e., "vector_shuffle v, undef".
7354/// Mask is e.g., <0, 0, 1, 1> instead of <0, 4, 1, 5>.
7355static bool isVZIP_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult){
7356 unsigned EltSz = VT.getScalarSizeInBits();
7357 if (EltSz == 64)
7358 return false;
7359
7360 unsigned NumElts = VT.getVectorNumElements();
7361 if ((M.size() != NumElts && M.size() != NumElts * 2) || NumElts % 2 != 0)
7362 return false;
7363
7364 for (unsigned i = 0; i < M.size(); i += NumElts) {
7365 WhichResult = SelectPairHalf(Elements: NumElts, Mask: M, Index: i);
7366 unsigned Idx = WhichResult * NumElts / 2;
7367 for (unsigned j = 0; j < NumElts; j += 2) {
7368 if ((M[i+j] >= 0 && (unsigned) M[i+j] != Idx) ||
7369 (M[i+j+1] >= 0 && (unsigned) M[i+j+1] != Idx))
7370 return false;
7371 Idx += 1;
7372 }
7373 }
7374
7375 if (M.size() == NumElts*2)
7376 WhichResult = 0;
7377
7378 // VZIP.32 for 64-bit vectors is a pseudo-instruction alias for VTRN.32.
7379 if (VT.is64BitVector() && EltSz == 32)
7380 return false;
7381
7382 return true;
7383}
7384
7385/// Check if \p ShuffleMask is a NEON two-result shuffle (VZIP, VUZP, VTRN),
7386/// and return the corresponding ARMISD opcode if it is, or 0 if it isn't.
7387static unsigned isNEONTwoResultShuffleMask(ArrayRef<int> ShuffleMask, EVT VT,
7388 unsigned &WhichResult,
7389 bool &isV_UNDEF) {
7390 isV_UNDEF = false;
7391 if (isVTRNMask(M: ShuffleMask, VT, WhichResult))
7392 return ARMISD::VTRN;
7393 if (isVUZPMask(M: ShuffleMask, VT, WhichResult))
7394 return ARMISD::VUZP;
7395 if (isVZIPMask(M: ShuffleMask, VT, WhichResult))
7396 return ARMISD::VZIP;
7397
7398 isV_UNDEF = true;
7399 if (isVTRN_v_undef_Mask(M: ShuffleMask, VT, WhichResult))
7400 return ARMISD::VTRN;
7401 if (isVUZP_v_undef_Mask(M: ShuffleMask, VT, WhichResult))
7402 return ARMISD::VUZP;
7403 if (isVZIP_v_undef_Mask(M: ShuffleMask, VT, WhichResult))
7404 return ARMISD::VZIP;
7405
7406 return 0;
7407}
7408
7409/// \return true if this is a reverse operation on an vector.
7410static bool isReverseMask(ArrayRef<int> M, EVT VT) {
7411 unsigned NumElts = VT.getVectorNumElements();
7412 // Make sure the mask has the right size.
7413 if (NumElts != M.size())
7414 return false;
7415
7416 // Look for <15, ..., 3, -1, 1, 0>.
7417 for (unsigned i = 0; i != NumElts; ++i)
7418 if (M[i] >= 0 && M[i] != (int) (NumElts - 1 - i))
7419 return false;
7420
7421 return true;
7422}
7423
7424static bool isTruncMask(ArrayRef<int> M, EVT VT, bool Top, bool SingleSource) {
7425 unsigned NumElts = VT.getVectorNumElements();
7426 // Make sure the mask has the right size.
7427 if (NumElts != M.size() || (VT != MVT::v8i16 && VT != MVT::v16i8))
7428 return false;
7429
7430 // Half-width truncation patterns (e.g. v4i32 -> v8i16):
7431 // !Top && SingleSource: <0, 2, 4, 6, 0, 2, 4, 6>
7432 // !Top && !SingleSource: <0, 2, 4, 6, 8, 10, 12, 14>
7433 // Top && SingleSource: <1, 3, 5, 7, 1, 3, 5, 7>
7434 // Top && !SingleSource: <1, 3, 5, 7, 9, 11, 13, 15>
7435 int Ofs = Top ? 1 : 0;
7436 int Upper = SingleSource ? 0 : NumElts;
7437 for (int i = 0, e = NumElts / 2; i != e; ++i) {
7438 if (M[i] >= 0 && M[i] != (i * 2) + Ofs)
7439 return false;
7440 if (M[i + e] >= 0 && M[i + e] != (i * 2) + Ofs + Upper)
7441 return false;
7442 }
7443 return true;
7444}
7445
7446static bool isVMOVNMask(ArrayRef<int> M, EVT VT, bool Top, bool SingleSource) {
7447 unsigned NumElts = VT.getVectorNumElements();
7448 // Make sure the mask has the right size.
7449 if (NumElts != M.size() || (VT != MVT::v8i16 && VT != MVT::v16i8))
7450 return false;
7451
7452 // If Top
7453 // Look for <0, N, 2, N+2, 4, N+4, ..>.
7454 // This inserts Input2 into Input1
7455 // else if not Top
7456 // Look for <0, N+1, 2, N+3, 4, N+5, ..>
7457 // This inserts Input1 into Input2
7458 unsigned Offset = Top ? 0 : 1;
7459 unsigned N = SingleSource ? 0 : NumElts;
7460 for (unsigned i = 0; i < NumElts; i += 2) {
7461 if (M[i] >= 0 && M[i] != (int)i)
7462 return false;
7463 if (M[i + 1] >= 0 && M[i + 1] != (int)(N + i + Offset))
7464 return false;
7465 }
7466
7467 return true;
7468}
7469
7470static bool isVMOVNTruncMask(ArrayRef<int> M, EVT ToVT, bool rev) {
7471 unsigned NumElts = ToVT.getVectorNumElements();
7472 if (NumElts != M.size())
7473 return false;
7474
7475 // Test if the Trunc can be convertible to a VMOVN with this shuffle. We are
7476 // looking for patterns of:
7477 // !rev: 0 N/2 1 N/2+1 2 N/2+2 ...
7478 // rev: N/2 0 N/2+1 1 N/2+2 2 ...
7479
7480 unsigned Off0 = rev ? NumElts / 2 : 0;
7481 unsigned Off1 = rev ? 0 : NumElts / 2;
7482 for (unsigned i = 0; i < NumElts; i += 2) {
7483 if (M[i] >= 0 && M[i] != (int)(Off0 + i / 2))
7484 return false;
7485 if (M[i + 1] >= 0 && M[i + 1] != (int)(Off1 + i / 2))
7486 return false;
7487 }
7488
7489 return true;
7490}
7491
7492// Reconstruct an MVE VCVT from a BuildVector of scalar fptrunc, all extracted
7493// from a pair of inputs. For example:
7494// BUILDVECTOR(FP_ROUND(EXTRACT_ELT(X, 0),
7495// FP_ROUND(EXTRACT_ELT(Y, 0),
7496// FP_ROUND(EXTRACT_ELT(X, 1),
7497// FP_ROUND(EXTRACT_ELT(Y, 1), ...)
7498static SDValue LowerBuildVectorOfFPTrunc(SDValue BV, SelectionDAG &DAG,
7499 const ARMSubtarget *ST) {
7500 assert(BV.getOpcode() == ISD::BUILD_VECTOR && "Unknown opcode!");
7501 if (!ST->hasMVEFloatOps())
7502 return SDValue();
7503
7504 SDLoc dl(BV);
7505 EVT VT = BV.getValueType();
7506 if (VT != MVT::v8f16)
7507 return SDValue();
7508
7509 // We are looking for a buildvector of fptrunc elements, where all the
7510 // elements are interleavingly extracted from two sources. Check the first two
7511 // items are valid enough and extract some info from them (they are checked
7512 // properly in the loop below).
7513 if (BV.getOperand(i: 0).getOpcode() != ISD::FP_ROUND ||
7514 BV.getOperand(i: 0).getOperand(i: 0).getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
7515 BV.getOperand(i: 0).getOperand(i: 0).getConstantOperandVal(i: 1) != 0)
7516 return SDValue();
7517 if (BV.getOperand(i: 1).getOpcode() != ISD::FP_ROUND ||
7518 BV.getOperand(i: 1).getOperand(i: 0).getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
7519 BV.getOperand(i: 1).getOperand(i: 0).getConstantOperandVal(i: 1) != 0)
7520 return SDValue();
7521 SDValue Op0 = BV.getOperand(i: 0).getOperand(i: 0).getOperand(i: 0);
7522 SDValue Op1 = BV.getOperand(i: 1).getOperand(i: 0).getOperand(i: 0);
7523 if (Op0.getValueType() != MVT::v4f32 || Op1.getValueType() != MVT::v4f32)
7524 return SDValue();
7525
7526 // Check all the values in the BuildVector line up with our expectations.
7527 for (unsigned i = 1; i < 4; i++) {
7528 auto Check = [](SDValue Trunc, SDValue Op, unsigned Idx) {
7529 return Trunc.getOpcode() == ISD::FP_ROUND &&
7530 Trunc.getOperand(i: 0).getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
7531 Trunc.getOperand(i: 0).getOperand(i: 0) == Op &&
7532 Trunc.getOperand(i: 0).getConstantOperandVal(i: 1) == Idx;
7533 };
7534 if (!Check(BV.getOperand(i: i * 2 + 0), Op0, i))
7535 return SDValue();
7536 if (!Check(BV.getOperand(i: i * 2 + 1), Op1, i))
7537 return SDValue();
7538 }
7539
7540 SDValue N1 = DAG.getNode(Opcode: ARMISD::VCVTN, DL: dl, VT, N1: DAG.getUNDEF(VT), N2: Op0,
7541 N3: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
7542 return DAG.getNode(Opcode: ARMISD::VCVTN, DL: dl, VT, N1, N2: Op1,
7543 N3: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
7544}
7545
7546// Reconstruct an MVE VCVT from a BuildVector of scalar fpext, all extracted
7547// from a single input on alternating lanes. For example:
7548// BUILDVECTOR(FP_ROUND(EXTRACT_ELT(X, 0),
7549// FP_ROUND(EXTRACT_ELT(X, 2),
7550// FP_ROUND(EXTRACT_ELT(X, 4), ...)
7551static SDValue LowerBuildVectorOfFPExt(SDValue BV, SelectionDAG &DAG,
7552 const ARMSubtarget *ST) {
7553 assert(BV.getOpcode() == ISD::BUILD_VECTOR && "Unknown opcode!");
7554 if (!ST->hasMVEFloatOps())
7555 return SDValue();
7556
7557 SDLoc dl(BV);
7558 EVT VT = BV.getValueType();
7559 if (VT != MVT::v4f32)
7560 return SDValue();
7561
7562 // We are looking for a buildvector of fptext elements, where all the
7563 // elements are alternating lanes from a single source. For example <0,2,4,6>
7564 // or <1,3,5,7>. Check the first two items are valid enough and extract some
7565 // info from them (they are checked properly in the loop below).
7566 if (BV.getOperand(i: 0).getOpcode() != ISD::FP_EXTEND ||
7567 BV.getOperand(i: 0).getOperand(i: 0).getOpcode() != ISD::EXTRACT_VECTOR_ELT)
7568 return SDValue();
7569 SDValue Op0 = BV.getOperand(i: 0).getOperand(i: 0).getOperand(i: 0);
7570 int Offset = BV.getOperand(i: 0).getOperand(i: 0).getConstantOperandVal(i: 1);
7571 if (Op0.getValueType() != MVT::v8f16 || (Offset != 0 && Offset != 1))
7572 return SDValue();
7573
7574 // Check all the values in the BuildVector line up with our expectations.
7575 for (unsigned i = 1; i < 4; i++) {
7576 auto Check = [](SDValue Trunc, SDValue Op, unsigned Idx) {
7577 return Trunc.getOpcode() == ISD::FP_EXTEND &&
7578 Trunc.getOperand(i: 0).getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
7579 Trunc.getOperand(i: 0).getOperand(i: 0) == Op &&
7580 Trunc.getOperand(i: 0).getConstantOperandVal(i: 1) == Idx;
7581 };
7582 if (!Check(BV.getOperand(i), Op0, 2 * i + Offset))
7583 return SDValue();
7584 }
7585
7586 return DAG.getNode(Opcode: ARMISD::VCVTL, DL: dl, VT, N1: Op0,
7587 N2: DAG.getConstant(Val: Offset, DL: dl, VT: MVT::i32));
7588}
7589
7590// If N is an integer constant that can be moved into a register in one
7591// instruction, return an SDValue of such a constant (will become a MOV
7592// instruction). Otherwise return null.
7593static SDValue IsSingleInstrConstant(SDValue N, SelectionDAG &DAG,
7594 const ARMSubtarget *ST, const SDLoc &dl) {
7595 uint64_t Val;
7596 if (!isa<ConstantSDNode>(Val: N))
7597 return SDValue();
7598 Val = N->getAsZExtVal();
7599
7600 if (ST->isThumb1Only()) {
7601 if (Val <= 255 || ~Val <= 255)
7602 return DAG.getConstant(Val, DL: dl, VT: MVT::i32);
7603 } else {
7604 if (ARM_AM::getSOImmVal(Arg: Val) != -1 || ARM_AM::getSOImmVal(Arg: ~Val) != -1)
7605 return DAG.getConstant(Val, DL: dl, VT: MVT::i32);
7606 }
7607 return SDValue();
7608}
7609
7610static SDValue LowerBUILD_VECTOR_i1(SDValue Op, SelectionDAG &DAG,
7611 const ARMSubtarget *ST) {
7612 SDLoc dl(Op);
7613 EVT VT = Op.getValueType();
7614
7615 assert(ST->hasMVEIntegerOps() && "LowerBUILD_VECTOR_i1 called without MVE!");
7616
7617 unsigned NumElts = VT.getVectorNumElements();
7618 unsigned BoolMask;
7619 unsigned BitsPerBool;
7620 if (NumElts == 2) {
7621 BitsPerBool = 8;
7622 BoolMask = 0xff;
7623 } else if (NumElts == 4) {
7624 BitsPerBool = 4;
7625 BoolMask = 0xf;
7626 } else if (NumElts == 8) {
7627 BitsPerBool = 2;
7628 BoolMask = 0x3;
7629 } else if (NumElts == 16) {
7630 BitsPerBool = 1;
7631 BoolMask = 0x1;
7632 } else
7633 return SDValue();
7634
7635 // If this is a single value copied into all lanes (a splat), we can just sign
7636 // extend that single value
7637 SDValue FirstOp = Op.getOperand(i: 0);
7638 if (!isa<ConstantSDNode>(Val: FirstOp) &&
7639 llvm::all_of(Range: llvm::drop_begin(RangeOrContainer: Op->ops()), P: [&FirstOp](const SDUse &U) {
7640 return U.get().isUndef() || U.get() == FirstOp;
7641 })) {
7642 SDValue Ext = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl, VT: MVT::i32, N1: FirstOp,
7643 N2: DAG.getValueType(MVT::i1));
7644 return DAG.getNode(Opcode: ARMISD::PREDICATE_CAST, DL: dl, VT: Op.getValueType(), Operand: Ext);
7645 }
7646
7647 // First create base with bits set where known
7648 unsigned Bits32 = 0;
7649 for (unsigned i = 0; i < NumElts; ++i) {
7650 SDValue V = Op.getOperand(i);
7651 if (!isa<ConstantSDNode>(Val: V) && !V.isUndef())
7652 continue;
7653 bool BitSet = V.isUndef() ? false : V->getAsZExtVal();
7654 if (BitSet)
7655 Bits32 |= BoolMask << (i * BitsPerBool);
7656 }
7657
7658 // Add in unknown nodes
7659 SDValue Base = DAG.getNode(Opcode: ARMISD::PREDICATE_CAST, DL: dl, VT,
7660 Operand: DAG.getConstant(Val: Bits32, DL: dl, VT: MVT::i32));
7661 for (unsigned i = 0; i < NumElts; ++i) {
7662 SDValue V = Op.getOperand(i);
7663 if (isa<ConstantSDNode>(Val: V) || V.isUndef())
7664 continue;
7665 Base = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT, N1: Base, N2: V,
7666 N3: DAG.getConstant(Val: i, DL: dl, VT: MVT::i32));
7667 }
7668
7669 return Base;
7670}
7671
7672static SDValue LowerBUILD_VECTORToVIDUP(SDValue Op, SelectionDAG &DAG,
7673 const ARMSubtarget *ST) {
7674 if (!ST->hasMVEIntegerOps())
7675 return SDValue();
7676
7677 // We are looking for a buildvector where each element is Op[0] + i*N
7678 EVT VT = Op.getValueType();
7679 SDValue Op0 = Op.getOperand(i: 0);
7680 unsigned NumElts = VT.getVectorNumElements();
7681
7682 // Get the increment value from operand 1
7683 SDValue Op1 = Op.getOperand(i: 1);
7684 if (Op1.getOpcode() != ISD::ADD || Op1.getOperand(i: 0) != Op0 ||
7685 !isa<ConstantSDNode>(Val: Op1.getOperand(i: 1)))
7686 return SDValue();
7687 unsigned N = Op1.getConstantOperandVal(i: 1);
7688 if (N != 1 && N != 2 && N != 4 && N != 8)
7689 return SDValue();
7690
7691 // Check that each other operand matches
7692 for (unsigned I = 2; I < NumElts; I++) {
7693 SDValue OpI = Op.getOperand(i: I);
7694 if (OpI.getOpcode() != ISD::ADD || OpI.getOperand(i: 0) != Op0 ||
7695 !isa<ConstantSDNode>(Val: OpI.getOperand(i: 1)) ||
7696 OpI.getConstantOperandVal(i: 1) != I * N)
7697 return SDValue();
7698 }
7699
7700 SDLoc DL(Op);
7701 return DAG.getNode(Opcode: ARMISD::VIDUP, DL, VTList: DAG.getVTList(VT1: VT, VT2: MVT::i32), N1: Op0,
7702 N2: DAG.getConstant(Val: N, DL, VT: MVT::i32));
7703}
7704
7705// Returns true if the operation N can be treated as qr instruction variant at
7706// operand Op.
7707static bool IsQRMVEInstruction(const SDNode *N, const SDNode *Op) {
7708 switch (N->getOpcode()) {
7709 case ISD::ADD:
7710 case ISD::MUL:
7711 case ISD::SADDSAT:
7712 case ISD::UADDSAT:
7713 case ISD::AVGFLOORS:
7714 case ISD::AVGFLOORU:
7715 return true;
7716 case ISD::SUB:
7717 case ISD::SSUBSAT:
7718 case ISD::USUBSAT:
7719 return N->getOperand(Num: 1).getNode() == Op;
7720 case ISD::INTRINSIC_WO_CHAIN:
7721 switch (N->getConstantOperandVal(Num: 0)) {
7722 case Intrinsic::arm_mve_add_predicated:
7723 case Intrinsic::arm_mve_mul_predicated:
7724 case Intrinsic::arm_mve_qadd_predicated:
7725 case Intrinsic::arm_mve_vhadd:
7726 case Intrinsic::arm_mve_hadd_predicated:
7727 case Intrinsic::arm_mve_vqdmulh:
7728 case Intrinsic::arm_mve_qdmulh_predicated:
7729 case Intrinsic::arm_mve_vqrdmulh:
7730 case Intrinsic::arm_mve_qrdmulh_predicated:
7731 case Intrinsic::arm_mve_vqdmull:
7732 case Intrinsic::arm_mve_vqdmull_predicated:
7733 return true;
7734 case Intrinsic::arm_mve_sub_predicated:
7735 case Intrinsic::arm_mve_qsub_predicated:
7736 case Intrinsic::arm_mve_vhsub:
7737 case Intrinsic::arm_mve_hsub_predicated:
7738 return N->getOperand(Num: 2).getNode() == Op;
7739 default:
7740 return false;
7741 }
7742 default:
7743 return false;
7744 }
7745}
7746
7747// If this is a case we can't handle, return null and let the default
7748// expansion code take care of it.
7749SDValue ARMTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
7750 const ARMSubtarget *ST) const {
7751 BuildVectorSDNode *BVN = cast<BuildVectorSDNode>(Val: Op.getNode());
7752 SDLoc dl(Op);
7753 EVT VT = Op.getValueType();
7754
7755 if (ST->hasMVEIntegerOps() && VT.getScalarSizeInBits() == 1)
7756 return LowerBUILD_VECTOR_i1(Op, DAG, ST);
7757
7758 if (SDValue R = LowerBUILD_VECTORToVIDUP(Op, DAG, ST))
7759 return R;
7760
7761 APInt SplatBits, SplatUndef;
7762 unsigned SplatBitSize;
7763 bool HasAnyUndefs;
7764 if (BVN->isConstantSplat(SplatValue&: SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
7765 if (SplatUndef.isAllOnes())
7766 return DAG.getUNDEF(VT);
7767
7768 // If all the users of this constant splat are qr instruction variants,
7769 // generate a vdup of the constant.
7770 if (ST->hasMVEIntegerOps() && VT.getScalarSizeInBits() == SplatBitSize &&
7771 (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32) &&
7772 all_of(Range: BVN->users(),
7773 P: [BVN](const SDNode *U) { return IsQRMVEInstruction(N: U, Op: BVN); })) {
7774 EVT DupVT = SplatBitSize == 32 ? MVT::v4i32
7775 : SplatBitSize == 16 ? MVT::v8i16
7776 : MVT::v16i8;
7777 SDValue Const = DAG.getConstant(Val: SplatBits.getZExtValue(), DL: dl, VT: MVT::i32);
7778 SDValue VDup = DAG.getNode(Opcode: ARMISD::VDUP, DL: dl, VT: DupVT, Operand: Const);
7779 return DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT, Operand: VDup);
7780 }
7781
7782 if ((ST->hasNEON() && SplatBitSize <= 64) ||
7783 (ST->hasMVEIntegerOps() && SplatBitSize <= 64)) {
7784 // Check if an immediate VMOV works.
7785 EVT VmovVT;
7786 SDValue Val =
7787 isVMOVModifiedImm(SplatBits: SplatBits.getZExtValue(), SplatUndef: SplatUndef.getZExtValue(),
7788 SplatBitSize, DAG, dl, VT&: VmovVT, VectorVT: VT, type: VMOVModImm);
7789
7790 if (Val.getNode()) {
7791 SDValue Vmov = DAG.getNode(Opcode: ARMISD::VMOVIMM, DL: dl, VT: VmovVT, Operand: Val);
7792 return DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT, Operand: Vmov);
7793 }
7794
7795 // Try an immediate VMVN.
7796 uint64_t NegatedImm = (~SplatBits).getZExtValue();
7797 Val = isVMOVModifiedImm(
7798 SplatBits: NegatedImm, SplatUndef: SplatUndef.getZExtValue(), SplatBitSize, DAG, dl, VT&: VmovVT,
7799 VectorVT: VT, type: ST->hasMVEIntegerOps() ? MVEVMVNModImm : VMVNModImm);
7800 if (Val.getNode()) {
7801 SDValue Vmov = DAG.getNode(Opcode: ARMISD::VMVNIMM, DL: dl, VT: VmovVT, Operand: Val);
7802 return DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT, Operand: Vmov);
7803 }
7804
7805 // Use vmov.f32 to materialize other v2f32 and v4f32 splats.
7806 if ((VT == MVT::v2f32 || VT == MVT::v4f32) && SplatBitSize == 32) {
7807 int ImmVal = ARM_AM::getFP32Imm(Imm: SplatBits);
7808 if (ImmVal != -1) {
7809 SDValue Val = DAG.getTargetConstant(Val: ImmVal, DL: dl, VT: MVT::i32);
7810 return DAG.getNode(Opcode: ARMISD::VMOVFPIMM, DL: dl, VT, Operand: Val);
7811 }
7812 }
7813
7814 // If we are under MVE, generate a VDUP(constant), bitcast to the original
7815 // type.
7816 if (ST->hasMVEIntegerOps() &&
7817 (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32)) {
7818 EVT DupVT = SplatBitSize == 32 ? MVT::v4i32
7819 : SplatBitSize == 16 ? MVT::v8i16
7820 : MVT::v16i8;
7821 SDValue Const = DAG.getConstant(Val: SplatBits.getZExtValue(), DL: dl, VT: MVT::i32);
7822 SDValue VDup = DAG.getNode(Opcode: ARMISD::VDUP, DL: dl, VT: DupVT, Operand: Const);
7823 return DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT, Operand: VDup);
7824 }
7825 }
7826 }
7827
7828 // Scan through the operands to see if only one value is used.
7829 //
7830 // As an optimisation, even if more than one value is used it may be more
7831 // profitable to splat with one value then change some lanes.
7832 //
7833 // Heuristically we decide to do this if the vector has a "dominant" value,
7834 // defined as splatted to more than half of the lanes.
7835 unsigned NumElts = VT.getVectorNumElements();
7836 bool isOnlyLowElement = true;
7837 bool usesOnlyOneValue = true;
7838 bool hasDominantValue = false;
7839 bool isConstant = true;
7840
7841 // Map of the number of times a particular SDValue appears in the
7842 // element list.
7843 DenseMap<SDValue, unsigned> ValueCounts;
7844 SDValue Value;
7845 for (unsigned i = 0; i < NumElts; ++i) {
7846 SDValue V = Op.getOperand(i);
7847 if (V.isUndef())
7848 continue;
7849 if (i > 0)
7850 isOnlyLowElement = false;
7851 if (!isa<ConstantFPSDNode>(Val: V) && !isa<ConstantSDNode>(Val: V))
7852 isConstant = false;
7853
7854 unsigned &Count = ValueCounts[V];
7855
7856 // Is this value dominant? (takes up more than half of the lanes)
7857 if (++Count > (NumElts / 2)) {
7858 hasDominantValue = true;
7859 Value = V;
7860 }
7861 }
7862 if (ValueCounts.size() != 1)
7863 usesOnlyOneValue = false;
7864 if (!Value.getNode() && !ValueCounts.empty())
7865 Value = ValueCounts.begin()->first;
7866
7867 if (ValueCounts.empty())
7868 return DAG.getUNDEF(VT);
7869
7870 // Loads are better lowered with insert_vector_elt/ARMISD::BUILD_VECTOR.
7871 // Keep going if we are hitting this case.
7872 if (isOnlyLowElement && !ISD::isNormalLoad(N: Value.getNode()) &&
7873 (VT != MVT::v8f16 || ST->hasFullFP16()))
7874 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT, Operand: Value);
7875
7876 unsigned EltSize = VT.getScalarSizeInBits();
7877
7878 // Use VDUP for non-constant splats. For f32 constant splats, reduce to
7879 // i32 and try again.
7880 if (hasDominantValue && EltSize <= 32) {
7881 if (!isConstant) {
7882 SDValue N;
7883
7884 // If we are VDUPing a value that comes directly from a vector, that will
7885 // cause an unnecessary move to and from a GPR, where instead we could
7886 // just use VDUPLANE. We can only do this if the lane being extracted
7887 // is at a constant index, as the VDUP from lane instructions only have
7888 // constant-index forms.
7889 ConstantSDNode *constIndex;
7890 if (Value->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
7891 (constIndex = dyn_cast<ConstantSDNode>(Val: Value->getOperand(Num: 1)))) {
7892 // We need to create a new undef vector to use for the VDUPLANE if the
7893 // size of the vector from which we get the value is different than the
7894 // size of the vector that we need to create. We will insert the element
7895 // such that the register coalescer will remove unnecessary copies.
7896 if (VT != Value->getOperand(Num: 0).getValueType()) {
7897 unsigned index = constIndex->getAPIntValue().getLimitedValue() %
7898 VT.getVectorNumElements();
7899 N = DAG.getNode(Opcode: ARMISD::VDUPLANE, DL: dl, VT,
7900 N1: DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT, N1: DAG.getUNDEF(VT),
7901 N2: Value, N3: DAG.getConstant(Val: index, DL: dl, VT: MVT::i32)),
7902 N2: DAG.getConstant(Val: index, DL: dl, VT: MVT::i32));
7903 } else
7904 N = DAG.getNode(Opcode: ARMISD::VDUPLANE, DL: dl, VT,
7905 N1: Value->getOperand(Num: 0), N2: Value->getOperand(Num: 1));
7906 } else
7907 N = DAG.getNode(Opcode: ARMISD::VDUP, DL: dl, VT, Operand: Value);
7908
7909 if (!usesOnlyOneValue) {
7910 // The dominant value was splatted as 'N', but we now have to insert
7911 // all differing elements.
7912 for (unsigned I = 0; I < NumElts; ++I) {
7913 if (Op.getOperand(i: I) == Value)
7914 continue;
7915 SmallVector<SDValue, 3> Ops;
7916 Ops.push_back(Elt: N);
7917 Ops.push_back(Elt: Op.getOperand(i: I));
7918 Ops.push_back(Elt: DAG.getConstant(Val: I, DL: dl, VT: MVT::i32));
7919 N = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT, Ops);
7920 }
7921 }
7922 return N;
7923 }
7924 if (VT.getVectorElementType().isFloatingPoint()) {
7925 SmallVector<SDValue, 8> Ops;
7926 MVT FVT = VT.getVectorElementType().getSimpleVT();
7927 assert(FVT == MVT::f32 || FVT == MVT::f16);
7928 MVT IVT = (FVT == MVT::f32) ? MVT::i32 : MVT::i16;
7929 for (unsigned i = 0; i < NumElts; ++i)
7930 Ops.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: IVT,
7931 Operand: Op.getOperand(i)));
7932 EVT VecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: IVT, NumElements: NumElts);
7933 SDValue Val = DAG.getBuildVector(VT: VecVT, DL: dl, Ops);
7934 Val = LowerBUILD_VECTOR(Op: Val, DAG, ST);
7935 if (Val.getNode())
7936 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: Val);
7937 }
7938 if (usesOnlyOneValue) {
7939 SDValue Val = IsSingleInstrConstant(N: Value, DAG, ST, dl);
7940 if (isConstant && Val.getNode())
7941 return DAG.getNode(Opcode: ARMISD::VDUP, DL: dl, VT, Operand: Val);
7942 }
7943 }
7944
7945 // If all elements are constants and the case above didn't get hit, fall back
7946 // to the default expansion, which will generate a load from the constant
7947 // pool.
7948 if (isConstant)
7949 return SDValue();
7950
7951 // Reconstruct the BUILDVECTOR to one of the legal shuffles (such as vext and
7952 // vmovn). Empirical tests suggest this is rarely worth it for vectors of
7953 // length <= 2.
7954 if (NumElts >= 4)
7955 if (SDValue shuffle = ReconstructShuffle(Op, DAG))
7956 return shuffle;
7957
7958 // Attempt to turn a buildvector of scalar fptrunc's or fpext's back into
7959 // VCVT's
7960 if (SDValue VCVT = LowerBuildVectorOfFPTrunc(BV: Op, DAG, ST: Subtarget))
7961 return VCVT;
7962 if (SDValue VCVT = LowerBuildVectorOfFPExt(BV: Op, DAG, ST: Subtarget))
7963 return VCVT;
7964
7965 if (ST->hasNEON() && VT.is128BitVector() && VT != MVT::v2f64 && VT != MVT::v4f32) {
7966 // If we haven't found an efficient lowering, try splitting a 128-bit vector
7967 // into two 64-bit vectors; we might discover a better way to lower it.
7968 SmallVector<SDValue, 64> Ops(Op->op_begin(), Op->op_begin() + NumElts);
7969 EVT ExtVT = VT.getVectorElementType();
7970 EVT HVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ExtVT, NumElements: NumElts / 2);
7971 SDValue Lower = DAG.getBuildVector(VT: HVT, DL: dl, Ops: ArrayRef(&Ops[0], NumElts / 2));
7972 if (Lower.getOpcode() == ISD::BUILD_VECTOR)
7973 Lower = LowerBUILD_VECTOR(Op: Lower, DAG, ST);
7974 SDValue Upper =
7975 DAG.getBuildVector(VT: HVT, DL: dl, Ops: ArrayRef(&Ops[NumElts / 2], NumElts / 2));
7976 if (Upper.getOpcode() == ISD::BUILD_VECTOR)
7977 Upper = LowerBUILD_VECTOR(Op: Upper, DAG, ST);
7978 if (Lower && Upper)
7979 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT, N1: Lower, N2: Upper);
7980 }
7981
7982 // Vectors with 32- or 64-bit elements can be built by directly assigning
7983 // the subregisters. Lower it to an ARMISD::BUILD_VECTOR so the operands
7984 // will be legalized.
7985 if (EltSize >= 32) {
7986 // Do the expansion with floating-point types, since that is what the VFP
7987 // registers are defined to use, and since i64 is not legal.
7988 EVT EltVT = EVT::getFloatingPointVT(BitWidth: EltSize);
7989 EVT VecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, NumElements: NumElts);
7990 SmallVector<SDValue, 8> Ops;
7991 for (unsigned i = 0; i < NumElts; ++i)
7992 Ops.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: EltVT, Operand: Op.getOperand(i)));
7993 SDValue Val = DAG.getNode(Opcode: ARMISD::BUILD_VECTOR, DL: dl, VT: VecVT, Ops);
7994 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: Val);
7995 }
7996
7997 // If all else fails, just use a sequence of INSERT_VECTOR_ELT when we
7998 // know the default expansion would otherwise fall back on something even
7999 // worse. For a vector with one or two non-undef values, that's
8000 // scalar_to_vector for the elements followed by a shuffle (provided the
8001 // shuffle is valid for the target) and materialization element by element
8002 // on the stack followed by a load for everything else.
8003 if ((!isConstant && !usesOnlyOneValue) ||
8004 (VT == MVT::v8f16 && !ST->hasFullFP16())) {
8005 SDValue Vec = DAG.getUNDEF(VT);
8006 for (unsigned i = 0 ; i < NumElts; ++i) {
8007 SDValue V = Op.getOperand(i);
8008 if (V.isUndef())
8009 continue;
8010 SDValue LaneIdx = DAG.getConstant(Val: i, DL: dl, VT: MVT::i32);
8011 Vec = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT, N1: Vec, N2: V, N3: LaneIdx);
8012 }
8013 return Vec;
8014 }
8015
8016 return SDValue();
8017}
8018
8019// Gather data to see if the operation can be modelled as a
8020// shuffle in combination with VEXTs.
8021SDValue ARMTargetLowering::ReconstructShuffle(SDValue Op,
8022 SelectionDAG &DAG) const {
8023 assert(Op.getOpcode() == ISD::BUILD_VECTOR && "Unknown opcode!");
8024 SDLoc dl(Op);
8025 EVT VT = Op.getValueType();
8026 unsigned NumElts = VT.getVectorNumElements();
8027
8028 struct ShuffleSourceInfo {
8029 SDValue Vec;
8030 unsigned MinElt = std::numeric_limits<unsigned>::max();
8031 unsigned MaxElt = 0;
8032
8033 // We may insert some combination of BITCASTs and VEXT nodes to force Vec to
8034 // be compatible with the shuffle we intend to construct. As a result
8035 // ShuffleVec will be some sliding window into the original Vec.
8036 SDValue ShuffleVec;
8037
8038 // Code should guarantee that element i in Vec starts at element "WindowBase
8039 // + i * WindowScale in ShuffleVec".
8040 int WindowBase = 0;
8041 int WindowScale = 1;
8042
8043 ShuffleSourceInfo(SDValue Vec) : Vec(Vec), ShuffleVec(Vec) {}
8044
8045 bool operator ==(SDValue OtherVec) { return Vec == OtherVec; }
8046 };
8047
8048 // First gather all vectors used as an immediate source for this BUILD_VECTOR
8049 // node.
8050 SmallVector<ShuffleSourceInfo, 2> Sources;
8051 for (unsigned i = 0; i < NumElts; ++i) {
8052 SDValue V = Op.getOperand(i);
8053 if (V.isUndef())
8054 continue;
8055 else if (V.getOpcode() != ISD::EXTRACT_VECTOR_ELT) {
8056 // A shuffle can only come from building a vector from various
8057 // elements of other vectors.
8058 return SDValue();
8059 } else if (!isa<ConstantSDNode>(Val: V.getOperand(i: 1))) {
8060 // Furthermore, shuffles require a constant mask, whereas extractelts
8061 // accept variable indices.
8062 return SDValue();
8063 }
8064
8065 // Add this element source to the list if it's not already there.
8066 SDValue SourceVec = V.getOperand(i: 0);
8067 auto Source = llvm::find(Range&: Sources, Val: SourceVec);
8068 if (Source == Sources.end())
8069 Source = Sources.insert(I: Sources.end(), Elt: ShuffleSourceInfo(SourceVec));
8070
8071 // Update the minimum and maximum lane number seen.
8072 unsigned EltNo = V.getConstantOperandVal(i: 1);
8073 Source->MinElt = std::min(a: Source->MinElt, b: EltNo);
8074 Source->MaxElt = std::max(a: Source->MaxElt, b: EltNo);
8075 }
8076
8077 // Currently only do something sane when at most two source vectors
8078 // are involved.
8079 if (Sources.size() > 2)
8080 return SDValue();
8081
8082 // Find out the smallest element size among result and two sources, and use
8083 // it as element size to build the shuffle_vector.
8084 EVT SmallestEltTy = VT.getVectorElementType();
8085 for (auto &Source : Sources) {
8086 EVT SrcEltTy = Source.Vec.getValueType().getVectorElementType();
8087 if (SrcEltTy.bitsLT(VT: SmallestEltTy))
8088 SmallestEltTy = SrcEltTy;
8089 }
8090 unsigned ResMultiplier =
8091 VT.getScalarSizeInBits() / SmallestEltTy.getSizeInBits();
8092 NumElts = VT.getSizeInBits() / SmallestEltTy.getSizeInBits();
8093 EVT ShuffleVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: SmallestEltTy, NumElements: NumElts);
8094
8095 // If the source vector is too wide or too narrow, we may nevertheless be able
8096 // to construct a compatible shuffle either by concatenating it with UNDEF or
8097 // extracting a suitable range of elements.
8098 for (auto &Src : Sources) {
8099 EVT SrcVT = Src.ShuffleVec.getValueType();
8100
8101 uint64_t SrcVTSize = SrcVT.getFixedSizeInBits();
8102 uint64_t VTSize = VT.getFixedSizeInBits();
8103 if (SrcVTSize == VTSize)
8104 continue;
8105
8106 // This stage of the search produces a source with the same element type as
8107 // the original, but with a total width matching the BUILD_VECTOR output.
8108 EVT EltVT = SrcVT.getVectorElementType();
8109 unsigned NumSrcElts = VTSize / EltVT.getFixedSizeInBits();
8110 EVT DestVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, NumElements: NumSrcElts);
8111
8112 if (SrcVTSize < VTSize) {
8113 if (2 * SrcVTSize != VTSize)
8114 return SDValue();
8115 // We can pad out the smaller vector for free, so if it's part of a
8116 // shuffle...
8117 Src.ShuffleVec =
8118 DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: DestVT, N1: Src.ShuffleVec,
8119 N2: DAG.getUNDEF(VT: Src.ShuffleVec.getValueType()));
8120 continue;
8121 }
8122
8123 if (SrcVTSize != 2 * VTSize)
8124 return SDValue();
8125
8126 if (Src.MaxElt - Src.MinElt >= NumSrcElts) {
8127 // Span too large for a VEXT to cope
8128 return SDValue();
8129 }
8130
8131 if (Src.MinElt >= NumSrcElts) {
8132 // The extraction can just take the second half
8133 Src.ShuffleVec =
8134 DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: DestVT, N1: Src.ShuffleVec,
8135 N2: DAG.getConstant(Val: NumSrcElts, DL: dl, VT: MVT::i32));
8136 Src.WindowBase = -NumSrcElts;
8137 } else if (Src.MaxElt < NumSrcElts) {
8138 // The extraction can just take the first half
8139 Src.ShuffleVec =
8140 DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: DestVT, N1: Src.ShuffleVec,
8141 N2: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
8142 } else {
8143 // An actual VEXT is needed
8144 SDValue VEXTSrc1 =
8145 DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: DestVT, N1: Src.ShuffleVec,
8146 N2: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
8147 SDValue VEXTSrc2 =
8148 DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: DestVT, N1: Src.ShuffleVec,
8149 N2: DAG.getConstant(Val: NumSrcElts, DL: dl, VT: MVT::i32));
8150
8151 Src.ShuffleVec = DAG.getNode(Opcode: ARMISD::VEXT, DL: dl, VT: DestVT, N1: VEXTSrc1,
8152 N2: VEXTSrc2,
8153 N3: DAG.getConstant(Val: Src.MinElt, DL: dl, VT: MVT::i32));
8154 Src.WindowBase = -Src.MinElt;
8155 }
8156 }
8157
8158 // Another possible incompatibility occurs from the vector element types. We
8159 // can fix this by bitcasting the source vectors to the same type we intend
8160 // for the shuffle.
8161 for (auto &Src : Sources) {
8162 EVT SrcEltTy = Src.ShuffleVec.getValueType().getVectorElementType();
8163 if (SrcEltTy == SmallestEltTy)
8164 continue;
8165 assert(ShuffleVT.getVectorElementType() == SmallestEltTy);
8166 Src.ShuffleVec = DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT: ShuffleVT, Operand: Src.ShuffleVec);
8167 Src.WindowScale = SrcEltTy.getSizeInBits() / SmallestEltTy.getSizeInBits();
8168 Src.WindowBase *= Src.WindowScale;
8169 }
8170
8171 // Final check before we try to actually produce a shuffle.
8172 LLVM_DEBUG({
8173 for (auto Src : Sources)
8174 assert(Src.ShuffleVec.getValueType() == ShuffleVT);
8175 });
8176
8177 // The stars all align, our next step is to produce the mask for the shuffle.
8178 SmallVector<int, 8> Mask(ShuffleVT.getVectorNumElements(), -1);
8179 int BitsPerShuffleLane = ShuffleVT.getScalarSizeInBits();
8180 for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) {
8181 SDValue Entry = Op.getOperand(i);
8182 if (Entry.isUndef())
8183 continue;
8184
8185 auto Src = llvm::find(Range&: Sources, Val: Entry.getOperand(i: 0));
8186 int EltNo = cast<ConstantSDNode>(Val: Entry.getOperand(i: 1))->getSExtValue();
8187
8188 // EXTRACT_VECTOR_ELT performs an implicit any_ext; BUILD_VECTOR an implicit
8189 // trunc. So only std::min(SrcBits, DestBits) actually get defined in this
8190 // segment.
8191 EVT OrigEltTy = Entry.getOperand(i: 0).getValueType().getVectorElementType();
8192 int BitsDefined = std::min(a: OrigEltTy.getScalarSizeInBits(),
8193 b: VT.getScalarSizeInBits());
8194 int LanesDefined = BitsDefined / BitsPerShuffleLane;
8195
8196 // This source is expected to fill ResMultiplier lanes of the final shuffle,
8197 // starting at the appropriate offset.
8198 int *LaneMask = &Mask[i * ResMultiplier];
8199
8200 int ExtractBase = EltNo * Src->WindowScale + Src->WindowBase;
8201 ExtractBase += NumElts * (Src - Sources.begin());
8202 for (int j = 0; j < LanesDefined; ++j)
8203 LaneMask[j] = ExtractBase + j;
8204 }
8205
8206
8207 // We can't handle more than two sources. This should have already
8208 // been checked before this point.
8209 assert(Sources.size() <= 2 && "Too many sources!");
8210
8211 SDValue ShuffleOps[] = { DAG.getUNDEF(VT: ShuffleVT), DAG.getUNDEF(VT: ShuffleVT) };
8212 for (unsigned i = 0; i < Sources.size(); ++i)
8213 ShuffleOps[i] = Sources[i].ShuffleVec;
8214
8215 SDValue Shuffle = buildLegalVectorShuffle(VT: ShuffleVT, DL: dl, N0: ShuffleOps[0],
8216 N1: ShuffleOps[1], Mask, DAG);
8217 if (!Shuffle)
8218 return SDValue();
8219 return DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT, Operand: Shuffle);
8220}
8221
8222enum ShuffleOpCodes {
8223 OP_COPY = 0, // Copy, used for things like <u,u,u,3> to say it is <0,1,2,3>
8224 OP_VREV,
8225 OP_VDUP0,
8226 OP_VDUP1,
8227 OP_VDUP2,
8228 OP_VDUP3,
8229 OP_VEXT1,
8230 OP_VEXT2,
8231 OP_VEXT3,
8232 OP_VUZPL, // VUZP, left result
8233 OP_VUZPR, // VUZP, right result
8234 OP_VZIPL, // VZIP, left result
8235 OP_VZIPR, // VZIP, right result
8236 OP_VTRNL, // VTRN, left result
8237 OP_VTRNR // VTRN, right result
8238};
8239
8240static bool isLegalMVEShuffleOp(unsigned PFEntry) {
8241 unsigned OpNum = (PFEntry >> 26) & 0x0F;
8242 switch (OpNum) {
8243 case OP_COPY:
8244 case OP_VREV:
8245 case OP_VDUP0:
8246 case OP_VDUP1:
8247 case OP_VDUP2:
8248 case OP_VDUP3:
8249 return true;
8250 }
8251 return false;
8252}
8253
8254/// isShuffleMaskLegal - Targets can use this to indicate that they only
8255/// support *some* VECTOR_SHUFFLE operations, those with specific masks.
8256/// By default, if a target supports the VECTOR_SHUFFLE node, all mask values
8257/// are assumed to be legal.
8258bool ARMTargetLowering::isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const {
8259 if (VT.getVectorNumElements() == 4 &&
8260 (VT.is128BitVector() || VT.is64BitVector())) {
8261 unsigned PFIndexes[4];
8262 for (unsigned i = 0; i != 4; ++i) {
8263 if (M[i] < 0)
8264 PFIndexes[i] = 8;
8265 else
8266 PFIndexes[i] = M[i];
8267 }
8268
8269 // Compute the index in the perfect shuffle table.
8270 unsigned PFTableIndex =
8271 PFIndexes[0]*9*9*9+PFIndexes[1]*9*9+PFIndexes[2]*9+PFIndexes[3];
8272 unsigned PFEntry = PerfectShuffleTable[PFTableIndex];
8273 unsigned Cost = (PFEntry >> 30);
8274
8275 if (Cost <= 4 && (Subtarget->hasNEON() || isLegalMVEShuffleOp(PFEntry)))
8276 return true;
8277 }
8278
8279 bool ReverseVEXT, isV_UNDEF;
8280 unsigned Imm, WhichResult;
8281
8282 unsigned EltSize = VT.getScalarSizeInBits();
8283 if (EltSize >= 32 ||
8284 ShuffleVectorSDNode::isSplatMask(Mask: M) ||
8285 ShuffleVectorInst::isIdentityMask(Mask: M, NumSrcElts: M.size()) ||
8286 isVREVMask(M, VT, BlockSize: 64) ||
8287 isVREVMask(M, VT, BlockSize: 32) ||
8288 isVREVMask(M, VT, BlockSize: 16))
8289 return true;
8290 else if (Subtarget->hasNEON() &&
8291 (isVEXTMask(M, VT, ReverseVEXT, Imm) ||
8292 isVTBLMask(M, VT) ||
8293 isNEONTwoResultShuffleMask(ShuffleMask: M, VT, WhichResult, isV_UNDEF)))
8294 return true;
8295 else if ((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) &&
8296 isReverseMask(M, VT))
8297 return true;
8298 else if (Subtarget->hasMVEIntegerOps() &&
8299 (isVMOVNMask(M, VT, Top: true, SingleSource: false) ||
8300 isVMOVNMask(M, VT, Top: false, SingleSource: false) || isVMOVNMask(M, VT, Top: true, SingleSource: true)))
8301 return true;
8302 else if (Subtarget->hasMVEIntegerOps() &&
8303 (isTruncMask(M, VT, Top: false, SingleSource: false) ||
8304 isTruncMask(M, VT, Top: false, SingleSource: true) ||
8305 isTruncMask(M, VT, Top: true, SingleSource: false) || isTruncMask(M, VT, Top: true, SingleSource: true)))
8306 return true;
8307 else
8308 return false;
8309}
8310
8311/// GeneratePerfectShuffle - Given an entry in the perfect-shuffle table, emit
8312/// the specified operations to build the shuffle.
8313static SDValue GeneratePerfectShuffle(unsigned PFEntry, SDValue LHS,
8314 SDValue RHS, SelectionDAG &DAG,
8315 const SDLoc &dl) {
8316 unsigned OpNum = (PFEntry >> 26) & 0x0F;
8317 unsigned LHSID = (PFEntry >> 13) & ((1 << 13)-1);
8318 unsigned RHSID = (PFEntry >> 0) & ((1 << 13)-1);
8319
8320 if (OpNum == OP_COPY) {
8321 if (LHSID == (1*9+2)*9+3) return LHS;
8322 assert(LHSID == ((4*9+5)*9+6)*9+7 && "Illegal OP_COPY!");
8323 return RHS;
8324 }
8325
8326 SDValue OpLHS, OpRHS;
8327 OpLHS = GeneratePerfectShuffle(PFEntry: PerfectShuffleTable[LHSID], LHS, RHS, DAG, dl);
8328 OpRHS = GeneratePerfectShuffle(PFEntry: PerfectShuffleTable[RHSID], LHS, RHS, DAG, dl);
8329 EVT VT = OpLHS.getValueType();
8330
8331 switch (OpNum) {
8332 default: llvm_unreachable("Unknown shuffle opcode!");
8333 case OP_VREV:
8334 // VREV divides the vector in half and swaps within the half.
8335 if (VT.getScalarSizeInBits() == 32)
8336 return DAG.getNode(Opcode: ARMISD::VREV64, DL: dl, VT, Operand: OpLHS);
8337 // vrev <4 x i16> -> VREV32
8338 if (VT.getScalarSizeInBits() == 16)
8339 return DAG.getNode(Opcode: ARMISD::VREV32, DL: dl, VT, Operand: OpLHS);
8340 // vrev <4 x i8> -> VREV16
8341 assert(VT.getScalarSizeInBits() == 8);
8342 return DAG.getNode(Opcode: ARMISD::VREV16, DL: dl, VT, Operand: OpLHS);
8343 case OP_VDUP0:
8344 case OP_VDUP1:
8345 case OP_VDUP2:
8346 case OP_VDUP3:
8347 return DAG.getNode(Opcode: ARMISD::VDUPLANE, DL: dl, VT,
8348 N1: OpLHS, N2: DAG.getConstant(Val: OpNum-OP_VDUP0, DL: dl, VT: MVT::i32));
8349 case OP_VEXT1:
8350 case OP_VEXT2:
8351 case OP_VEXT3:
8352 return DAG.getNode(Opcode: ARMISD::VEXT, DL: dl, VT,
8353 N1: OpLHS, N2: OpRHS,
8354 N3: DAG.getConstant(Val: OpNum - OP_VEXT1 + 1, DL: dl, VT: MVT::i32));
8355 case OP_VUZPL:
8356 case OP_VUZPR:
8357 return DAG.getNode(Opcode: ARMISD::VUZP, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: VT),
8358 N1: OpLHS, N2: OpRHS).getValue(R: OpNum-OP_VUZPL);
8359 case OP_VZIPL:
8360 case OP_VZIPR:
8361 return DAG.getNode(Opcode: ARMISD::VZIP, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: VT),
8362 N1: OpLHS, N2: OpRHS).getValue(R: OpNum-OP_VZIPL);
8363 case OP_VTRNL:
8364 case OP_VTRNR:
8365 return DAG.getNode(Opcode: ARMISD::VTRN, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: VT),
8366 N1: OpLHS, N2: OpRHS).getValue(R: OpNum-OP_VTRNL);
8367 }
8368}
8369
8370static SDValue LowerVECTOR_SHUFFLEv8i8(SDValue Op,
8371 ArrayRef<int> ShuffleMask,
8372 SelectionDAG &DAG) {
8373 // Check to see if we can use the VTBL instruction.
8374 SDValue V1 = Op.getOperand(i: 0);
8375 SDValue V2 = Op.getOperand(i: 1);
8376 SDLoc DL(Op);
8377
8378 SmallVector<SDValue, 8> VTBLMask;
8379 for (int I : ShuffleMask)
8380 VTBLMask.push_back(Elt: DAG.getSignedConstant(Val: I, DL, VT: MVT::i32));
8381
8382 if (V2.getNode()->isUndef())
8383 return DAG.getNode(Opcode: ARMISD::VTBL1, DL, VT: MVT::v8i8, N1: V1,
8384 N2: DAG.getBuildVector(VT: MVT::v8i8, DL, Ops: VTBLMask));
8385
8386 return DAG.getNode(Opcode: ARMISD::VTBL2, DL, VT: MVT::v8i8, N1: V1, N2: V2,
8387 N3: DAG.getBuildVector(VT: MVT::v8i8, DL, Ops: VTBLMask));
8388}
8389
8390static SDValue LowerReverse_VECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) {
8391 SDLoc DL(Op);
8392 EVT VT = Op.getValueType();
8393
8394 assert((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) &&
8395 "Expect an v8i16/v16i8 type");
8396 SDValue OpLHS = DAG.getNode(Opcode: ARMISD::VREV64, DL, VT, Operand: Op.getOperand(i: 0));
8397 // For a v16i8 type: After the VREV, we have got <7, ..., 0, 15, ..., 8>. Now,
8398 // extract the first 8 bytes into the top double word and the last 8 bytes
8399 // into the bottom double word, through a new vector shuffle that will be
8400 // turned into a VEXT on Neon, or a couple of VMOVDs on MVE.
8401 std::vector<int> NewMask;
8402 for (unsigned i = 0; i < VT.getVectorNumElements() / 2; i++)
8403 NewMask.push_back(x: VT.getVectorNumElements() / 2 + i);
8404 for (unsigned i = 0; i < VT.getVectorNumElements() / 2; i++)
8405 NewMask.push_back(x: i);
8406 return DAG.getVectorShuffle(VT, dl: DL, N1: OpLHS, N2: OpLHS, Mask: NewMask);
8407}
8408
8409static EVT getVectorTyFromPredicateVector(EVT VT) {
8410 switch (VT.getSimpleVT().SimpleTy) {
8411 case MVT::v2i1:
8412 return MVT::v2f64;
8413 case MVT::v4i1:
8414 return MVT::v4i32;
8415 case MVT::v8i1:
8416 return MVT::v8i16;
8417 case MVT::v16i1:
8418 return MVT::v16i8;
8419 default:
8420 llvm_unreachable("Unexpected vector predicate type");
8421 }
8422}
8423
8424static SDValue PromoteMVEPredVector(SDLoc dl, SDValue Pred, EVT VT,
8425 SelectionDAG &DAG) {
8426 // Converting from boolean predicates to integers involves creating a vector
8427 // of all ones or all zeroes and selecting the lanes based upon the real
8428 // predicate.
8429 SDValue AllOnes =
8430 DAG.getTargetConstant(Val: ARM_AM::createVMOVModImm(OpCmode: 0xe, Val: 0xff), DL: dl, VT: MVT::i32);
8431 AllOnes = DAG.getNode(Opcode: ARMISD::VMOVIMM, DL: dl, VT: MVT::v16i8, Operand: AllOnes);
8432
8433 SDValue AllZeroes =
8434 DAG.getTargetConstant(Val: ARM_AM::createVMOVModImm(OpCmode: 0xe, Val: 0x0), DL: dl, VT: MVT::i32);
8435 AllZeroes = DAG.getNode(Opcode: ARMISD::VMOVIMM, DL: dl, VT: MVT::v16i8, Operand: AllZeroes);
8436
8437 // Get full vector type from predicate type
8438 EVT NewVT = getVectorTyFromPredicateVector(VT);
8439
8440 SDValue RecastV1;
8441 // If the real predicate is an v8i1 or v4i1 (not v16i1) then we need to recast
8442 // this to a v16i1. This cannot be done with an ordinary bitcast because the
8443 // sizes are not the same. We have to use a MVE specific PREDICATE_CAST node,
8444 // since we know in hardware the sizes are really the same.
8445 if (VT != MVT::v16i1)
8446 RecastV1 = DAG.getNode(Opcode: ARMISD::PREDICATE_CAST, DL: dl, VT: MVT::v16i1, Operand: Pred);
8447 else
8448 RecastV1 = Pred;
8449
8450 // Select either all ones or zeroes depending upon the real predicate bits.
8451 SDValue PredAsVector =
8452 DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: MVT::v16i8, N1: RecastV1, N2: AllOnes, N3: AllZeroes);
8453
8454 // Recast our new predicate-as-integer v16i8 vector into something
8455 // appropriate for the shuffle, i.e. v4i32 for a real v4i1 predicate.
8456 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVT, Operand: PredAsVector);
8457}
8458
8459static SDValue LowerVECTOR_SHUFFLE_i1(SDValue Op, SelectionDAG &DAG,
8460 const ARMSubtarget *ST) {
8461 EVT VT = Op.getValueType();
8462 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val: Op.getNode());
8463 ArrayRef<int> ShuffleMask = SVN->getMask();
8464
8465 assert(ST->hasMVEIntegerOps() &&
8466 "No support for vector shuffle of boolean predicates");
8467
8468 SDValue V1 = Op.getOperand(i: 0);
8469 SDValue V2 = Op.getOperand(i: 1);
8470 SDLoc dl(Op);
8471 if (isReverseMask(M: ShuffleMask, VT)) {
8472 SDValue cast = DAG.getNode(Opcode: ARMISD::PREDICATE_CAST, DL: dl, VT: MVT::i32, Operand: V1);
8473 SDValue rbit = DAG.getNode(Opcode: ISD::BITREVERSE, DL: dl, VT: MVT::i32, Operand: cast);
8474 SDValue srl = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i32, N1: rbit,
8475 N2: DAG.getConstant(Val: 16, DL: dl, VT: MVT::i32));
8476 return DAG.getNode(Opcode: ARMISD::PREDICATE_CAST, DL: dl, VT, Operand: srl);
8477 }
8478
8479 // Until we can come up with optimised cases for every single vector
8480 // shuffle in existence we have chosen the least painful strategy. This is
8481 // to essentially promote the boolean predicate to a 8-bit integer, where
8482 // each predicate represents a byte. Then we fall back on a normal integer
8483 // vector shuffle and convert the result back into a predicate vector. In
8484 // many cases the generated code might be even better than scalar code
8485 // operating on bits. Just imagine trying to shuffle 8 arbitrary 2-bit
8486 // fields in a register into 8 other arbitrary 2-bit fields!
8487 SDValue PredAsVector1 = PromoteMVEPredVector(dl, Pred: V1, VT, DAG);
8488 EVT NewVT = PredAsVector1.getValueType();
8489 SDValue PredAsVector2 = V2.isUndef() ? DAG.getUNDEF(VT: NewVT)
8490 : PromoteMVEPredVector(dl, Pred: V2, VT, DAG);
8491 assert(PredAsVector2.getValueType() == NewVT &&
8492 "Expected identical vector type in expanded i1 shuffle!");
8493
8494 // Do the shuffle!
8495 SDValue Shuffled = DAG.getVectorShuffle(VT: NewVT, dl, N1: PredAsVector1,
8496 N2: PredAsVector2, Mask: ShuffleMask);
8497
8498 // Now return the result of comparing the shuffled vector with zero,
8499 // which will generate a real predicate, i.e. v4i1, v8i1 or v16i1. For a v2i1
8500 // we convert to a v4i1 compare to fill in the two halves of the i64 as i32s.
8501 if (VT == MVT::v2i1) {
8502 SDValue BC = DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT: MVT::v4i32, Operand: Shuffled);
8503 SDValue Cmp = DAG.getNode(Opcode: ARMISD::VCMPZ, DL: dl, VT: MVT::v4i1, N1: BC,
8504 N2: DAG.getConstant(Val: ARMCC::NE, DL: dl, VT: MVT::i32));
8505 return DAG.getNode(Opcode: ARMISD::PREDICATE_CAST, DL: dl, VT: MVT::v2i1, Operand: Cmp);
8506 }
8507 return DAG.getNode(Opcode: ARMISD::VCMPZ, DL: dl, VT, N1: Shuffled,
8508 N2: DAG.getConstant(Val: ARMCC::NE, DL: dl, VT: MVT::i32));
8509}
8510
8511static SDValue LowerVECTOR_SHUFFLEUsingMovs(SDValue Op,
8512 ArrayRef<int> ShuffleMask,
8513 SelectionDAG &DAG) {
8514 // Attempt to lower the vector shuffle using as many whole register movs as
8515 // possible. This is useful for types smaller than 32bits, which would
8516 // often otherwise become a series for grp movs.
8517 SDLoc dl(Op);
8518 EVT VT = Op.getValueType();
8519 if (VT.getScalarSizeInBits() >= 32)
8520 return SDValue();
8521
8522 assert((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) &&
8523 "Unexpected vector type");
8524 int NumElts = VT.getVectorNumElements();
8525 int QuarterSize = NumElts / 4;
8526 // The four final parts of the vector, as i32's
8527 SDValue Parts[4];
8528
8529 // Look for full lane vmovs like <0,1,2,3> or <u,5,6,7> etc, (but not
8530 // <u,u,u,u>), returning the vmov lane index
8531 auto getMovIdx = [](ArrayRef<int> ShuffleMask, int Start, int Length) {
8532 // Detect which mov lane this would be from the first non-undef element.
8533 int MovIdx = -1;
8534 for (int i = 0; i < Length; i++) {
8535 if (ShuffleMask[Start + i] >= 0) {
8536 if (ShuffleMask[Start + i] % Length != i)
8537 return -1;
8538 MovIdx = ShuffleMask[Start + i] / Length;
8539 break;
8540 }
8541 }
8542 // If all items are undef, leave this for other combines
8543 if (MovIdx == -1)
8544 return -1;
8545 // Check the remaining values are the correct part of the same mov
8546 for (int i = 1; i < Length; i++) {
8547 if (ShuffleMask[Start + i] >= 0 &&
8548 (ShuffleMask[Start + i] / Length != MovIdx ||
8549 ShuffleMask[Start + i] % Length != i))
8550 return -1;
8551 }
8552 return MovIdx;
8553 };
8554
8555 for (int Part = 0; Part < 4; ++Part) {
8556 // Does this part look like a mov
8557 int Elt = getMovIdx(ShuffleMask, Part * QuarterSize, QuarterSize);
8558 if (Elt != -1) {
8559 SDValue Input = Op->getOperand(Num: 0);
8560 if (Elt >= 4) {
8561 Input = Op->getOperand(Num: 1);
8562 Elt -= 4;
8563 }
8564 SDValue BitCast = DAG.getBitcast(VT: MVT::v4f32, V: Input);
8565 Parts[Part] = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: MVT::f32, N1: BitCast,
8566 N2: DAG.getConstant(Val: Elt, DL: dl, VT: MVT::i32));
8567 }
8568 }
8569
8570 // Nothing interesting found, just return
8571 if (!Parts[0] && !Parts[1] && !Parts[2] && !Parts[3])
8572 return SDValue();
8573
8574 // The other parts need to be built with the old shuffle vector, cast to a
8575 // v4i32 and extract_vector_elts
8576 if (!Parts[0] || !Parts[1] || !Parts[2] || !Parts[3]) {
8577 SmallVector<int, 16> NewShuffleMask;
8578 for (int Part = 0; Part < 4; ++Part)
8579 for (int i = 0; i < QuarterSize; i++)
8580 NewShuffleMask.push_back(
8581 Elt: Parts[Part] ? -1 : ShuffleMask[Part * QuarterSize + i]);
8582 SDValue NewShuffle = DAG.getVectorShuffle(
8583 VT, dl, N1: Op->getOperand(Num: 0), N2: Op->getOperand(Num: 1), Mask: NewShuffleMask);
8584 SDValue BitCast = DAG.getBitcast(VT: MVT::v4f32, V: NewShuffle);
8585
8586 for (int Part = 0; Part < 4; ++Part)
8587 if (!Parts[Part])
8588 Parts[Part] = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: MVT::f32,
8589 N1: BitCast, N2: DAG.getConstant(Val: Part, DL: dl, VT: MVT::i32));
8590 }
8591 // Build a vector out of the various parts and bitcast it back to the original
8592 // type.
8593 SDValue NewVec = DAG.getNode(Opcode: ARMISD::BUILD_VECTOR, DL: dl, VT: MVT::v4f32, Ops: Parts);
8594 return DAG.getBitcast(VT, V: NewVec);
8595}
8596
8597static SDValue LowerVECTOR_SHUFFLEUsingOneOff(SDValue Op,
8598 ArrayRef<int> ShuffleMask,
8599 SelectionDAG &DAG) {
8600 SDValue V1 = Op.getOperand(i: 0);
8601 SDValue V2 = Op.getOperand(i: 1);
8602 EVT VT = Op.getValueType();
8603 unsigned NumElts = VT.getVectorNumElements();
8604
8605 // An One-Off Identity mask is one that is mostly an identity mask from as
8606 // single source but contains a single element out-of-place, either from a
8607 // different vector or from another position in the same vector. As opposed to
8608 // lowering this via a ARMISD::BUILD_VECTOR we can generate an extract/insert
8609 // pair directly.
8610 auto isOneOffIdentityMask = [](ArrayRef<int> Mask, EVT VT, int BaseOffset,
8611 int &OffElement) {
8612 OffElement = -1;
8613 int NonUndef = 0;
8614 for (int i = 0, NumMaskElts = Mask.size(); i < NumMaskElts; ++i) {
8615 if (Mask[i] == -1)
8616 continue;
8617 NonUndef++;
8618 if (Mask[i] != i + BaseOffset) {
8619 if (OffElement == -1)
8620 OffElement = i;
8621 else
8622 return false;
8623 }
8624 }
8625 return NonUndef > 2 && OffElement != -1;
8626 };
8627 int OffElement;
8628 SDValue VInput;
8629 if (isOneOffIdentityMask(ShuffleMask, VT, 0, OffElement))
8630 VInput = V1;
8631 else if (isOneOffIdentityMask(ShuffleMask, VT, NumElts, OffElement))
8632 VInput = V2;
8633 else
8634 return SDValue();
8635
8636 SDLoc dl(Op);
8637 EVT SVT = VT.getScalarType() == MVT::i8 || VT.getScalarType() == MVT::i16
8638 ? MVT::i32
8639 : VT.getScalarType();
8640 SDValue Elt = DAG.getNode(
8641 Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: SVT,
8642 N1: ShuffleMask[OffElement] < (int)NumElts ? V1 : V2,
8643 N2: DAG.getVectorIdxConstant(Val: ShuffleMask[OffElement] % NumElts, DL: dl));
8644 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT, N1: VInput, N2: Elt,
8645 N3: DAG.getVectorIdxConstant(Val: OffElement % NumElts, DL: dl));
8646}
8647
8648static SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG,
8649 const ARMSubtarget *ST) {
8650 SDValue V1 = Op.getOperand(i: 0);
8651 SDValue V2 = Op.getOperand(i: 1);
8652 SDLoc dl(Op);
8653 EVT VT = Op.getValueType();
8654 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val: Op.getNode());
8655 unsigned EltSize = VT.getScalarSizeInBits();
8656
8657 if (ST->hasMVEIntegerOps() && EltSize == 1)
8658 return LowerVECTOR_SHUFFLE_i1(Op, DAG, ST);
8659
8660 // Convert shuffles that are directly supported on NEON to target-specific
8661 // DAG nodes, instead of keeping them as shuffles and matching them again
8662 // during code selection. This is more efficient and avoids the possibility
8663 // of inconsistencies between legalization and selection.
8664 // FIXME: floating-point vectors should be canonicalized to integer vectors
8665 // of the same time so that they get CSEd properly.
8666 ArrayRef<int> ShuffleMask = SVN->getMask();
8667
8668 if (EltSize <= 32) {
8669 if (SVN->isSplat()) {
8670 int Lane = SVN->getSplatIndex();
8671 // If this is undef splat, generate it via "just" vdup, if possible.
8672 if (Lane == -1) Lane = 0;
8673
8674 // Test if V1 is a SCALAR_TO_VECTOR.
8675 if (Lane == 0 && V1.getOpcode() == ISD::SCALAR_TO_VECTOR) {
8676 return DAG.getNode(Opcode: ARMISD::VDUP, DL: dl, VT, Operand: V1.getOperand(i: 0));
8677 }
8678 // Test if V1 is a BUILD_VECTOR which is equivalent to a SCALAR_TO_VECTOR
8679 // (and probably will turn into a SCALAR_TO_VECTOR once legalization
8680 // reaches it).
8681 if (Lane == 0 && V1.getOpcode() == ISD::BUILD_VECTOR &&
8682 !isa<ConstantSDNode>(Val: V1.getOperand(i: 0))) {
8683 bool IsScalarToVector = true;
8684 for (unsigned i = 1, e = V1.getNumOperands(); i != e; ++i)
8685 if (!V1.getOperand(i).isUndef()) {
8686 IsScalarToVector = false;
8687 break;
8688 }
8689 if (IsScalarToVector)
8690 return DAG.getNode(Opcode: ARMISD::VDUP, DL: dl, VT, Operand: V1.getOperand(i: 0));
8691 }
8692 return DAG.getNode(Opcode: ARMISD::VDUPLANE, DL: dl, VT, N1: V1,
8693 N2: DAG.getConstant(Val: Lane, DL: dl, VT: MVT::i32));
8694 }
8695
8696 bool ReverseVEXT = false;
8697 unsigned Imm = 0;
8698 if (ST->hasNEON() && isVEXTMask(M: ShuffleMask, VT, ReverseVEXT, Imm)) {
8699 if (ReverseVEXT)
8700 std::swap(a&: V1, b&: V2);
8701 return DAG.getNode(Opcode: ARMISD::VEXT, DL: dl, VT, N1: V1, N2: V2,
8702 N3: DAG.getConstant(Val: Imm, DL: dl, VT: MVT::i32));
8703 }
8704
8705 if (isVREVMask(M: ShuffleMask, VT, BlockSize: 64))
8706 return DAG.getNode(Opcode: ARMISD::VREV64, DL: dl, VT, Operand: V1);
8707 if (isVREVMask(M: ShuffleMask, VT, BlockSize: 32))
8708 return DAG.getNode(Opcode: ARMISD::VREV32, DL: dl, VT, Operand: V1);
8709 if (isVREVMask(M: ShuffleMask, VT, BlockSize: 16))
8710 return DAG.getNode(Opcode: ARMISD::VREV16, DL: dl, VT, Operand: V1);
8711
8712 if (ST->hasNEON() && V2->isUndef() && isSingletonVEXTMask(M: ShuffleMask, VT, Imm)) {
8713 return DAG.getNode(Opcode: ARMISD::VEXT, DL: dl, VT, N1: V1, N2: V1,
8714 N3: DAG.getConstant(Val: Imm, DL: dl, VT: MVT::i32));
8715 }
8716
8717 // Check for Neon shuffles that modify both input vectors in place.
8718 // If both results are used, i.e., if there are two shuffles with the same
8719 // source operands and with masks corresponding to both results of one of
8720 // these operations, DAG memoization will ensure that a single node is
8721 // used for both shuffles.
8722 unsigned WhichResult = 0;
8723 bool isV_UNDEF = false;
8724 if (ST->hasNEON()) {
8725 if (unsigned ShuffleOpc = isNEONTwoResultShuffleMask(
8726 ShuffleMask, VT, WhichResult, isV_UNDEF)) {
8727 if (isV_UNDEF)
8728 V2 = V1;
8729 return DAG.getNode(Opcode: ShuffleOpc, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: VT), N1: V1, N2: V2)
8730 .getValue(R: WhichResult);
8731 }
8732 }
8733 if (ST->hasMVEIntegerOps()) {
8734 if (isVMOVNMask(M: ShuffleMask, VT, Top: false, SingleSource: false))
8735 return DAG.getNode(Opcode: ARMISD::VMOVN, DL: dl, VT, N1: V2, N2: V1,
8736 N3: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
8737 if (isVMOVNMask(M: ShuffleMask, VT, Top: true, SingleSource: false))
8738 return DAG.getNode(Opcode: ARMISD::VMOVN, DL: dl, VT, N1: V1, N2: V2,
8739 N3: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
8740 if (isVMOVNMask(M: ShuffleMask, VT, Top: true, SingleSource: true))
8741 return DAG.getNode(Opcode: ARMISD::VMOVN, DL: dl, VT, N1: V1, N2: V1,
8742 N3: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
8743 }
8744
8745 // Also check for these shuffles through CONCAT_VECTORS: we canonicalize
8746 // shuffles that produce a result larger than their operands with:
8747 // shuffle(concat(v1, undef), concat(v2, undef))
8748 // ->
8749 // shuffle(concat(v1, v2), undef)
8750 // because we can access quad vectors (see PerformVECTOR_SHUFFLECombine).
8751 //
8752 // This is useful in the general case, but there are special cases where
8753 // native shuffles produce larger results: the two-result ops.
8754 //
8755 // Look through the concat when lowering them:
8756 // shuffle(concat(v1, v2), undef)
8757 // ->
8758 // concat(VZIP(v1, v2):0, :1)
8759 //
8760 if (ST->hasNEON() && V1->getOpcode() == ISD::CONCAT_VECTORS && V2->isUndef()) {
8761 SDValue SubV1 = V1->getOperand(Num: 0);
8762 SDValue SubV2 = V1->getOperand(Num: 1);
8763 EVT SubVT = SubV1.getValueType();
8764
8765 // We expect these to have been canonicalized to -1.
8766 assert(llvm::all_of(ShuffleMask, [&](int i) {
8767 return i < (int)VT.getVectorNumElements();
8768 }) && "Unexpected shuffle index into UNDEF operand!");
8769
8770 if (unsigned ShuffleOpc = isNEONTwoResultShuffleMask(
8771 ShuffleMask, VT: SubVT, WhichResult, isV_UNDEF)) {
8772 if (isV_UNDEF)
8773 SubV2 = SubV1;
8774 assert((WhichResult == 0) &&
8775 "In-place shuffle of concat can only have one result!");
8776 SDValue Res = DAG.getNode(Opcode: ShuffleOpc, DL: dl, VTList: DAG.getVTList(VT1: SubVT, VT2: SubVT),
8777 N1: SubV1, N2: SubV2);
8778 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT, N1: Res.getValue(R: 0),
8779 N2: Res.getValue(R: 1));
8780 }
8781 }
8782 }
8783
8784 if (ST->hasMVEIntegerOps() && EltSize <= 32 &&
8785 (ST->hasFullFP16() || VT != MVT::v8f16)) {
8786 if (SDValue V = LowerVECTOR_SHUFFLEUsingOneOff(Op, ShuffleMask, DAG))
8787 return V;
8788
8789 for (bool Top : {false, true}) {
8790 for (bool SingleSource : {false, true}) {
8791 if (isTruncMask(M: ShuffleMask, VT, Top, SingleSource)) {
8792 MVT FromSVT = MVT::getIntegerVT(BitWidth: EltSize * 2);
8793 MVT FromVT = MVT::getVectorVT(VT: FromSVT, NumElements: ShuffleMask.size() / 2);
8794 SDValue Lo = DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT: FromVT, Operand: V1);
8795 SDValue Hi = DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT: FromVT,
8796 Operand: SingleSource ? V1 : V2);
8797 if (Top) {
8798 SDValue Amt = DAG.getConstant(Val: EltSize, DL: dl, VT: FromVT);
8799 Lo = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: FromVT, N1: Lo, N2: Amt);
8800 Hi = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: FromVT, N1: Hi, N2: Amt);
8801 }
8802 return DAG.getNode(Opcode: ARMISD::MVETRUNC, DL: dl, VT, N1: Lo, N2: Hi);
8803 }
8804 }
8805 }
8806 }
8807
8808 // If the shuffle is not directly supported and it has 4 elements, use
8809 // the PerfectShuffle-generated table to synthesize it from other shuffles.
8810 unsigned NumElts = VT.getVectorNumElements();
8811 if (NumElts == 4) {
8812 unsigned PFIndexes[4];
8813 for (unsigned i = 0; i != 4; ++i) {
8814 if (ShuffleMask[i] < 0)
8815 PFIndexes[i] = 8;
8816 else
8817 PFIndexes[i] = ShuffleMask[i];
8818 }
8819
8820 // Compute the index in the perfect shuffle table.
8821 unsigned PFTableIndex =
8822 PFIndexes[0]*9*9*9+PFIndexes[1]*9*9+PFIndexes[2]*9+PFIndexes[3];
8823 unsigned PFEntry = PerfectShuffleTable[PFTableIndex];
8824 unsigned Cost = (PFEntry >> 30);
8825
8826 if (Cost <= 4) {
8827 if (ST->hasNEON())
8828 return GeneratePerfectShuffle(PFEntry, LHS: V1, RHS: V2, DAG, dl);
8829 else if (isLegalMVEShuffleOp(PFEntry)) {
8830 unsigned LHSID = (PFEntry >> 13) & ((1 << 13)-1);
8831 unsigned RHSID = (PFEntry >> 0) & ((1 << 13)-1);
8832 unsigned PFEntryLHS = PerfectShuffleTable[LHSID];
8833 unsigned PFEntryRHS = PerfectShuffleTable[RHSID];
8834 if (isLegalMVEShuffleOp(PFEntry: PFEntryLHS) && isLegalMVEShuffleOp(PFEntry: PFEntryRHS))
8835 return GeneratePerfectShuffle(PFEntry, LHS: V1, RHS: V2, DAG, dl);
8836 }
8837 }
8838 }
8839
8840 // Implement shuffles with 32- or 64-bit elements as ARMISD::BUILD_VECTORs.
8841 if (EltSize >= 32) {
8842 // Do the expansion with floating-point types, since that is what the VFP
8843 // registers are defined to use, and since i64 is not legal.
8844 EVT EltVT = EVT::getFloatingPointVT(BitWidth: EltSize);
8845 EVT VecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, NumElements: NumElts);
8846 V1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: VecVT, Operand: V1);
8847 V2 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: VecVT, Operand: V2);
8848 SmallVector<SDValue, 8> Ops;
8849 for (unsigned i = 0; i < NumElts; ++i) {
8850 if (ShuffleMask[i] < 0)
8851 Ops.push_back(Elt: DAG.getUNDEF(VT: EltVT));
8852 else
8853 Ops.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT,
8854 N1: ShuffleMask[i] < (int)NumElts ? V1 : V2,
8855 N2: DAG.getConstant(Val: ShuffleMask[i] & (NumElts-1),
8856 DL: dl, VT: MVT::i32)));
8857 }
8858 SDValue Val = DAG.getNode(Opcode: ARMISD::BUILD_VECTOR, DL: dl, VT: VecVT, Ops);
8859 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: Val);
8860 }
8861
8862 if ((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) &&
8863 isReverseMask(M: ShuffleMask, VT))
8864 return LowerReverse_VECTOR_SHUFFLE(Op, DAG);
8865
8866 if (ST->hasNEON() && VT == MVT::v8i8)
8867 if (SDValue NewOp = LowerVECTOR_SHUFFLEv8i8(Op, ShuffleMask, DAG))
8868 return NewOp;
8869
8870 if (ST->hasMVEIntegerOps())
8871 if (SDValue NewOp = LowerVECTOR_SHUFFLEUsingMovs(Op, ShuffleMask, DAG))
8872 return NewOp;
8873
8874 // Lower v8f16 via v8i16 to avoid invalid f16 nodes.
8875 if (VT == MVT::v8f16 && !ST->hasFullFP16()) {
8876 SDValue BC0 =
8877 DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT: MVT::v8i16, Operand: Op.getOperand(i: 0));
8878 SDValue BC1 =
8879 DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT: MVT::v8i16, Operand: Op.getOperand(i: 1));
8880 SDValue Shuf = DAG.getVectorShuffle(VT: MVT::v8i16, dl, N1: BC0, N2: BC1, Mask: ShuffleMask);
8881 return DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT, Operand: Shuf);
8882 }
8883
8884 return SDValue();
8885}
8886
8887static SDValue LowerINSERT_VECTOR_ELT_i1(SDValue Op, SelectionDAG &DAG,
8888 const ARMSubtarget *ST) {
8889 EVT VecVT = Op.getOperand(i: 0).getValueType();
8890 SDLoc dl(Op);
8891
8892 assert(ST->hasMVEIntegerOps() &&
8893 "LowerINSERT_VECTOR_ELT_i1 called without MVE!");
8894
8895 SDValue Conv =
8896 DAG.getNode(Opcode: ARMISD::PREDICATE_CAST, DL: dl, VT: MVT::i32, Operand: Op->getOperand(Num: 0));
8897 unsigned Lane = Op.getConstantOperandVal(i: 2);
8898 unsigned LaneWidth =
8899 getVectorTyFromPredicateVector(VT: VecVT).getScalarSizeInBits() / 8;
8900 unsigned Mask = ((1 << LaneWidth) - 1) << Lane * LaneWidth;
8901 SDValue Ext = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl, VT: MVT::i32,
8902 N1: Op.getOperand(i: 1), N2: DAG.getValueType(MVT::i1));
8903 SDValue BFI = DAG.getNode(Opcode: ARMISD::BFI, DL: dl, VT: MVT::i32, N1: Conv, N2: Ext,
8904 N3: DAG.getConstant(Val: ~Mask, DL: dl, VT: MVT::i32));
8905 return DAG.getNode(Opcode: ARMISD::PREDICATE_CAST, DL: dl, VT: Op.getValueType(), Operand: BFI);
8906}
8907
8908SDValue ARMTargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
8909 SelectionDAG &DAG) const {
8910 // INSERT_VECTOR_ELT is legal only for immediate indexes.
8911 SDValue Lane = Op.getOperand(i: 2);
8912 if (!isa<ConstantSDNode>(Val: Lane))
8913 return SDValue();
8914
8915 SDValue Elt = Op.getOperand(i: 1);
8916 EVT EltVT = Elt.getValueType();
8917
8918 if (Subtarget->hasMVEIntegerOps() &&
8919 Op.getValueType().getScalarSizeInBits() == 1)
8920 return LowerINSERT_VECTOR_ELT_i1(Op, DAG, ST: Subtarget);
8921
8922 if (getTypeAction(Context&: *DAG.getContext(), VT: EltVT) ==
8923 TargetLowering::TypeSoftPromoteHalf) {
8924 // INSERT_VECTOR_ELT doesn't want f16 operands promoting to f32,
8925 // but the type system will try to do that if we don't intervene.
8926 // Reinterpret any such vector-element insertion as one with the
8927 // corresponding integer types.
8928
8929 SDLoc dl(Op);
8930
8931 EVT IEltVT = MVT::getIntegerVT(BitWidth: EltVT.getScalarSizeInBits());
8932 assert(getTypeAction(*DAG.getContext(), IEltVT) !=
8933 TargetLowering::TypeSoftPromoteHalf);
8934
8935 SDValue VecIn = Op.getOperand(i: 0);
8936 EVT VecVT = VecIn.getValueType();
8937 EVT IVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: IEltVT,
8938 NumElements: VecVT.getVectorNumElements());
8939
8940 SDValue IElt = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: IEltVT, Operand: Elt);
8941 SDValue IVecIn = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: IVecVT, Operand: VecIn);
8942 SDValue IVecOut = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: IVecVT,
8943 N1: IVecIn, N2: IElt, N3: Lane);
8944 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: VecVT, Operand: IVecOut);
8945 }
8946
8947 return Op;
8948}
8949
8950static SDValue LowerEXTRACT_VECTOR_ELT_i1(SDValue Op, SelectionDAG &DAG,
8951 const ARMSubtarget *ST) {
8952 EVT VecVT = Op.getOperand(i: 0).getValueType();
8953 SDLoc dl(Op);
8954
8955 assert(ST->hasMVEIntegerOps() &&
8956 "LowerINSERT_VECTOR_ELT_i1 called without MVE!");
8957
8958 SDValue Conv =
8959 DAG.getNode(Opcode: ARMISD::PREDICATE_CAST, DL: dl, VT: MVT::i32, Operand: Op->getOperand(Num: 0));
8960 unsigned Lane = Op.getConstantOperandVal(i: 1);
8961 unsigned LaneWidth =
8962 getVectorTyFromPredicateVector(VT: VecVT).getScalarSizeInBits() / 8;
8963 SDValue Shift = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i32, N1: Conv,
8964 N2: DAG.getConstant(Val: Lane * LaneWidth, DL: dl, VT: MVT::i32));
8965 return Shift;
8966}
8967
8968static SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG,
8969 const ARMSubtarget *ST) {
8970 // EXTRACT_VECTOR_ELT is legal only for immediate indexes.
8971 SDValue Lane = Op.getOperand(i: 1);
8972 if (!isa<ConstantSDNode>(Val: Lane))
8973 return SDValue();
8974
8975 SDValue Vec = Op.getOperand(i: 0);
8976 EVT VT = Vec.getValueType();
8977
8978 if (ST->hasMVEIntegerOps() && VT.getScalarSizeInBits() == 1)
8979 return LowerEXTRACT_VECTOR_ELT_i1(Op, DAG, ST);
8980
8981 if (Op.getValueType() == MVT::i32 && Vec.getScalarValueSizeInBits() < 32) {
8982 SDLoc dl(Op);
8983 return DAG.getNode(Opcode: ARMISD::VGETLANEu, DL: dl, VT: MVT::i32, N1: Vec, N2: Lane);
8984 }
8985
8986 return Op;
8987}
8988
8989static SDValue LowerCONCAT_VECTORS_i1(SDValue Op, SelectionDAG &DAG,
8990 const ARMSubtarget *ST) {
8991 SDLoc dl(Op);
8992 assert(Op.getValueType().getScalarSizeInBits() == 1 &&
8993 "Unexpected custom CONCAT_VECTORS lowering");
8994 assert(isPowerOf2_32(Op.getNumOperands()) &&
8995 "Unexpected custom CONCAT_VECTORS lowering");
8996 assert(ST->hasMVEIntegerOps() &&
8997 "CONCAT_VECTORS lowering only supported for MVE");
8998
8999 auto ConcatPair = [&](SDValue V1, SDValue V2) {
9000 EVT Op1VT = V1.getValueType();
9001 EVT Op2VT = V2.getValueType();
9002 assert(Op1VT == Op2VT && "Operand types don't match!");
9003 assert((Op1VT == MVT::v2i1 || Op1VT == MVT::v4i1 || Op1VT == MVT::v8i1) &&
9004 "Unexpected i1 concat operations!");
9005 EVT VT = Op1VT.getDoubleNumVectorElementsVT(Context&: *DAG.getContext());
9006
9007 SDValue NewV1 = PromoteMVEPredVector(dl, Pred: V1, VT: Op1VT, DAG);
9008 SDValue NewV2 = PromoteMVEPredVector(dl, Pred: V2, VT: Op2VT, DAG);
9009
9010 // We now have Op1 + Op2 promoted to vectors of integers, where v8i1 gets
9011 // promoted to v8i16, etc.
9012 MVT ElType =
9013 getVectorTyFromPredicateVector(VT).getScalarType().getSimpleVT();
9014 unsigned NumElts = 2 * Op1VT.getVectorNumElements();
9015
9016 EVT ConcatVT = MVT::getVectorVT(VT: ElType, NumElements: NumElts);
9017 if (Op1VT == MVT::v4i1 || Op1VT == MVT::v8i1) {
9018 // Use MVETRUNC to truncate the combined NewV1::NewV2 into the smaller
9019 // ConcatVT.
9020 SDValue ConVec =
9021 DAG.getNode(Opcode: ARMISD::MVETRUNC, DL: dl, VT: ConcatVT, N1: NewV1, N2: NewV2);
9022 return DAG.getNode(Opcode: ARMISD::VCMPZ, DL: dl, VT, N1: ConVec,
9023 N2: DAG.getConstant(Val: ARMCC::NE, DL: dl, VT: MVT::i32));
9024 }
9025
9026 // Extract the vector elements from Op1 and Op2 one by one and truncate them
9027 // to be the right size for the destination. For example, if Op1 is v4i1
9028 // then the promoted vector is v4i32. The result of concatenation gives a
9029 // v8i1, which when promoted is v8i16. That means each i32 element from Op1
9030 // needs truncating to i16 and inserting in the result.
9031 auto ExtractInto = [&DAG, &dl](SDValue NewV, SDValue ConVec, unsigned &j) {
9032 EVT NewVT = NewV.getValueType();
9033 EVT ConcatVT = ConVec.getValueType();
9034 unsigned ExtScale = 1;
9035 if (NewVT == MVT::v2f64) {
9036 NewV = DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT: MVT::v4i32, Operand: NewV);
9037 ExtScale = 2;
9038 }
9039 for (unsigned i = 0, e = NewVT.getVectorNumElements(); i < e; i++, j++) {
9040 SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: MVT::i32, N1: NewV,
9041 N2: DAG.getIntPtrConstant(Val: i * ExtScale, DL: dl));
9042 ConVec = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: ConcatVT, N1: ConVec, N2: Elt,
9043 N3: DAG.getConstant(Val: j, DL: dl, VT: MVT::i32));
9044 }
9045 return ConVec;
9046 };
9047 unsigned j = 0;
9048 SDValue ConVec = DAG.getNode(Opcode: ISD::UNDEF, DL: dl, VT: ConcatVT);
9049 ConVec = ExtractInto(NewV1, ConVec, j);
9050 ConVec = ExtractInto(NewV2, ConVec, j);
9051
9052 // Now return the result of comparing the subvector with zero, which will
9053 // generate a real predicate, i.e. v4i1, v8i1 or v16i1.
9054 return DAG.getNode(Opcode: ARMISD::VCMPZ, DL: dl, VT, N1: ConVec,
9055 N2: DAG.getConstant(Val: ARMCC::NE, DL: dl, VT: MVT::i32));
9056 };
9057
9058 // Concat each pair of subvectors and pack into the lower half of the array.
9059 SmallVector<SDValue> ConcatOps(Op->ops());
9060 while (ConcatOps.size() > 1) {
9061 for (unsigned I = 0, E = ConcatOps.size(); I != E; I += 2) {
9062 SDValue V1 = ConcatOps[I];
9063 SDValue V2 = ConcatOps[I + 1];
9064 ConcatOps[I / 2] = ConcatPair(V1, V2);
9065 }
9066 ConcatOps.resize(N: ConcatOps.size() / 2);
9067 }
9068 return ConcatOps[0];
9069}
9070
9071static SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG,
9072 const ARMSubtarget *ST) {
9073 EVT VT = Op->getValueType(ResNo: 0);
9074 if (ST->hasMVEIntegerOps() && VT.getScalarSizeInBits() == 1)
9075 return LowerCONCAT_VECTORS_i1(Op, DAG, ST);
9076
9077 // The only time a CONCAT_VECTORS operation can have legal types is when
9078 // two 64-bit vectors are concatenated to a 128-bit vector.
9079 assert(Op.getValueType().is128BitVector() && Op.getNumOperands() == 2 &&
9080 "unexpected CONCAT_VECTORS");
9081 SDLoc dl(Op);
9082 SDValue Val = DAG.getUNDEF(VT: MVT::v2f64);
9083 SDValue Op0 = Op.getOperand(i: 0);
9084 SDValue Op1 = Op.getOperand(i: 1);
9085 if (!Op0.isUndef())
9086 Val = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: MVT::v2f64, N1: Val,
9087 N2: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::f64, Operand: Op0),
9088 N3: DAG.getIntPtrConstant(Val: 0, DL: dl));
9089 if (!Op1.isUndef())
9090 Val = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: MVT::v2f64, N1: Val,
9091 N2: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::f64, Operand: Op1),
9092 N3: DAG.getIntPtrConstant(Val: 1, DL: dl));
9093 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: Op.getValueType(), Operand: Val);
9094}
9095
9096static SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, SelectionDAG &DAG,
9097 const ARMSubtarget *ST) {
9098 SDValue V1 = Op.getOperand(i: 0);
9099 SDValue V2 = Op.getOperand(i: 1);
9100 SDLoc dl(Op);
9101 EVT VT = Op.getValueType();
9102 EVT Op1VT = V1.getValueType();
9103 unsigned NumElts = VT.getVectorNumElements();
9104 unsigned Index = V2->getAsZExtVal();
9105
9106 assert(VT.getScalarSizeInBits() == 1 &&
9107 "Unexpected custom EXTRACT_SUBVECTOR lowering");
9108 assert(ST->hasMVEIntegerOps() &&
9109 "EXTRACT_SUBVECTOR lowering only supported for MVE");
9110
9111 SDValue NewV1 = PromoteMVEPredVector(dl, Pred: V1, VT: Op1VT, DAG);
9112
9113 // We now have Op1 promoted to a vector of integers, where v8i1 gets
9114 // promoted to v8i16, etc.
9115
9116 MVT ElType = getVectorTyFromPredicateVector(VT).getScalarType().getSimpleVT();
9117
9118 if (NumElts == 2) {
9119 EVT SubVT = MVT::v4i32;
9120 SDValue SubVec = DAG.getNode(Opcode: ISD::UNDEF, DL: dl, VT: SubVT);
9121 for (unsigned i = Index, j = 0; i < (Index + NumElts); i++, j += 2) {
9122 SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: MVT::i32, N1: NewV1,
9123 N2: DAG.getIntPtrConstant(Val: i, DL: dl));
9124 SubVec = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: SubVT, N1: SubVec, N2: Elt,
9125 N3: DAG.getConstant(Val: j, DL: dl, VT: MVT::i32));
9126 SubVec = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: SubVT, N1: SubVec, N2: Elt,
9127 N3: DAG.getConstant(Val: j + 1, DL: dl, VT: MVT::i32));
9128 }
9129 SDValue Cmp = DAG.getNode(Opcode: ARMISD::VCMPZ, DL: dl, VT: MVT::v4i1, N1: SubVec,
9130 N2: DAG.getConstant(Val: ARMCC::NE, DL: dl, VT: MVT::i32));
9131 return DAG.getNode(Opcode: ARMISD::PREDICATE_CAST, DL: dl, VT: MVT::v2i1, Operand: Cmp);
9132 }
9133
9134 EVT SubVT = MVT::getVectorVT(VT: ElType, NumElements: NumElts);
9135 SDValue SubVec = DAG.getNode(Opcode: ISD::UNDEF, DL: dl, VT: SubVT);
9136 for (unsigned i = Index, j = 0; i < (Index + NumElts); i++, j++) {
9137 SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: MVT::i32, N1: NewV1,
9138 N2: DAG.getIntPtrConstant(Val: i, DL: dl));
9139 SubVec = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: SubVT, N1: SubVec, N2: Elt,
9140 N3: DAG.getConstant(Val: j, DL: dl, VT: MVT::i32));
9141 }
9142
9143 // Now return the result of comparing the subvector with zero,
9144 // which will generate a real predicate, i.e. v4i1, v8i1 or v16i1.
9145 return DAG.getNode(Opcode: ARMISD::VCMPZ, DL: dl, VT, N1: SubVec,
9146 N2: DAG.getConstant(Val: ARMCC::NE, DL: dl, VT: MVT::i32));
9147}
9148
9149// Turn a truncate into a predicate (an i1 vector) into icmp(and(x, 1), 0).
9150static SDValue LowerTruncatei1(SDNode *N, SelectionDAG &DAG,
9151 const ARMSubtarget *ST) {
9152 assert(ST->hasMVEIntegerOps() && "Expected MVE!");
9153 EVT VT = N->getValueType(ResNo: 0);
9154 assert((VT == MVT::v16i1 || VT == MVT::v8i1 || VT == MVT::v4i1) &&
9155 "Expected a vector i1 type!");
9156 SDValue Op = N->getOperand(Num: 0);
9157 EVT FromVT = Op.getValueType();
9158 SDLoc DL(N);
9159
9160 SDValue And =
9161 DAG.getNode(Opcode: ISD::AND, DL, VT: FromVT, N1: Op, N2: DAG.getConstant(Val: 1, DL, VT: FromVT));
9162 return DAG.getNode(Opcode: ISD::SETCC, DL, VT, N1: And, N2: DAG.getConstant(Val: 0, DL, VT: FromVT),
9163 N3: DAG.getCondCode(Cond: ISD::SETNE));
9164}
9165
9166static SDValue LowerTruncate(SDNode *N, SelectionDAG &DAG,
9167 const ARMSubtarget *Subtarget) {
9168 if (!Subtarget->hasMVEIntegerOps())
9169 return SDValue();
9170
9171 EVT ToVT = N->getValueType(ResNo: 0);
9172 if (ToVT.getScalarType() == MVT::i1)
9173 return LowerTruncatei1(N, DAG, ST: Subtarget);
9174
9175 // MVE does not have a single instruction to perform the truncation of a v4i32
9176 // into the lower half of a v8i16, in the same way that a NEON vmovn would.
9177 // Most of the instructions in MVE follow the 'Beats' system, where moving
9178 // values from different lanes is usually something that the instructions
9179 // avoid.
9180 //
9181 // Instead it has top/bottom instructions such as VMOVLT/B and VMOVNT/B,
9182 // which take a the top/bottom half of a larger lane and extend it (or do the
9183 // opposite, truncating into the top/bottom lane from a larger lane). Note
9184 // that because of the way we widen lanes, a v4i16 is really a v4i32 using the
9185 // bottom 16bits from each vector lane. This works really well with T/B
9186 // instructions, but that doesn't extend to v8i32->v8i16 where the lanes need
9187 // to move order.
9188 //
9189 // But truncates and sext/zext are always going to be fairly common from llvm.
9190 // We have several options for how to deal with them:
9191 // - Wherever possible combine them into an instruction that makes them
9192 // "free". This includes loads/stores, which can perform the trunc as part
9193 // of the memory operation. Or certain shuffles that can be turned into
9194 // VMOVN/VMOVL.
9195 // - Lane Interleaving to transform blocks surrounded by ext/trunc. So
9196 // trunc(mul(sext(a), sext(b))) may become
9197 // VMOVNT(VMUL(VMOVLB(a), VMOVLB(b)), VMUL(VMOVLT(a), VMOVLT(b))). (Which in
9198 // this case can use VMULL). This is performed in the
9199 // MVELaneInterleavingPass.
9200 // - Otherwise we have an option. By default we would expand the
9201 // zext/sext/trunc into a series of lane extract/inserts going via GPR
9202 // registers. One for each vector lane in the vector. This can obviously be
9203 // very expensive.
9204 // - The other option is to use the fact that loads/store can extend/truncate
9205 // to turn a trunc into two truncating stack stores and a stack reload. This
9206 // becomes 3 back-to-back memory operations, but at least that is less than
9207 // all the insert/extracts.
9208 //
9209 // In order to do the last, we convert certain trunc's into MVETRUNC, which
9210 // are either optimized where they can be, or eventually lowered into stack
9211 // stores/loads. This prevents us from splitting a v8i16 trunc into two stores
9212 // two early, where other instructions would be better, and stops us from
9213 // having to reconstruct multiple buildvector shuffles into loads/stores.
9214 if (ToVT != MVT::v8i16 && ToVT != MVT::v16i8)
9215 return SDValue();
9216 EVT FromVT = N->getOperand(Num: 0).getValueType();
9217 if (FromVT != MVT::v8i32 && FromVT != MVT::v16i16)
9218 return SDValue();
9219
9220 SDValue Lo, Hi;
9221 std::tie(args&: Lo, args&: Hi) = DAG.SplitVectorOperand(N, OpNo: 0);
9222 SDLoc DL(N);
9223 return DAG.getNode(Opcode: ARMISD::MVETRUNC, DL, VT: ToVT, N1: Lo, N2: Hi);
9224}
9225
9226static SDValue LowerVectorExtend(SDNode *N, SelectionDAG &DAG,
9227 const ARMSubtarget *Subtarget) {
9228 if (!Subtarget->hasMVEIntegerOps())
9229 return SDValue();
9230
9231 // See LowerTruncate above for an explanation of MVEEXT/MVETRUNC.
9232
9233 EVT ToVT = N->getValueType(ResNo: 0);
9234 if (ToVT != MVT::v16i32 && ToVT != MVT::v8i32 && ToVT != MVT::v16i16)
9235 return SDValue();
9236 SDValue Op = N->getOperand(Num: 0);
9237 EVT FromVT = Op.getValueType();
9238 if (FromVT != MVT::v8i16 && FromVT != MVT::v16i8)
9239 return SDValue();
9240
9241 SDLoc DL(N);
9242 EVT ExtVT = ToVT.getHalfNumVectorElementsVT(Context&: *DAG.getContext());
9243 if (ToVT.getScalarType() == MVT::i32 && FromVT.getScalarType() == MVT::i8)
9244 ExtVT = MVT::v8i16;
9245
9246 unsigned Opcode =
9247 N->getOpcode() == ISD::SIGN_EXTEND ? ARMISD::MVESEXT : ARMISD::MVEZEXT;
9248 SDValue Ext = DAG.getNode(Opcode, DL, VTList: DAG.getVTList(VT1: ExtVT, VT2: ExtVT), N: Op);
9249 SDValue Ext1 = Ext.getValue(R: 1);
9250
9251 if (ToVT.getScalarType() == MVT::i32 && FromVT.getScalarType() == MVT::i8) {
9252 Ext = DAG.getNode(Opcode: N->getOpcode(), DL, VT: MVT::v8i32, Operand: Ext);
9253 Ext1 = DAG.getNode(Opcode: N->getOpcode(), DL, VT: MVT::v8i32, Operand: Ext1);
9254 }
9255
9256 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: ToVT, N1: Ext, N2: Ext1);
9257}
9258
9259/// isExtendedBUILD_VECTOR - Check if N is a constant BUILD_VECTOR where each
9260/// element has been zero/sign-extended, depending on the isSigned parameter,
9261/// from an integer type half its size.
9262static bool isExtendedBUILD_VECTOR(SDNode *N, SelectionDAG &DAG,
9263 bool isSigned) {
9264 // A v2i64 BUILD_VECTOR will have been legalized to a BITCAST from v4i32.
9265 EVT VT = N->getValueType(ResNo: 0);
9266 if (VT == MVT::v2i64 && N->getOpcode() == ISD::BITCAST) {
9267 SDNode *BVN = N->getOperand(Num: 0).getNode();
9268 if (BVN->getValueType(ResNo: 0) != MVT::v4i32 ||
9269 BVN->getOpcode() != ISD::BUILD_VECTOR)
9270 return false;
9271 unsigned LoElt = DAG.getDataLayout().isBigEndian() ? 1 : 0;
9272 unsigned HiElt = 1 - LoElt;
9273 ConstantSDNode *Lo0 = dyn_cast<ConstantSDNode>(Val: BVN->getOperand(Num: LoElt));
9274 ConstantSDNode *Hi0 = dyn_cast<ConstantSDNode>(Val: BVN->getOperand(Num: HiElt));
9275 ConstantSDNode *Lo1 = dyn_cast<ConstantSDNode>(Val: BVN->getOperand(Num: LoElt+2));
9276 ConstantSDNode *Hi1 = dyn_cast<ConstantSDNode>(Val: BVN->getOperand(Num: HiElt+2));
9277 if (!Lo0 || !Hi0 || !Lo1 || !Hi1)
9278 return false;
9279 if (isSigned) {
9280 if (Hi0->getSExtValue() == Lo0->getSExtValue() >> 32 &&
9281 Hi1->getSExtValue() == Lo1->getSExtValue() >> 32)
9282 return true;
9283 } else {
9284 if (Hi0->isZero() && Hi1->isZero())
9285 return true;
9286 }
9287 return false;
9288 }
9289
9290 if (N->getOpcode() != ISD::BUILD_VECTOR)
9291 return false;
9292
9293 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
9294 SDNode *Elt = N->getOperand(Num: i).getNode();
9295 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val: Elt)) {
9296 unsigned EltSize = VT.getScalarSizeInBits();
9297 unsigned HalfSize = EltSize / 2;
9298 if (isSigned) {
9299 if (!isIntN(N: HalfSize, x: C->getSExtValue()))
9300 return false;
9301 } else {
9302 if (!isUIntN(N: HalfSize, x: C->getZExtValue()))
9303 return false;
9304 }
9305 continue;
9306 }
9307 return false;
9308 }
9309
9310 return true;
9311}
9312
9313/// isSignExtended - Check if a node is a vector value that is sign-extended
9314/// or a constant BUILD_VECTOR with sign-extended elements.
9315static bool isSignExtended(SDNode *N, SelectionDAG &DAG) {
9316 if (N->getOpcode() == ISD::SIGN_EXTEND || ISD::isSEXTLoad(N))
9317 return true;
9318 if (isExtendedBUILD_VECTOR(N, DAG, isSigned: true))
9319 return true;
9320 return false;
9321}
9322
9323/// isZeroExtended - Check if a node is a vector value that is zero-extended (or
9324/// any-extended) or a constant BUILD_VECTOR with zero-extended elements.
9325static bool isZeroExtended(SDNode *N, SelectionDAG &DAG) {
9326 if (N->getOpcode() == ISD::ZERO_EXTEND || N->getOpcode() == ISD::ANY_EXTEND ||
9327 ISD::isZEXTLoad(N))
9328 return true;
9329 if (isExtendedBUILD_VECTOR(N, DAG, isSigned: false))
9330 return true;
9331 return false;
9332}
9333
9334static EVT getExtensionTo64Bits(const EVT &OrigVT) {
9335 if (OrigVT.getSizeInBits() >= 64)
9336 return OrigVT;
9337
9338 assert(OrigVT.isSimple() && "Expecting a simple value type");
9339
9340 MVT::SimpleValueType OrigSimpleTy = OrigVT.getSimpleVT().SimpleTy;
9341 switch (OrigSimpleTy) {
9342 default: llvm_unreachable("Unexpected Vector Type");
9343 case MVT::v2i8:
9344 case MVT::v2i16:
9345 return MVT::v2i32;
9346 case MVT::v4i8:
9347 return MVT::v4i16;
9348 }
9349}
9350
9351/// AddRequiredExtensionForVMULL - Add a sign/zero extension to extend the total
9352/// value size to 64 bits. We need a 64-bit D register as an operand to VMULL.
9353/// We insert the required extension here to get the vector to fill a D register.
9354static SDValue AddRequiredExtensionForVMULL(SDValue N, SelectionDAG &DAG,
9355 const EVT &OrigTy,
9356 const EVT &ExtTy,
9357 unsigned ExtOpcode) {
9358 // The vector originally had a size of OrigTy. It was then extended to ExtTy.
9359 // We expect the ExtTy to be 128-bits total. If the OrigTy is less than
9360 // 64-bits we need to insert a new extension so that it will be 64-bits.
9361 assert(ExtTy.is128BitVector() && "Unexpected extension size");
9362 if (OrigTy.getSizeInBits() >= 64)
9363 return N;
9364
9365 // Must extend size to at least 64 bits to be used as an operand for VMULL.
9366 EVT NewVT = getExtensionTo64Bits(OrigVT: OrigTy);
9367
9368 return DAG.getNode(Opcode: ExtOpcode, DL: SDLoc(N), VT: NewVT, Operand: N);
9369}
9370
9371/// SkipLoadExtensionForVMULL - return a load of the original vector size that
9372/// does not do any sign/zero extension. If the original vector is less
9373/// than 64 bits, an appropriate extension will be added after the load to
9374/// reach a total size of 64 bits. We have to add the extension separately
9375/// because ARM does not have a sign/zero extending load for vectors.
9376static SDValue SkipLoadExtensionForVMULL(LoadSDNode *LD, SelectionDAG& DAG) {
9377 EVT ExtendedTy = getExtensionTo64Bits(OrigVT: LD->getMemoryVT());
9378
9379 // The load already has the right type.
9380 if (ExtendedTy == LD->getMemoryVT())
9381 return DAG.getLoad(VT: LD->getMemoryVT(), dl: SDLoc(LD), Chain: LD->getChain(),
9382 Ptr: LD->getBasePtr(), PtrInfo: LD->getPointerInfo(), Alignment: LD->getAlign(),
9383 MMOFlags: LD->getMemOperand()->getFlags());
9384
9385 // We need to create a zextload/sextload. We cannot just create a load
9386 // followed by a zext/zext node because LowerMUL is also run during normal
9387 // operation legalization where we can't create illegal types.
9388 return DAG.getExtLoad(ExtType: LD->getExtensionType(), dl: SDLoc(LD), VT: ExtendedTy,
9389 Chain: LD->getChain(), Ptr: LD->getBasePtr(), PtrInfo: LD->getPointerInfo(),
9390 MemVT: LD->getMemoryVT(), Alignment: LD->getAlign(),
9391 MMOFlags: LD->getMemOperand()->getFlags());
9392}
9393
9394/// SkipExtensionForVMULL - For a node that is a SIGN_EXTEND, ZERO_EXTEND,
9395/// ANY_EXTEND, extending load, or BUILD_VECTOR with extended elements, return
9396/// the unextended value. The unextended vector should be 64 bits so that it can
9397/// be used as an operand to a VMULL instruction. If the original vector size
9398/// before extension is less than 64 bits we add a an extension to resize
9399/// the vector to 64 bits.
9400static SDValue SkipExtensionForVMULL(SDNode *N, SelectionDAG &DAG) {
9401 if (N->getOpcode() == ISD::SIGN_EXTEND ||
9402 N->getOpcode() == ISD::ZERO_EXTEND || N->getOpcode() == ISD::ANY_EXTEND)
9403 return AddRequiredExtensionForVMULL(N: N->getOperand(Num: 0), DAG,
9404 OrigTy: N->getOperand(Num: 0)->getValueType(ResNo: 0),
9405 ExtTy: N->getValueType(ResNo: 0),
9406 ExtOpcode: N->getOpcode());
9407
9408 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Val: N)) {
9409 assert((ISD::isSEXTLoad(LD) || ISD::isZEXTLoad(LD)) &&
9410 "Expected extending load");
9411
9412 SDValue newLoad = SkipLoadExtensionForVMULL(LD, DAG);
9413 DAG.ReplaceAllUsesOfValueWith(From: SDValue(LD, 1), To: newLoad.getValue(R: 1));
9414 unsigned Opcode = ISD::isSEXTLoad(N: LD) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
9415 SDValue extLoad =
9416 DAG.getNode(Opcode, DL: SDLoc(newLoad), VT: LD->getValueType(ResNo: 0), Operand: newLoad);
9417 DAG.ReplaceAllUsesOfValueWith(From: SDValue(LD, 0), To: extLoad);
9418
9419 return newLoad;
9420 }
9421
9422 // Otherwise, the value must be a BUILD_VECTOR. For v2i64, it will
9423 // have been legalized as a BITCAST from v4i32.
9424 if (N->getOpcode() == ISD::BITCAST) {
9425 SDNode *BVN = N->getOperand(Num: 0).getNode();
9426 assert(BVN->getOpcode() == ISD::BUILD_VECTOR &&
9427 BVN->getValueType(0) == MVT::v4i32 && "expected v4i32 BUILD_VECTOR");
9428 unsigned LowElt = DAG.getDataLayout().isBigEndian() ? 1 : 0;
9429 return DAG.getBuildVector(
9430 VT: MVT::v2i32, DL: SDLoc(N),
9431 Ops: {BVN->getOperand(Num: LowElt), BVN->getOperand(Num: LowElt + 2)});
9432 }
9433 // Construct a new BUILD_VECTOR with elements truncated to half the size.
9434 assert(N->getOpcode() == ISD::BUILD_VECTOR && "expected BUILD_VECTOR");
9435 EVT VT = N->getValueType(ResNo: 0);
9436 unsigned EltSize = VT.getScalarSizeInBits() / 2;
9437 unsigned NumElts = VT.getVectorNumElements();
9438 MVT TruncVT = MVT::getIntegerVT(BitWidth: EltSize);
9439 SmallVector<SDValue, 8> Ops;
9440 SDLoc dl(N);
9441 for (unsigned i = 0; i != NumElts; ++i) {
9442 const APInt &CInt = N->getConstantOperandAPInt(Num: i);
9443 // Element types smaller than 32 bits are not legal, so use i32 elements.
9444 // The values are implicitly truncated so sext vs. zext doesn't matter.
9445 Ops.push_back(Elt: DAG.getConstant(Val: CInt.zextOrTrunc(width: 32), DL: dl, VT: MVT::i32));
9446 }
9447 return DAG.getBuildVector(VT: MVT::getVectorVT(VT: TruncVT, NumElements: NumElts), DL: dl, Ops);
9448}
9449
9450static bool isAddSubSExt(SDNode *N, SelectionDAG &DAG) {
9451 unsigned Opcode = N->getOpcode();
9452 if (Opcode == ISD::ADD || Opcode == ISD::SUB) {
9453 SDNode *N0 = N->getOperand(Num: 0).getNode();
9454 SDNode *N1 = N->getOperand(Num: 1).getNode();
9455 return N0->hasOneUse() && N1->hasOneUse() &&
9456 isSignExtended(N: N0, DAG) && isSignExtended(N: N1, DAG);
9457 }
9458 return false;
9459}
9460
9461static bool isAddSubZExt(SDNode *N, SelectionDAG &DAG) {
9462 unsigned Opcode = N->getOpcode();
9463 if (Opcode == ISD::ADD || Opcode == ISD::SUB) {
9464 SDNode *N0 = N->getOperand(Num: 0).getNode();
9465 SDNode *N1 = N->getOperand(Num: 1).getNode();
9466 return N0->hasOneUse() && N1->hasOneUse() &&
9467 isZeroExtended(N: N0, DAG) && isZeroExtended(N: N1, DAG);
9468 }
9469 return false;
9470}
9471
9472static SDValue LowerMUL(SDValue Op, SelectionDAG &DAG) {
9473 // Multiplications are only custom-lowered for 128-bit vectors so that
9474 // VMULL can be detected. Otherwise v2i64 multiplications are not legal.
9475 EVT VT = Op.getValueType();
9476 assert(VT.is128BitVector() && VT.isInteger() &&
9477 "unexpected type for custom-lowering ISD::MUL");
9478 SDNode *N0 = Op.getOperand(i: 0).getNode();
9479 SDNode *N1 = Op.getOperand(i: 1).getNode();
9480 unsigned NewOpc = 0;
9481 bool isMLA = false;
9482 bool isN0SExt = isSignExtended(N: N0, DAG);
9483 bool isN1SExt = isSignExtended(N: N1, DAG);
9484 if (isN0SExt && isN1SExt)
9485 NewOpc = ARMISD::VMULLs;
9486 else {
9487 bool isN0ZExt = isZeroExtended(N: N0, DAG);
9488 bool isN1ZExt = isZeroExtended(N: N1, DAG);
9489 if (isN0ZExt && isN1ZExt)
9490 NewOpc = ARMISD::VMULLu;
9491 else if (isN1SExt || isN1ZExt) {
9492 // Look for (s/zext A + s/zext B) * (s/zext C). We want to turn these
9493 // into (s/zext A * s/zext C) + (s/zext B * s/zext C)
9494 if (isN1SExt && isAddSubSExt(N: N0, DAG)) {
9495 NewOpc = ARMISD::VMULLs;
9496 isMLA = true;
9497 } else if (isN1ZExt && isAddSubZExt(N: N0, DAG)) {
9498 NewOpc = ARMISD::VMULLu;
9499 isMLA = true;
9500 } else if (isN0ZExt && isAddSubZExt(N: N1, DAG)) {
9501 std::swap(a&: N0, b&: N1);
9502 NewOpc = ARMISD::VMULLu;
9503 isMLA = true;
9504 }
9505 }
9506
9507 if (!NewOpc) {
9508 if (VT == MVT::v2i64)
9509 // Fall through to expand this. It is not legal.
9510 return SDValue();
9511 else
9512 // Other vector multiplications are legal.
9513 return Op;
9514 }
9515 }
9516
9517 // Legalize to a VMULL instruction.
9518 SDLoc DL(Op);
9519 SDValue Op0;
9520 SDValue Op1 = SkipExtensionForVMULL(N: N1, DAG);
9521 if (!isMLA) {
9522 Op0 = SkipExtensionForVMULL(N: N0, DAG);
9523 assert(Op0.getValueType().is64BitVector() &&
9524 Op1.getValueType().is64BitVector() &&
9525 "unexpected types for extended operands to VMULL");
9526 return DAG.getNode(Opcode: NewOpc, DL, VT, N1: Op0, N2: Op1);
9527 }
9528
9529 // Optimizing (zext A + zext B) * C, to (VMULL A, C) + (VMULL B, C) during
9530 // isel lowering to take advantage of no-stall back to back vmul + vmla.
9531 // vmull q0, d4, d6
9532 // vmlal q0, d5, d6
9533 // is faster than
9534 // vaddl q0, d4, d5
9535 // vmovl q1, d6
9536 // vmul q0, q0, q1
9537 SDValue N00 = SkipExtensionForVMULL(N: N0->getOperand(Num: 0).getNode(), DAG);
9538 SDValue N01 = SkipExtensionForVMULL(N: N0->getOperand(Num: 1).getNode(), DAG);
9539 EVT Op1VT = Op1.getValueType();
9540 return DAG.getNode(Opcode: N0->getOpcode(), DL, VT,
9541 N1: DAG.getNode(Opcode: NewOpc, DL, VT,
9542 N1: DAG.getNode(Opcode: ISD::BITCAST, DL, VT: Op1VT, Operand: N00), N2: Op1),
9543 N2: DAG.getNode(Opcode: NewOpc, DL, VT,
9544 N1: DAG.getNode(Opcode: ISD::BITCAST, DL, VT: Op1VT, Operand: N01), N2: Op1));
9545}
9546
9547static SDValue LowerSDIV_v4i8(SDValue X, SDValue Y, const SDLoc &dl,
9548 SelectionDAG &DAG) {
9549 // TODO: Should this propagate fast-math-flags?
9550
9551 // Convert to float
9552 // float4 xf = vcvt_f32_s32(vmovl_s16(a.lo));
9553 // float4 yf = vcvt_f32_s32(vmovl_s16(b.lo));
9554 X = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: MVT::v4i32, Operand: X);
9555 Y = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: MVT::v4i32, Operand: Y);
9556 X = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: MVT::v4f32, Operand: X);
9557 Y = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: MVT::v4f32, Operand: Y);
9558 // Get reciprocal estimate.
9559 // float4 recip = vrecpeq_f32(yf);
9560 Y = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: MVT::v4f32,
9561 N1: DAG.getConstant(Val: Intrinsic::arm_neon_vrecpe, DL: dl, VT: MVT::i32),
9562 N2: Y);
9563 // Because char has a smaller range than uchar, we can actually get away
9564 // without any newton steps. This requires that we use a weird bias
9565 // of 0xb000, however (again, this has been exhaustively tested).
9566 // float4 result = as_float4(as_int4(xf*recip) + 0xb000);
9567 X = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::v4f32, N1: X, N2: Y);
9568 X = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v4i32, Operand: X);
9569 Y = DAG.getConstant(Val: 0xb000, DL: dl, VT: MVT::v4i32);
9570 X = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::v4i32, N1: X, N2: Y);
9571 X = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v4f32, Operand: X);
9572 // Convert back to short.
9573 X = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: dl, VT: MVT::v4i32, Operand: X);
9574 X = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::v4i16, Operand: X);
9575 return X;
9576}
9577
9578static SDValue LowerSDIV_v4i16(SDValue N0, SDValue N1, const SDLoc &dl,
9579 SelectionDAG &DAG) {
9580 // TODO: Should this propagate fast-math-flags?
9581
9582 SDValue N2;
9583 // Convert to float.
9584 // float4 yf = vcvt_f32_s32(vmovl_s16(y));
9585 // float4 xf = vcvt_f32_s32(vmovl_s16(x));
9586 N0 = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: MVT::v4i32, Operand: N0);
9587 N1 = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: MVT::v4i32, Operand: N1);
9588 N0 = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: MVT::v4f32, Operand: N0);
9589 N1 = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: MVT::v4f32, Operand: N1);
9590
9591 // Use reciprocal estimate and one refinement step.
9592 // float4 recip = vrecpeq_f32(yf);
9593 // recip *= vrecpsq_f32(yf, recip);
9594 N2 = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: MVT::v4f32,
9595 N1: DAG.getConstant(Val: Intrinsic::arm_neon_vrecpe, DL: dl, VT: MVT::i32),
9596 N2: N1);
9597 N1 = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: MVT::v4f32,
9598 N1: DAG.getConstant(Val: Intrinsic::arm_neon_vrecps, DL: dl, VT: MVT::i32),
9599 N2: N1, N3: N2);
9600 N2 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::v4f32, N1, N2);
9601 // Because short has a smaller range than ushort, we can actually get away
9602 // with only a single newton step. This requires that we use a weird bias
9603 // of 89, however (again, this has been exhaustively tested).
9604 // float4 result = as_float4(as_int4(xf*recip) + 0x89);
9605 N0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::v4f32, N1: N0, N2);
9606 N0 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v4i32, Operand: N0);
9607 N1 = DAG.getConstant(Val: 0x89, DL: dl, VT: MVT::v4i32);
9608 N0 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::v4i32, N1: N0, N2: N1);
9609 N0 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v4f32, Operand: N0);
9610 // Convert back to integer and return.
9611 // return vmovn_s32(vcvt_s32_f32(result));
9612 N0 = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: dl, VT: MVT::v4i32, Operand: N0);
9613 N0 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::v4i16, Operand: N0);
9614 return N0;
9615}
9616
9617static SDValue LowerSDIV(SDValue Op, SelectionDAG &DAG,
9618 const ARMSubtarget *ST) {
9619 EVT VT = Op.getValueType();
9620 assert((VT == MVT::v4i16 || VT == MVT::v8i8) &&
9621 "unexpected type for custom-lowering ISD::SDIV");
9622
9623 SDLoc dl(Op);
9624 SDValue N0 = Op.getOperand(i: 0);
9625 SDValue N1 = Op.getOperand(i: 1);
9626 SDValue N2, N3;
9627
9628 if (VT == MVT::v8i8) {
9629 N0 = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: MVT::v8i16, Operand: N0);
9630 N1 = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: MVT::v8i16, Operand: N1);
9631
9632 N2 = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: MVT::v4i16, N1: N0,
9633 N2: DAG.getIntPtrConstant(Val: 4, DL: dl));
9634 N3 = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: MVT::v4i16, N1,
9635 N2: DAG.getIntPtrConstant(Val: 4, DL: dl));
9636 N0 = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: MVT::v4i16, N1: N0,
9637 N2: DAG.getIntPtrConstant(Val: 0, DL: dl));
9638 N1 = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: MVT::v4i16, N1,
9639 N2: DAG.getIntPtrConstant(Val: 0, DL: dl));
9640
9641 N0 = LowerSDIV_v4i8(X: N0, Y: N1, dl, DAG); // v4i16
9642 N2 = LowerSDIV_v4i8(X: N2, Y: N3, dl, DAG); // v4i16
9643
9644 N0 = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: MVT::v8i16, N1: N0, N2);
9645 N0 = LowerCONCAT_VECTORS(Op: N0, DAG, ST);
9646
9647 N0 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::v8i8, Operand: N0);
9648 return N0;
9649 }
9650 return LowerSDIV_v4i16(N0, N1, dl, DAG);
9651}
9652
9653static SDValue LowerUDIV(SDValue Op, SelectionDAG &DAG,
9654 const ARMSubtarget *ST) {
9655 // TODO: Should this propagate fast-math-flags?
9656 EVT VT = Op.getValueType();
9657 assert((VT == MVT::v4i16 || VT == MVT::v8i8) &&
9658 "unexpected type for custom-lowering ISD::UDIV");
9659
9660 SDLoc dl(Op);
9661 SDValue N0 = Op.getOperand(i: 0);
9662 SDValue N1 = Op.getOperand(i: 1);
9663 SDValue N2, N3;
9664
9665 if (VT == MVT::v8i8) {
9666 N0 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: MVT::v8i16, Operand: N0);
9667 N1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: MVT::v8i16, Operand: N1);
9668
9669 N2 = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: MVT::v4i16, N1: N0,
9670 N2: DAG.getIntPtrConstant(Val: 4, DL: dl));
9671 N3 = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: MVT::v4i16, N1,
9672 N2: DAG.getIntPtrConstant(Val: 4, DL: dl));
9673 N0 = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: MVT::v4i16, N1: N0,
9674 N2: DAG.getIntPtrConstant(Val: 0, DL: dl));
9675 N1 = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: MVT::v4i16, N1,
9676 N2: DAG.getIntPtrConstant(Val: 0, DL: dl));
9677
9678 N0 = LowerSDIV_v4i16(N0, N1, dl, DAG); // v4i16
9679 N2 = LowerSDIV_v4i16(N0: N2, N1: N3, dl, DAG); // v4i16
9680
9681 N0 = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: MVT::v8i16, N1: N0, N2);
9682 N0 = LowerCONCAT_VECTORS(Op: N0, DAG, ST);
9683
9684 N0 = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: MVT::v8i8,
9685 N1: DAG.getConstant(Val: Intrinsic::arm_neon_vqmovnsu, DL: dl,
9686 VT: MVT::i32),
9687 N2: N0);
9688 return N0;
9689 }
9690
9691 // v4i16 sdiv ... Convert to float.
9692 // float4 yf = vcvt_f32_s32(vmovl_u16(y));
9693 // float4 xf = vcvt_f32_s32(vmovl_u16(x));
9694 N0 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: MVT::v4i32, Operand: N0);
9695 N1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: MVT::v4i32, Operand: N1);
9696 N0 = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: MVT::v4f32, Operand: N0);
9697 SDValue BN1 = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: MVT::v4f32, Operand: N1);
9698
9699 // Use reciprocal estimate and two refinement steps.
9700 // float4 recip = vrecpeq_f32(yf);
9701 // recip *= vrecpsq_f32(yf, recip);
9702 // recip *= vrecpsq_f32(yf, recip);
9703 N2 = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: MVT::v4f32,
9704 N1: DAG.getConstant(Val: Intrinsic::arm_neon_vrecpe, DL: dl, VT: MVT::i32),
9705 N2: BN1);
9706 N1 = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: MVT::v4f32,
9707 N1: DAG.getConstant(Val: Intrinsic::arm_neon_vrecps, DL: dl, VT: MVT::i32),
9708 N2: BN1, N3: N2);
9709 N2 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::v4f32, N1, N2);
9710 N1 = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: MVT::v4f32,
9711 N1: DAG.getConstant(Val: Intrinsic::arm_neon_vrecps, DL: dl, VT: MVT::i32),
9712 N2: BN1, N3: N2);
9713 N2 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::v4f32, N1, N2);
9714 // Simply multiplying by the reciprocal estimate can leave us a few ulps
9715 // too low, so we add 2 ulps (exhaustive testing shows that this is enough,
9716 // and that it will never cause us to return an answer too large).
9717 // float4 result = as_float4(as_int4(xf*recip) + 2);
9718 N0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::v4f32, N1: N0, N2);
9719 N0 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v4i32, Operand: N0);
9720 N1 = DAG.getConstant(Val: 2, DL: dl, VT: MVT::v4i32);
9721 N0 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::v4i32, N1: N0, N2: N1);
9722 N0 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v4f32, Operand: N0);
9723 // Convert back to integer and return.
9724 // return vmovn_u32(vcvt_s32_f32(result));
9725 N0 = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: dl, VT: MVT::v4i32, Operand: N0);
9726 N0 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::v4i16, Operand: N0);
9727 return N0;
9728}
9729
9730static SDValue LowerADDSUBO_CARRY(SDValue Op, SelectionDAG &DAG,
9731 unsigned Opcode, bool IsSigned) {
9732 EVT VT0 = Op.getValue(R: 0).getValueType();
9733 EVT VT1 = Op.getValue(R: 1).getValueType();
9734
9735 bool InvertCarry = Opcode == ARMISD::SUBE;
9736 SDValue OpLHS = Op.getOperand(i: 0);
9737 SDValue OpRHS = Op.getOperand(i: 1);
9738 SDValue OpCarryIn = valueToCarryFlag(Value: Op.getOperand(i: 2), DAG, Invert: InvertCarry);
9739
9740 SDLoc DL(Op);
9741
9742 SDValue Result = DAG.getNode(Opcode, DL, VTList: DAG.getVTList(VT1: VT0, VT2: MVT::i32), N1: OpLHS,
9743 N2: OpRHS, N3: OpCarryIn);
9744
9745 SDValue OutFlag =
9746 IsSigned ? overflowFlagToValue(Flags: Result.getValue(R: 1), VT: VT1, DAG)
9747 : carryFlagToValue(Flags: Result.getValue(R: 1), VT: VT1, DAG, Invert: InvertCarry);
9748
9749 return DAG.getMergeValues(Ops: {Result, OutFlag}, dl: DL);
9750}
9751
9752SDValue ARMTargetLowering::LowerWindowsDIVLibCall(SDValue Op, SelectionDAG &DAG,
9753 bool Signed,
9754 SDValue &Chain) const {
9755 EVT VT = Op.getValueType();
9756 assert((VT == MVT::i32 || VT == MVT::i64) &&
9757 "unexpected type for custom lowering DIV");
9758 SDLoc dl(Op);
9759
9760 const auto &DL = DAG.getDataLayout();
9761 RTLIB::Libcall LC;
9762 if (Signed)
9763 LC = VT == MVT::i32 ? RTLIB::SDIVREM_I32 : RTLIB::SDIVREM_I64;
9764 else
9765 LC = VT == MVT::i32 ? RTLIB::UDIVREM_I32 : RTLIB::UDIVREM_I64;
9766
9767 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
9768 SDValue ES = DAG.getExternalSymbol(LCImpl, VT: getPointerTy(DL));
9769
9770 ARMTargetLowering::ArgListTy Args;
9771
9772 for (auto AI : {1, 0}) {
9773 SDValue Operand = Op.getOperand(i: AI);
9774 Args.emplace_back(args&: Operand,
9775 args: Operand.getValueType().getTypeForEVT(Context&: *DAG.getContext()));
9776 }
9777
9778 CallLoweringInfo CLI(DAG);
9779 CLI.setDebugLoc(dl).setChain(Chain).setCallee(
9780 CC: DAG.getLibcalls().getLibcallImplCallingConv(Call: LCImpl),
9781 ResultType: VT.getTypeForEVT(Context&: *DAG.getContext()), Target: ES, ArgsList: std::move(Args));
9782
9783 return LowerCallTo(CLI).first;
9784}
9785
9786// This is a code size optimisation: return the original SDIV node to
9787// DAGCombiner when we don't want to expand SDIV into a sequence of
9788// instructions, and an empty node otherwise which will cause the
9789// SDIV to be expanded in DAGCombine.
9790SDValue
9791ARMTargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
9792 SelectionDAG &DAG,
9793 SmallVectorImpl<SDNode *> &Created) const {
9794 // TODO: Support SREM
9795 if (N->getOpcode() != ISD::SDIV)
9796 return SDValue();
9797
9798 const auto &ST = DAG.getSubtarget<ARMSubtarget>();
9799 const bool MinSize = ST.hasMinSize();
9800 const bool HasDivide = ST.isThumb() ? ST.hasDivideInThumbMode()
9801 : ST.hasDivideInARMMode();
9802
9803 // Don't touch vector types; rewriting this may lead to scalarizing
9804 // the int divs.
9805 if (N->getOperand(Num: 0).getValueType().isVector())
9806 return SDValue();
9807
9808 // Bail if MinSize is not set, and also for both ARM and Thumb mode we need
9809 // hwdiv support for this to be really profitable.
9810 if (!(MinSize && HasDivide))
9811 return SDValue();
9812
9813 // ARM mode is a bit simpler than Thumb: we can handle large power
9814 // of 2 immediates with 1 mov instruction; no further checks required,
9815 // just return the sdiv node.
9816 if (!ST.isThumb())
9817 return SDValue(N, 0);
9818
9819 // In Thumb mode, immediates larger than 128 need a wide 4-byte MOV,
9820 // and thus lose the code size benefits of a MOVS that requires only 2.
9821 // TargetTransformInfo and 'getIntImmCodeSizeCost' could be helpful here,
9822 // but as it's doing exactly this, it's not worth the trouble to get TTI.
9823 if (Divisor.sgt(RHS: 128))
9824 return SDValue();
9825
9826 return SDValue(N, 0);
9827}
9828
9829SDValue ARMTargetLowering::LowerDIV_Windows(SDValue Op, SelectionDAG &DAG,
9830 bool Signed) const {
9831 assert(Op.getValueType() == MVT::i32 &&
9832 "unexpected type for custom lowering DIV");
9833 SDLoc dl(Op);
9834
9835 SDValue DBZCHK = DAG.getNode(Opcode: ARMISD::WIN__DBZCHK, DL: dl, VT: MVT::Other,
9836 N1: DAG.getEntryNode(), N2: Op.getOperand(i: 1));
9837
9838 return LowerWindowsDIVLibCall(Op, DAG, Signed, Chain&: DBZCHK);
9839}
9840
9841static SDValue WinDBZCheckDenominator(SelectionDAG &DAG, SDNode *N, SDValue InChain) {
9842 SDLoc DL(N);
9843 SDValue Op = N->getOperand(Num: 1);
9844 if (N->getValueType(ResNo: 0) == MVT::i32)
9845 return DAG.getNode(Opcode: ARMISD::WIN__DBZCHK, DL, VT: MVT::Other, N1: InChain, N2: Op);
9846 SDValue Lo, Hi;
9847 std::tie(args&: Lo, args&: Hi) = DAG.SplitScalar(N: Op, DL, LoVT: MVT::i32, HiVT: MVT::i32);
9848 return DAG.getNode(Opcode: ARMISD::WIN__DBZCHK, DL, VT: MVT::Other, N1: InChain,
9849 N2: DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: Lo, N2: Hi));
9850}
9851
9852void ARMTargetLowering::ExpandDIV_Windows(
9853 SDValue Op, SelectionDAG &DAG, bool Signed,
9854 SmallVectorImpl<SDValue> &Results) const {
9855 const auto &DL = DAG.getDataLayout();
9856
9857 assert(Op.getValueType() == MVT::i64 &&
9858 "unexpected type for custom lowering DIV");
9859 SDLoc dl(Op);
9860
9861 SDValue DBZCHK = WinDBZCheckDenominator(DAG, N: Op.getNode(), InChain: DAG.getEntryNode());
9862
9863 SDValue Result = LowerWindowsDIVLibCall(Op, DAG, Signed, Chain&: DBZCHK);
9864
9865 SDValue Lower = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i32, Operand: Result);
9866 SDValue Upper = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i64, N1: Result,
9867 N2: DAG.getConstant(Val: 32, DL: dl, VT: getPointerTy(DL)));
9868 Upper = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i32, Operand: Upper);
9869
9870 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: MVT::i64, N1: Lower, N2: Upper));
9871}
9872
9873std::pair<SDValue, SDValue>
9874ARMTargetLowering::LowerAEABIUnalignedLoad(SDValue Op,
9875 SelectionDAG &DAG) const {
9876 // If we have an unaligned load from a i32 or i64 that would normally be
9877 // split into separate ldrb's, we can use the __aeabi_uread4/__aeabi_uread8
9878 // functions instead.
9879 LoadSDNode *LD = cast<LoadSDNode>(Val: Op.getNode());
9880 EVT MemVT = LD->getMemoryVT();
9881 if (MemVT != MVT::i32 && MemVT != MVT::i64)
9882 return std::make_pair(x: SDValue(), y: SDValue());
9883
9884 const auto &MF = DAG.getMachineFunction();
9885 unsigned AS = LD->getAddressSpace();
9886 Align Alignment = LD->getAlign();
9887 const DataLayout &DL = DAG.getDataLayout();
9888 bool AllowsUnaligned = Subtarget->allowsUnalignedMem();
9889 RTLIB::Libcall LC =
9890 (MemVT == MVT::i32) ? RTLIB::AEABI_UREAD4 : RTLIB::AEABI_UREAD8;
9891
9892 if (MF.getFunction().hasMinSize() && !AllowsUnaligned &&
9893 Alignment <= llvm::Align(2) && DAG.getLibcalls().getLibcallImpl(Call: LC)) {
9894 MakeLibCallOptions Opts;
9895 SDLoc dl(Op);
9896
9897 auto Pair = makeLibCall(DAG, LC, RetVT: MemVT.getSimpleVT(), Ops: LD->getBasePtr(),
9898 CallOptions: Opts, dl, Chain: LD->getChain());
9899
9900 // If necessary, extend the node to 64bit
9901 if (LD->getExtensionType() != ISD::NON_EXTLOAD) {
9902 unsigned ExtType = LD->getExtensionType() == ISD::SEXTLOAD
9903 ? ISD::SIGN_EXTEND
9904 : ISD::ZERO_EXTEND;
9905 SDValue EN = DAG.getNode(Opcode: ExtType, DL: dl, VT: LD->getValueType(ResNo: 0), Operand: Pair.first);
9906 Pair.first = EN;
9907 }
9908 return Pair;
9909 }
9910
9911 // Default expand to individual loads
9912 if (!allowsMemoryAccess(Context&: *DAG.getContext(), DL, VT: MemVT, AddrSpace: AS, Alignment))
9913 return expandUnalignedLoad(LD, DAG);
9914 return std::make_pair(x: SDValue(), y: SDValue());
9915}
9916
9917SDValue ARMTargetLowering::LowerAEABIUnalignedStore(SDValue Op,
9918 SelectionDAG &DAG) const {
9919 // If we have an unaligned store to a i32 or i64 that would normally be
9920 // split into separate ldrb's, we can use the __aeabi_uwrite4/__aeabi_uwrite8
9921 // functions instead.
9922 StoreSDNode *ST = cast<StoreSDNode>(Val: Op.getNode());
9923 EVT MemVT = ST->getMemoryVT();
9924 if (MemVT != MVT::i32 && MemVT != MVT::i64)
9925 return SDValue();
9926
9927 const auto &MF = DAG.getMachineFunction();
9928 unsigned AS = ST->getAddressSpace();
9929 Align Alignment = ST->getAlign();
9930 const DataLayout &DL = DAG.getDataLayout();
9931 bool AllowsUnaligned = Subtarget->allowsUnalignedMem();
9932 RTLIB::Libcall LC =
9933 (MemVT == MVT::i32) ? RTLIB::AEABI_UWRITE4 : RTLIB::AEABI_UWRITE8;
9934
9935 if (MF.getFunction().hasMinSize() && !AllowsUnaligned &&
9936 Alignment <= llvm::Align(2) && DAG.getLibcalls().getLibcallImpl(Call: LC)) {
9937
9938 SDLoc dl(Op);
9939
9940 // If necessary, trunc the value to 32bit
9941 SDValue StoreVal = ST->getOperand(Num: 1);
9942 if (ST->isTruncatingStore())
9943 StoreVal = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MemVT, Operand: ST->getOperand(Num: 1));
9944
9945 MakeLibCallOptions Opts;
9946 auto CallResult =
9947 makeLibCall(DAG, LC, RetVT: MVT::isVoid, Ops: {StoreVal, ST->getBasePtr()}, CallOptions: Opts,
9948 dl, Chain: ST->getChain());
9949
9950 return CallResult.second;
9951 }
9952
9953 // Default expand to individual stores
9954 if (!allowsMemoryAccess(Context&: *DAG.getContext(), DL, VT: MemVT, AddrSpace: AS, Alignment))
9955 return expandUnalignedStore(ST, DAG);
9956 return SDValue();
9957}
9958
9959static SDValue LowerPredicateLoad(SDValue Op, SelectionDAG &DAG) {
9960 LoadSDNode *LD = cast<LoadSDNode>(Val: Op.getNode());
9961 EVT MemVT = LD->getMemoryVT();
9962 assert((MemVT == MVT::v2i1 || MemVT == MVT::v4i1 || MemVT == MVT::v8i1 ||
9963 MemVT == MVT::v16i1) &&
9964 "Expected a predicate type!");
9965 assert(MemVT == Op.getValueType());
9966 assert(LD->getExtensionType() == ISD::NON_EXTLOAD &&
9967 "Expected a non-extending load");
9968 assert(LD->isUnindexed() && "Expected a unindexed load");
9969
9970 // The basic MVE VLDR on a v2i1/v4i1/v8i1 actually loads the entire 16bit
9971 // predicate, with the "v4i1" bits spread out over the 16 bits loaded. We
9972 // need to make sure that 8/4/2 bits are actually loaded into the correct
9973 // place, which means loading the value and then shuffling the values into
9974 // the bottom bits of the predicate.
9975 // Equally, VLDR for an v16i1 will actually load 32bits (so will be incorrect
9976 // for BE).
9977 // Speaking of BE, apparently the rest of llvm will assume a reverse order to
9978 // a natural VMSR(load), so needs to be reversed.
9979
9980 SDLoc dl(Op);
9981 SDValue Load = DAG.getExtLoad(
9982 ExtType: ISD::EXTLOAD, dl, VT: MVT::i32, Chain: LD->getChain(), Ptr: LD->getBasePtr(),
9983 MemVT: EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: MemVT.getSizeInBits()),
9984 MMO: LD->getMemOperand());
9985 SDValue Val = Load;
9986 if (DAG.getDataLayout().isBigEndian())
9987 Val = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i32,
9988 N1: DAG.getNode(Opcode: ISD::BITREVERSE, DL: dl, VT: MVT::i32, Operand: Load),
9989 N2: DAG.getConstant(Val: 32 - MemVT.getSizeInBits(), DL: dl, VT: MVT::i32));
9990 SDValue Pred = DAG.getNode(Opcode: ARMISD::PREDICATE_CAST, DL: dl, VT: MVT::v16i1, Operand: Val);
9991 if (MemVT != MVT::v16i1)
9992 Pred = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: MemVT, N1: Pred,
9993 N2: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
9994 return DAG.getMergeValues(Ops: {Pred, Load.getValue(R: 1)}, dl);
9995}
9996
9997void ARMTargetLowering::LowerLOAD(SDNode *N, SmallVectorImpl<SDValue> &Results,
9998 SelectionDAG &DAG) const {
9999 LoadSDNode *LD = cast<LoadSDNode>(Val: N);
10000 EVT MemVT = LD->getMemoryVT();
10001
10002 if (MemVT == MVT::i64 && Subtarget->hasV5TEOps() &&
10003 !Subtarget->isThumb1Only() && LD->isVolatile() &&
10004 LD->getAlign() >= Subtarget->getDualLoadStoreAlignment()) {
10005 assert(LD->isUnindexed() && "Loads should be unindexed at this point.");
10006 SDLoc dl(N);
10007 SDValue Result = DAG.getMemIntrinsicNode(
10008 Opcode: ARMISD::LDRD, dl, VTList: DAG.getVTList(VTs: {MVT::i32, MVT::i32, MVT::Other}),
10009 Ops: {LD->getChain(), LD->getBasePtr()}, MemVT, MMO: LD->getMemOperand());
10010 SDValue Lo = Result.getValue(R: DAG.getDataLayout().isLittleEndian() ? 0 : 1);
10011 SDValue Hi = Result.getValue(R: DAG.getDataLayout().isLittleEndian() ? 1 : 0);
10012 SDValue Pair = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: MVT::i64, N1: Lo, N2: Hi);
10013 Results.append(IL: {Pair, Result.getValue(R: 2)});
10014 } else if (MemVT == MVT::i32 || MemVT == MVT::i64) {
10015 auto Pair = LowerAEABIUnalignedLoad(Op: SDValue(N, 0), DAG);
10016 if (Pair.first) {
10017 Results.push_back(Elt: Pair.first);
10018 Results.push_back(Elt: Pair.second);
10019 }
10020 }
10021}
10022
10023static SDValue LowerPredicateStore(SDValue Op, SelectionDAG &DAG) {
10024 StoreSDNode *ST = cast<StoreSDNode>(Val: Op.getNode());
10025 EVT MemVT = ST->getMemoryVT();
10026 assert((MemVT == MVT::v2i1 || MemVT == MVT::v4i1 || MemVT == MVT::v8i1 ||
10027 MemVT == MVT::v16i1) &&
10028 "Expected a predicate type!");
10029 assert(MemVT == ST->getValue().getValueType());
10030 assert(!ST->isTruncatingStore() && "Expected a non-extending store");
10031 assert(ST->isUnindexed() && "Expected a unindexed store");
10032
10033 // Only store the v2i1 or v4i1 or v8i1 worth of bits, via a buildvector with
10034 // top bits unset and a scalar store.
10035 SDLoc dl(Op);
10036 SDValue Build = ST->getValue();
10037 if (MemVT != MVT::v16i1) {
10038 SmallVector<SDValue, 16> Ops;
10039 for (unsigned I = 0; I < MemVT.getVectorNumElements(); I++) {
10040 unsigned Elt = DAG.getDataLayout().isBigEndian()
10041 ? MemVT.getVectorNumElements() - I - 1
10042 : I;
10043 Ops.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: MVT::i32, N1: Build,
10044 N2: DAG.getConstant(Val: Elt, DL: dl, VT: MVT::i32)));
10045 }
10046 for (unsigned I = MemVT.getVectorNumElements(); I < 16; I++)
10047 Ops.push_back(Elt: DAG.getUNDEF(VT: MVT::i32));
10048 Build = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: dl, VT: MVT::v16i1, Ops);
10049 }
10050 SDValue GRP = DAG.getNode(Opcode: ARMISD::PREDICATE_CAST, DL: dl, VT: MVT::i32, Operand: Build);
10051 if (MemVT == MVT::v16i1 && DAG.getDataLayout().isBigEndian())
10052 GRP = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i32,
10053 N1: DAG.getNode(Opcode: ISD::BITREVERSE, DL: dl, VT: MVT::i32, Operand: GRP),
10054 N2: DAG.getConstant(Val: 16, DL: dl, VT: MVT::i32));
10055 return DAG.getTruncStore(
10056 Chain: ST->getChain(), dl, Val: GRP, Ptr: ST->getBasePtr(),
10057 SVT: EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: MemVT.getSizeInBits()),
10058 MMO: ST->getMemOperand());
10059}
10060
10061SDValue ARMTargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG,
10062 const ARMSubtarget *Subtarget) const {
10063 StoreSDNode *ST = cast<StoreSDNode>(Val: Op.getNode());
10064 EVT MemVT = ST->getMemoryVT();
10065
10066 if (MemVT == MVT::i64 && Subtarget->hasV5TEOps() &&
10067 !Subtarget->isThumb1Only() && ST->isVolatile() &&
10068 ST->getAlign() >= Subtarget->getDualLoadStoreAlignment()) {
10069 assert(ST->isUnindexed() && "Stores should be unindexed at this point.");
10070 SDNode *N = Op.getNode();
10071 SDLoc dl(N);
10072
10073 SDValue Lo = DAG.getNode(
10074 Opcode: ISD::EXTRACT_ELEMENT, DL: dl, VT: MVT::i32, N1: ST->getValue(),
10075 N2: DAG.getTargetConstant(Val: DAG.getDataLayout().isLittleEndian() ? 0 : 1, DL: dl,
10076 VT: MVT::i32));
10077 SDValue Hi = DAG.getNode(
10078 Opcode: ISD::EXTRACT_ELEMENT, DL: dl, VT: MVT::i32, N1: ST->getValue(),
10079 N2: DAG.getTargetConstant(Val: DAG.getDataLayout().isLittleEndian() ? 1 : 0, DL: dl,
10080 VT: MVT::i32));
10081
10082 return DAG.getMemIntrinsicNode(Opcode: ARMISD::STRD, dl, VTList: DAG.getVTList(VT: MVT::Other),
10083 Ops: {ST->getChain(), Lo, Hi, ST->getBasePtr()},
10084 MemVT, MMO: ST->getMemOperand());
10085 } else if (Subtarget->hasMVEIntegerOps() &&
10086 ((MemVT == MVT::v2i1 || MemVT == MVT::v4i1 || MemVT == MVT::v8i1 ||
10087 MemVT == MVT::v16i1))) {
10088 return LowerPredicateStore(Op, DAG);
10089 } else if (MemVT == MVT::i32 || MemVT == MVT::i64) {
10090 return LowerAEABIUnalignedStore(Op, DAG);
10091 }
10092 return SDValue();
10093}
10094
10095static bool isZeroVector(SDValue N) {
10096 return (ISD::isBuildVectorAllZeros(N: N.getNode()) ||
10097 (N->getOpcode() == ARMISD::VMOVIMM &&
10098 isNullConstant(V: N->getOperand(Num: 0))));
10099}
10100
10101static SDValue LowerMLOAD(SDValue Op, SelectionDAG &DAG) {
10102 MaskedLoadSDNode *N = cast<MaskedLoadSDNode>(Val: Op.getNode());
10103 MVT VT = Op.getSimpleValueType();
10104 SDValue Mask = N->getMask();
10105 SDValue PassThru = N->getPassThru();
10106 SDLoc dl(Op);
10107
10108 if (isZeroVector(N: PassThru))
10109 return Op;
10110
10111 // MVE Masked loads use zero as the passthru value. Here we convert undef to
10112 // zero too, and other values are lowered to a select.
10113 SDValue ZeroVec = DAG.getNode(Opcode: ARMISD::VMOVIMM, DL: dl, VT,
10114 Operand: DAG.getTargetConstant(Val: 0, DL: dl, VT: MVT::i32));
10115 SDValue NewLoad = DAG.getMaskedLoad(
10116 VT, dl, Chain: N->getChain(), Base: N->getBasePtr(), Offset: N->getOffset(), Mask, Src0: ZeroVec,
10117 MemVT: N->getMemoryVT(), MMO: N->getMemOperand(), AM: N->getAddressingMode(),
10118 N->getExtensionType(), IsExpanding: N->isExpandingLoad());
10119 SDValue Combo = NewLoad;
10120 bool PassThruIsCastZero = (PassThru.getOpcode() == ISD::BITCAST ||
10121 PassThru.getOpcode() == ARMISD::VECTOR_REG_CAST) &&
10122 isZeroVector(N: PassThru->getOperand(Num: 0));
10123 if (!PassThru.isUndef() && !PassThruIsCastZero)
10124 Combo = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT, N1: Mask, N2: NewLoad, N3: PassThru);
10125 return DAG.getMergeValues(Ops: {Combo, NewLoad.getValue(R: 1)}, dl);
10126}
10127
10128static SDValue LowerVecReduce(SDValue Op, SelectionDAG &DAG,
10129 const ARMSubtarget *ST) {
10130 if (!ST->hasMVEIntegerOps())
10131 return SDValue();
10132
10133 SDLoc dl(Op);
10134 unsigned BaseOpcode = 0;
10135 switch (Op->getOpcode()) {
10136 default: llvm_unreachable("Expected VECREDUCE opcode");
10137 case ISD::VECREDUCE_FADD: BaseOpcode = ISD::FADD; break;
10138 case ISD::VECREDUCE_FMUL: BaseOpcode = ISD::FMUL; break;
10139 case ISD::VECREDUCE_MUL: BaseOpcode = ISD::MUL; break;
10140 case ISD::VECREDUCE_AND: BaseOpcode = ISD::AND; break;
10141 case ISD::VECREDUCE_OR: BaseOpcode = ISD::OR; break;
10142 case ISD::VECREDUCE_XOR: BaseOpcode = ISD::XOR; break;
10143 case ISD::VECREDUCE_FMAX: BaseOpcode = ISD::FMAXNUM; break;
10144 case ISD::VECREDUCE_FMIN: BaseOpcode = ISD::FMINNUM; break;
10145 }
10146
10147 SDValue Op0 = Op->getOperand(Num: 0);
10148 EVT VT = Op0.getValueType();
10149 EVT EltVT = VT.getVectorElementType();
10150 unsigned NumElts = VT.getVectorNumElements();
10151 unsigned NumActiveLanes = NumElts;
10152
10153 assert((NumActiveLanes == 16 || NumActiveLanes == 8 || NumActiveLanes == 4 ||
10154 NumActiveLanes == 2) &&
10155 "Only expected a power 2 vector size");
10156
10157 // Use Mul(X, Rev(X)) until 4 items remain. Going down to 4 vector elements
10158 // allows us to easily extract vector elements from the lanes.
10159 while (NumActiveLanes > 4) {
10160 unsigned RevOpcode = NumActiveLanes == 16 ? ARMISD::VREV16 : ARMISD::VREV32;
10161 SDValue Rev = DAG.getNode(Opcode: RevOpcode, DL: dl, VT, Operand: Op0);
10162 Op0 = DAG.getNode(Opcode: BaseOpcode, DL: dl, VT, N1: Op0, N2: Rev);
10163 NumActiveLanes /= 2;
10164 }
10165
10166 SDValue Res;
10167 if (NumActiveLanes == 4) {
10168 // The remaining 4 elements are summed sequentially
10169 SDValue Ext0 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Op0,
10170 N2: DAG.getConstant(Val: 0 * NumElts / 4, DL: dl, VT: MVT::i32));
10171 SDValue Ext1 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Op0,
10172 N2: DAG.getConstant(Val: 1 * NumElts / 4, DL: dl, VT: MVT::i32));
10173 SDValue Ext2 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Op0,
10174 N2: DAG.getConstant(Val: 2 * NumElts / 4, DL: dl, VT: MVT::i32));
10175 SDValue Ext3 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Op0,
10176 N2: DAG.getConstant(Val: 3 * NumElts / 4, DL: dl, VT: MVT::i32));
10177 SDValue Res0 = DAG.getNode(Opcode: BaseOpcode, DL: dl, VT: EltVT, N1: Ext0, N2: Ext1, Flags: Op->getFlags());
10178 SDValue Res1 = DAG.getNode(Opcode: BaseOpcode, DL: dl, VT: EltVT, N1: Ext2, N2: Ext3, Flags: Op->getFlags());
10179 Res = DAG.getNode(Opcode: BaseOpcode, DL: dl, VT: EltVT, N1: Res0, N2: Res1, Flags: Op->getFlags());
10180 } else {
10181 SDValue Ext0 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Op0,
10182 N2: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
10183 SDValue Ext1 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Op0,
10184 N2: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
10185 Res = DAG.getNode(Opcode: BaseOpcode, DL: dl, VT: EltVT, N1: Ext0, N2: Ext1, Flags: Op->getFlags());
10186 }
10187
10188 // Result type may be wider than element type.
10189 if (EltVT != Op->getValueType(ResNo: 0))
10190 Res = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: Op->getValueType(ResNo: 0), Operand: Res);
10191 return Res;
10192}
10193
10194static SDValue LowerVecReduceF(SDValue Op, SelectionDAG &DAG,
10195 const ARMSubtarget *ST) {
10196 if (!ST->hasMVEFloatOps())
10197 return SDValue();
10198 return LowerVecReduce(Op, DAG, ST);
10199}
10200
10201static SDValue LowerVecReduceMinMax(SDValue Op, SelectionDAG &DAG,
10202 const ARMSubtarget *ST) {
10203 if (!ST->hasNEON())
10204 return SDValue();
10205
10206 SDLoc dl(Op);
10207 SDValue Op0 = Op->getOperand(Num: 0);
10208 EVT VT = Op0.getValueType();
10209 EVT EltVT = VT.getVectorElementType();
10210
10211 unsigned PairwiseIntrinsic = 0;
10212 switch (Op->getOpcode()) {
10213 default:
10214 llvm_unreachable("Expected VECREDUCE opcode");
10215 case ISD::VECREDUCE_UMIN:
10216 PairwiseIntrinsic = Intrinsic::arm_neon_vpminu;
10217 break;
10218 case ISD::VECREDUCE_UMAX:
10219 PairwiseIntrinsic = Intrinsic::arm_neon_vpmaxu;
10220 break;
10221 case ISD::VECREDUCE_SMIN:
10222 PairwiseIntrinsic = Intrinsic::arm_neon_vpmins;
10223 break;
10224 case ISD::VECREDUCE_SMAX:
10225 PairwiseIntrinsic = Intrinsic::arm_neon_vpmaxs;
10226 break;
10227 }
10228 SDValue PairwiseOp = DAG.getConstant(Val: PairwiseIntrinsic, DL: dl, VT: MVT::i32);
10229
10230 unsigned NumElts = VT.getVectorNumElements();
10231 unsigned NumActiveLanes = NumElts;
10232
10233 assert((NumActiveLanes == 16 || NumActiveLanes == 8 || NumActiveLanes == 4 ||
10234 NumActiveLanes == 2) &&
10235 "Only expected a power 2 vector size");
10236
10237 // Split 128-bit vectors, since vpmin/max takes 2 64-bit vectors.
10238 if (VT.is128BitVector()) {
10239 SDValue Lo, Hi;
10240 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Op0, DL: dl);
10241 VT = Lo.getValueType();
10242 Op0 = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT, Ops: {PairwiseOp, Lo, Hi});
10243 NumActiveLanes /= 2;
10244 }
10245
10246 // Use pairwise reductions until one lane remains
10247 while (NumActiveLanes > 1) {
10248 Op0 = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT, Ops: {PairwiseOp, Op0, Op0});
10249 NumActiveLanes /= 2;
10250 }
10251
10252 SDValue Res = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Op0,
10253 N2: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
10254
10255 // Result type may be wider than element type.
10256 if (EltVT != Op.getValueType()) {
10257 unsigned Extend = 0;
10258 switch (Op->getOpcode()) {
10259 default:
10260 llvm_unreachable("Expected VECREDUCE opcode");
10261 case ISD::VECREDUCE_UMIN:
10262 case ISD::VECREDUCE_UMAX:
10263 Extend = ISD::ZERO_EXTEND;
10264 break;
10265 case ISD::VECREDUCE_SMIN:
10266 case ISD::VECREDUCE_SMAX:
10267 Extend = ISD::SIGN_EXTEND;
10268 break;
10269 }
10270 Res = DAG.getNode(Opcode: Extend, DL: dl, VT: Op.getValueType(), Operand: Res);
10271 }
10272 return Res;
10273}
10274
10275static SDValue LowerAtomicLoadStore(SDValue Op, SelectionDAG &DAG) {
10276 if (isStrongerThanMonotonic(AO: cast<AtomicSDNode>(Val&: Op)->getSuccessOrdering()))
10277 // Acquire/Release load/store is not legal for targets without a dmb or
10278 // equivalent available.
10279 return SDValue();
10280
10281 // Monotonic load/store is legal for all targets.
10282 return Op;
10283}
10284
10285static void ReplaceREADCYCLECOUNTER(SDNode *N,
10286 SmallVectorImpl<SDValue> &Results,
10287 SelectionDAG &DAG,
10288 const ARMSubtarget *Subtarget) {
10289 SDLoc DL(N);
10290 // Under Power Management extensions, the cycle-count is:
10291 // mrc p15, #0, <Rt>, c9, c13, #0
10292 SDValue Ops[] = { N->getOperand(Num: 0), // Chain
10293 DAG.getTargetConstant(Val: Intrinsic::arm_mrc, DL, VT: MVT::i32),
10294 DAG.getTargetConstant(Val: 15, DL, VT: MVT::i32),
10295 DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32),
10296 DAG.getTargetConstant(Val: 9, DL, VT: MVT::i32),
10297 DAG.getTargetConstant(Val: 13, DL, VT: MVT::i32),
10298 DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32)
10299 };
10300
10301 SDValue Cycles32 = DAG.getNode(Opcode: ISD::INTRINSIC_W_CHAIN, DL,
10302 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other), Ops);
10303 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: MVT::i64, N1: Cycles32,
10304 N2: DAG.getConstant(Val: 0, DL, VT: MVT::i32)));
10305 Results.push_back(Elt: Cycles32.getValue(R: 1));
10306}
10307
10308static SDValue createGPRPairNode2xi32(SelectionDAG &DAG, SDValue V0,
10309 SDValue V1) {
10310 SDLoc dl(V0.getNode());
10311 SDValue RegClass =
10312 DAG.getTargetConstant(Val: ARM::GPRPairRegClassID, DL: dl, VT: MVT::i32);
10313 SDValue SubReg0 = DAG.getTargetConstant(Val: ARM::gsub_0, DL: dl, VT: MVT::i32);
10314 SDValue SubReg1 = DAG.getTargetConstant(Val: ARM::gsub_1, DL: dl, VT: MVT::i32);
10315 const SDValue Ops[] = {RegClass, V0, SubReg0, V1, SubReg1};
10316 return SDValue(
10317 DAG.getMachineNode(Opcode: TargetOpcode::REG_SEQUENCE, dl, VT: MVT::Untyped, Ops), 0);
10318}
10319
10320static SDValue createGPRPairNodei64(SelectionDAG &DAG, SDValue V) {
10321 SDLoc dl(V.getNode());
10322 auto [VLo, VHi] = DAG.SplitScalar(N: V, DL: dl, LoVT: MVT::i32, HiVT: MVT::i32);
10323 bool isBigEndian = DAG.getDataLayout().isBigEndian();
10324 if (isBigEndian)
10325 std::swap(a&: VLo, b&: VHi);
10326 return createGPRPairNode2xi32(DAG, V0: VLo, V1: VHi);
10327}
10328
10329static void ReplaceCMP_SWAP_64Results(SDNode *N,
10330 SmallVectorImpl<SDValue> &Results,
10331 SelectionDAG &DAG) {
10332 assert(N->getValueType(0) == MVT::i64 &&
10333 "AtomicCmpSwap on types less than 64 should be legal");
10334 SDValue Ops[] = {
10335 createGPRPairNode2xi32(DAG, V0: N->getOperand(Num: 1),
10336 V1: DAG.getUNDEF(VT: MVT::i32)), // pointer, temp
10337 createGPRPairNodei64(DAG, V: N->getOperand(Num: 2)), // expected
10338 createGPRPairNodei64(DAG, V: N->getOperand(Num: 3)), // new
10339 N->getOperand(Num: 0), // chain in
10340 };
10341 SDNode *CmpSwap = DAG.getMachineNode(
10342 Opcode: ARM::CMP_SWAP_64, dl: SDLoc(N),
10343 VTs: DAG.getVTList(VT1: MVT::Untyped, VT2: MVT::Untyped, VT3: MVT::Other), Ops);
10344
10345 MachineMemOperand *MemOp = cast<MemSDNode>(Val: N)->getMemOperand();
10346 DAG.setNodeMemRefs(N: cast<MachineSDNode>(Val: CmpSwap), NewMemRefs: {MemOp});
10347
10348 bool isBigEndian = DAG.getDataLayout().isBigEndian();
10349
10350 SDValue Lo =
10351 DAG.getTargetExtractSubreg(SRIdx: isBigEndian ? ARM::gsub_1 : ARM::gsub_0,
10352 DL: SDLoc(N), VT: MVT::i32, Operand: SDValue(CmpSwap, 0));
10353 SDValue Hi =
10354 DAG.getTargetExtractSubreg(SRIdx: isBigEndian ? ARM::gsub_0 : ARM::gsub_1,
10355 DL: SDLoc(N), VT: MVT::i32, Operand: SDValue(CmpSwap, 0));
10356 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: SDLoc(N), VT: MVT::i64, N1: Lo, N2: Hi));
10357 Results.push_back(Elt: SDValue(CmpSwap, 2));
10358}
10359
10360SDValue ARMTargetLowering::LowerFSETCC(SDValue Op, SelectionDAG &DAG) const {
10361 SDLoc dl(Op);
10362 EVT VT = Op.getValueType();
10363 SDValue Chain = Op.getOperand(i: 0);
10364 SDValue LHS = Op.getOperand(i: 1);
10365 SDValue RHS = Op.getOperand(i: 2);
10366 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 3))->get();
10367 bool IsSignaling = Op.getOpcode() == ISD::STRICT_FSETCCS;
10368
10369 // If we don't have instructions of this float type then soften to a libcall
10370 // and use SETCC instead.
10371 if (isUnsupportedFloatingType(VT: LHS.getValueType())) {
10372 softenSetCCOperands(DAG, VT: LHS.getValueType(), NewLHS&: LHS, NewRHS&: RHS, CCCode&: CC, DL: dl, OldLHS: LHS, OldRHS: RHS,
10373 Chain, IsSignaling);
10374 if (!RHS.getNode()) {
10375 RHS = DAG.getConstant(Val: 0, DL: dl, VT: LHS.getValueType());
10376 CC = ISD::SETNE;
10377 }
10378 SDValue Result = DAG.getNode(Opcode: ISD::SETCC, DL: dl, VT, N1: LHS, N2: RHS,
10379 N3: DAG.getCondCode(Cond: CC));
10380 return DAG.getMergeValues(Ops: {Result, Chain}, dl);
10381 }
10382
10383 ARMCC::CondCodes CondCode, CondCode2;
10384 FPCCToARMCC(CC, CondCode, CondCode2);
10385
10386 SDValue True = DAG.getConstant(Val: 1, DL: dl, VT);
10387 SDValue False = DAG.getConstant(Val: 0, DL: dl, VT);
10388 SDValue ARMcc = DAG.getConstant(Val: CondCode, DL: dl, VT: MVT::i32);
10389 SDValue Cmp = getVFPCmp(LHS, RHS, DAG, dl, Signaling: IsSignaling);
10390 SDValue Result = getCMOV(dl, VT, FalseVal: False, TrueVal: True, ARMcc, Flags: Cmp, DAG);
10391 if (CondCode2 != ARMCC::AL) {
10392 ARMcc = DAG.getConstant(Val: CondCode2, DL: dl, VT: MVT::i32);
10393 Result = getCMOV(dl, VT, FalseVal: Result, TrueVal: True, ARMcc, Flags: Cmp, DAG);
10394 }
10395 return DAG.getMergeValues(Ops: {Result, Chain}, dl);
10396}
10397
10398SDValue ARMTargetLowering::LowerSPONENTRY(SDValue Op, SelectionDAG &DAG) const {
10399 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
10400
10401 EVT VT = getPointerTy(DL: DAG.getDataLayout());
10402 int FI = MFI.CreateFixedObject(Size: 4, SPOffset: 0, IsImmutable: false);
10403 return DAG.getFrameIndex(FI, VT);
10404}
10405
10406SDValue ARMTargetLowering::LowerFP_TO_BF16(SDValue Op,
10407 SelectionDAG &DAG) const {
10408 SDLoc DL(Op);
10409 MakeLibCallOptions CallOptions;
10410 MVT SVT = Op.getOperand(i: 0).getSimpleValueType();
10411 RTLIB::Libcall LC = RTLIB::getFPROUND(OpVT: SVT, RetVT: MVT::bf16);
10412 SDValue Res =
10413 makeLibCall(DAG, LC, RetVT: MVT::f32, Ops: Op.getOperand(i: 0), CallOptions, dl: DL).first;
10414 return DAG.getBitcast(VT: MVT::i32, V: Res);
10415}
10416
10417SDValue ARMTargetLowering::LowerCMP(SDValue Op, SelectionDAG &DAG) const {
10418 SDLoc dl(Op);
10419 SDValue LHS = Op.getOperand(i: 0);
10420 SDValue RHS = Op.getOperand(i: 1);
10421
10422 // Determine if this is signed or unsigned comparison
10423 bool IsSigned = (Op.getOpcode() == ISD::SCMP);
10424
10425 // Special case for Thumb1 UCMP only
10426 if (!IsSigned && Subtarget->isThumb1Only()) {
10427 // For Thumb unsigned comparison, use this sequence:
10428 // subs r2, r0, r1 ; r2 = LHS - RHS, sets flags
10429 // sbc r2, r2 ; r2 = r2 - r2 - !carry
10430 // cmp r1, r0 ; compare RHS with LHS
10431 // sbc r1, r1 ; r1 = r1 - r1 - !carry
10432 // subs r0, r2, r1 ; r0 = r2 - r1 (final result)
10433
10434 // First subtraction: LHS - RHS
10435 SDValue Sub1WithFlags = DAG.getNode(
10436 Opcode: ARMISD::SUBC, DL: dl, VTList: DAG.getVTList(VT1: MVT::i32, VT2: FlagsVT), N1: LHS, N2: RHS);
10437 SDValue Sub1Result = Sub1WithFlags.getValue(R: 0);
10438 SDValue Flags1 = Sub1WithFlags.getValue(R: 1);
10439
10440 // SUBE: Sub1Result - Sub1Result - !carry
10441 // This gives 0 if LHS >= RHS (unsigned), -1 if LHS < RHS (unsigned)
10442 SDValue Sbc1 =
10443 DAG.getNode(Opcode: ARMISD::SUBE, DL: dl, VTList: DAG.getVTList(VT1: MVT::i32, VT2: FlagsVT),
10444 N1: Sub1Result, N2: Sub1Result, N3: Flags1);
10445 SDValue Sbc1Result = Sbc1.getValue(R: 0);
10446
10447 // Second comparison: RHS vs LHS (reverse comparison)
10448 SDValue CmpFlags = DAG.getNode(Opcode: ARMISD::CMP, DL: dl, VT: FlagsVT, N1: RHS, N2: LHS);
10449
10450 // SUBE: RHS - RHS - !carry
10451 // This gives 0 if RHS <= LHS (unsigned), -1 if RHS > LHS (unsigned)
10452 SDValue Sbc2 = DAG.getNode(
10453 Opcode: ARMISD::SUBE, DL: dl, VTList: DAG.getVTList(VT1: MVT::i32, VT2: FlagsVT), N1: RHS, N2: RHS, N3: CmpFlags);
10454 SDValue Sbc2Result = Sbc2.getValue(R: 0);
10455
10456 // Final subtraction: Sbc1Result - Sbc2Result (no flags needed)
10457 SDValue Result =
10458 DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32, N1: Sbc1Result, N2: Sbc2Result);
10459 if (Op.getValueType() != MVT::i32)
10460 Result = DAG.getSExtOrTrunc(Op: Result, DL: dl, VT: Op.getValueType());
10461
10462 return Result;
10463 }
10464
10465 // For the ARM assembly pattern:
10466 // subs r0, r0, r1 ; subtract RHS from LHS and set flags
10467 // movgt r0, #1 ; if LHS > RHS, set result to 1 (GT for signed, HI for
10468 // unsigned) mvnlt r0, #0 ; if LHS < RHS, set result to -1 (LT for
10469 // signed, LO for unsigned)
10470 // ; if LHS == RHS, result remains 0 from the subs
10471
10472 // Optimization: if RHS is a subtraction against 0, use ADDC instead of SUBC
10473 unsigned Opcode = ARMISD::SUBC;
10474
10475 // Check if RHS is a subtraction against 0: (0 - X)
10476 if (RHS.getOpcode() == ISD::SUB) {
10477 SDValue SubLHS = RHS.getOperand(i: 0);
10478 SDValue SubRHS = RHS.getOperand(i: 1);
10479
10480 // Check if it's 0 - X
10481 if (isNullConstant(V: SubLHS)) {
10482 bool CanUseAdd = false;
10483 if (IsSigned) {
10484 // For SCMP: only if X is known to never be INT_MIN (to avoid overflow)
10485 if (RHS->getFlags().hasNoSignedWrap() || !DAG.computeKnownBits(Op: SubRHS)
10486 .getSignedMinValue()
10487 .isMinSignedValue()) {
10488 CanUseAdd = true;
10489 }
10490 } else {
10491 // For UCMP: only if X is known to never be zero
10492 if (DAG.isKnownNeverZero(Op: SubRHS)) {
10493 CanUseAdd = true;
10494 }
10495 }
10496
10497 if (CanUseAdd) {
10498 Opcode = ARMISD::ADDC;
10499 RHS = SubRHS; // Replace RHS with X, so we do LHS + X instead of
10500 // LHS - (0 - X)
10501 }
10502 }
10503 }
10504
10505 // Generate the operation with flags
10506 SDValue OpWithFlags =
10507 DAG.getNode(Opcode, DL: dl, VTList: DAG.getVTList(VT1: MVT::i32, VT2: FlagsVT), N1: LHS, N2: RHS);
10508
10509 SDValue OpResult = OpWithFlags.getValue(R: 0);
10510 SDValue Flags = OpWithFlags.getValue(R: 1);
10511
10512 // Constants for conditional moves
10513 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32);
10514 SDValue MinusOne = DAG.getAllOnesConstant(DL: dl, VT: MVT::i32);
10515
10516 // Select condition codes based on signed vs unsigned
10517 ARMCC::CondCodes GTCond = IsSigned ? ARMCC::GT : ARMCC::HI;
10518 ARMCC::CondCodes LTCond = IsSigned ? ARMCC::LT : ARMCC::LO;
10519
10520 // First conditional move: if greater than, set to 1
10521 SDValue GTCondValue = DAG.getConstant(Val: GTCond, DL: dl, VT: MVT::i32);
10522 SDValue Result1 = DAG.getNode(Opcode: ARMISD::CMOV, DL: dl, VT: MVT::i32, N1: OpResult, N2: One,
10523 N3: GTCondValue, N4: Flags);
10524
10525 // Second conditional move: if less than, set to -1
10526 SDValue LTCondValue = DAG.getConstant(Val: LTCond, DL: dl, VT: MVT::i32);
10527 SDValue Result2 = DAG.getNode(Opcode: ARMISD::CMOV, DL: dl, VT: MVT::i32, N1: Result1, N2: MinusOne,
10528 N3: LTCondValue, N4: Flags);
10529
10530 if (Op.getValueType() != MVT::i32)
10531 Result2 = DAG.getSExtOrTrunc(Op: Result2, DL: dl, VT: Op.getValueType());
10532
10533 return Result2;
10534}
10535
10536SDValue ARMTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
10537 LLVM_DEBUG(dbgs() << "Lowering node: "; Op.dump());
10538 switch (Op.getOpcode()) {
10539 default: llvm_unreachable("Don't know how to custom lower this!");
10540 case ISD::WRITE_REGISTER: return LowerWRITE_REGISTER(Op, DAG);
10541 case ISD::ConstantPool: return LowerConstantPool(Op, DAG);
10542 case ISD::BlockAddress: return LowerBlockAddress(Op, DAG);
10543 case ISD::GlobalAddress: return LowerGlobalAddress(Op, DAG);
10544 case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG);
10545 case ISD::SELECT: return LowerSELECT(Op, DAG);
10546 case ISD::SELECT_CC: return LowerSELECT_CC(Op, DAG);
10547 case ISD::BRCOND: return LowerBRCOND(Op, DAG);
10548 case ISD::BR_CC: return LowerBR_CC(Op, DAG);
10549 case ISD::BR_JT: return LowerBR_JT(Op, DAG);
10550 case ISD::VASTART: return LowerVASTART(Op, DAG);
10551 case ISD::ATOMIC_FENCE: return LowerATOMIC_FENCE(Op, DAG, Subtarget);
10552 case ISD::PREFETCH: return LowerPREFETCH(Op, DAG, Subtarget);
10553 case ISD::STRICT_UINT_TO_FP:
10554 case ISD::STRICT_SINT_TO_FP:
10555 case ISD::SINT_TO_FP:
10556 case ISD::UINT_TO_FP: return LowerINT_TO_FP(Op, DAG);
10557 case ISD::STRICT_FP_TO_SINT:
10558 case ISD::STRICT_FP_TO_UINT:
10559 case ISD::FP_TO_SINT:
10560 case ISD::FP_TO_UINT: return LowerFP_TO_INT(Op, DAG);
10561 case ISD::FP_TO_SINT_SAT:
10562 case ISD::FP_TO_UINT_SAT: return LowerFP_TO_INT_SAT(Op, DAG, Subtarget);
10563 case ISD::FCOPYSIGN: return LowerFCOPYSIGN(Op, DAG);
10564 case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG);
10565 case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG);
10566 case ISD::EH_SJLJ_SETJMP: return LowerEH_SJLJ_SETJMP(Op, DAG);
10567 case ISD::EH_SJLJ_LONGJMP: return LowerEH_SJLJ_LONGJMP(Op, DAG);
10568 case ISD::EH_SJLJ_SETUP_DISPATCH: return LowerEH_SJLJ_SETUP_DISPATCH(Op, DAG);
10569 case ISD::INTRINSIC_VOID: return LowerINTRINSIC_VOID(Op, DAG, Subtarget);
10570 case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG,
10571 Subtarget);
10572 case ISD::BITCAST: return ExpandBITCAST(N: Op.getNode(), DAG, Subtarget);
10573 case ISD::SHL:
10574 case ISD::SRL:
10575 case ISD::SRA: return LowerShift(N: Op.getNode(), DAG, ST: Subtarget);
10576 case ISD::SREM: return LowerREM(N: Op.getNode(), DAG);
10577 case ISD::UREM: return LowerREM(N: Op.getNode(), DAG);
10578 case ISD::SHL_PARTS: return LowerShiftLeftParts(Op, DAG);
10579 case ISD::SRL_PARTS:
10580 case ISD::SRA_PARTS: return LowerShiftRightParts(Op, DAG);
10581 case ISD::CTTZ:
10582 case ISD::CTTZ_ZERO_POISON: return LowerCTTZ(N: Op.getNode(), DAG, ST: Subtarget);
10583 case ISD::CTPOP: return LowerCTPOP(N: Op.getNode(), DAG, ST: Subtarget);
10584 case ISD::SETCC: return LowerVSETCC(Op, DAG, ST: Subtarget);
10585 case ISD::SETCCCARRY: return LowerSETCCCARRY(Op, DAG);
10586 case ISD::ConstantFP: return LowerConstantFP(Op, DAG, ST: Subtarget);
10587 case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG, ST: Subtarget);
10588 case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG, ST: Subtarget);
10589 case ISD::EXTRACT_SUBVECTOR: return LowerEXTRACT_SUBVECTOR(Op, DAG, ST: Subtarget);
10590 case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG);
10591 case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG, ST: Subtarget);
10592 case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG, ST: Subtarget);
10593 case ISD::TRUNCATE: return LowerTruncate(N: Op.getNode(), DAG, Subtarget);
10594 case ISD::SIGN_EXTEND:
10595 case ISD::ZERO_EXTEND: return LowerVectorExtend(N: Op.getNode(), DAG, Subtarget);
10596 case ISD::GET_ROUNDING: return LowerGET_ROUNDING(Op, DAG);
10597 case ISD::SET_ROUNDING: return LowerSET_ROUNDING(Op, DAG);
10598 case ISD::SET_FPMODE:
10599 return LowerSET_FPMODE(Op, DAG);
10600 case ISD::RESET_FPMODE:
10601 return LowerRESET_FPMODE(Op, DAG);
10602 case ISD::MUL: return LowerMUL(Op, DAG);
10603 case ISD::SDIV:
10604 if (getTargetMachine().getTargetTriple().isOSWindows() &&
10605 !Op.getValueType().isVector())
10606 return LowerDIV_Windows(Op, DAG, /* Signed */ true);
10607 return LowerSDIV(Op, DAG, ST: Subtarget);
10608 case ISD::UDIV:
10609 if (getTargetMachine().getTargetTriple().isOSWindows() &&
10610 !Op.getValueType().isVector())
10611 return LowerDIV_Windows(Op, DAG, /* Signed */ false);
10612 return LowerUDIV(Op, DAG, ST: Subtarget);
10613 case ISD::UADDO_CARRY:
10614 return LowerADDSUBO_CARRY(Op, DAG, Opcode: ARMISD::ADDE, IsSigned: false /*unsigned*/);
10615 case ISD::USUBO_CARRY:
10616 return LowerADDSUBO_CARRY(Op, DAG, Opcode: ARMISD::SUBE, IsSigned: false /*unsigned*/);
10617 case ISD::SADDO_CARRY:
10618 return LowerADDSUBO_CARRY(Op, DAG, Opcode: ARMISD::ADDE, IsSigned: true /*signed*/);
10619 case ISD::SSUBO_CARRY:
10620 return LowerADDSUBO_CARRY(Op, DAG, Opcode: ARMISD::SUBE, IsSigned: true /*signed*/);
10621 case ISD::UADDO:
10622 case ISD::USUBO:
10623 case ISD::UMULO:
10624 case ISD::SADDO:
10625 case ISD::SSUBO:
10626 case ISD::SMULO:
10627 return LowerALUO(Op, DAG);
10628 case ISD::SADDSAT:
10629 case ISD::SSUBSAT:
10630 case ISD::UADDSAT:
10631 case ISD::USUBSAT:
10632 return LowerADDSUBSAT(Op, DAG, Subtarget);
10633 case ISD::LOAD: {
10634 auto *LD = cast<LoadSDNode>(Val&: Op);
10635 EVT MemVT = LD->getMemoryVT();
10636 if (Subtarget->hasMVEIntegerOps() &&
10637 (MemVT == MVT::v2i1 || MemVT == MVT::v4i1 || MemVT == MVT::v8i1 ||
10638 MemVT == MVT::v16i1))
10639 return LowerPredicateLoad(Op, DAG);
10640
10641 auto Pair = LowerAEABIUnalignedLoad(Op, DAG);
10642 if (Pair.first)
10643 return DAG.getMergeValues(Ops: {Pair.first, Pair.second}, dl: SDLoc(Pair.first));
10644 return SDValue();
10645 }
10646 case ISD::STORE:
10647 return LowerSTORE(Op, DAG, Subtarget);
10648 case ISD::MLOAD:
10649 return LowerMLOAD(Op, DAG);
10650 case ISD::VECREDUCE_MUL:
10651 case ISD::VECREDUCE_AND:
10652 case ISD::VECREDUCE_OR:
10653 case ISD::VECREDUCE_XOR:
10654 return LowerVecReduce(Op, DAG, ST: Subtarget);
10655 case ISD::VECREDUCE_FADD:
10656 case ISD::VECREDUCE_FMUL:
10657 case ISD::VECREDUCE_FMIN:
10658 case ISD::VECREDUCE_FMAX:
10659 return LowerVecReduceF(Op, DAG, ST: Subtarget);
10660 case ISD::VECREDUCE_UMIN:
10661 case ISD::VECREDUCE_UMAX:
10662 case ISD::VECREDUCE_SMIN:
10663 case ISD::VECREDUCE_SMAX:
10664 return LowerVecReduceMinMax(Op, DAG, ST: Subtarget);
10665 case ISD::ATOMIC_LOAD:
10666 case ISD::ATOMIC_STORE:
10667 return LowerAtomicLoadStore(Op, DAG);
10668 case ISD::SDIVREM:
10669 case ISD::UDIVREM: return LowerDivRem(Op, DAG);
10670 case ISD::DYNAMIC_STACKALLOC:
10671 if (getTargetMachine().getTargetTriple().isOSWindows())
10672 return LowerDYNAMIC_STACKALLOC(Op, DAG);
10673 llvm_unreachable("Don't know how to custom lower this!");
10674 case ISD::STRICT_FP_ROUND:
10675 case ISD::FP_ROUND: return LowerFP_ROUND(Op, DAG);
10676 case ISD::STRICT_FP_EXTEND:
10677 case ISD::FP_EXTEND: return LowerFP_EXTEND(Op, DAG);
10678 case ISD::STRICT_FSETCC:
10679 case ISD::STRICT_FSETCCS: return LowerFSETCC(Op, DAG);
10680 case ISD::SPONENTRY:
10681 return LowerSPONENTRY(Op, DAG);
10682 case ISD::FP_TO_BF16:
10683 return LowerFP_TO_BF16(Op, DAG);
10684 case ARMISD::WIN__DBZCHK: return SDValue();
10685 case ISD::UCMP:
10686 case ISD::SCMP:
10687 return LowerCMP(Op, DAG);
10688 case ISD::ABS:
10689 return LowerABS(Op, DAG);
10690 case ISD::STRICT_LROUND:
10691 case ISD::STRICT_LLROUND:
10692 case ISD::STRICT_LRINT:
10693 case ISD::STRICT_LLRINT: {
10694 assert((Op.getOperand(1).getValueType() == MVT::f16 ||
10695 Op.getOperand(1).getValueType() == MVT::bf16) &&
10696 "Expected custom lowering of rounding operations only for f16");
10697 SDLoc DL(Op);
10698 SDValue Ext = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL, ResultTys: {MVT::f32, MVT::Other},
10699 Ops: {Op.getOperand(i: 0), Op.getOperand(i: 1)});
10700 return DAG.getNode(Opcode: Op.getOpcode(), DL, ResultTys: {Op.getValueType(), MVT::Other},
10701 Ops: {Ext.getValue(R: 1), Ext.getValue(R: 0)});
10702 }
10703 }
10704}
10705
10706static void ReplaceLongIntrinsic(SDNode *N, SmallVectorImpl<SDValue> &Results,
10707 SelectionDAG &DAG) {
10708 unsigned IntNo = N->getConstantOperandVal(Num: 0);
10709 unsigned Opc = 0;
10710 if (IntNo == Intrinsic::arm_smlald)
10711 Opc = ARMISD::SMLALD;
10712 else if (IntNo == Intrinsic::arm_smlaldx)
10713 Opc = ARMISD::SMLALDX;
10714 else if (IntNo == Intrinsic::arm_smlsld)
10715 Opc = ARMISD::SMLSLD;
10716 else if (IntNo == Intrinsic::arm_smlsldx)
10717 Opc = ARMISD::SMLSLDX;
10718 else
10719 return;
10720
10721 SDLoc dl(N);
10722 SDValue Lo, Hi;
10723 std::tie(args&: Lo, args&: Hi) = DAG.SplitScalar(N: N->getOperand(Num: 3), DL: dl, LoVT: MVT::i32, HiVT: MVT::i32);
10724
10725 SDValue LongMul = DAG.getNode(Opcode: Opc, DL: dl,
10726 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
10727 N1: N->getOperand(Num: 1), N2: N->getOperand(Num: 2),
10728 N3: Lo, N4: Hi);
10729 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: MVT::i64,
10730 N1: LongMul.getValue(R: 0), N2: LongMul.getValue(R: 1)));
10731}
10732
10733/// ReplaceNodeResults - Replace the results of node with an illegal result
10734/// type with new values built out of custom code.
10735void ARMTargetLowering::ReplaceNodeResults(SDNode *N,
10736 SmallVectorImpl<SDValue> &Results,
10737 SelectionDAG &DAG) const {
10738 SDValue Res;
10739 switch (N->getOpcode()) {
10740 default:
10741 llvm_unreachable("Don't know how to custom expand this!");
10742 case ISD::READ_REGISTER:
10743 ExpandREAD_REGISTER(N, Results, DAG);
10744 break;
10745 case ISD::BITCAST:
10746 Res = ExpandBITCAST(N, DAG, Subtarget);
10747 break;
10748 case ISD::SRL:
10749 case ISD::SRA:
10750 case ISD::SHL:
10751 Res = Expand64BitShift(N, DAG, ST: Subtarget);
10752 break;
10753 case ISD::SREM:
10754 case ISD::UREM:
10755 Res = LowerREM(N, DAG);
10756 break;
10757 case ISD::SDIVREM:
10758 case ISD::UDIVREM:
10759 Res = LowerDivRem(Op: SDValue(N, 0), DAG);
10760 assert(Res.getNumOperands() == 2 && "DivRem needs two values");
10761 Results.push_back(Elt: Res.getValue(R: 0));
10762 Results.push_back(Elt: Res.getValue(R: 1));
10763 return;
10764 case ISD::SADDSAT:
10765 case ISD::SSUBSAT:
10766 case ISD::UADDSAT:
10767 case ISD::USUBSAT:
10768 Res = LowerADDSUBSAT(Op: SDValue(N, 0), DAG, Subtarget);
10769 break;
10770 case ISD::READCYCLECOUNTER:
10771 ReplaceREADCYCLECOUNTER(N, Results, DAG, Subtarget);
10772 return;
10773 case ISD::UDIV:
10774 case ISD::SDIV:
10775 assert(getTargetMachine().getTargetTriple().isOSWindows() &&
10776 "can only expand DIV on Windows");
10777 return ExpandDIV_Windows(Op: SDValue(N, 0), DAG, Signed: N->getOpcode() == ISD::SDIV,
10778 Results);
10779 case ISD::ATOMIC_CMP_SWAP:
10780 ReplaceCMP_SWAP_64Results(N, Results, DAG);
10781 return;
10782 case ISD::INTRINSIC_WO_CHAIN:
10783 return ReplaceLongIntrinsic(N, Results, DAG);
10784 case ISD::LOAD:
10785 LowerLOAD(N, Results, DAG);
10786 break;
10787 case ISD::STORE:
10788 Res = LowerAEABIUnalignedStore(Op: SDValue(N, 0), DAG);
10789 break;
10790 case ISD::TRUNCATE:
10791 Res = LowerTruncate(N, DAG, Subtarget);
10792 break;
10793 case ISD::SIGN_EXTEND:
10794 case ISD::ZERO_EXTEND:
10795 Res = LowerVectorExtend(N, DAG, Subtarget);
10796 break;
10797 case ISD::FP_TO_SINT_SAT:
10798 case ISD::FP_TO_UINT_SAT:
10799 Res = LowerFP_TO_INT_SAT(Op: SDValue(N, 0), DAG, Subtarget);
10800 break;
10801 }
10802 if (Res.getNode())
10803 Results.push_back(Elt: Res);
10804}
10805
10806//===----------------------------------------------------------------------===//
10807// ARM Scheduler Hooks
10808//===----------------------------------------------------------------------===//
10809
10810/// SetupEntryBlockForSjLj - Insert code into the entry block that creates and
10811/// registers the function context.
10812void ARMTargetLowering::SetupEntryBlockForSjLj(MachineInstr &MI,
10813 MachineBasicBlock *MBB,
10814 MachineBasicBlock *DispatchBB,
10815 int FI) const {
10816 assert(!Subtarget->isROPI() && !Subtarget->isRWPI() &&
10817 "ROPI/RWPI not currently supported with SjLj");
10818 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
10819 DebugLoc dl = MI.getDebugLoc();
10820 MachineFunction *MF = MBB->getParent();
10821 MachineRegisterInfo *MRI = &MF->getRegInfo();
10822 MachineConstantPool *MCP = MF->getConstantPool();
10823 ARMFunctionInfo *AFI = MF->getInfo<ARMFunctionInfo>();
10824 const Function &F = MF->getFunction();
10825
10826 bool isThumb = Subtarget->isThumb();
10827 bool isThumb2 = Subtarget->isThumb2();
10828
10829 unsigned PCLabelId = AFI->createPICLabelUId();
10830 unsigned PCAdj = (isThumb || isThumb2) ? 4 : 8;
10831 ARMConstantPoolValue *CPV =
10832 ARMConstantPoolMBB::Create(C&: F.getContext(), mbb: DispatchBB, ID: PCLabelId, PCAdj);
10833 unsigned CPI = MCP->getConstantPoolIndex(V: CPV, Alignment: Align(4));
10834
10835 const TargetRegisterClass *TRC = isThumb ? &ARM::tGPRRegClass
10836 : &ARM::GPRRegClass;
10837
10838 // Grab constant pool and fixed stack memory operands.
10839 MachineMemOperand *CPMMO =
10840 MF->getMachineMemOperand(PtrInfo: MachinePointerInfo::getConstantPool(MF&: *MF),
10841 F: MachineMemOperand::MOLoad, Size: 4, BaseAlignment: Align(4));
10842
10843 MachineMemOperand *FIMMOSt =
10844 MF->getMachineMemOperand(PtrInfo: MachinePointerInfo::getFixedStack(MF&: *MF, FI),
10845 F: MachineMemOperand::MOStore, Size: 4, BaseAlignment: Align(4));
10846
10847 // Load the address of the dispatch MBB into the jump buffer.
10848 if (isThumb2) {
10849 // Incoming value: jbuf
10850 // ldr.n r5, LCPI1_1
10851 // orr r5, r5, #1
10852 // add r5, pc
10853 // str r5, [$jbuf, #+4] ; &jbuf[1]
10854 Register NewVReg1 = MRI->createVirtualRegister(RegClass: TRC);
10855 BuildMI(BB&: *MBB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: ARM::t2LDRpci), DestReg: NewVReg1)
10856 .addConstantPoolIndex(Idx: CPI)
10857 .addMemOperand(MMO: CPMMO)
10858 .add(MOs: predOps(Pred: ARMCC::AL));
10859 // Set the low bit because of thumb mode.
10860 Register NewVReg2 = MRI->createVirtualRegister(RegClass: TRC);
10861 BuildMI(BB&: *MBB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: ARM::t2ORRri), DestReg: NewVReg2)
10862 .addReg(RegNo: NewVReg1, Flags: RegState::Kill)
10863 .addImm(Val: 0x01)
10864 .add(MOs: predOps(Pred: ARMCC::AL))
10865 .add(MO: condCodeOp());
10866 Register NewVReg3 = MRI->createVirtualRegister(RegClass: TRC);
10867 BuildMI(BB&: *MBB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: ARM::tPICADD), DestReg: NewVReg3)
10868 .addReg(RegNo: NewVReg2, Flags: RegState::Kill)
10869 .addImm(Val: PCLabelId);
10870 BuildMI(BB&: *MBB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: ARM::t2STRi12))
10871 .addReg(RegNo: NewVReg3, Flags: RegState::Kill)
10872 .addFrameIndex(Idx: FI)
10873 .addImm(Val: 36) // &jbuf[1] :: pc
10874 .addMemOperand(MMO: FIMMOSt)
10875 .add(MOs: predOps(Pred: ARMCC::AL));
10876 } else if (isThumb) {
10877 // Incoming value: jbuf
10878 // ldr.n r1, LCPI1_4
10879 // add r1, pc
10880 // mov r2, #1
10881 // orrs r1, r2
10882 // add r2, $jbuf, #+4 ; &jbuf[1]
10883 // str r1, [r2]
10884 Register NewVReg1 = MRI->createVirtualRegister(RegClass: TRC);
10885 BuildMI(BB&: *MBB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: ARM::tLDRpci), DestReg: NewVReg1)
10886 .addConstantPoolIndex(Idx: CPI)
10887 .addMemOperand(MMO: CPMMO)
10888 .add(MOs: predOps(Pred: ARMCC::AL));
10889 Register NewVReg2 = MRI->createVirtualRegister(RegClass: TRC);
10890 BuildMI(BB&: *MBB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: ARM::tPICADD), DestReg: NewVReg2)
10891 .addReg(RegNo: NewVReg1, Flags: RegState::Kill)
10892 .addImm(Val: PCLabelId);
10893 // Set the low bit because of thumb mode.
10894 Register NewVReg3 = MRI->createVirtualRegister(RegClass: TRC);
10895 BuildMI(BB&: *MBB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: ARM::tMOVi8), DestReg: NewVReg3)
10896 .addReg(RegNo: ARM::CPSR, Flags: RegState::Define)
10897 .addImm(Val: 1)
10898 .add(MOs: predOps(Pred: ARMCC::AL));
10899 Register NewVReg4 = MRI->createVirtualRegister(RegClass: TRC);
10900 BuildMI(BB&: *MBB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: ARM::tORR), DestReg: NewVReg4)
10901 .addReg(RegNo: ARM::CPSR, Flags: RegState::Define)
10902 .addReg(RegNo: NewVReg2, Flags: RegState::Kill)
10903 .addReg(RegNo: NewVReg3, Flags: RegState::Kill)
10904 .add(MOs: predOps(Pred: ARMCC::AL));
10905 Register NewVReg5 = MRI->createVirtualRegister(RegClass: TRC);
10906 BuildMI(BB&: *MBB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: ARM::tADDframe), DestReg: NewVReg5)
10907 .addFrameIndex(Idx: FI)
10908 .addImm(Val: 36); // &jbuf[1] :: pc
10909 BuildMI(BB&: *MBB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: ARM::tSTRi))
10910 .addReg(RegNo: NewVReg4, Flags: RegState::Kill)
10911 .addReg(RegNo: NewVReg5, Flags: RegState::Kill)
10912 .addImm(Val: 0)
10913 .addMemOperand(MMO: FIMMOSt)
10914 .add(MOs: predOps(Pred: ARMCC::AL));
10915 } else {
10916 // Incoming value: jbuf
10917 // ldr r1, LCPI1_1
10918 // add r1, pc, r1
10919 // str r1, [$jbuf, #+4] ; &jbuf[1]
10920 Register NewVReg1 = MRI->createVirtualRegister(RegClass: TRC);
10921 BuildMI(BB&: *MBB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: ARM::LDRi12), DestReg: NewVReg1)
10922 .addConstantPoolIndex(Idx: CPI)
10923 .addImm(Val: 0)
10924 .addMemOperand(MMO: CPMMO)
10925 .add(MOs: predOps(Pred: ARMCC::AL));
10926 Register NewVReg2 = MRI->createVirtualRegister(RegClass: TRC);
10927 BuildMI(BB&: *MBB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: ARM::PICADD), DestReg: NewVReg2)
10928 .addReg(RegNo: NewVReg1, Flags: RegState::Kill)
10929 .addImm(Val: PCLabelId)
10930 .add(MOs: predOps(Pred: ARMCC::AL));
10931 BuildMI(BB&: *MBB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: ARM::STRi12))
10932 .addReg(RegNo: NewVReg2, Flags: RegState::Kill)
10933 .addFrameIndex(Idx: FI)
10934 .addImm(Val: 36) // &jbuf[1] :: pc
10935 .addMemOperand(MMO: FIMMOSt)
10936 .add(MOs: predOps(Pred: ARMCC::AL));
10937 }
10938}
10939
10940void ARMTargetLowering::EmitSjLjDispatchBlock(MachineInstr &MI,
10941 MachineBasicBlock *MBB) const {
10942 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
10943 DebugLoc dl = MI.getDebugLoc();
10944 MachineFunction *MF = MBB->getParent();
10945 MachineRegisterInfo *MRI = &MF->getRegInfo();
10946 MachineFrameInfo &MFI = MF->getFrameInfo();
10947 int FI = MFI.getFunctionContextIndex();
10948
10949 const TargetRegisterClass *TRC = Subtarget->isThumb() ? &ARM::tGPRRegClass
10950 : &ARM::GPRnopcRegClass;
10951
10952 // Get a mapping of the call site numbers to all of the landing pads they're
10953 // associated with.
10954 DenseMap<unsigned, SmallVector<MachineBasicBlock*, 2>> CallSiteNumToLPad;
10955 unsigned MaxCSNum = 0;
10956 for (MachineBasicBlock &BB : *MF) {
10957 if (!BB.isEHPad())
10958 continue;
10959
10960 // FIXME: We should assert that the EH_LABEL is the first MI in the landing
10961 // pad.
10962 for (MachineInstr &II : BB) {
10963 if (!II.isEHLabel())
10964 continue;
10965
10966 MCSymbol *Sym = II.getOperand(i: 0).getMCSymbol();
10967 if (!MF->hasCallSiteLandingPad(Sym)) continue;
10968
10969 SmallVectorImpl<unsigned> &CallSiteIdxs = MF->getCallSiteLandingPad(Sym);
10970 for (unsigned Idx : CallSiteIdxs) {
10971 CallSiteNumToLPad[Idx].push_back(Elt: &BB);
10972 MaxCSNum = std::max(a: MaxCSNum, b: Idx);
10973 }
10974 break;
10975 }
10976 }
10977
10978 // Get an ordered list of the machine basic blocks for the jump table.
10979 std::vector<MachineBasicBlock*> LPadList;
10980 SmallPtrSet<MachineBasicBlock*, 32> InvokeBBs;
10981 LPadList.reserve(n: CallSiteNumToLPad.size());
10982 for (unsigned I = 1; I <= MaxCSNum; ++I) {
10983 SmallVectorImpl<MachineBasicBlock*> &MBBList = CallSiteNumToLPad[I];
10984 for (MachineBasicBlock *MBB : MBBList) {
10985 LPadList.push_back(x: MBB);
10986 InvokeBBs.insert_range(R: MBB->predecessors());
10987 }
10988 }
10989
10990 assert(!LPadList.empty() &&
10991 "No landing pad destinations for the dispatch jump table!");
10992
10993 // Create the jump table and associated information.
10994 MachineJumpTableInfo *JTI =
10995 MF->getOrCreateJumpTableInfo(JTEntryKind: MachineJumpTableInfo::EK_Inline);
10996 unsigned MJTI = JTI->createJumpTableIndex(DestBBs: LPadList);
10997
10998 // Create the MBBs for the dispatch code.
10999
11000 // Shove the dispatch's address into the return slot in the function context.
11001 MachineBasicBlock *DispatchBB = MF->CreateMachineBasicBlock();
11002 DispatchBB->setIsEHPad();
11003
11004 MachineBasicBlock *TrapBB = MF->CreateMachineBasicBlock();
11005
11006 BuildMI(BB: TrapBB, MIMD: dl, MCID: TII->get(Opcode: Subtarget->isThumb() ? ARM::tTRAP : ARM::TRAP));
11007 DispatchBB->addSuccessor(Succ: TrapBB);
11008
11009 MachineBasicBlock *DispContBB = MF->CreateMachineBasicBlock();
11010 DispatchBB->addSuccessor(Succ: DispContBB);
11011
11012 // Insert and MBBs.
11013 MF->insert(MBBI: MF->end(), MBB: DispatchBB);
11014 MF->insert(MBBI: MF->end(), MBB: DispContBB);
11015 MF->insert(MBBI: MF->end(), MBB: TrapBB);
11016
11017 // Insert code into the entry block that creates and registers the function
11018 // context.
11019 SetupEntryBlockForSjLj(MI, MBB, DispatchBB, FI);
11020
11021 MachineMemOperand *FIMMOLd = MF->getMachineMemOperand(
11022 PtrInfo: MachinePointerInfo::getFixedStack(MF&: *MF, FI),
11023 F: MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile, Size: 4, BaseAlignment: Align(4));
11024
11025 MachineInstrBuilder MIB;
11026 MIB = BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::Int_eh_sjlj_dispatchsetup));
11027
11028 const ARMBaseInstrInfo *AII = static_cast<const ARMBaseInstrInfo*>(TII);
11029 const ARMBaseRegisterInfo &RI = AII->getRegisterInfo();
11030
11031 // Add a register mask with no preserved registers. This results in all
11032 // registers being marked as clobbered. This can't work if the dispatch block
11033 // is in a Thumb1 function and is linked with ARM code which uses the FP
11034 // registers, as there is no way to preserve the FP registers in Thumb1 mode.
11035 MIB.addRegMask(Mask: RI.getSjLjDispatchPreservedMask(MF: *MF));
11036
11037 bool IsPositionIndependent = isPositionIndependent();
11038 unsigned NumLPads = LPadList.size();
11039 if (Subtarget->isThumb2()) {
11040 Register NewVReg1 = MRI->createVirtualRegister(RegClass: TRC);
11041 BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::t2LDRi12), DestReg: NewVReg1)
11042 .addFrameIndex(Idx: FI)
11043 .addImm(Val: 4)
11044 .addMemOperand(MMO: FIMMOLd)
11045 .add(MOs: predOps(Pred: ARMCC::AL));
11046
11047 if (NumLPads < 256) {
11048 BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::t2CMPri))
11049 .addReg(RegNo: NewVReg1)
11050 .addImm(Val: LPadList.size())
11051 .add(MOs: predOps(Pred: ARMCC::AL));
11052 } else {
11053 Register VReg1 = MRI->createVirtualRegister(RegClass: TRC);
11054 BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::t2MOVi16), DestReg: VReg1)
11055 .addImm(Val: NumLPads & 0xFFFF)
11056 .add(MOs: predOps(Pred: ARMCC::AL));
11057
11058 unsigned VReg2 = VReg1;
11059 if ((NumLPads & 0xFFFF0000) != 0) {
11060 VReg2 = MRI->createVirtualRegister(RegClass: TRC);
11061 BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::t2MOVTi16), DestReg: VReg2)
11062 .addReg(RegNo: VReg1)
11063 .addImm(Val: NumLPads >> 16)
11064 .add(MOs: predOps(Pred: ARMCC::AL));
11065 }
11066
11067 BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::t2CMPrr))
11068 .addReg(RegNo: NewVReg1)
11069 .addReg(RegNo: VReg2)
11070 .add(MOs: predOps(Pred: ARMCC::AL));
11071 }
11072
11073 BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::t2Bcc))
11074 .addMBB(MBB: TrapBB)
11075 .addImm(Val: ARMCC::HI)
11076 .addReg(RegNo: ARM::CPSR);
11077
11078 Register NewVReg3 = MRI->createVirtualRegister(RegClass: TRC);
11079 BuildMI(BB: DispContBB, MIMD: dl, MCID: TII->get(Opcode: ARM::t2LEApcrelJT), DestReg: NewVReg3)
11080 .addJumpTableIndex(Idx: MJTI)
11081 .add(MOs: predOps(Pred: ARMCC::AL));
11082
11083 Register NewVReg4 = MRI->createVirtualRegister(RegClass: TRC);
11084 BuildMI(BB: DispContBB, MIMD: dl, MCID: TII->get(Opcode: ARM::t2ADDrs), DestReg: NewVReg4)
11085 .addReg(RegNo: NewVReg3, Flags: RegState::Kill)
11086 .addReg(RegNo: NewVReg1)
11087 .addImm(Val: ARM_AM::getSORegOpc(ShOp: ARM_AM::lsl, Imm: 2))
11088 .add(MOs: predOps(Pred: ARMCC::AL))
11089 .add(MO: condCodeOp());
11090
11091 BuildMI(BB: DispContBB, MIMD: dl, MCID: TII->get(Opcode: ARM::t2BR_JT))
11092 .addReg(RegNo: NewVReg4, Flags: RegState::Kill)
11093 .addReg(RegNo: NewVReg1)
11094 .addJumpTableIndex(Idx: MJTI);
11095 } else if (Subtarget->isThumb()) {
11096 Register NewVReg1 = MRI->createVirtualRegister(RegClass: TRC);
11097 BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::tLDRspi), DestReg: NewVReg1)
11098 .addFrameIndex(Idx: FI)
11099 .addImm(Val: 1)
11100 .addMemOperand(MMO: FIMMOLd)
11101 .add(MOs: predOps(Pred: ARMCC::AL));
11102
11103 if (NumLPads < 256) {
11104 BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::tCMPi8))
11105 .addReg(RegNo: NewVReg1)
11106 .addImm(Val: NumLPads)
11107 .add(MOs: predOps(Pred: ARMCC::AL));
11108 } else {
11109 MachineConstantPool *ConstantPool = MF->getConstantPool();
11110 Type *Int32Ty = Type::getInt32Ty(C&: MF->getFunction().getContext());
11111 const Constant *C = ConstantInt::get(Ty: Int32Ty, V: NumLPads);
11112
11113 // MachineConstantPool wants an explicit alignment.
11114 Align Alignment = MF->getDataLayout().getPrefTypeAlign(Ty: Int32Ty);
11115 unsigned Idx = ConstantPool->getConstantPoolIndex(C, Alignment);
11116
11117 Register VReg1 = MRI->createVirtualRegister(RegClass: TRC);
11118 BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::tLDRpci))
11119 .addReg(RegNo: VReg1, Flags: RegState::Define)
11120 .addConstantPoolIndex(Idx)
11121 .add(MOs: predOps(Pred: ARMCC::AL));
11122 BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::tCMPr))
11123 .addReg(RegNo: NewVReg1)
11124 .addReg(RegNo: VReg1)
11125 .add(MOs: predOps(Pred: ARMCC::AL));
11126 }
11127
11128 BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::tBcc))
11129 .addMBB(MBB: TrapBB)
11130 .addImm(Val: ARMCC::HI)
11131 .addReg(RegNo: ARM::CPSR);
11132
11133 Register NewVReg2 = MRI->createVirtualRegister(RegClass: TRC);
11134 BuildMI(BB: DispContBB, MIMD: dl, MCID: TII->get(Opcode: ARM::tLSLri), DestReg: NewVReg2)
11135 .addReg(RegNo: ARM::CPSR, Flags: RegState::Define)
11136 .addReg(RegNo: NewVReg1)
11137 .addImm(Val: 2)
11138 .add(MOs: predOps(Pred: ARMCC::AL));
11139
11140 Register NewVReg3 = MRI->createVirtualRegister(RegClass: TRC);
11141 BuildMI(BB: DispContBB, MIMD: dl, MCID: TII->get(Opcode: ARM::tLEApcrelJT), DestReg: NewVReg3)
11142 .addJumpTableIndex(Idx: MJTI)
11143 .add(MOs: predOps(Pred: ARMCC::AL));
11144
11145 Register NewVReg4 = MRI->createVirtualRegister(RegClass: TRC);
11146 BuildMI(BB: DispContBB, MIMD: dl, MCID: TII->get(Opcode: ARM::tADDrr), DestReg: NewVReg4)
11147 .addReg(RegNo: ARM::CPSR, Flags: RegState::Define)
11148 .addReg(RegNo: NewVReg2, Flags: RegState::Kill)
11149 .addReg(RegNo: NewVReg3)
11150 .add(MOs: predOps(Pred: ARMCC::AL));
11151
11152 MachineMemOperand *JTMMOLd =
11153 MF->getMachineMemOperand(PtrInfo: MachinePointerInfo::getJumpTable(MF&: *MF),
11154 F: MachineMemOperand::MOLoad, Size: 4, BaseAlignment: Align(4));
11155
11156 Register NewVReg5 = MRI->createVirtualRegister(RegClass: TRC);
11157 BuildMI(BB: DispContBB, MIMD: dl, MCID: TII->get(Opcode: ARM::tLDRi), DestReg: NewVReg5)
11158 .addReg(RegNo: NewVReg4, Flags: RegState::Kill)
11159 .addImm(Val: 0)
11160 .addMemOperand(MMO: JTMMOLd)
11161 .add(MOs: predOps(Pred: ARMCC::AL));
11162
11163 unsigned NewVReg6 = NewVReg5;
11164 if (IsPositionIndependent) {
11165 NewVReg6 = MRI->createVirtualRegister(RegClass: TRC);
11166 BuildMI(BB: DispContBB, MIMD: dl, MCID: TII->get(Opcode: ARM::tADDrr), DestReg: NewVReg6)
11167 .addReg(RegNo: ARM::CPSR, Flags: RegState::Define)
11168 .addReg(RegNo: NewVReg5, Flags: RegState::Kill)
11169 .addReg(RegNo: NewVReg3)
11170 .add(MOs: predOps(Pred: ARMCC::AL));
11171 }
11172
11173 BuildMI(BB: DispContBB, MIMD: dl, MCID: TII->get(Opcode: ARM::tBR_JTr))
11174 .addReg(RegNo: NewVReg6, Flags: RegState::Kill)
11175 .addJumpTableIndex(Idx: MJTI);
11176 } else {
11177 Register NewVReg1 = MRI->createVirtualRegister(RegClass: TRC);
11178 BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::LDRi12), DestReg: NewVReg1)
11179 .addFrameIndex(Idx: FI)
11180 .addImm(Val: 4)
11181 .addMemOperand(MMO: FIMMOLd)
11182 .add(MOs: predOps(Pred: ARMCC::AL));
11183
11184 if (NumLPads < 256) {
11185 BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::CMPri))
11186 .addReg(RegNo: NewVReg1)
11187 .addImm(Val: NumLPads)
11188 .add(MOs: predOps(Pred: ARMCC::AL));
11189 } else if (Subtarget->hasV6T2Ops() && isUInt<16>(x: NumLPads)) {
11190 Register VReg1 = MRI->createVirtualRegister(RegClass: TRC);
11191 BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::MOVi16), DestReg: VReg1)
11192 .addImm(Val: NumLPads & 0xFFFF)
11193 .add(MOs: predOps(Pred: ARMCC::AL));
11194
11195 unsigned VReg2 = VReg1;
11196 if ((NumLPads & 0xFFFF0000) != 0) {
11197 VReg2 = MRI->createVirtualRegister(RegClass: TRC);
11198 BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::MOVTi16), DestReg: VReg2)
11199 .addReg(RegNo: VReg1)
11200 .addImm(Val: NumLPads >> 16)
11201 .add(MOs: predOps(Pred: ARMCC::AL));
11202 }
11203
11204 BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::CMPrr))
11205 .addReg(RegNo: NewVReg1)
11206 .addReg(RegNo: VReg2)
11207 .add(MOs: predOps(Pred: ARMCC::AL));
11208 } else {
11209 MachineConstantPool *ConstantPool = MF->getConstantPool();
11210 Type *Int32Ty = Type::getInt32Ty(C&: MF->getFunction().getContext());
11211 const Constant *C = ConstantInt::get(Ty: Int32Ty, V: NumLPads);
11212
11213 // MachineConstantPool wants an explicit alignment.
11214 Align Alignment = MF->getDataLayout().getPrefTypeAlign(Ty: Int32Ty);
11215 unsigned Idx = ConstantPool->getConstantPoolIndex(C, Alignment);
11216
11217 Register VReg1 = MRI->createVirtualRegister(RegClass: TRC);
11218 BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::LDRcp))
11219 .addReg(RegNo: VReg1, Flags: RegState::Define)
11220 .addConstantPoolIndex(Idx)
11221 .addImm(Val: 0)
11222 .add(MOs: predOps(Pred: ARMCC::AL));
11223 BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::CMPrr))
11224 .addReg(RegNo: NewVReg1)
11225 .addReg(RegNo: VReg1, Flags: RegState::Kill)
11226 .add(MOs: predOps(Pred: ARMCC::AL));
11227 }
11228
11229 BuildMI(BB: DispatchBB, MIMD: dl, MCID: TII->get(Opcode: ARM::Bcc))
11230 .addMBB(MBB: TrapBB)
11231 .addImm(Val: ARMCC::HI)
11232 .addReg(RegNo: ARM::CPSR);
11233
11234 Register NewVReg3 = MRI->createVirtualRegister(RegClass: TRC);
11235 BuildMI(BB: DispContBB, MIMD: dl, MCID: TII->get(Opcode: ARM::MOVsi), DestReg: NewVReg3)
11236 .addReg(RegNo: NewVReg1)
11237 .addImm(Val: ARM_AM::getSORegOpc(ShOp: ARM_AM::lsl, Imm: 2))
11238 .add(MOs: predOps(Pred: ARMCC::AL))
11239 .add(MO: condCodeOp());
11240 Register NewVReg4 = MRI->createVirtualRegister(RegClass: TRC);
11241 BuildMI(BB: DispContBB, MIMD: dl, MCID: TII->get(Opcode: ARM::LEApcrelJT), DestReg: NewVReg4)
11242 .addJumpTableIndex(Idx: MJTI)
11243 .add(MOs: predOps(Pred: ARMCC::AL));
11244
11245 MachineMemOperand *JTMMOLd =
11246 MF->getMachineMemOperand(PtrInfo: MachinePointerInfo::getJumpTable(MF&: *MF),
11247 F: MachineMemOperand::MOLoad, Size: 4, BaseAlignment: Align(4));
11248 Register NewVReg5 = MRI->createVirtualRegister(RegClass: TRC);
11249 BuildMI(BB: DispContBB, MIMD: dl, MCID: TII->get(Opcode: ARM::LDRrs), DestReg: NewVReg5)
11250 .addReg(RegNo: NewVReg3, Flags: RegState::Kill)
11251 .addReg(RegNo: NewVReg4)
11252 .addImm(Val: 0)
11253 .addMemOperand(MMO: JTMMOLd)
11254 .add(MOs: predOps(Pred: ARMCC::AL));
11255
11256 if (IsPositionIndependent) {
11257 BuildMI(BB: DispContBB, MIMD: dl, MCID: TII->get(Opcode: ARM::BR_JTadd))
11258 .addReg(RegNo: NewVReg5, Flags: RegState::Kill)
11259 .addReg(RegNo: NewVReg4)
11260 .addJumpTableIndex(Idx: MJTI);
11261 } else {
11262 BuildMI(BB: DispContBB, MIMD: dl, MCID: TII->get(Opcode: ARM::BR_JTr))
11263 .addReg(RegNo: NewVReg5, Flags: RegState::Kill)
11264 .addJumpTableIndex(Idx: MJTI);
11265 }
11266 }
11267
11268 // Add the jump table entries as successors to the MBB.
11269 SmallPtrSet<MachineBasicBlock*, 8> SeenMBBs;
11270 for (MachineBasicBlock *CurMBB : LPadList) {
11271 if (SeenMBBs.insert(Ptr: CurMBB).second)
11272 DispContBB->addSuccessor(Succ: CurMBB);
11273 }
11274
11275 // N.B. the order the invoke BBs are processed in doesn't matter here.
11276 const MCPhysReg *SavedRegs = RI.getCalleeSavedRegs(MF);
11277 SmallVector<MachineBasicBlock*, 64> MBBLPads;
11278 for (MachineBasicBlock *BB : InvokeBBs) {
11279
11280 // Remove the landing pad successor from the invoke block and replace it
11281 // with the new dispatch block.
11282 SmallVector<MachineBasicBlock*, 4> Successors(BB->successors());
11283 while (!Successors.empty()) {
11284 MachineBasicBlock *SMBB = Successors.pop_back_val();
11285 if (SMBB->isEHPad()) {
11286 BB->removeSuccessor(Succ: SMBB);
11287 MBBLPads.push_back(Elt: SMBB);
11288 }
11289 }
11290
11291 BB->addSuccessor(Succ: DispatchBB, Prob: BranchProbability::getZero());
11292 BB->normalizeSuccProbs();
11293
11294 // Find the invoke call and mark all of the callee-saved registers as
11295 // 'implicit defined' so that they're spilled. This prevents code from
11296 // moving instructions to before the EH block, where they will never be
11297 // executed.
11298 for (MachineBasicBlock::reverse_iterator
11299 II = BB->rbegin(), IE = BB->rend(); II != IE; ++II) {
11300 if (!II->isCall()) continue;
11301
11302 DenseSet<unsigned> DefRegs;
11303 for (MachineInstr::mop_iterator
11304 OI = II->operands_begin(), OE = II->operands_end();
11305 OI != OE; ++OI) {
11306 if (!OI->isReg()) continue;
11307 DefRegs.insert(V: OI->getReg());
11308 }
11309
11310 MachineInstrBuilder MIB(*MF, &*II);
11311
11312 for (unsigned i = 0; SavedRegs[i] != 0; ++i) {
11313 unsigned Reg = SavedRegs[i];
11314 if (Subtarget->isThumb2() &&
11315 !ARM::tGPRRegClass.contains(Reg) &&
11316 !ARM::hGPRRegClass.contains(Reg))
11317 continue;
11318 if (Subtarget->isThumb1Only() && !ARM::tGPRRegClass.contains(Reg))
11319 continue;
11320 if (!Subtarget->isThumb() && !ARM::GPRRegClass.contains(Reg))
11321 continue;
11322 if (!DefRegs.contains(V: Reg))
11323 MIB.addReg(RegNo: Reg, Flags: RegState::ImplicitDefine | RegState::Dead);
11324 }
11325
11326 break;
11327 }
11328 }
11329
11330 // Mark all former landing pads as non-landing pads. The dispatch is the only
11331 // landing pad now.
11332 for (MachineBasicBlock *MBBLPad : MBBLPads)
11333 MBBLPad->setIsEHPad(false);
11334
11335 // The instruction is gone now.
11336 MI.eraseFromParent();
11337}
11338
11339static
11340MachineBasicBlock *OtherSucc(MachineBasicBlock *MBB, MachineBasicBlock *Succ) {
11341 for (MachineBasicBlock *S : MBB->successors())
11342 if (S != Succ)
11343 return S;
11344 llvm_unreachable("Expecting a BB with two successors!");
11345}
11346
11347/// Return the load opcode for a given load size. If load size >= 8,
11348/// neon opcode will be returned.
11349static unsigned getLdOpcode(unsigned LdSize, bool IsThumb1, bool IsThumb2) {
11350 if (LdSize >= 8)
11351 return LdSize == 16 ? ARM::VLD1q32wb_fixed
11352 : LdSize == 8 ? ARM::VLD1d32wb_fixed : 0;
11353 if (IsThumb1)
11354 return LdSize == 4 ? ARM::tLDRi
11355 : LdSize == 2 ? ARM::tLDRHi
11356 : LdSize == 1 ? ARM::tLDRBi : 0;
11357 if (IsThumb2)
11358 return LdSize == 4 ? ARM::t2LDR_POST
11359 : LdSize == 2 ? ARM::t2LDRH_POST
11360 : LdSize == 1 ? ARM::t2LDRB_POST : 0;
11361 return LdSize == 4 ? ARM::LDR_POST_IMM
11362 : LdSize == 2 ? ARM::LDRH_POST
11363 : LdSize == 1 ? ARM::LDRB_POST_IMM : 0;
11364}
11365
11366/// Return the store opcode for a given store size. If store size >= 8,
11367/// neon opcode will be returned.
11368static unsigned getStOpcode(unsigned StSize, bool IsThumb1, bool IsThumb2) {
11369 if (StSize >= 8)
11370 return StSize == 16 ? ARM::VST1q32wb_fixed
11371 : StSize == 8 ? ARM::VST1d32wb_fixed : 0;
11372 if (IsThumb1)
11373 return StSize == 4 ? ARM::tSTRi
11374 : StSize == 2 ? ARM::tSTRHi
11375 : StSize == 1 ? ARM::tSTRBi : 0;
11376 if (IsThumb2)
11377 return StSize == 4 ? ARM::t2STR_POST
11378 : StSize == 2 ? ARM::t2STRH_POST
11379 : StSize == 1 ? ARM::t2STRB_POST : 0;
11380 return StSize == 4 ? ARM::STR_POST_IMM
11381 : StSize == 2 ? ARM::STRH_POST
11382 : StSize == 1 ? ARM::STRB_POST_IMM : 0;
11383}
11384
11385/// Emit a post-increment load operation with given size. The instructions
11386/// will be added to BB at Pos.
11387static void emitPostLd(MachineBasicBlock *BB, MachineBasicBlock::iterator Pos,
11388 const TargetInstrInfo *TII, const DebugLoc &dl,
11389 unsigned LdSize, unsigned Data, unsigned AddrIn,
11390 unsigned AddrOut, bool IsThumb1, bool IsThumb2) {
11391 unsigned LdOpc = getLdOpcode(LdSize, IsThumb1, IsThumb2);
11392 assert(LdOpc != 0 && "Should have a load opcode");
11393 if (LdSize >= 8) {
11394 BuildMI(BB&: *BB, I: Pos, MIMD: dl, MCID: TII->get(Opcode: LdOpc), DestReg: Data)
11395 .addReg(RegNo: AddrOut, Flags: RegState::Define)
11396 .addReg(RegNo: AddrIn)
11397 .addImm(Val: 0)
11398 .add(MOs: predOps(Pred: ARMCC::AL));
11399 } else if (IsThumb1) {
11400 // load + update AddrIn
11401 BuildMI(BB&: *BB, I: Pos, MIMD: dl, MCID: TII->get(Opcode: LdOpc), DestReg: Data)
11402 .addReg(RegNo: AddrIn)
11403 .addImm(Val: 0)
11404 .add(MOs: predOps(Pred: ARMCC::AL));
11405 BuildMI(BB&: *BB, I: Pos, MIMD: dl, MCID: TII->get(Opcode: ARM::tADDi8), DestReg: AddrOut)
11406 .add(MO: t1CondCodeOp())
11407 .addReg(RegNo: AddrIn)
11408 .addImm(Val: LdSize)
11409 .add(MOs: predOps(Pred: ARMCC::AL));
11410 } else if (IsThumb2) {
11411 BuildMI(BB&: *BB, I: Pos, MIMD: dl, MCID: TII->get(Opcode: LdOpc), DestReg: Data)
11412 .addReg(RegNo: AddrOut, Flags: RegState::Define)
11413 .addReg(RegNo: AddrIn)
11414 .addImm(Val: LdSize)
11415 .add(MOs: predOps(Pred: ARMCC::AL));
11416 } else { // arm
11417 BuildMI(BB&: *BB, I: Pos, MIMD: dl, MCID: TII->get(Opcode: LdOpc), DestReg: Data)
11418 .addReg(RegNo: AddrOut, Flags: RegState::Define)
11419 .addReg(RegNo: AddrIn)
11420 .addReg(RegNo: 0)
11421 .addImm(Val: LdSize)
11422 .add(MOs: predOps(Pred: ARMCC::AL));
11423 }
11424}
11425
11426/// Emit a post-increment store operation with given size. The instructions
11427/// will be added to BB at Pos.
11428static void emitPostSt(MachineBasicBlock *BB, MachineBasicBlock::iterator Pos,
11429 const TargetInstrInfo *TII, const DebugLoc &dl,
11430 unsigned StSize, unsigned Data, unsigned AddrIn,
11431 unsigned AddrOut, bool IsThumb1, bool IsThumb2) {
11432 unsigned StOpc = getStOpcode(StSize, IsThumb1, IsThumb2);
11433 assert(StOpc != 0 && "Should have a store opcode");
11434 if (StSize >= 8) {
11435 BuildMI(BB&: *BB, I: Pos, MIMD: dl, MCID: TII->get(Opcode: StOpc), DestReg: AddrOut)
11436 .addReg(RegNo: AddrIn)
11437 .addImm(Val: 0)
11438 .addReg(RegNo: Data)
11439 .add(MOs: predOps(Pred: ARMCC::AL));
11440 } else if (IsThumb1) {
11441 // store + update AddrIn
11442 BuildMI(BB&: *BB, I: Pos, MIMD: dl, MCID: TII->get(Opcode: StOpc))
11443 .addReg(RegNo: Data)
11444 .addReg(RegNo: AddrIn)
11445 .addImm(Val: 0)
11446 .add(MOs: predOps(Pred: ARMCC::AL));
11447 BuildMI(BB&: *BB, I: Pos, MIMD: dl, MCID: TII->get(Opcode: ARM::tADDi8), DestReg: AddrOut)
11448 .add(MO: t1CondCodeOp())
11449 .addReg(RegNo: AddrIn)
11450 .addImm(Val: StSize)
11451 .add(MOs: predOps(Pred: ARMCC::AL));
11452 } else if (IsThumb2) {
11453 BuildMI(BB&: *BB, I: Pos, MIMD: dl, MCID: TII->get(Opcode: StOpc), DestReg: AddrOut)
11454 .addReg(RegNo: Data)
11455 .addReg(RegNo: AddrIn)
11456 .addImm(Val: StSize)
11457 .add(MOs: predOps(Pred: ARMCC::AL));
11458 } else { // arm
11459 BuildMI(BB&: *BB, I: Pos, MIMD: dl, MCID: TII->get(Opcode: StOpc), DestReg: AddrOut)
11460 .addReg(RegNo: Data)
11461 .addReg(RegNo: AddrIn)
11462 .addReg(RegNo: 0)
11463 .addImm(Val: StSize)
11464 .add(MOs: predOps(Pred: ARMCC::AL));
11465 }
11466}
11467
11468MachineBasicBlock *
11469ARMTargetLowering::EmitStructByval(MachineInstr &MI,
11470 MachineBasicBlock *BB) const {
11471 // This pseudo instruction has 3 operands: dst, src, size
11472 // We expand it to a loop if size > Subtarget->getMaxInlineSizeThreshold().
11473 // Otherwise, we will generate unrolled scalar copies.
11474 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
11475 const BasicBlock *LLVM_BB = BB->getBasicBlock();
11476 MachineFunction::iterator It = ++BB->getIterator();
11477
11478 Register dest = MI.getOperand(i: 0).getReg();
11479 Register src = MI.getOperand(i: 1).getReg();
11480 unsigned SizeVal = MI.getOperand(i: 2).getImm();
11481 unsigned Alignment = MI.getOperand(i: 3).getImm();
11482 DebugLoc dl = MI.getDebugLoc();
11483
11484 MachineFunction *MF = BB->getParent();
11485 MachineRegisterInfo &MRI = MF->getRegInfo();
11486 unsigned UnitSize = 0;
11487 const TargetRegisterClass *TRC = nullptr;
11488 const TargetRegisterClass *VecTRC = nullptr;
11489
11490 bool IsThumb1 = Subtarget->isThumb1Only();
11491 bool IsThumb2 = Subtarget->isThumb2();
11492 bool IsThumb = Subtarget->isThumb();
11493
11494 if (Alignment & 1) {
11495 UnitSize = 1;
11496 } else if (Alignment & 2) {
11497 UnitSize = 2;
11498 } else {
11499 // Check whether we can use NEON instructions.
11500 if (!MF->getFunction().hasFnAttribute(Kind: Attribute::NoImplicitFloat) &&
11501 Subtarget->hasNEON()) {
11502 if ((Alignment % 16 == 0) && SizeVal >= 16)
11503 UnitSize = 16;
11504 else if ((Alignment % 8 == 0) && SizeVal >= 8)
11505 UnitSize = 8;
11506 }
11507 // Can't use NEON instructions.
11508 if (UnitSize == 0)
11509 UnitSize = 4;
11510 }
11511
11512 // Select the correct opcode and register class for unit size load/store
11513 bool IsNeon = UnitSize >= 8;
11514 TRC = IsThumb ? &ARM::tGPRRegClass : &ARM::GPRRegClass;
11515 if (IsNeon)
11516 VecTRC = UnitSize == 16 ? &ARM::DPairRegClass
11517 : UnitSize == 8 ? &ARM::DPRRegClass
11518 : nullptr;
11519
11520 unsigned BytesLeft = SizeVal % UnitSize;
11521 unsigned LoopSize = SizeVal - BytesLeft;
11522
11523 if (SizeVal <= Subtarget->getMaxInlineSizeThreshold()) {
11524 // Use LDR and STR to copy.
11525 // [scratch, srcOut] = LDR_POST(srcIn, UnitSize)
11526 // [destOut] = STR_POST(scratch, destIn, UnitSize)
11527 unsigned srcIn = src;
11528 unsigned destIn = dest;
11529 for (unsigned i = 0; i < LoopSize; i+=UnitSize) {
11530 Register srcOut = MRI.createVirtualRegister(RegClass: TRC);
11531 Register destOut = MRI.createVirtualRegister(RegClass: TRC);
11532 Register scratch = MRI.createVirtualRegister(RegClass: IsNeon ? VecTRC : TRC);
11533 emitPostLd(BB, Pos: MI, TII, dl, LdSize: UnitSize, Data: scratch, AddrIn: srcIn, AddrOut: srcOut,
11534 IsThumb1, IsThumb2);
11535 emitPostSt(BB, Pos: MI, TII, dl, StSize: UnitSize, Data: scratch, AddrIn: destIn, AddrOut: destOut,
11536 IsThumb1, IsThumb2);
11537 srcIn = srcOut;
11538 destIn = destOut;
11539 }
11540
11541 // Handle the leftover bytes with LDRB and STRB.
11542 // [scratch, srcOut] = LDRB_POST(srcIn, 1)
11543 // [destOut] = STRB_POST(scratch, destIn, 1)
11544 for (unsigned i = 0; i < BytesLeft; i++) {
11545 Register srcOut = MRI.createVirtualRegister(RegClass: TRC);
11546 Register destOut = MRI.createVirtualRegister(RegClass: TRC);
11547 Register scratch = MRI.createVirtualRegister(RegClass: TRC);
11548 emitPostLd(BB, Pos: MI, TII, dl, LdSize: 1, Data: scratch, AddrIn: srcIn, AddrOut: srcOut,
11549 IsThumb1, IsThumb2);
11550 emitPostSt(BB, Pos: MI, TII, dl, StSize: 1, Data: scratch, AddrIn: destIn, AddrOut: destOut,
11551 IsThumb1, IsThumb2);
11552 srcIn = srcOut;
11553 destIn = destOut;
11554 }
11555 MI.eraseFromParent(); // The instruction is gone now.
11556 return BB;
11557 }
11558
11559 // Expand the pseudo op to a loop.
11560 // thisMBB:
11561 // ...
11562 // movw varEnd, # --> with thumb2
11563 // movt varEnd, #
11564 // ldrcp varEnd, idx --> without thumb2
11565 // fallthrough --> loopMBB
11566 // loopMBB:
11567 // PHI varPhi, varEnd, varLoop
11568 // PHI srcPhi, src, srcLoop
11569 // PHI destPhi, dst, destLoop
11570 // [scratch, srcLoop] = LDR_POST(srcPhi, UnitSize)
11571 // [destLoop] = STR_POST(scratch, destPhi, UnitSize)
11572 // subs varLoop, varPhi, #UnitSize
11573 // bne loopMBB
11574 // fallthrough --> exitMBB
11575 // exitMBB:
11576 // epilogue to handle left-over bytes
11577 // [scratch, srcOut] = LDRB_POST(srcLoop, 1)
11578 // [destOut] = STRB_POST(scratch, destLoop, 1)
11579 MachineBasicBlock *loopMBB = MF->CreateMachineBasicBlock(BB: LLVM_BB);
11580 MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(BB: LLVM_BB);
11581 MF->insert(MBBI: It, MBB: loopMBB);
11582 MF->insert(MBBI: It, MBB: exitMBB);
11583
11584 // Set the call frame size on entry to the new basic blocks.
11585 unsigned CallFrameSize = TII->getCallFrameSizeAt(MI);
11586 loopMBB->setCallFrameSize(CallFrameSize);
11587 exitMBB->setCallFrameSize(CallFrameSize);
11588
11589 // Transfer the remainder of BB and its successor edges to exitMBB.
11590 exitMBB->splice(Where: exitMBB->begin(), Other: BB,
11591 From: std::next(x: MachineBasicBlock::iterator(MI)), To: BB->end());
11592 exitMBB->transferSuccessorsAndUpdatePHIs(FromMBB: BB);
11593
11594 // Load an immediate to varEnd.
11595 Register varEnd = MRI.createVirtualRegister(RegClass: TRC);
11596 if (Subtarget->useMovt()) {
11597 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: IsThumb ? ARM::t2MOVi32imm : ARM::MOVi32imm),
11598 DestReg: varEnd)
11599 .addImm(Val: LoopSize);
11600 } else if (Subtarget->genExecuteOnly()) {
11601 assert(IsThumb && "Non-thumb expected to have used movt");
11602 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: ARM::tMOVi32imm), DestReg: varEnd).addImm(Val: LoopSize);
11603 } else {
11604 MachineConstantPool *ConstantPool = MF->getConstantPool();
11605 Type *Int32Ty = Type::getInt32Ty(C&: MF->getFunction().getContext());
11606 const Constant *C = ConstantInt::get(Ty: Int32Ty, V: LoopSize);
11607
11608 // MachineConstantPool wants an explicit alignment.
11609 Align Alignment = MF->getDataLayout().getPrefTypeAlign(Ty: Int32Ty);
11610 unsigned Idx = ConstantPool->getConstantPoolIndex(C, Alignment);
11611 MachineMemOperand *CPMMO =
11612 MF->getMachineMemOperand(PtrInfo: MachinePointerInfo::getConstantPool(MF&: *MF),
11613 F: MachineMemOperand::MOLoad, Size: 4, BaseAlignment: Align(4));
11614
11615 if (IsThumb)
11616 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: ARM::tLDRpci))
11617 .addReg(RegNo: varEnd, Flags: RegState::Define)
11618 .addConstantPoolIndex(Idx)
11619 .add(MOs: predOps(Pred: ARMCC::AL))
11620 .addMemOperand(MMO: CPMMO);
11621 else
11622 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: ARM::LDRcp))
11623 .addReg(RegNo: varEnd, Flags: RegState::Define)
11624 .addConstantPoolIndex(Idx)
11625 .addImm(Val: 0)
11626 .add(MOs: predOps(Pred: ARMCC::AL))
11627 .addMemOperand(MMO: CPMMO);
11628 }
11629 BB->addSuccessor(Succ: loopMBB);
11630
11631 // Generate the loop body:
11632 // varPhi = PHI(varLoop, varEnd)
11633 // srcPhi = PHI(srcLoop, src)
11634 // destPhi = PHI(destLoop, dst)
11635 MachineBasicBlock *entryBB = BB;
11636 BB = loopMBB;
11637 Register varLoop = MRI.createVirtualRegister(RegClass: TRC);
11638 Register varPhi = MRI.createVirtualRegister(RegClass: TRC);
11639 Register srcLoop = MRI.createVirtualRegister(RegClass: TRC);
11640 Register srcPhi = MRI.createVirtualRegister(RegClass: TRC);
11641 Register destLoop = MRI.createVirtualRegister(RegClass: TRC);
11642 Register destPhi = MRI.createVirtualRegister(RegClass: TRC);
11643
11644 BuildMI(BB&: *BB, I: BB->begin(), MIMD: dl, MCID: TII->get(Opcode: ARM::PHI), DestReg: varPhi)
11645 .addReg(RegNo: varLoop).addMBB(MBB: loopMBB)
11646 .addReg(RegNo: varEnd).addMBB(MBB: entryBB);
11647 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: ARM::PHI), DestReg: srcPhi)
11648 .addReg(RegNo: srcLoop).addMBB(MBB: loopMBB)
11649 .addReg(RegNo: src).addMBB(MBB: entryBB);
11650 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: ARM::PHI), DestReg: destPhi)
11651 .addReg(RegNo: destLoop).addMBB(MBB: loopMBB)
11652 .addReg(RegNo: dest).addMBB(MBB: entryBB);
11653
11654 // [scratch, srcLoop] = LDR_POST(srcPhi, UnitSize)
11655 // [destLoop] = STR_POST(scratch, destPhi, UnitSiz)
11656 Register scratch = MRI.createVirtualRegister(RegClass: IsNeon ? VecTRC : TRC);
11657 emitPostLd(BB, Pos: BB->end(), TII, dl, LdSize: UnitSize, Data: scratch, AddrIn: srcPhi, AddrOut: srcLoop,
11658 IsThumb1, IsThumb2);
11659 emitPostSt(BB, Pos: BB->end(), TII, dl, StSize: UnitSize, Data: scratch, AddrIn: destPhi, AddrOut: destLoop,
11660 IsThumb1, IsThumb2);
11661
11662 // Decrement loop variable by UnitSize.
11663 if (IsThumb1) {
11664 BuildMI(BB&: *BB, I: BB->end(), MIMD: dl, MCID: TII->get(Opcode: ARM::tSUBi8), DestReg: varLoop)
11665 .add(MO: t1CondCodeOp())
11666 .addReg(RegNo: varPhi)
11667 .addImm(Val: UnitSize)
11668 .add(MOs: predOps(Pred: ARMCC::AL));
11669 } else {
11670 MachineInstrBuilder MIB =
11671 BuildMI(BB&: *BB, I: BB->end(), MIMD: dl,
11672 MCID: TII->get(Opcode: IsThumb2 ? ARM::t2SUBri : ARM::SUBri), DestReg: varLoop);
11673 MIB.addReg(RegNo: varPhi)
11674 .addImm(Val: UnitSize)
11675 .add(MOs: predOps(Pred: ARMCC::AL))
11676 .add(MO: condCodeOp());
11677 MIB->getOperand(i: 5).setReg(ARM::CPSR);
11678 MIB->getOperand(i: 5).setIsDef(true);
11679 }
11680 BuildMI(BB&: *BB, I: BB->end(), MIMD: dl,
11681 MCID: TII->get(Opcode: IsThumb1 ? ARM::tBcc : IsThumb2 ? ARM::t2Bcc : ARM::Bcc))
11682 .addMBB(MBB: loopMBB).addImm(Val: ARMCC::NE).addReg(RegNo: ARM::CPSR);
11683
11684 // loopMBB can loop back to loopMBB or fall through to exitMBB.
11685 BB->addSuccessor(Succ: loopMBB);
11686 BB->addSuccessor(Succ: exitMBB);
11687
11688 // Add epilogue to handle BytesLeft.
11689 BB = exitMBB;
11690 auto StartOfExit = exitMBB->begin();
11691
11692 // [scratch, srcOut] = LDRB_POST(srcLoop, 1)
11693 // [destOut] = STRB_POST(scratch, destLoop, 1)
11694 unsigned srcIn = srcLoop;
11695 unsigned destIn = destLoop;
11696 for (unsigned i = 0; i < BytesLeft; i++) {
11697 Register srcOut = MRI.createVirtualRegister(RegClass: TRC);
11698 Register destOut = MRI.createVirtualRegister(RegClass: TRC);
11699 Register scratch = MRI.createVirtualRegister(RegClass: TRC);
11700 emitPostLd(BB, Pos: StartOfExit, TII, dl, LdSize: 1, Data: scratch, AddrIn: srcIn, AddrOut: srcOut,
11701 IsThumb1, IsThumb2);
11702 emitPostSt(BB, Pos: StartOfExit, TII, dl, StSize: 1, Data: scratch, AddrIn: destIn, AddrOut: destOut,
11703 IsThumb1, IsThumb2);
11704 srcIn = srcOut;
11705 destIn = destOut;
11706 }
11707
11708 MI.eraseFromParent(); // The instruction is gone now.
11709 return BB;
11710}
11711
11712MachineBasicBlock *
11713ARMTargetLowering::EmitLowered__chkstk(MachineInstr &MI,
11714 MachineBasicBlock *MBB) const {
11715 const TargetMachine &TM = getTargetMachine();
11716 const TargetInstrInfo &TII = *Subtarget->getInstrInfo();
11717 DebugLoc DL = MI.getDebugLoc();
11718
11719 assert(TM.getTargetTriple().isOSWindows() &&
11720 "__chkstk is only supported on Windows");
11721 assert(Subtarget->isThumb2() && "Windows on ARM requires Thumb-2 mode");
11722
11723 // __chkstk takes the number of words to allocate on the stack in R4, and
11724 // returns the stack adjustment in number of bytes in R4. This will not
11725 // clober any other registers (other than the obvious lr).
11726 //
11727 // Although, technically, IP should be considered a register which may be
11728 // clobbered, the call itself will not touch it. Windows on ARM is a pure
11729 // thumb-2 environment, so there is no interworking required. As a result, we
11730 // do not expect a veneer to be emitted by the linker, clobbering IP.
11731 //
11732 // Each module receives its own copy of __chkstk, so no import thunk is
11733 // required, again, ensuring that IP is not clobbered.
11734 //
11735 // Finally, although some linkers may theoretically provide a trampoline for
11736 // out of range calls (which is quite common due to a 32M range limitation of
11737 // branches for Thumb), we can generate the long-call version via
11738 // -mcmodel=large, alleviating the need for the trampoline which may clobber
11739 // IP.
11740
11741 RTLIB::LibcallImpl ChkStkLibcall = getLibcallImpl(Call: RTLIB::STACK_PROBE);
11742 if (ChkStkLibcall == RTLIB::Unsupported)
11743 reportFatalUsageError(reason: "no available implementation of __chkstk");
11744
11745 const char *ChkStk = getLibcallImplName(Call: ChkStkLibcall).data();
11746 switch (TM.getCodeModel()) {
11747 case CodeModel::Tiny:
11748 llvm_unreachable("Tiny code model not available on ARM.");
11749 case CodeModel::Small:
11750 case CodeModel::Medium:
11751 case CodeModel::Kernel:
11752 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: ARM::tBL))
11753 .add(MOs: predOps(Pred: ARMCC::AL))
11754 .addExternalSymbol(FnName: ChkStk)
11755 .addReg(RegNo: ARM::R4, Flags: RegState::Implicit | RegState::Kill)
11756 .addReg(RegNo: ARM::R4, Flags: RegState::Implicit | RegState::Define)
11757 .addReg(RegNo: ARM::R12,
11758 Flags: RegState::Implicit | RegState::Define | RegState::Dead)
11759 .addReg(RegNo: ARM::CPSR,
11760 Flags: RegState::Implicit | RegState::Define | RegState::Dead);
11761 break;
11762 case CodeModel::Large: {
11763 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
11764 Register Reg = MRI.createVirtualRegister(RegClass: &ARM::rGPRRegClass);
11765
11766 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: ARM::t2MOVi32imm), DestReg: Reg)
11767 .addExternalSymbol(FnName: ChkStk);
11768 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: gettBLXrOpcode(MF: *MBB->getParent())))
11769 .add(MOs: predOps(Pred: ARMCC::AL))
11770 .addReg(RegNo: Reg, Flags: RegState::Kill)
11771 .addReg(RegNo: ARM::R4, Flags: RegState::Implicit | RegState::Kill)
11772 .addReg(RegNo: ARM::R4, Flags: RegState::Implicit | RegState::Define)
11773 .addReg(RegNo: ARM::R12,
11774 Flags: RegState::Implicit | RegState::Define | RegState::Dead)
11775 .addReg(RegNo: ARM::CPSR,
11776 Flags: RegState::Implicit | RegState::Define | RegState::Dead);
11777 break;
11778 }
11779 }
11780
11781 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: ARM::t2SUBrr), DestReg: ARM::SP)
11782 .addReg(RegNo: ARM::SP, Flags: RegState::Kill)
11783 .addReg(RegNo: ARM::R4, Flags: RegState::Kill)
11784 .setMIFlags(MachineInstr::FrameSetup)
11785 .add(MOs: predOps(Pred: ARMCC::AL))
11786 .add(MO: condCodeOp());
11787
11788 MI.eraseFromParent();
11789 return MBB;
11790}
11791
11792MachineBasicBlock *
11793ARMTargetLowering::EmitLowered__dbzchk(MachineInstr &MI,
11794 MachineBasicBlock *MBB) const {
11795 DebugLoc DL = MI.getDebugLoc();
11796 MachineFunction *MF = MBB->getParent();
11797 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
11798
11799 MachineBasicBlock *ContBB = MF->CreateMachineBasicBlock();
11800 MF->insert(MBBI: ++MBB->getIterator(), MBB: ContBB);
11801 ContBB->splice(Where: ContBB->begin(), Other: MBB,
11802 From: std::next(x: MachineBasicBlock::iterator(MI)), To: MBB->end());
11803 ContBB->transferSuccessorsAndUpdatePHIs(FromMBB: MBB);
11804 MBB->addSuccessor(Succ: ContBB);
11805
11806 MachineBasicBlock *TrapBB = MF->CreateMachineBasicBlock();
11807 BuildMI(BB: TrapBB, MIMD: DL, MCID: TII->get(Opcode: ARM::t__brkdiv0));
11808 MF->push_back(MBB: TrapBB);
11809 MBB->addSuccessor(Succ: TrapBB);
11810
11811 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: ARM::tCMPi8))
11812 .addReg(RegNo: MI.getOperand(i: 0).getReg())
11813 .addImm(Val: 0)
11814 .add(MOs: predOps(Pred: ARMCC::AL));
11815 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: ARM::t2Bcc))
11816 .addMBB(MBB: TrapBB)
11817 .addImm(Val: ARMCC::EQ)
11818 .addReg(RegNo: ARM::CPSR);
11819
11820 MI.eraseFromParent();
11821 return ContBB;
11822}
11823
11824// The CPSR operand of SelectItr might be missing a kill marker
11825// because there were multiple uses of CPSR, and ISel didn't know
11826// which to mark. Figure out whether SelectItr should have had a
11827// kill marker, and set it if it should. Returns the correct kill
11828// marker value.
11829static bool checkAndUpdateCPSRKill(MachineBasicBlock::iterator SelectItr,
11830 MachineBasicBlock* BB,
11831 const TargetRegisterInfo* TRI) {
11832 // Scan forward through BB for a use/def of CPSR.
11833 MachineBasicBlock::iterator miI(std::next(x: SelectItr));
11834 for (MachineBasicBlock::iterator miE = BB->end(); miI != miE; ++miI) {
11835 const MachineInstr& mi = *miI;
11836 if (mi.readsRegister(Reg: ARM::CPSR, /*TRI=*/nullptr))
11837 return false;
11838 if (mi.definesRegister(Reg: ARM::CPSR, /*TRI=*/nullptr))
11839 break; // Should have kill-flag - update below.
11840 }
11841
11842 // If we hit the end of the block, check whether CPSR is live into a
11843 // successor.
11844 if (miI == BB->end()) {
11845 for (MachineBasicBlock *Succ : BB->successors())
11846 if (Succ->isLiveIn(Reg: ARM::CPSR))
11847 return false;
11848 }
11849
11850 // We found a def, or hit the end of the basic block and CPSR wasn't live
11851 // out. SelectMI should have a kill flag on CPSR.
11852 SelectItr->addRegisterKilled(IncomingReg: ARM::CPSR, RegInfo: TRI);
11853 return true;
11854}
11855
11856/// Adds logic in loop entry MBB to calculate loop iteration count and adds
11857/// t2WhileLoopSetup and t2WhileLoopStart to generate WLS loop
11858static Register genTPEntry(MachineBasicBlock *TpEntry,
11859 MachineBasicBlock *TpLoopBody,
11860 MachineBasicBlock *TpExit, Register OpSizeReg,
11861 const TargetInstrInfo *TII, DebugLoc Dl,
11862 MachineRegisterInfo &MRI) {
11863 // Calculates loop iteration count = ceil(n/16) = (n + 15) >> 4.
11864 Register AddDestReg = MRI.createVirtualRegister(RegClass: &ARM::rGPRRegClass);
11865 BuildMI(BB: TpEntry, MIMD: Dl, MCID: TII->get(Opcode: ARM::t2ADDri), DestReg: AddDestReg)
11866 .addUse(RegNo: OpSizeReg)
11867 .addImm(Val: 15)
11868 .add(MOs: predOps(Pred: ARMCC::AL))
11869 .addReg(RegNo: 0);
11870
11871 Register LsrDestReg = MRI.createVirtualRegister(RegClass: &ARM::rGPRRegClass);
11872 BuildMI(BB: TpEntry, MIMD: Dl, MCID: TII->get(Opcode: ARM::t2LSRri), DestReg: LsrDestReg)
11873 .addUse(RegNo: AddDestReg, Flags: RegState::Kill)
11874 .addImm(Val: 4)
11875 .add(MOs: predOps(Pred: ARMCC::AL))
11876 .addReg(RegNo: 0);
11877
11878 Register TotalIterationsReg = MRI.createVirtualRegister(RegClass: &ARM::GPRlrRegClass);
11879 BuildMI(BB: TpEntry, MIMD: Dl, MCID: TII->get(Opcode: ARM::t2WhileLoopSetup), DestReg: TotalIterationsReg)
11880 .addUse(RegNo: LsrDestReg, Flags: RegState::Kill);
11881
11882 BuildMI(BB: TpEntry, MIMD: Dl, MCID: TII->get(Opcode: ARM::t2WhileLoopStart))
11883 .addUse(RegNo: TotalIterationsReg)
11884 .addMBB(MBB: TpExit);
11885
11886 BuildMI(BB: TpEntry, MIMD: Dl, MCID: TII->get(Opcode: ARM::t2B))
11887 .addMBB(MBB: TpLoopBody)
11888 .add(MOs: predOps(Pred: ARMCC::AL));
11889
11890 return TotalIterationsReg;
11891}
11892
11893/// Adds logic in the loopBody MBB to generate MVE_VCTP, t2DoLoopDec and
11894/// t2DoLoopEnd. These are used by later passes to generate tail predicated
11895/// loops.
11896static void genTPLoopBody(MachineBasicBlock *TpLoopBody,
11897 MachineBasicBlock *TpEntry, MachineBasicBlock *TpExit,
11898 const TargetInstrInfo *TII, DebugLoc Dl,
11899 MachineRegisterInfo &MRI, Register OpSrcReg,
11900 Register OpDestReg, Register ElementCountReg,
11901 Register TotalIterationsReg, bool IsMemcpy) {
11902 // First insert 4 PHI nodes for: Current pointer to Src (if memcpy), Dest
11903 // array, loop iteration counter, predication counter.
11904
11905 Register SrcPhiReg, CurrSrcReg;
11906 if (IsMemcpy) {
11907 // Current position in the src array
11908 SrcPhiReg = MRI.createVirtualRegister(RegClass: &ARM::rGPRRegClass);
11909 CurrSrcReg = MRI.createVirtualRegister(RegClass: &ARM::rGPRRegClass);
11910 BuildMI(BB: TpLoopBody, MIMD: Dl, MCID: TII->get(Opcode: ARM::PHI), DestReg: SrcPhiReg)
11911 .addUse(RegNo: OpSrcReg)
11912 .addMBB(MBB: TpEntry)
11913 .addUse(RegNo: CurrSrcReg)
11914 .addMBB(MBB: TpLoopBody);
11915 }
11916
11917 // Current position in the dest array
11918 Register DestPhiReg = MRI.createVirtualRegister(RegClass: &ARM::rGPRRegClass);
11919 Register CurrDestReg = MRI.createVirtualRegister(RegClass: &ARM::rGPRRegClass);
11920 BuildMI(BB: TpLoopBody, MIMD: Dl, MCID: TII->get(Opcode: ARM::PHI), DestReg: DestPhiReg)
11921 .addUse(RegNo: OpDestReg)
11922 .addMBB(MBB: TpEntry)
11923 .addUse(RegNo: CurrDestReg)
11924 .addMBB(MBB: TpLoopBody);
11925
11926 // Current loop counter
11927 Register LoopCounterPhiReg = MRI.createVirtualRegister(RegClass: &ARM::GPRlrRegClass);
11928 Register RemainingLoopIterationsReg =
11929 MRI.createVirtualRegister(RegClass: &ARM::GPRlrRegClass);
11930 BuildMI(BB: TpLoopBody, MIMD: Dl, MCID: TII->get(Opcode: ARM::PHI), DestReg: LoopCounterPhiReg)
11931 .addUse(RegNo: TotalIterationsReg)
11932 .addMBB(MBB: TpEntry)
11933 .addUse(RegNo: RemainingLoopIterationsReg)
11934 .addMBB(MBB: TpLoopBody);
11935
11936 // Predication counter
11937 Register PredCounterPhiReg = MRI.createVirtualRegister(RegClass: &ARM::rGPRRegClass);
11938 Register RemainingElementsReg = MRI.createVirtualRegister(RegClass: &ARM::rGPRRegClass);
11939 BuildMI(BB: TpLoopBody, MIMD: Dl, MCID: TII->get(Opcode: ARM::PHI), DestReg: PredCounterPhiReg)
11940 .addUse(RegNo: ElementCountReg)
11941 .addMBB(MBB: TpEntry)
11942 .addUse(RegNo: RemainingElementsReg)
11943 .addMBB(MBB: TpLoopBody);
11944
11945 // Pass predication counter to VCTP
11946 Register VccrReg = MRI.createVirtualRegister(RegClass: &ARM::VCCRRegClass);
11947 BuildMI(BB: TpLoopBody, MIMD: Dl, MCID: TII->get(Opcode: ARM::MVE_VCTP8), DestReg: VccrReg)
11948 .addUse(RegNo: PredCounterPhiReg)
11949 .addImm(Val: ARMVCC::None)
11950 .addReg(RegNo: 0)
11951 .addReg(RegNo: 0);
11952
11953 BuildMI(BB: TpLoopBody, MIMD: Dl, MCID: TII->get(Opcode: ARM::t2SUBri), DestReg: RemainingElementsReg)
11954 .addUse(RegNo: PredCounterPhiReg)
11955 .addImm(Val: 16)
11956 .add(MOs: predOps(Pred: ARMCC::AL))
11957 .addReg(RegNo: 0);
11958
11959 // VLDRB (only if memcpy) and VSTRB instructions, predicated using VPR
11960 Register SrcValueReg;
11961 if (IsMemcpy) {
11962 SrcValueReg = MRI.createVirtualRegister(RegClass: &ARM::MQPRRegClass);
11963 BuildMI(BB: TpLoopBody, MIMD: Dl, MCID: TII->get(Opcode: ARM::MVE_VLDRBU8_post))
11964 .addDef(RegNo: CurrSrcReg)
11965 .addDef(RegNo: SrcValueReg)
11966 .addReg(RegNo: SrcPhiReg)
11967 .addImm(Val: 16)
11968 .addImm(Val: ARMVCC::Then)
11969 .addUse(RegNo: VccrReg)
11970 .addReg(RegNo: 0);
11971 } else
11972 SrcValueReg = OpSrcReg;
11973
11974 BuildMI(BB: TpLoopBody, MIMD: Dl, MCID: TII->get(Opcode: ARM::MVE_VSTRBU8_post))
11975 .addDef(RegNo: CurrDestReg)
11976 .addUse(RegNo: SrcValueReg)
11977 .addReg(RegNo: DestPhiReg)
11978 .addImm(Val: 16)
11979 .addImm(Val: ARMVCC::Then)
11980 .addUse(RegNo: VccrReg)
11981 .addReg(RegNo: 0);
11982
11983 // Add the pseudoInstrs for decrementing the loop counter and marking the
11984 // end:t2DoLoopDec and t2DoLoopEnd
11985 BuildMI(BB: TpLoopBody, MIMD: Dl, MCID: TII->get(Opcode: ARM::t2LoopDec), DestReg: RemainingLoopIterationsReg)
11986 .addUse(RegNo: LoopCounterPhiReg)
11987 .addImm(Val: 1);
11988
11989 BuildMI(BB: TpLoopBody, MIMD: Dl, MCID: TII->get(Opcode: ARM::t2LoopEnd))
11990 .addUse(RegNo: RemainingLoopIterationsReg)
11991 .addMBB(MBB: TpLoopBody);
11992
11993 BuildMI(BB: TpLoopBody, MIMD: Dl, MCID: TII->get(Opcode: ARM::t2B))
11994 .addMBB(MBB: TpExit)
11995 .add(MOs: predOps(Pred: ARMCC::AL));
11996}
11997
11998bool ARMTargetLowering::supportKCFIBundles() const {
11999 // KCFI is supported in all ARM/Thumb modes
12000 return true;
12001}
12002
12003MachineInstr *
12004ARMTargetLowering::EmitKCFICheck(MachineBasicBlock &MBB,
12005 MachineBasicBlock::instr_iterator &MBBI,
12006 const TargetInstrInfo *TII) const {
12007 assert(MBBI->isCall() && MBBI->getCFIType() &&
12008 "Invalid call instruction for a KCFI check");
12009
12010 MachineOperand *TargetOp = nullptr;
12011 switch (MBBI->getOpcode()) {
12012 // ARM mode opcodes
12013 case ARM::BLX:
12014 case ARM::BLX_pred:
12015 case ARM::BLX_noip:
12016 case ARM::BLX_pred_noip:
12017 case ARM::BX_CALL:
12018 TargetOp = &MBBI->getOperand(i: 0);
12019 break;
12020 case ARM::TCRETURNri:
12021 case ARM::TCRETURNrinotr12:
12022 case ARM::TAILJMPr:
12023 case ARM::TAILJMPr4:
12024 TargetOp = &MBBI->getOperand(i: 0);
12025 break;
12026 // Thumb mode opcodes (Thumb1 and Thumb2)
12027 // Note: Most Thumb call instructions have predicate operands before the
12028 // target register Format: tBLXr pred, predreg, target_register, ...
12029 case ARM::tBLXr: // Thumb1/Thumb2: BLX register (requires V5T)
12030 case ARM::tBLXr_noip: // Thumb1/Thumb2: BLX register, no IP clobber
12031 case ARM::tBX_CALL: // Thumb1 only: BX call (push LR, BX)
12032 TargetOp = &MBBI->getOperand(i: 2);
12033 break;
12034 // Tail call instructions don't have predicates, target is operand 0
12035 case ARM::tTAILJMPr: // Thumb1/Thumb2: Tail call via register
12036 TargetOp = &MBBI->getOperand(i: 0);
12037 break;
12038 default:
12039 llvm_unreachable("Unexpected CFI call opcode");
12040 }
12041
12042 assert(TargetOp && TargetOp->isReg() && "Invalid target operand");
12043 TargetOp->setIsRenamable(false);
12044
12045 // Select the appropriate KCFI_CHECK variant based on the instruction set
12046 unsigned KCFICheckOpcode;
12047 if (Subtarget->isThumb()) {
12048 if (Subtarget->isThumb2()) {
12049 KCFICheckOpcode = ARM::KCFI_CHECK_Thumb2;
12050 } else {
12051 KCFICheckOpcode = ARM::KCFI_CHECK_Thumb1;
12052 }
12053 } else {
12054 KCFICheckOpcode = ARM::KCFI_CHECK_ARM;
12055 }
12056
12057 return BuildMI(BB&: MBB, I: MBBI, MIMD: MBBI->getDebugLoc(), MCID: TII->get(Opcode: KCFICheckOpcode))
12058 .addReg(RegNo: TargetOp->getReg())
12059 .addImm(Val: MBBI->getCFIType())
12060 .getInstr();
12061}
12062
12063MachineBasicBlock *
12064ARMTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
12065 MachineBasicBlock *BB) const {
12066 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
12067 DebugLoc dl = MI.getDebugLoc();
12068 bool isThumb2 = Subtarget->isThumb2();
12069 switch (MI.getOpcode()) {
12070 default: {
12071 MI.print(OS&: errs());
12072 llvm_unreachable("Unexpected instr type to insert");
12073 }
12074
12075 // Thumb1 post-indexed loads are really just single-register LDMs.
12076 case ARM::tLDR_postidx: {
12077 MachineOperand Def(MI.getOperand(i: 1));
12078 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: ARM::tLDMIA_UPD))
12079 .add(MO: Def) // Rn_wb
12080 .add(MO: MI.getOperand(i: 2)) // Rn
12081 .add(MO: MI.getOperand(i: 3)) // PredImm
12082 .add(MO: MI.getOperand(i: 4)) // PredReg
12083 .add(MO: MI.getOperand(i: 0)) // Rt
12084 .cloneMemRefs(OtherMI: MI);
12085 MI.eraseFromParent();
12086 return BB;
12087 }
12088
12089 case ARM::MVE_MEMCPYLOOPINST:
12090 case ARM::MVE_MEMSETLOOPINST: {
12091
12092 // Transformation below expands MVE_MEMCPYLOOPINST/MVE_MEMSETLOOPINST Pseudo
12093 // into a Tail Predicated (TP) Loop. It adds the instructions to calculate
12094 // the iteration count =ceil(size_in_bytes/16)) in the TP entry block and
12095 // adds the relevant instructions in the TP loop Body for generation of a
12096 // WLSTP loop.
12097
12098 // Below is relevant portion of the CFG after the transformation.
12099 // The Machine Basic Blocks are shown along with branch conditions (in
12100 // brackets). Note that TP entry/exit MBBs depict the entry/exit of this
12101 // portion of the CFG and may not necessarily be the entry/exit of the
12102 // function.
12103
12104 // (Relevant) CFG after transformation:
12105 // TP entry MBB
12106 // |
12107 // |-----------------|
12108 // (n <= 0) (n > 0)
12109 // | |
12110 // | TP loop Body MBB<--|
12111 // | | |
12112 // \ |___________|
12113 // \ /
12114 // TP exit MBB
12115
12116 MachineFunction *MF = BB->getParent();
12117 MachineFunctionProperties &Properties = MF->getProperties();
12118 MachineRegisterInfo &MRI = MF->getRegInfo();
12119
12120 Register OpDestReg = MI.getOperand(i: 0).getReg();
12121 Register OpSrcReg = MI.getOperand(i: 1).getReg();
12122 Register OpSizeReg = MI.getOperand(i: 2).getReg();
12123
12124 // Allocate the required MBBs and add to parent function.
12125 MachineBasicBlock *TpEntry = BB;
12126 MachineBasicBlock *TpLoopBody = MF->CreateMachineBasicBlock();
12127 MachineBasicBlock *TpExit;
12128
12129 MF->push_back(MBB: TpLoopBody);
12130
12131 // If any instructions are present in the current block after
12132 // MVE_MEMCPYLOOPINST or MVE_MEMSETLOOPINST, split the current block and
12133 // move the instructions into the newly created exit block. If there are no
12134 // instructions add an explicit branch to the FallThrough block and then
12135 // split.
12136 //
12137 // The split is required for two reasons:
12138 // 1) A terminator(t2WhileLoopStart) will be placed at that site.
12139 // 2) Since a TPLoopBody will be added later, any phis in successive blocks
12140 // need to be updated. splitAt() already handles this.
12141 TpExit = BB->splitAt(SplitInst&: MI, UpdateLiveIns: false);
12142 if (TpExit == BB) {
12143 assert(BB->canFallThrough() && "Exit Block must be Fallthrough of the "
12144 "block containing memcpy/memset Pseudo");
12145 TpExit = BB->getFallThrough();
12146 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: ARM::t2B))
12147 .addMBB(MBB: TpExit)
12148 .add(MOs: predOps(Pred: ARMCC::AL));
12149 TpExit = BB->splitAt(SplitInst&: MI, UpdateLiveIns: false);
12150 }
12151
12152 // Add logic for iteration count
12153 Register TotalIterationsReg =
12154 genTPEntry(TpEntry, TpLoopBody, TpExit, OpSizeReg, TII, Dl: dl, MRI);
12155
12156 // Add the vectorized (and predicated) loads/store instructions
12157 bool IsMemcpy = MI.getOpcode() == ARM::MVE_MEMCPYLOOPINST;
12158 genTPLoopBody(TpLoopBody, TpEntry, TpExit, TII, Dl: dl, MRI, OpSrcReg,
12159 OpDestReg, ElementCountReg: OpSizeReg, TotalIterationsReg, IsMemcpy);
12160
12161 // Required to avoid conflict with the MachineVerifier during testing.
12162 Properties.resetNoPHIs();
12163
12164 // Connect the blocks
12165 TpEntry->addSuccessor(Succ: TpLoopBody);
12166 TpLoopBody->addSuccessor(Succ: TpLoopBody);
12167 TpLoopBody->addSuccessor(Succ: TpExit);
12168
12169 // Reorder for a more natural layout
12170 TpLoopBody->moveAfter(NewBefore: TpEntry);
12171 TpExit->moveAfter(NewBefore: TpLoopBody);
12172
12173 // Finally, remove the memcpy Pseudo Instruction
12174 MI.eraseFromParent();
12175
12176 // Return the exit block as it may contain other instructions requiring a
12177 // custom inserter
12178 return TpExit;
12179 }
12180
12181 // The Thumb2 pre-indexed stores have the same MI operands, they just
12182 // define them differently in the .td files from the isel patterns, so
12183 // they need pseudos.
12184 case ARM::t2STR_preidx:
12185 MI.setDesc(TII->get(Opcode: ARM::t2STR_PRE));
12186 return BB;
12187 case ARM::t2STRB_preidx:
12188 MI.setDesc(TII->get(Opcode: ARM::t2STRB_PRE));
12189 return BB;
12190 case ARM::t2STRH_preidx:
12191 MI.setDesc(TII->get(Opcode: ARM::t2STRH_PRE));
12192 return BB;
12193
12194 case ARM::STRi_preidx:
12195 case ARM::STRBi_preidx: {
12196 unsigned NewOpc = MI.getOpcode() == ARM::STRi_preidx ? ARM::STR_PRE_IMM
12197 : ARM::STRB_PRE_IMM;
12198 // Decode the offset.
12199 unsigned Offset = MI.getOperand(i: 4).getImm();
12200 bool isSub = ARM_AM::getAM2Op(AM2Opc: Offset) == ARM_AM::sub;
12201 Offset = ARM_AM::getAM2Offset(AM2Opc: Offset);
12202 if (isSub)
12203 Offset = -Offset;
12204
12205 MachineMemOperand *MMO = *MI.memoperands_begin();
12206 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: NewOpc))
12207 .add(MO: MI.getOperand(i: 0)) // Rn_wb
12208 .add(MO: MI.getOperand(i: 1)) // Rt
12209 .add(MO: MI.getOperand(i: 2)) // Rn
12210 .addImm(Val: Offset) // offset (skip GPR==zero_reg)
12211 .add(MO: MI.getOperand(i: 5)) // pred
12212 .add(MO: MI.getOperand(i: 6))
12213 .addMemOperand(MMO);
12214 MI.eraseFromParent();
12215 return BB;
12216 }
12217 case ARM::STRr_preidx:
12218 case ARM::STRBr_preidx:
12219 case ARM::STRH_preidx: {
12220 unsigned NewOpc;
12221 switch (MI.getOpcode()) {
12222 default: llvm_unreachable("unexpected opcode!");
12223 case ARM::STRr_preidx: NewOpc = ARM::STR_PRE_REG; break;
12224 case ARM::STRBr_preidx: NewOpc = ARM::STRB_PRE_REG; break;
12225 case ARM::STRH_preidx: NewOpc = ARM::STRH_PRE; break;
12226 }
12227 MachineInstrBuilder MIB = BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: NewOpc));
12228 for (const MachineOperand &MO : MI.operands())
12229 MIB.add(MO);
12230 MI.eraseFromParent();
12231 return BB;
12232 }
12233
12234 case ARM::tMOVCCr_pseudo: {
12235 // To "insert" a SELECT_CC instruction, we actually have to insert the
12236 // diamond control-flow pattern. The incoming instruction knows the
12237 // destination vreg to set, the condition code register to branch on, the
12238 // true/false values to select between, and a branch opcode to use.
12239 const BasicBlock *LLVM_BB = BB->getBasicBlock();
12240 MachineFunction::iterator It = ++BB->getIterator();
12241
12242 // thisMBB:
12243 // ...
12244 // TrueVal = ...
12245 // cmpTY ccX, r1, r2
12246 // bCC copy1MBB
12247 // fallthrough --> copy0MBB
12248 MachineBasicBlock *thisMBB = BB;
12249 MachineFunction *F = BB->getParent();
12250 MachineBasicBlock *copy0MBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
12251 MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
12252 F->insert(MBBI: It, MBB: copy0MBB);
12253 F->insert(MBBI: It, MBB: sinkMBB);
12254
12255 // Set the call frame size on entry to the new basic blocks.
12256 unsigned CallFrameSize = TII->getCallFrameSizeAt(MI);
12257 copy0MBB->setCallFrameSize(CallFrameSize);
12258 sinkMBB->setCallFrameSize(CallFrameSize);
12259
12260 // Check whether CPSR is live past the tMOVCCr_pseudo.
12261 const TargetRegisterInfo *TRI = Subtarget->getRegisterInfo();
12262 if (!MI.killsRegister(Reg: ARM::CPSR, /*TRI=*/nullptr) &&
12263 !checkAndUpdateCPSRKill(SelectItr: MI, BB: thisMBB, TRI)) {
12264 copy0MBB->addLiveIn(PhysReg: ARM::CPSR);
12265 sinkMBB->addLiveIn(PhysReg: ARM::CPSR);
12266 }
12267
12268 // Transfer the remainder of BB and its successor edges to sinkMBB.
12269 sinkMBB->splice(Where: sinkMBB->begin(), Other: BB,
12270 From: std::next(x: MachineBasicBlock::iterator(MI)), To: BB->end());
12271 sinkMBB->transferSuccessorsAndUpdatePHIs(FromMBB: BB);
12272
12273 BB->addSuccessor(Succ: copy0MBB);
12274 BB->addSuccessor(Succ: sinkMBB);
12275
12276 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: ARM::tBcc))
12277 .addMBB(MBB: sinkMBB)
12278 .addImm(Val: MI.getOperand(i: 3).getImm())
12279 .addReg(RegNo: MI.getOperand(i: 4).getReg());
12280
12281 // copy0MBB:
12282 // %FalseValue = ...
12283 // # fallthrough to sinkMBB
12284 BB = copy0MBB;
12285
12286 // Update machine-CFG edges
12287 BB->addSuccessor(Succ: sinkMBB);
12288
12289 // sinkMBB:
12290 // %Result = phi [ %FalseValue, copy0MBB ], [ %TrueValue, thisMBB ]
12291 // ...
12292 BB = sinkMBB;
12293 BuildMI(BB&: *BB, I: BB->begin(), MIMD: dl, MCID: TII->get(Opcode: ARM::PHI), DestReg: MI.getOperand(i: 0).getReg())
12294 .addReg(RegNo: MI.getOperand(i: 1).getReg())
12295 .addMBB(MBB: copy0MBB)
12296 .addReg(RegNo: MI.getOperand(i: 2).getReg())
12297 .addMBB(MBB: thisMBB);
12298
12299 MI.eraseFromParent(); // The pseudo instruction is gone now.
12300 return BB;
12301 }
12302
12303 case ARM::BCCi64:
12304 case ARM::BCCZi64: {
12305 // If there is an unconditional branch to the other successor, remove it.
12306 BB->erase(I: std::next(x: MachineBasicBlock::iterator(MI)), E: BB->end());
12307
12308 // Compare both parts that make up the double comparison separately for
12309 // equality.
12310 bool RHSisZero = MI.getOpcode() == ARM::BCCZi64;
12311
12312 Register LHS1 = MI.getOperand(i: 1).getReg();
12313 Register LHS2 = MI.getOperand(i: 2).getReg();
12314 if (RHSisZero) {
12315 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: isThumb2 ? ARM::t2CMPri : ARM::CMPri))
12316 .addReg(RegNo: LHS1)
12317 .addImm(Val: 0)
12318 .add(MOs: predOps(Pred: ARMCC::AL));
12319 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: isThumb2 ? ARM::t2CMPri : ARM::CMPri))
12320 .addReg(RegNo: LHS2).addImm(Val: 0)
12321 .addImm(Val: ARMCC::EQ).addReg(RegNo: ARM::CPSR);
12322 } else {
12323 Register RHS1 = MI.getOperand(i: 3).getReg();
12324 Register RHS2 = MI.getOperand(i: 4).getReg();
12325 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: isThumb2 ? ARM::t2CMPrr : ARM::CMPrr))
12326 .addReg(RegNo: LHS1)
12327 .addReg(RegNo: RHS1)
12328 .add(MOs: predOps(Pred: ARMCC::AL));
12329 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: isThumb2 ? ARM::t2CMPrr : ARM::CMPrr))
12330 .addReg(RegNo: LHS2).addReg(RegNo: RHS2)
12331 .addImm(Val: ARMCC::EQ).addReg(RegNo: ARM::CPSR);
12332 }
12333
12334 MachineBasicBlock *destMBB = MI.getOperand(i: RHSisZero ? 3 : 5).getMBB();
12335 MachineBasicBlock *exitMBB = OtherSucc(MBB: BB, Succ: destMBB);
12336 if (MI.getOperand(i: 0).getImm() == ARMCC::NE)
12337 std::swap(a&: destMBB, b&: exitMBB);
12338
12339 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: isThumb2 ? ARM::t2Bcc : ARM::Bcc))
12340 .addMBB(MBB: destMBB).addImm(Val: ARMCC::EQ).addReg(RegNo: ARM::CPSR);
12341 if (isThumb2)
12342 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: ARM::t2B))
12343 .addMBB(MBB: exitMBB)
12344 .add(MOs: predOps(Pred: ARMCC::AL));
12345 else
12346 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: ARM::B)) .addMBB(MBB: exitMBB);
12347
12348 MI.eraseFromParent(); // The pseudo instruction is gone now.
12349 return BB;
12350 }
12351
12352 case ARM::Int_eh_sjlj_setjmp:
12353 case ARM::Int_eh_sjlj_setjmp_nofp:
12354 case ARM::tInt_eh_sjlj_setjmp:
12355 case ARM::t2Int_eh_sjlj_setjmp:
12356 case ARM::t2Int_eh_sjlj_setjmp_nofp:
12357 return BB;
12358
12359 case ARM::Int_eh_sjlj_setup_dispatch:
12360 EmitSjLjDispatchBlock(MI, MBB: BB);
12361 return BB;
12362 case ARM::COPY_STRUCT_BYVAL_I32:
12363 ++NumLoopByVals;
12364 return EmitStructByval(MI, BB);
12365 case ARM::WIN__CHKSTK:
12366 return EmitLowered__chkstk(MI, MBB: BB);
12367 case ARM::WIN__DBZCHK:
12368 return EmitLowered__dbzchk(MI, MBB: BB);
12369 }
12370}
12371
12372/// Attaches vregs to MEMCPY that it will use as scratch registers
12373/// when it is expanded into LDM/STM. This is done as a post-isel lowering
12374/// instead of as a custom inserter because we need the use list from the SDNode.
12375static void attachMEMCPYScratchRegs(const ARMSubtarget *Subtarget,
12376 MachineInstr &MI, const SDNode *Node) {
12377 bool isThumb1 = Subtarget->isThumb1Only();
12378
12379 MachineFunction *MF = MI.getParent()->getParent();
12380 MachineRegisterInfo &MRI = MF->getRegInfo();
12381 MachineInstrBuilder MIB(*MF, MI);
12382
12383 // If the new dst/src is unused mark it as dead.
12384 if (!Node->hasAnyUseOfValue(Value: 0)) {
12385 MI.getOperand(i: 0).setIsDead(true);
12386 }
12387 if (!Node->hasAnyUseOfValue(Value: 1)) {
12388 MI.getOperand(i: 1).setIsDead(true);
12389 }
12390
12391 // The MEMCPY both defines and kills the scratch registers.
12392 for (unsigned I = 0; I != MI.getOperand(i: 4).getImm(); ++I) {
12393 Register TmpReg = MRI.createVirtualRegister(RegClass: isThumb1 ? &ARM::tGPRRegClass
12394 : &ARM::GPRRegClass);
12395 MIB.addReg(RegNo: TmpReg, Flags: RegState::Define|RegState::Dead);
12396 }
12397}
12398
12399void ARMTargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI,
12400 SDNode *Node) const {
12401 if (MI.getOpcode() == ARM::MEMCPY) {
12402 attachMEMCPYScratchRegs(Subtarget, MI, Node);
12403 return;
12404 }
12405
12406 const MCInstrDesc *MCID = &MI.getDesc();
12407 // Adjust potentially 's' setting instructions after isel, i.e. ADC, SBC, RSB,
12408 // RSC. Coming out of isel, they have an implicit CPSR def, but the optional
12409 // operand is still set to noreg. If needed, set the optional operand's
12410 // register to CPSR, and remove the redundant implicit def.
12411 //
12412 // e.g. ADCS (..., implicit-def CPSR) -> ADC (... opt:def CPSR).
12413
12414 // Rename pseudo opcodes.
12415 unsigned NewOpc = convertAddSubFlagsOpcode(OldOpc: MI.getOpcode());
12416 unsigned ccOutIdx;
12417 if (NewOpc) {
12418 const ARMBaseInstrInfo *TII = Subtarget->getInstrInfo();
12419 MCID = &TII->get(Opcode: NewOpc);
12420
12421 assert(MCID->getNumOperands() ==
12422 MI.getDesc().getNumOperands() + 5 - MI.getDesc().getSize()
12423 && "converted opcode should be the same except for cc_out"
12424 " (and, on Thumb1, pred)");
12425
12426 MI.setDesc(*MCID);
12427
12428 // Add the optional cc_out operand
12429 MI.addOperand(Op: MachineOperand::CreateReg(Reg: 0, /*isDef=*/true));
12430
12431 // On Thumb1, move all input operands to the end, then add the predicate
12432 if (Subtarget->isThumb1Only()) {
12433 for (unsigned c = MCID->getNumOperands() - 4; c--;) {
12434 MI.addOperand(Op: MI.getOperand(i: 1));
12435 MI.removeOperand(OpNo: 1);
12436 }
12437
12438 // Restore the ties
12439 for (unsigned i = MI.getNumOperands(); i--;) {
12440 const MachineOperand& op = MI.getOperand(i);
12441 if (op.isReg() && op.isUse()) {
12442 int DefIdx = MCID->getOperandConstraint(OpNum: i, Constraint: MCOI::TIED_TO);
12443 if (DefIdx != -1)
12444 MI.tieOperands(DefIdx, UseIdx: i);
12445 }
12446 }
12447
12448 MI.addOperand(Op: MachineOperand::CreateImm(Val: ARMCC::AL));
12449 MI.addOperand(Op: MachineOperand::CreateReg(Reg: 0, /*isDef=*/false));
12450 ccOutIdx = 1;
12451 } else
12452 ccOutIdx = MCID->getNumOperands() - 1;
12453 } else
12454 ccOutIdx = MCID->getNumOperands() - 1;
12455
12456 // Any ARM instruction that sets the 's' bit should specify an optional
12457 // "cc_out" operand in the last operand position.
12458 if (!MI.hasOptionalDef() || !MCID->operands()[ccOutIdx].isOptionalDef()) {
12459 assert(!NewOpc && "Optional cc_out operand required");
12460 return;
12461 }
12462 // Look for an implicit def of CPSR added by MachineInstr ctor. Remove it
12463 // since we already have an optional CPSR def.
12464 bool definesCPSR = false;
12465 bool deadCPSR = false;
12466 for (unsigned i = MCID->getNumOperands(), e = MI.getNumOperands(); i != e;
12467 ++i) {
12468 const MachineOperand &MO = MI.getOperand(i);
12469 if (MO.isReg() && MO.isDef() && MO.getReg() == ARM::CPSR) {
12470 definesCPSR = true;
12471 if (MO.isDead())
12472 deadCPSR = true;
12473 MI.removeOperand(OpNo: i);
12474 break;
12475 }
12476 }
12477 if (!definesCPSR) {
12478 assert(!NewOpc && "Optional cc_out operand required");
12479 return;
12480 }
12481 assert(deadCPSR == !Node->hasAnyUseOfValue(1) && "inconsistent dead flag");
12482 if (deadCPSR) {
12483 assert(!MI.getOperand(ccOutIdx).getReg() &&
12484 "expect uninitialized optional cc_out operand");
12485 // Thumb1 instructions must have the S bit even if the CPSR is dead.
12486 if (!Subtarget->isThumb1Only())
12487 return;
12488 }
12489
12490 // If this instruction was defined with an optional CPSR def and its dag node
12491 // had a live implicit CPSR def, then activate the optional CPSR def.
12492 MachineOperand &MO = MI.getOperand(i: ccOutIdx);
12493 MO.setReg(ARM::CPSR);
12494 MO.setIsDef(true);
12495}
12496
12497//===----------------------------------------------------------------------===//
12498// ARM Optimization Hooks
12499//===----------------------------------------------------------------------===//
12500
12501// Helper function that checks if N is a null or all ones constant.
12502static inline bool isZeroOrAllOnes(SDValue N, bool AllOnes) {
12503 return AllOnes ? isAllOnesConstant(V: N) : isNullConstant(V: N);
12504}
12505
12506// Return true if N is conditionally 0 or all ones.
12507// Detects these expressions where cc is an i1 value:
12508//
12509// (select cc 0, y) [AllOnes=0]
12510// (select cc y, 0) [AllOnes=0]
12511// (zext cc) [AllOnes=0]
12512// (sext cc) [AllOnes=0/1]
12513// (select cc -1, y) [AllOnes=1]
12514// (select cc y, -1) [AllOnes=1]
12515//
12516// Invert is set when N is the null/all ones constant when CC is false.
12517// OtherOp is set to the alternative value of N.
12518static bool isConditionalZeroOrAllOnes(SDNode *N, bool AllOnes,
12519 SDValue &CC, bool &Invert,
12520 SDValue &OtherOp,
12521 SelectionDAG &DAG) {
12522 switch (N->getOpcode()) {
12523 default: return false;
12524 case ISD::SELECT: {
12525 CC = N->getOperand(Num: 0);
12526 SDValue N1 = N->getOperand(Num: 1);
12527 SDValue N2 = N->getOperand(Num: 2);
12528 if (isZeroOrAllOnes(N: N1, AllOnes)) {
12529 Invert = false;
12530 OtherOp = N2;
12531 return true;
12532 }
12533 if (isZeroOrAllOnes(N: N2, AllOnes)) {
12534 Invert = true;
12535 OtherOp = N1;
12536 return true;
12537 }
12538 return false;
12539 }
12540 case ISD::ZERO_EXTEND:
12541 // (zext cc) can never be the all ones value.
12542 if (AllOnes)
12543 return false;
12544 [[fallthrough]];
12545 case ISD::SIGN_EXTEND: {
12546 SDLoc dl(N);
12547 EVT VT = N->getValueType(ResNo: 0);
12548 CC = N->getOperand(Num: 0);
12549 if (CC.getValueType() != MVT::i1 || CC.getOpcode() != ISD::SETCC)
12550 return false;
12551 Invert = !AllOnes;
12552 if (AllOnes)
12553 // When looking for an AllOnes constant, N is an sext, and the 'other'
12554 // value is 0.
12555 OtherOp = DAG.getConstant(Val: 0, DL: dl, VT);
12556 else if (N->getOpcode() == ISD::ZERO_EXTEND)
12557 // When looking for a 0 constant, N can be zext or sext.
12558 OtherOp = DAG.getConstant(Val: 1, DL: dl, VT);
12559 else
12560 OtherOp = DAG.getAllOnesConstant(DL: dl, VT);
12561 return true;
12562 }
12563 }
12564}
12565
12566// Combine a constant select operand into its use:
12567//
12568// (add (select cc, 0, c), x) -> (select cc, x, (add, x, c))
12569// (sub x, (select cc, 0, c)) -> (select cc, x, (sub, x, c))
12570// (and (select cc, -1, c), x) -> (select cc, x, (and, x, c)) [AllOnes=1]
12571// (or (select cc, 0, c), x) -> (select cc, x, (or, x, c))
12572// (xor (select cc, 0, c), x) -> (select cc, x, (xor, x, c))
12573//
12574// The transform is rejected if the select doesn't have a constant operand that
12575// is null, or all ones when AllOnes is set.
12576//
12577// Also recognize sext/zext from i1:
12578//
12579// (add (zext cc), x) -> (select cc (add x, 1), x)
12580// (add (sext cc), x) -> (select cc (add x, -1), x)
12581//
12582// These transformations eventually create predicated instructions.
12583//
12584// @param N The node to transform.
12585// @param Slct The N operand that is a select.
12586// @param OtherOp The other N operand (x above).
12587// @param DCI Context.
12588// @param AllOnes Require the select constant to be all ones instead of null.
12589// @returns The new node, or SDValue() on failure.
12590static
12591SDValue combineSelectAndUse(SDNode *N, SDValue Slct, SDValue OtherOp,
12592 TargetLowering::DAGCombinerInfo &DCI,
12593 bool AllOnes = false) {
12594 SelectionDAG &DAG = DCI.DAG;
12595 EVT VT = N->getValueType(ResNo: 0);
12596 SDValue NonConstantVal;
12597 SDValue CCOp;
12598 bool SwapSelectOps;
12599 if (!isConditionalZeroOrAllOnes(N: Slct.getNode(), AllOnes, CC&: CCOp, Invert&: SwapSelectOps,
12600 OtherOp&: NonConstantVal, DAG))
12601 return SDValue();
12602
12603 // Slct is now know to be the desired identity constant when CC is true.
12604 SDValue TrueVal = OtherOp;
12605 SDValue FalseVal = DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT,
12606 N1: OtherOp, N2: NonConstantVal);
12607 // Unless SwapSelectOps says CC should be false.
12608 if (SwapSelectOps)
12609 std::swap(a&: TrueVal, b&: FalseVal);
12610
12611 return DAG.getNode(Opcode: ISD::SELECT, DL: SDLoc(N), VT,
12612 N1: CCOp, N2: TrueVal, N3: FalseVal);
12613}
12614
12615// Attempt combineSelectAndUse on each operand of a commutative operator N.
12616static
12617SDValue combineSelectAndUseCommutative(SDNode *N, bool AllOnes,
12618 TargetLowering::DAGCombinerInfo &DCI) {
12619 SDValue N0 = N->getOperand(Num: 0);
12620 SDValue N1 = N->getOperand(Num: 1);
12621 if (N0.getNode()->hasOneUse())
12622 if (SDValue Result = combineSelectAndUse(N, Slct: N0, OtherOp: N1, DCI, AllOnes))
12623 return Result;
12624 if (N1.getNode()->hasOneUse())
12625 if (SDValue Result = combineSelectAndUse(N, Slct: N1, OtherOp: N0, DCI, AllOnes))
12626 return Result;
12627 return SDValue();
12628}
12629
12630static bool IsVUZPShuffleNode(SDNode *N) {
12631 // VUZP shuffle node.
12632 if (N->getOpcode() == ARMISD::VUZP)
12633 return true;
12634
12635 // "VUZP" on i32 is an alias for VTRN.
12636 if (N->getOpcode() == ARMISD::VTRN && N->getValueType(ResNo: 0) == MVT::v2i32)
12637 return true;
12638
12639 return false;
12640}
12641
12642static SDValue AddCombineToVPADD(SDNode *N, SDValue N0, SDValue N1,
12643 TargetLowering::DAGCombinerInfo &DCI,
12644 const ARMSubtarget *Subtarget) {
12645 // Look for ADD(VUZP.0, VUZP.1).
12646 if (!IsVUZPShuffleNode(N: N0.getNode()) || N0.getNode() != N1.getNode() ||
12647 N0 == N1)
12648 return SDValue();
12649
12650 // Make sure the ADD is a 64-bit add; there is no 128-bit VPADD.
12651 if (!N->getValueType(ResNo: 0).is64BitVector())
12652 return SDValue();
12653
12654 // Generate vpadd.
12655 SelectionDAG &DAG = DCI.DAG;
12656 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
12657 SDLoc dl(N);
12658 SDNode *Unzip = N0.getNode();
12659 EVT VT = N->getValueType(ResNo: 0);
12660
12661 SmallVector<SDValue, 8> Ops;
12662 Ops.push_back(Elt: DAG.getConstant(Val: Intrinsic::arm_neon_vpadd, DL: dl,
12663 VT: TLI.getPointerTy(DL: DAG.getDataLayout())));
12664 Ops.push_back(Elt: Unzip->getOperand(Num: 0));
12665 Ops.push_back(Elt: Unzip->getOperand(Num: 1));
12666
12667 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT, Ops);
12668}
12669
12670static SDValue AddCombineVUZPToVPADDL(SDNode *N, SDValue N0, SDValue N1,
12671 TargetLowering::DAGCombinerInfo &DCI,
12672 const ARMSubtarget *Subtarget) {
12673 // Check for two extended operands.
12674 if (!(N0.getOpcode() == ISD::SIGN_EXTEND &&
12675 N1.getOpcode() == ISD::SIGN_EXTEND) &&
12676 !(N0.getOpcode() == ISD::ZERO_EXTEND &&
12677 N1.getOpcode() == ISD::ZERO_EXTEND))
12678 return SDValue();
12679
12680 SDValue N00 = N0.getOperand(i: 0);
12681 SDValue N10 = N1.getOperand(i: 0);
12682
12683 // Look for ADD(SEXT(VUZP.0), SEXT(VUZP.1))
12684 if (!IsVUZPShuffleNode(N: N00.getNode()) || N00.getNode() != N10.getNode() ||
12685 N00 == N10)
12686 return SDValue();
12687
12688 // We only recognize Q register paddl here; this can't be reached until
12689 // after type legalization.
12690 if (!N00.getValueType().is64BitVector() ||
12691 !N0.getValueType().is128BitVector())
12692 return SDValue();
12693
12694 // Generate vpaddl.
12695 SelectionDAG &DAG = DCI.DAG;
12696 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
12697 SDLoc dl(N);
12698 EVT VT = N->getValueType(ResNo: 0);
12699
12700 SmallVector<SDValue, 8> Ops;
12701 // Form vpaddl.sN or vpaddl.uN depending on the kind of extension.
12702 unsigned Opcode;
12703 if (N0.getOpcode() == ISD::SIGN_EXTEND)
12704 Opcode = Intrinsic::arm_neon_vpaddls;
12705 else
12706 Opcode = Intrinsic::arm_neon_vpaddlu;
12707 Ops.push_back(Elt: DAG.getConstant(Val: Opcode, DL: dl,
12708 VT: TLI.getPointerTy(DL: DAG.getDataLayout())));
12709 EVT ElemTy = N00.getValueType().getVectorElementType();
12710 unsigned NumElts = VT.getVectorNumElements();
12711 EVT ConcatVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ElemTy, NumElements: NumElts * 2);
12712 SDValue Concat = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(N), VT: ConcatVT,
12713 N1: N00.getOperand(i: 0), N2: N00.getOperand(i: 1));
12714 Ops.push_back(Elt: Concat);
12715
12716 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT, Ops);
12717}
12718
12719// FIXME: This function shouldn't be necessary; if we lower BUILD_VECTOR in
12720// an appropriate manner, we end up with ADD(VUZP(ZEXT(N))), which is
12721// much easier to match.
12722static SDValue
12723AddCombineBUILD_VECTORToVPADDL(SDNode *N, SDValue N0, SDValue N1,
12724 TargetLowering::DAGCombinerInfo &DCI,
12725 const ARMSubtarget *Subtarget) {
12726 // Only perform optimization if after legalize, and if NEON is available. We
12727 // also expected both operands to be BUILD_VECTORs.
12728 if (DCI.isBeforeLegalize() || !Subtarget->hasNEON()
12729 || N0.getOpcode() != ISD::BUILD_VECTOR
12730 || N1.getOpcode() != ISD::BUILD_VECTOR)
12731 return SDValue();
12732
12733 // Check output type since VPADDL operand elements can only be 8, 16, or 32.
12734 EVT VT = N->getValueType(ResNo: 0);
12735 if (!VT.isInteger() || VT.getVectorElementType() == MVT::i64)
12736 return SDValue();
12737
12738 // Check that the vector operands are of the right form.
12739 // N0 and N1 are BUILD_VECTOR nodes with N number of EXTRACT_VECTOR
12740 // operands, where N is the size of the formed vector.
12741 // Each EXTRACT_VECTOR should have the same input vector and odd or even
12742 // index such that we have a pair wise add pattern.
12743
12744 // Grab the vector that all EXTRACT_VECTOR nodes should be referencing.
12745 if (N0->getOperand(Num: 0)->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
12746 return SDValue();
12747 SDValue Vec = N0->getOperand(Num: 0)->getOperand(Num: 0);
12748 SDNode *V = Vec.getNode();
12749 unsigned nextIndex = 0;
12750
12751 // For each operands to the ADD which are BUILD_VECTORs,
12752 // check to see if each of their operands are an EXTRACT_VECTOR with
12753 // the same vector and appropriate index.
12754 for (unsigned i = 0, e = N0->getNumOperands(); i != e; ++i) {
12755 if (N0->getOperand(Num: i)->getOpcode() == ISD::EXTRACT_VECTOR_ELT
12756 && N1->getOperand(Num: i)->getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
12757
12758 SDValue ExtVec0 = N0->getOperand(Num: i);
12759 SDValue ExtVec1 = N1->getOperand(Num: i);
12760
12761 // First operand is the vector, verify its the same.
12762 if (V != ExtVec0->getOperand(Num: 0).getNode() ||
12763 V != ExtVec1->getOperand(Num: 0).getNode())
12764 return SDValue();
12765
12766 // Second is the constant, verify its correct.
12767 ConstantSDNode *C0 = dyn_cast<ConstantSDNode>(Val: ExtVec0->getOperand(Num: 1));
12768 ConstantSDNode *C1 = dyn_cast<ConstantSDNode>(Val: ExtVec1->getOperand(Num: 1));
12769
12770 // For the constant, we want to see all the even or all the odd.
12771 if (!C0 || !C1 || C0->getZExtValue() != nextIndex
12772 || C1->getZExtValue() != nextIndex+1)
12773 return SDValue();
12774
12775 // Increment index.
12776 nextIndex+=2;
12777 } else
12778 return SDValue();
12779 }
12780
12781 // Don't generate vpaddl+vmovn; we'll match it to vpadd later. Also make sure
12782 // we're using the entire input vector, otherwise there's a size/legality
12783 // mismatch somewhere.
12784 if (nextIndex != Vec.getValueType().getVectorNumElements() ||
12785 Vec.getValueType().getVectorElementType() == VT.getVectorElementType())
12786 return SDValue();
12787
12788 // Create VPADDL node.
12789 SelectionDAG &DAG = DCI.DAG;
12790 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
12791
12792 SDLoc dl(N);
12793
12794 // Build operand list.
12795 SmallVector<SDValue, 8> Ops;
12796 Ops.push_back(Elt: DAG.getConstant(Val: Intrinsic::arm_neon_vpaddls, DL: dl,
12797 VT: TLI.getPointerTy(DL: DAG.getDataLayout())));
12798
12799 // Input is the vector.
12800 Ops.push_back(Elt: Vec);
12801
12802 // Get widened type and narrowed type.
12803 MVT widenType;
12804 unsigned numElem = VT.getVectorNumElements();
12805
12806 EVT inputLaneType = Vec.getValueType().getVectorElementType();
12807 switch (inputLaneType.getSimpleVT().SimpleTy) {
12808 case MVT::i8: widenType = MVT::getVectorVT(VT: MVT::i16, NumElements: numElem); break;
12809 case MVT::i16: widenType = MVT::getVectorVT(VT: MVT::i32, NumElements: numElem); break;
12810 case MVT::i32: widenType = MVT::getVectorVT(VT: MVT::i64, NumElements: numElem); break;
12811 default:
12812 llvm_unreachable("Invalid vector element type for padd optimization.");
12813 }
12814
12815 SDValue tmp = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: widenType, Ops);
12816 unsigned ExtOp = VT.bitsGT(VT: tmp.getValueType()) ? ISD::ANY_EXTEND : ISD::TRUNCATE;
12817 return DAG.getNode(Opcode: ExtOp, DL: dl, VT, Operand: tmp);
12818}
12819
12820static SDValue findMUL_LOHI(SDValue V) {
12821 if (V->getOpcode() == ISD::UMUL_LOHI ||
12822 V->getOpcode() == ISD::SMUL_LOHI)
12823 return V;
12824 return SDValue();
12825}
12826
12827static SDValue AddCombineTo64BitSMLAL16(SDNode *AddcNode, SDNode *AddeNode,
12828 TargetLowering::DAGCombinerInfo &DCI,
12829 const ARMSubtarget *Subtarget) {
12830 if (!Subtarget->hasBaseDSP())
12831 return SDValue();
12832
12833 // SMLALBB, SMLALBT, SMLALTB, SMLALTT multiply two 16-bit values and
12834 // accumulates the product into a 64-bit value. The 16-bit values will
12835 // be sign extended somehow or SRA'd into 32-bit values
12836 // (addc (adde (mul 16bit, 16bit), lo), hi)
12837 SDValue Mul = AddcNode->getOperand(Num: 0);
12838 SDValue Lo = AddcNode->getOperand(Num: 1);
12839 if (Mul.getOpcode() != ISD::MUL) {
12840 Lo = AddcNode->getOperand(Num: 0);
12841 Mul = AddcNode->getOperand(Num: 1);
12842 if (Mul.getOpcode() != ISD::MUL)
12843 return SDValue();
12844 }
12845
12846 SDValue SRA = AddeNode->getOperand(Num: 0);
12847 SDValue Hi = AddeNode->getOperand(Num: 1);
12848 if (SRA.getOpcode() != ISD::SRA) {
12849 SRA = AddeNode->getOperand(Num: 1);
12850 Hi = AddeNode->getOperand(Num: 0);
12851 if (SRA.getOpcode() != ISD::SRA)
12852 return SDValue();
12853 }
12854 if (auto Const = dyn_cast<ConstantSDNode>(Val: SRA.getOperand(i: 1))) {
12855 if (Const->getZExtValue() != 31)
12856 return SDValue();
12857 } else
12858 return SDValue();
12859
12860 if (SRA.getOperand(i: 0) != Mul)
12861 return SDValue();
12862
12863 SelectionDAG &DAG = DCI.DAG;
12864 SDLoc dl(AddcNode);
12865 unsigned Opcode = 0;
12866 SDValue Op0;
12867 SDValue Op1;
12868
12869 if (isS16(Op: Mul.getOperand(i: 0), DAG) && isS16(Op: Mul.getOperand(i: 1), DAG)) {
12870 Opcode = ARMISD::SMLALBB;
12871 Op0 = Mul.getOperand(i: 0);
12872 Op1 = Mul.getOperand(i: 1);
12873 } else if (isS16(Op: Mul.getOperand(i: 0), DAG) && isSRA16(Op: Mul.getOperand(i: 1))) {
12874 Opcode = ARMISD::SMLALBT;
12875 Op0 = Mul.getOperand(i: 0);
12876 Op1 = Mul.getOperand(i: 1).getOperand(i: 0);
12877 } else if (isSRA16(Op: Mul.getOperand(i: 0)) && isS16(Op: Mul.getOperand(i: 1), DAG)) {
12878 Opcode = ARMISD::SMLALTB;
12879 Op0 = Mul.getOperand(i: 0).getOperand(i: 0);
12880 Op1 = Mul.getOperand(i: 1);
12881 } else if (isSRA16(Op: Mul.getOperand(i: 0)) && isSRA16(Op: Mul.getOperand(i: 1))) {
12882 Opcode = ARMISD::SMLALTT;
12883 Op0 = Mul->getOperand(Num: 0).getOperand(i: 0);
12884 Op1 = Mul->getOperand(Num: 1).getOperand(i: 0);
12885 }
12886
12887 if (!Op0 || !Op1)
12888 return SDValue();
12889
12890 SDValue SMLAL = DAG.getNode(Opcode, DL: dl, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
12891 N1: Op0, N2: Op1, N3: Lo, N4: Hi);
12892 // Replace the ADDs' nodes uses by the MLA node's values.
12893 SDValue HiMLALResult(SMLAL.getNode(), 1);
12894 SDValue LoMLALResult(SMLAL.getNode(), 0);
12895
12896 DAG.ReplaceAllUsesOfValueWith(From: SDValue(AddcNode, 0), To: LoMLALResult);
12897 DAG.ReplaceAllUsesOfValueWith(From: SDValue(AddeNode, 0), To: HiMLALResult);
12898
12899 // Return original node to notify the driver to stop replacing.
12900 SDValue resNode(AddcNode, 0);
12901 return resNode;
12902}
12903
12904static SDValue AddCombineTo64bitMLAL(SDNode *AddeSubeNode,
12905 TargetLowering::DAGCombinerInfo &DCI,
12906 const ARMSubtarget *Subtarget) {
12907 // Look for multiply add opportunities.
12908 // The pattern is a ISD::UMUL_LOHI followed by two add nodes, where
12909 // each add nodes consumes a value from ISD::UMUL_LOHI and there is
12910 // a glue link from the first add to the second add.
12911 // If we find this pattern, we can replace the U/SMUL_LOHI, ADDC, and ADDE by
12912 // a S/UMLAL instruction.
12913 // UMUL_LOHI
12914 // / :lo \ :hi
12915 // V \ [no multiline comment]
12916 // loAdd -> ADDC |
12917 // \ :carry /
12918 // V V
12919 // ADDE <- hiAdd
12920 //
12921 // In the special case where only the higher part of a signed result is used
12922 // and the add to the low part of the result of ISD::UMUL_LOHI adds or subtracts
12923 // a constant with the exact value of 0x80000000, we recognize we are dealing
12924 // with a "rounded multiply and add" (or subtract) and transform it into
12925 // either a ARMISD::SMMLAR or ARMISD::SMMLSR respectively.
12926
12927 assert((AddeSubeNode->getOpcode() == ARMISD::ADDE ||
12928 AddeSubeNode->getOpcode() == ARMISD::SUBE) &&
12929 "Expect an ADDE or SUBE");
12930
12931 assert(AddeSubeNode->getNumOperands() == 3 &&
12932 AddeSubeNode->getOperand(2).getValueType() == MVT::i32 &&
12933 "ADDE node has the wrong inputs");
12934
12935 // Check that we are chained to the right ADDC or SUBC node.
12936 SDNode *AddcSubcNode = AddeSubeNode->getOperand(Num: 2).getNode();
12937 if ((AddeSubeNode->getOpcode() == ARMISD::ADDE &&
12938 AddcSubcNode->getOpcode() != ARMISD::ADDC) ||
12939 (AddeSubeNode->getOpcode() == ARMISD::SUBE &&
12940 AddcSubcNode->getOpcode() != ARMISD::SUBC))
12941 return SDValue();
12942
12943 SDValue AddcSubcOp0 = AddcSubcNode->getOperand(Num: 0);
12944 SDValue AddcSubcOp1 = AddcSubcNode->getOperand(Num: 1);
12945
12946 // Check if the two operands are from the same mul_lohi node.
12947 if (AddcSubcOp0.getNode() == AddcSubcOp1.getNode())
12948 return SDValue();
12949
12950 assert(AddcSubcNode->getNumValues() == 2 &&
12951 AddcSubcNode->getValueType(0) == MVT::i32 &&
12952 "Expect ADDC with two result values. First: i32");
12953
12954 // Check that the ADDC adds the low result of the S/UMUL_LOHI. If not, it
12955 // maybe a SMLAL which multiplies two 16-bit values.
12956 if (AddeSubeNode->getOpcode() == ARMISD::ADDE &&
12957 AddcSubcOp0->getOpcode() != ISD::UMUL_LOHI &&
12958 AddcSubcOp0->getOpcode() != ISD::SMUL_LOHI &&
12959 AddcSubcOp1->getOpcode() != ISD::UMUL_LOHI &&
12960 AddcSubcOp1->getOpcode() != ISD::SMUL_LOHI)
12961 return AddCombineTo64BitSMLAL16(AddcNode: AddcSubcNode, AddeNode: AddeSubeNode, DCI, Subtarget);
12962
12963 // Check for the triangle shape.
12964 SDValue AddeSubeOp0 = AddeSubeNode->getOperand(Num: 0);
12965 SDValue AddeSubeOp1 = AddeSubeNode->getOperand(Num: 1);
12966
12967 // Make sure that the ADDE/SUBE operands are not coming from the same node.
12968 if (AddeSubeOp0.getNode() == AddeSubeOp1.getNode())
12969 return SDValue();
12970
12971 // Find the MUL_LOHI node walking up ADDE/SUBE's operands.
12972 bool IsLeftOperandMUL = false;
12973 SDValue MULOp = findMUL_LOHI(V: AddeSubeOp0);
12974 if (MULOp == SDValue())
12975 MULOp = findMUL_LOHI(V: AddeSubeOp1);
12976 else
12977 IsLeftOperandMUL = true;
12978 if (MULOp == SDValue())
12979 return SDValue();
12980
12981 // Figure out the right opcode.
12982 unsigned Opc = MULOp->getOpcode();
12983 unsigned FinalOpc = (Opc == ISD::SMUL_LOHI) ? ARMISD::SMLAL : ARMISD::UMLAL;
12984
12985 // Figure out the high and low input values to the MLAL node.
12986 SDValue *HiAddSub = nullptr;
12987 SDValue *LoMul = nullptr;
12988 SDValue *LowAddSub = nullptr;
12989
12990 // Ensure that ADDE/SUBE is from high result of ISD::xMUL_LOHI.
12991 if ((AddeSubeOp0 != MULOp.getValue(R: 1)) && (AddeSubeOp1 != MULOp.getValue(R: 1)))
12992 return SDValue();
12993
12994 if (IsLeftOperandMUL)
12995 HiAddSub = &AddeSubeOp1;
12996 else
12997 HiAddSub = &AddeSubeOp0;
12998
12999 // Ensure that LoMul and LowAddSub are taken from correct ISD::SMUL_LOHI node
13000 // whose low result is fed to the ADDC/SUBC we are checking.
13001
13002 if (AddcSubcOp0 == MULOp.getValue(R: 0)) {
13003 LoMul = &AddcSubcOp0;
13004 LowAddSub = &AddcSubcOp1;
13005 }
13006 if (AddcSubcOp1 == MULOp.getValue(R: 0)) {
13007 LoMul = &AddcSubcOp1;
13008 LowAddSub = &AddcSubcOp0;
13009 }
13010
13011 if (!LoMul)
13012 return SDValue();
13013
13014 // If HiAddSub is the same node as ADDC/SUBC or is a predecessor of ADDC/SUBC
13015 // the replacement below will create a cycle.
13016 if (AddcSubcNode == HiAddSub->getNode() ||
13017 AddcSubcNode->isPredecessorOf(N: HiAddSub->getNode()))
13018 return SDValue();
13019
13020 // Create the merged node.
13021 SelectionDAG &DAG = DCI.DAG;
13022
13023 // Start building operand list.
13024 SmallVector<SDValue, 8> Ops;
13025 Ops.push_back(Elt: LoMul->getOperand(i: 0));
13026 Ops.push_back(Elt: LoMul->getOperand(i: 1));
13027
13028 // Check whether we can use SMMLAR, SMMLSR or SMMULR instead. For this to be
13029 // the case, we must be doing signed multiplication and only use the higher
13030 // part of the result of the MLAL, furthermore the LowAddSub must be a constant
13031 // addition or subtraction with the value of 0x800000.
13032 if (Subtarget->hasV6Ops() && Subtarget->hasDSP() && Subtarget->useMulOps() &&
13033 FinalOpc == ARMISD::SMLAL && !AddeSubeNode->hasAnyUseOfValue(Value: 1) &&
13034 LowAddSub->getNode()->getOpcode() == ISD::Constant &&
13035 static_cast<ConstantSDNode *>(LowAddSub->getNode())->getZExtValue() ==
13036 0x80000000) {
13037 Ops.push_back(Elt: *HiAddSub);
13038 if (AddcSubcNode->getOpcode() == ARMISD::SUBC) {
13039 FinalOpc = ARMISD::SMMLSR;
13040 } else {
13041 FinalOpc = ARMISD::SMMLAR;
13042 }
13043 SDValue NewNode = DAG.getNode(Opcode: FinalOpc, DL: SDLoc(AddcSubcNode), VT: MVT::i32, Ops);
13044 DAG.ReplaceAllUsesOfValueWith(From: SDValue(AddeSubeNode, 0), To: NewNode);
13045
13046 return SDValue(AddeSubeNode, 0);
13047 } else if (AddcSubcNode->getOpcode() == ARMISD::SUBC)
13048 // SMMLS is generated during instruction selection and the rest of this
13049 // function can not handle the case where AddcSubcNode is a SUBC.
13050 return SDValue();
13051
13052 // Finish building the operand list for {U/S}MLAL
13053 Ops.push_back(Elt: *LowAddSub);
13054 Ops.push_back(Elt: *HiAddSub);
13055
13056 SDValue MLALNode = DAG.getNode(Opcode: FinalOpc, DL: SDLoc(AddcSubcNode),
13057 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), Ops);
13058
13059 // Replace the ADDs' nodes uses by the MLA node's values.
13060 SDValue HiMLALResult(MLALNode.getNode(), 1);
13061 DAG.ReplaceAllUsesOfValueWith(From: SDValue(AddeSubeNode, 0), To: HiMLALResult);
13062
13063 SDValue LoMLALResult(MLALNode.getNode(), 0);
13064 DAG.ReplaceAllUsesOfValueWith(From: SDValue(AddcSubcNode, 0), To: LoMLALResult);
13065
13066 // Return original node to notify the driver to stop replacing.
13067 return SDValue(AddeSubeNode, 0);
13068}
13069
13070static SDValue AddCombineTo64bitUMAAL(SDNode *AddeNode,
13071 TargetLowering::DAGCombinerInfo &DCI,
13072 const ARMSubtarget *Subtarget) {
13073 // UMAAL is similar to UMLAL except that it adds two unsigned values.
13074 // While trying to combine for the other MLAL nodes, first search for the
13075 // chance to use UMAAL. Check if Addc uses a node which has already
13076 // been combined into a UMLAL. The other pattern is UMLAL using Addc/Adde
13077 // as the addend, and it's handled in PerformUMLALCombine.
13078
13079 if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP())
13080 return AddCombineTo64bitMLAL(AddeSubeNode: AddeNode, DCI, Subtarget);
13081
13082 // Check that we have a glued ADDC node.
13083 SDNode* AddcNode = AddeNode->getOperand(Num: 2).getNode();
13084 if (AddcNode->getOpcode() != ARMISD::ADDC)
13085 return SDValue();
13086
13087 // Find the converted UMAAL or quit if it doesn't exist.
13088 SDNode *UmlalNode = nullptr;
13089 SDValue AddHi;
13090 if (AddcNode->getOperand(Num: 0).getOpcode() == ARMISD::UMLAL) {
13091 UmlalNode = AddcNode->getOperand(Num: 0).getNode();
13092 AddHi = AddcNode->getOperand(Num: 1);
13093 } else if (AddcNode->getOperand(Num: 1).getOpcode() == ARMISD::UMLAL) {
13094 UmlalNode = AddcNode->getOperand(Num: 1).getNode();
13095 AddHi = AddcNode->getOperand(Num: 0);
13096 } else {
13097 return AddCombineTo64bitMLAL(AddeSubeNode: AddeNode, DCI, Subtarget);
13098 }
13099
13100 // The ADDC should be glued to an ADDE node, which uses the same UMLAL as
13101 // the ADDC as well as Zero.
13102 if (!isNullConstant(V: UmlalNode->getOperand(Num: 3)))
13103 return SDValue();
13104
13105 if ((isNullConstant(V: AddeNode->getOperand(Num: 0)) &&
13106 AddeNode->getOperand(Num: 1).getNode() == UmlalNode) ||
13107 (AddeNode->getOperand(Num: 0).getNode() == UmlalNode &&
13108 isNullConstant(V: AddeNode->getOperand(Num: 1)))) {
13109 SelectionDAG &DAG = DCI.DAG;
13110 SDValue Ops[] = { UmlalNode->getOperand(Num: 0), UmlalNode->getOperand(Num: 1),
13111 UmlalNode->getOperand(Num: 2), AddHi };
13112 SDValue UMAAL = DAG.getNode(Opcode: ARMISD::UMAAL, DL: SDLoc(AddcNode),
13113 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), Ops);
13114
13115 // Replace the ADDs' nodes uses by the UMAAL node's values.
13116 DAG.ReplaceAllUsesOfValueWith(From: SDValue(AddeNode, 0), To: SDValue(UMAAL.getNode(), 1));
13117 DAG.ReplaceAllUsesOfValueWith(From: SDValue(AddcNode, 0), To: SDValue(UMAAL.getNode(), 0));
13118
13119 // Return original node to notify the driver to stop replacing.
13120 return SDValue(AddeNode, 0);
13121 }
13122 return SDValue();
13123}
13124
13125static SDValue PerformUMLALCombine(SDNode *N, SelectionDAG &DAG,
13126 const ARMSubtarget *Subtarget) {
13127 if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP())
13128 return SDValue();
13129
13130 // Check that we have a pair of ADDC and ADDE as operands.
13131 // Both addends of the ADDE must be zero.
13132 SDNode* AddcNode = N->getOperand(Num: 2).getNode();
13133 SDNode* AddeNode = N->getOperand(Num: 3).getNode();
13134 if ((AddcNode->getOpcode() == ARMISD::ADDC) &&
13135 (AddeNode->getOpcode() == ARMISD::ADDE) &&
13136 isNullConstant(V: AddeNode->getOperand(Num: 0)) &&
13137 isNullConstant(V: AddeNode->getOperand(Num: 1)) &&
13138 (AddeNode->getOperand(Num: 2).getNode() == AddcNode))
13139 return DAG.getNode(Opcode: ARMISD::UMAAL, DL: SDLoc(N),
13140 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
13141 Ops: {N->getOperand(Num: 0), N->getOperand(Num: 1),
13142 AddcNode->getOperand(Num: 0), AddcNode->getOperand(Num: 1)});
13143 else
13144 return SDValue();
13145}
13146
13147static SDValue PerformAddcSubcCombine(SDNode *N,
13148 TargetLowering::DAGCombinerInfo &DCI,
13149 const ARMSubtarget *Subtarget) {
13150 SelectionDAG &DAG(DCI.DAG);
13151
13152 if (N->getOpcode() == ARMISD::SUBC && N->hasAnyUseOfValue(Value: 1)) {
13153 // (SUBC (ADDE 0, 0, C), 1) -> C
13154 SDValue LHS = N->getOperand(Num: 0);
13155 SDValue RHS = N->getOperand(Num: 1);
13156 if (LHS->getOpcode() == ARMISD::ADDE &&
13157 isNullConstant(V: LHS->getOperand(Num: 0)) &&
13158 isNullConstant(V: LHS->getOperand(Num: 1)) && isOneConstant(V: RHS)) {
13159 return DCI.CombineTo(N, Res0: SDValue(N, 0), Res1: LHS->getOperand(Num: 2));
13160 }
13161 }
13162
13163 if (Subtarget->isThumb1Only()) {
13164 SDValue RHS = N->getOperand(Num: 1);
13165 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: RHS)) {
13166 int32_t imm = C->getSExtValue();
13167 if (imm < 0 && imm > std::numeric_limits<int>::min()) {
13168 SDLoc DL(N);
13169 RHS = DAG.getConstant(Val: -imm, DL, VT: MVT::i32);
13170 unsigned Opcode = (N->getOpcode() == ARMISD::ADDC) ? ARMISD::SUBC
13171 : ARMISD::ADDC;
13172 return DAG.getNode(Opcode, DL, VTList: N->getVTList(), N1: N->getOperand(Num: 0), N2: RHS);
13173 }
13174 }
13175 }
13176
13177 return SDValue();
13178}
13179
13180static SDValue PerformAddeSubeCombine(SDNode *N,
13181 TargetLowering::DAGCombinerInfo &DCI,
13182 const ARMSubtarget *Subtarget) {
13183 if (Subtarget->isThumb1Only()) {
13184 SelectionDAG &DAG = DCI.DAG;
13185 SDValue RHS = N->getOperand(Num: 1);
13186 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: RHS)) {
13187 int64_t imm = C->getSExtValue();
13188 if (imm < 0) {
13189 SDLoc DL(N);
13190
13191 // The with-carry-in form matches bitwise not instead of the negation.
13192 // Effectively, the inverse interpretation of the carry flag already
13193 // accounts for part of the negation.
13194 RHS = DAG.getConstant(Val: ~imm, DL, VT: MVT::i32);
13195
13196 unsigned Opcode = (N->getOpcode() == ARMISD::ADDE) ? ARMISD::SUBE
13197 : ARMISD::ADDE;
13198 return DAG.getNode(Opcode, DL, VTList: N->getVTList(),
13199 N1: N->getOperand(Num: 0), N2: RHS, N3: N->getOperand(Num: 2));
13200 }
13201 }
13202 } else if (N->getOperand(Num: 1)->getOpcode() == ISD::SMUL_LOHI) {
13203 return AddCombineTo64bitMLAL(AddeSubeNode: N, DCI, Subtarget);
13204 }
13205 return SDValue();
13206}
13207
13208static SDValue PerformSELECTCombine(SDNode *N,
13209 TargetLowering::DAGCombinerInfo &DCI,
13210 const ARMSubtarget *Subtarget) {
13211 if (!Subtarget->hasMVEIntegerOps())
13212 return SDValue();
13213
13214 SDLoc dl(N);
13215 SDValue SetCC;
13216 SDValue LHS;
13217 SDValue RHS;
13218 ISD::CondCode CC;
13219 SDValue TrueVal;
13220 SDValue FalseVal;
13221
13222 if (N->getOpcode() == ISD::SELECT &&
13223 N->getOperand(Num: 0)->getOpcode() == ISD::SETCC) {
13224 SetCC = N->getOperand(Num: 0);
13225 LHS = SetCC->getOperand(Num: 0);
13226 RHS = SetCC->getOperand(Num: 1);
13227 CC = cast<CondCodeSDNode>(Val: SetCC->getOperand(Num: 2))->get();
13228 TrueVal = N->getOperand(Num: 1);
13229 FalseVal = N->getOperand(Num: 2);
13230 } else if (N->getOpcode() == ISD::SELECT_CC) {
13231 LHS = N->getOperand(Num: 0);
13232 RHS = N->getOperand(Num: 1);
13233 CC = cast<CondCodeSDNode>(Val: N->getOperand(Num: 4))->get();
13234 TrueVal = N->getOperand(Num: 2);
13235 FalseVal = N->getOperand(Num: 3);
13236 } else {
13237 return SDValue();
13238 }
13239
13240 unsigned int Opcode = 0;
13241 if ((TrueVal->getOpcode() == ISD::VECREDUCE_UMIN ||
13242 FalseVal->getOpcode() == ISD::VECREDUCE_UMIN) &&
13243 (CC == ISD::SETULT || CC == ISD::SETUGT)) {
13244 Opcode = ARMISD::VMINVu;
13245 if (CC == ISD::SETUGT)
13246 std::swap(a&: TrueVal, b&: FalseVal);
13247 } else if ((TrueVal->getOpcode() == ISD::VECREDUCE_SMIN ||
13248 FalseVal->getOpcode() == ISD::VECREDUCE_SMIN) &&
13249 (CC == ISD::SETLT || CC == ISD::SETGT)) {
13250 Opcode = ARMISD::VMINVs;
13251 if (CC == ISD::SETGT)
13252 std::swap(a&: TrueVal, b&: FalseVal);
13253 } else if ((TrueVal->getOpcode() == ISD::VECREDUCE_UMAX ||
13254 FalseVal->getOpcode() == ISD::VECREDUCE_UMAX) &&
13255 (CC == ISD::SETUGT || CC == ISD::SETULT)) {
13256 Opcode = ARMISD::VMAXVu;
13257 if (CC == ISD::SETULT)
13258 std::swap(a&: TrueVal, b&: FalseVal);
13259 } else if ((TrueVal->getOpcode() == ISD::VECREDUCE_SMAX ||
13260 FalseVal->getOpcode() == ISD::VECREDUCE_SMAX) &&
13261 (CC == ISD::SETGT || CC == ISD::SETLT)) {
13262 Opcode = ARMISD::VMAXVs;
13263 if (CC == ISD::SETLT)
13264 std::swap(a&: TrueVal, b&: FalseVal);
13265 } else
13266 return SDValue();
13267
13268 // Normalise to the right hand side being the vector reduction
13269 switch (TrueVal->getOpcode()) {
13270 case ISD::VECREDUCE_UMIN:
13271 case ISD::VECREDUCE_SMIN:
13272 case ISD::VECREDUCE_UMAX:
13273 case ISD::VECREDUCE_SMAX:
13274 std::swap(a&: LHS, b&: RHS);
13275 std::swap(a&: TrueVal, b&: FalseVal);
13276 break;
13277 }
13278
13279 EVT VectorType = FalseVal->getOperand(Num: 0).getValueType();
13280
13281 if (VectorType != MVT::v16i8 && VectorType != MVT::v8i16 &&
13282 VectorType != MVT::v4i32)
13283 return SDValue();
13284
13285 EVT VectorScalarType = VectorType.getVectorElementType();
13286
13287 // The values being selected must also be the ones being compared
13288 if (TrueVal != LHS || FalseVal != RHS)
13289 return SDValue();
13290
13291 EVT LeftType = LHS->getValueType(ResNo: 0);
13292 EVT RightType = RHS->getValueType(ResNo: 0);
13293
13294 // The types must match the reduced type too
13295 if (LeftType != VectorScalarType || RightType != VectorScalarType)
13296 return SDValue();
13297
13298 // Legalise the scalar to an i32
13299 if (VectorScalarType != MVT::i32)
13300 LHS = DCI.DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: MVT::i32, Operand: LHS);
13301
13302 // Generate the reduction as an i32 for legalisation purposes
13303 auto Reduction =
13304 DCI.DAG.getNode(Opcode, DL: dl, VT: MVT::i32, N1: LHS, N2: RHS->getOperand(Num: 0));
13305
13306 // The result isn't actually an i32 so truncate it back to its original type
13307 if (VectorScalarType != MVT::i32)
13308 Reduction = DCI.DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: VectorScalarType, Operand: Reduction);
13309
13310 return Reduction;
13311}
13312
13313// A special combine for the vqdmulh family of instructions. This is one of the
13314// potential set of patterns that could patch this instruction. The base pattern
13315// you would expect to be min(max(ashr(mul(mul(sext(x), 2), sext(y)), 16))).
13316// This matches the different min(max(ashr(mul(mul(sext(x), sext(y)), 2), 16))),
13317// which llvm will have optimized to min(ashr(mul(sext(x), sext(y)), 15))) as
13318// the max is unnecessary.
13319static SDValue PerformVQDMULHCombine(SDNode *N, SelectionDAG &DAG) {
13320 EVT VT = N->getValueType(ResNo: 0);
13321 SDValue Shft;
13322 ConstantSDNode *Clamp;
13323
13324 if (!VT.isVector() || VT.getScalarSizeInBits() > 64)
13325 return SDValue();
13326
13327 if (N->getOpcode() == ISD::SMIN) {
13328 Shft = N->getOperand(Num: 0);
13329 Clamp = isConstOrConstSplat(N: N->getOperand(Num: 1));
13330 } else if (N->getOpcode() == ISD::VSELECT) {
13331 // Detect a SMIN, which for an i64 node will be a vselect/setcc, not a smin.
13332 SDValue Cmp = N->getOperand(Num: 0);
13333 if (Cmp.getOpcode() != ISD::SETCC ||
13334 cast<CondCodeSDNode>(Val: Cmp.getOperand(i: 2))->get() != ISD::SETLT ||
13335 Cmp.getOperand(i: 0) != N->getOperand(Num: 1) ||
13336 Cmp.getOperand(i: 1) != N->getOperand(Num: 2))
13337 return SDValue();
13338 Shft = N->getOperand(Num: 1);
13339 Clamp = isConstOrConstSplat(N: N->getOperand(Num: 2));
13340 } else
13341 return SDValue();
13342
13343 if (!Clamp)
13344 return SDValue();
13345
13346 MVT ScalarType;
13347 int ShftAmt = 0;
13348 switch (Clamp->getSExtValue()) {
13349 case (1 << 7) - 1:
13350 ScalarType = MVT::i8;
13351 ShftAmt = 7;
13352 break;
13353 case (1 << 15) - 1:
13354 ScalarType = MVT::i16;
13355 ShftAmt = 15;
13356 break;
13357 case (1ULL << 31) - 1:
13358 ScalarType = MVT::i32;
13359 ShftAmt = 31;
13360 break;
13361 default:
13362 return SDValue();
13363 }
13364
13365 if (Shft.getOpcode() != ISD::SRA)
13366 return SDValue();
13367 ConstantSDNode *N1 = isConstOrConstSplat(N: Shft.getOperand(i: 1));
13368 if (!N1 || N1->getSExtValue() != ShftAmt)
13369 return SDValue();
13370
13371 SDValue Mul = Shft.getOperand(i: 0);
13372 if (Mul.getOpcode() != ISD::MUL)
13373 return SDValue();
13374
13375 SDValue Ext0 = Mul.getOperand(i: 0);
13376 SDValue Ext1 = Mul.getOperand(i: 1);
13377 if (Ext0.getOpcode() != ISD::SIGN_EXTEND ||
13378 Ext1.getOpcode() != ISD::SIGN_EXTEND)
13379 return SDValue();
13380 EVT VecVT = Ext0.getOperand(i: 0).getValueType();
13381 if (!VecVT.isPow2VectorType() || VecVT.getVectorNumElements() == 1)
13382 return SDValue();
13383 if (Ext1.getOperand(i: 0).getValueType() != VecVT ||
13384 VecVT.getScalarType() != ScalarType ||
13385 VT.getScalarSizeInBits() < ScalarType.getScalarSizeInBits() * 2)
13386 return SDValue();
13387
13388 SDLoc DL(Mul);
13389 unsigned LegalLanes = 128 / (ShftAmt + 1);
13390 EVT LegalVecVT = MVT::getVectorVT(VT: ScalarType, NumElements: LegalLanes);
13391 // For types smaller than legal vectors extend to be legal and only use needed
13392 // lanes.
13393 if (VecVT.getSizeInBits() < 128) {
13394 EVT ExtVecVT =
13395 MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: 128 / VecVT.getVectorNumElements()),
13396 NumElements: VecVT.getVectorNumElements());
13397 SDValue Inp0 =
13398 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: ExtVecVT, Operand: Ext0.getOperand(i: 0));
13399 SDValue Inp1 =
13400 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: ExtVecVT, Operand: Ext1.getOperand(i: 0));
13401 Inp0 = DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL, VT: LegalVecVT, Operand: Inp0);
13402 Inp1 = DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL, VT: LegalVecVT, Operand: Inp1);
13403 SDValue VQDMULH = DAG.getNode(Opcode: ARMISD::VQDMULH, DL, VT: LegalVecVT, N1: Inp0, N2: Inp1);
13404 SDValue Trunc = DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL, VT: ExtVecVT, Operand: VQDMULH);
13405 Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: VecVT, Operand: Trunc);
13406 return DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT, Operand: Trunc);
13407 }
13408
13409 // For larger types, split into legal sized chunks.
13410 assert(VecVT.getSizeInBits() % 128 == 0 && "Expected a power2 type");
13411 unsigned NumParts = VecVT.getSizeInBits() / 128;
13412 SmallVector<SDValue> Parts;
13413 for (unsigned I = 0; I < NumParts; ++I) {
13414 SDValue Inp0 =
13415 DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: LegalVecVT, N1: Ext0.getOperand(i: 0),
13416 N2: DAG.getVectorIdxConstant(Val: I * LegalLanes, DL));
13417 SDValue Inp1 =
13418 DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: LegalVecVT, N1: Ext1.getOperand(i: 0),
13419 N2: DAG.getVectorIdxConstant(Val: I * LegalLanes, DL));
13420 SDValue VQDMULH = DAG.getNode(Opcode: ARMISD::VQDMULH, DL, VT: LegalVecVT, N1: Inp0, N2: Inp1);
13421 Parts.push_back(Elt: VQDMULH);
13422 }
13423 return DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT,
13424 Operand: DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: VecVT, Ops: Parts));
13425}
13426
13427static SDValue PerformVSELECTCombine(SDNode *N,
13428 TargetLowering::DAGCombinerInfo &DCI,
13429 const ARMSubtarget *Subtarget) {
13430 if (!Subtarget->hasMVEIntegerOps())
13431 return SDValue();
13432
13433 // Constant fold vselect 0, A, B -> B
13434 // and vselect 0xffff, A, B -> A
13435 if (N->getOperand(Num: 0).getOpcode() == ARMISD::PREDICATE_CAST &&
13436 isa<ConstantSDNode>(Val: N->getOperand(Num: 0).getOperand(i: 0))) {
13437 unsigned C = N->getOperand(Num: 0).getConstantOperandVal(i: 0);
13438 if (C == 0)
13439 return N->getOperand(Num: 2);
13440 if (C == 0xffff)
13441 return N->getOperand(Num: 1);
13442 }
13443
13444 if (SDValue V = PerformVQDMULHCombine(N, DAG&: DCI.DAG))
13445 return V;
13446
13447 // Transforms vselect(not(cond), lhs, rhs) into vselect(cond, rhs, lhs).
13448 //
13449 // We need to re-implement this optimization here as the implementation in the
13450 // Target-Independent DAGCombiner does not handle the kind of constant we make
13451 // (it calls isConstOrConstSplat with AllowTruncation set to false - and for
13452 // good reason, allowing truncation there would break other targets).
13453 //
13454 // Currently, this is only done for MVE, as it's the only target that benefits
13455 // from this transformation (e.g. VPNOT+VPSEL becomes a single VPSEL).
13456 if (N->getOperand(Num: 0).getOpcode() != ISD::XOR)
13457 return SDValue();
13458 SDValue XOR = N->getOperand(Num: 0);
13459
13460 // Check if the XOR's RHS is either a 1, or a BUILD_VECTOR of 1s.
13461 // It is important to check with truncation allowed as the BUILD_VECTORs we
13462 // generate in those situations will truncate their operands.
13463 ConstantSDNode *Const =
13464 isConstOrConstSplat(N: XOR->getOperand(Num: 1), /*AllowUndefs*/ false,
13465 /*AllowTruncation*/ true);
13466 if (!Const || !Const->isOne())
13467 return SDValue();
13468
13469 // Rewrite into vselect(cond, rhs, lhs).
13470 SDValue Cond = XOR->getOperand(Num: 0);
13471 SDValue LHS = N->getOperand(Num: 1);
13472 SDValue RHS = N->getOperand(Num: 2);
13473 EVT Type = N->getValueType(ResNo: 0);
13474 return DCI.DAG.getNode(Opcode: ISD::VSELECT, DL: SDLoc(N), VT: Type, N1: Cond, N2: RHS, N3: LHS);
13475}
13476
13477// Convert vsetcc([0,1,2,..], splat(n), ult) -> vctp n
13478static SDValue PerformVSetCCToVCTPCombine(SDNode *N,
13479 TargetLowering::DAGCombinerInfo &DCI,
13480 const ARMSubtarget *Subtarget) {
13481 SDValue Op0 = N->getOperand(Num: 0);
13482 SDValue Op1 = N->getOperand(Num: 1);
13483 ISD::CondCode CC = cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get();
13484 EVT VT = N->getValueType(ResNo: 0);
13485
13486 if (!Subtarget->hasMVEIntegerOps() ||
13487 !DCI.DAG.getTargetLoweringInfo().isTypeLegal(VT))
13488 return SDValue();
13489
13490 if (CC == ISD::SETUGE) {
13491 std::swap(a&: Op0, b&: Op1);
13492 CC = ISD::SETULT;
13493 }
13494
13495 if (CC != ISD::SETULT || VT.getScalarSizeInBits() != 1 ||
13496 Op0.getOpcode() != ISD::BUILD_VECTOR)
13497 return SDValue();
13498
13499 // Check first operand is BuildVector of 0,1,2,...
13500 for (unsigned I = 0; I < VT.getVectorNumElements(); I++) {
13501 if (!Op0.getOperand(i: I).isUndef() &&
13502 !(isa<ConstantSDNode>(Val: Op0.getOperand(i: I)) &&
13503 Op0.getConstantOperandVal(i: I) == I))
13504 return SDValue();
13505 }
13506
13507 // The second is a Splat of Op1S
13508 SDValue Op1S = DCI.DAG.getSplatValue(V: Op1);
13509 if (!Op1S)
13510 return SDValue();
13511
13512 unsigned Opc;
13513 switch (VT.getVectorNumElements()) {
13514 case 2:
13515 Opc = Intrinsic::arm_mve_vctp64;
13516 break;
13517 case 4:
13518 Opc = Intrinsic::arm_mve_vctp32;
13519 break;
13520 case 8:
13521 Opc = Intrinsic::arm_mve_vctp16;
13522 break;
13523 case 16:
13524 Opc = Intrinsic::arm_mve_vctp8;
13525 break;
13526 default:
13527 return SDValue();
13528 }
13529
13530 SDLoc DL(N);
13531 return DCI.DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT,
13532 N1: DCI.DAG.getConstant(Val: Opc, DL, VT: MVT::i32),
13533 N2: DCI.DAG.getZExtOrTrunc(Op: Op1S, DL, VT: MVT::i32));
13534}
13535
13536/// PerformADDECombine - Target-specific dag combine transform from
13537/// ARMISD::ADDC, ARMISD::ADDE, and ISD::MUL_LOHI to MLAL or
13538/// ARMISD::ADDC, ARMISD::ADDE and ARMISD::UMLAL to ARMISD::UMAAL
13539static SDValue PerformADDECombine(SDNode *N,
13540 TargetLowering::DAGCombinerInfo &DCI,
13541 const ARMSubtarget *Subtarget) {
13542 // Only ARM and Thumb2 support UMLAL/SMLAL.
13543 if (Subtarget->isThumb1Only())
13544 return PerformAddeSubeCombine(N, DCI, Subtarget);
13545
13546 // Only perform the checks after legalize when the pattern is available.
13547 if (DCI.isBeforeLegalize()) return SDValue();
13548
13549 return AddCombineTo64bitUMAAL(AddeNode: N, DCI, Subtarget);
13550}
13551
13552/// PerformADDCombineWithOperands - Try DAG combinations for an ADD with
13553/// operands N0 and N1. This is a helper for PerformADDCombine that is
13554/// called with the default operands, and if that fails, with commuted
13555/// operands.
13556static SDValue PerformADDCombineWithOperands(SDNode *N, SDValue N0, SDValue N1,
13557 TargetLowering::DAGCombinerInfo &DCI,
13558 const ARMSubtarget *Subtarget){
13559 // Attempt to create vpadd for this add.
13560 if (SDValue Result = AddCombineToVPADD(N, N0, N1, DCI, Subtarget))
13561 return Result;
13562
13563 // Attempt to create vpaddl for this add.
13564 if (SDValue Result = AddCombineVUZPToVPADDL(N, N0, N1, DCI, Subtarget))
13565 return Result;
13566 if (SDValue Result = AddCombineBUILD_VECTORToVPADDL(N, N0, N1, DCI,
13567 Subtarget))
13568 return Result;
13569
13570 // fold (add (select cc, 0, c), x) -> (select cc, x, (add, x, c))
13571 if (N0.getNode()->hasOneUse())
13572 if (SDValue Result = combineSelectAndUse(N, Slct: N0, OtherOp: N1, DCI))
13573 return Result;
13574 return SDValue();
13575}
13576
13577static SDValue TryDistrubutionADDVecReduce(SDNode *N, SelectionDAG &DAG) {
13578 EVT VT = N->getValueType(ResNo: 0);
13579 SDValue N0 = N->getOperand(Num: 0);
13580 SDValue N1 = N->getOperand(Num: 1);
13581 SDLoc dl(N);
13582
13583 auto IsVecReduce = [](SDValue Op) {
13584 switch (Op.getOpcode()) {
13585 case ISD::VECREDUCE_ADD:
13586 case ARMISD::VADDVs:
13587 case ARMISD::VADDVu:
13588 case ARMISD::VMLAVs:
13589 case ARMISD::VMLAVu:
13590 return true;
13591 }
13592 return false;
13593 };
13594
13595 auto DistrubuteAddAddVecReduce = [&](SDValue N0, SDValue N1) {
13596 // Distribute add(X, add(vecreduce(Y), vecreduce(Z))) ->
13597 // add(add(X, vecreduce(Y)), vecreduce(Z))
13598 // to make better use of vaddva style instructions.
13599 if (VT == MVT::i32 && N1.getOpcode() == ISD::ADD && !IsVecReduce(N0) &&
13600 IsVecReduce(N1.getOperand(i: 0)) && IsVecReduce(N1.getOperand(i: 1)) &&
13601 !isa<ConstantSDNode>(Val: N0) && N1->hasOneUse()) {
13602 SDValue Add0 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: N0, N2: N1.getOperand(i: 0));
13603 return DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Add0, N2: N1.getOperand(i: 1));
13604 }
13605 // And turn add(add(A, reduce(B)), add(C, reduce(D))) ->
13606 // add(add(add(A, C), reduce(B)), reduce(D))
13607 if (VT == MVT::i32 && N0.getOpcode() == ISD::ADD &&
13608 N1.getOpcode() == ISD::ADD && N0->hasOneUse() && N1->hasOneUse()) {
13609 unsigned N0RedOp = 0;
13610 if (!IsVecReduce(N0.getOperand(i: N0RedOp))) {
13611 N0RedOp = 1;
13612 if (!IsVecReduce(N0.getOperand(i: N0RedOp)))
13613 return SDValue();
13614 }
13615
13616 unsigned N1RedOp = 0;
13617 if (!IsVecReduce(N1.getOperand(i: N1RedOp)))
13618 N1RedOp = 1;
13619 if (!IsVecReduce(N1.getOperand(i: N1RedOp)))
13620 return SDValue();
13621
13622 SDValue Add0 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: N0.getOperand(i: 1 - N0RedOp),
13623 N2: N1.getOperand(i: 1 - N1RedOp));
13624 SDValue Add1 =
13625 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Add0, N2: N0.getOperand(i: N0RedOp));
13626 return DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Add1, N2: N1.getOperand(i: N1RedOp));
13627 }
13628 return SDValue();
13629 };
13630 if (SDValue R = DistrubuteAddAddVecReduce(N0, N1))
13631 return R;
13632 if (SDValue R = DistrubuteAddAddVecReduce(N1, N0))
13633 return R;
13634
13635 // Distribute add(vecreduce(load(Y)), vecreduce(load(Z)))
13636 // Or add(add(X, vecreduce(load(Y))), vecreduce(load(Z)))
13637 // by ascending load offsets. This can help cores prefetch if the order of
13638 // loads is more predictable.
13639 auto DistrubuteVecReduceLoad = [&](SDValue N0, SDValue N1, bool IsForward) {
13640 // Check if two reductions are known to load data where one is before/after
13641 // another. Return negative if N0 loads data before N1, positive if N1 is
13642 // before N0 and 0 otherwise if nothing is known.
13643 auto IsKnownOrderedLoad = [&](SDValue N0, SDValue N1) {
13644 // Look through to the first operand of a MUL, for the VMLA case.
13645 // Currently only looks at the first operand, in the hope they are equal.
13646 if (N0.getOpcode() == ISD::MUL)
13647 N0 = N0.getOperand(i: 0);
13648 if (N1.getOpcode() == ISD::MUL)
13649 N1 = N1.getOperand(i: 0);
13650
13651 // Return true if the two operands are loads to the same object and the
13652 // offset of the first is known to be less than the offset of the second.
13653 LoadSDNode *Load0 = dyn_cast<LoadSDNode>(Val&: N0);
13654 LoadSDNode *Load1 = dyn_cast<LoadSDNode>(Val&: N1);
13655 if (!Load0 || !Load1 || Load0->getChain() != Load1->getChain() ||
13656 !Load0->isSimple() || !Load1->isSimple() || Load0->isIndexed() ||
13657 Load1->isIndexed())
13658 return 0;
13659
13660 auto BaseLocDecomp0 = BaseIndexOffset::match(N: Load0, DAG);
13661 auto BaseLocDecomp1 = BaseIndexOffset::match(N: Load1, DAG);
13662
13663 if (!BaseLocDecomp0.getBase() ||
13664 BaseLocDecomp0.getBase() != BaseLocDecomp1.getBase() ||
13665 !BaseLocDecomp0.hasValidOffset() || !BaseLocDecomp1.hasValidOffset())
13666 return 0;
13667 if (BaseLocDecomp0.getOffset() < BaseLocDecomp1.getOffset())
13668 return -1;
13669 if (BaseLocDecomp0.getOffset() > BaseLocDecomp1.getOffset())
13670 return 1;
13671 return 0;
13672 };
13673
13674 SDValue X;
13675 if (N0.getOpcode() == ISD::ADD && N0->hasOneUse()) {
13676 if (IsVecReduce(N0.getOperand(i: 0)) && IsVecReduce(N0.getOperand(i: 1))) {
13677 int IsBefore = IsKnownOrderedLoad(N0.getOperand(i: 0).getOperand(i: 0),
13678 N0.getOperand(i: 1).getOperand(i: 0));
13679 if (IsBefore < 0) {
13680 X = N0.getOperand(i: 0);
13681 N0 = N0.getOperand(i: 1);
13682 } else if (IsBefore > 0) {
13683 X = N0.getOperand(i: 1);
13684 N0 = N0.getOperand(i: 0);
13685 } else
13686 return SDValue();
13687 } else if (IsVecReduce(N0.getOperand(i: 0))) {
13688 X = N0.getOperand(i: 1);
13689 N0 = N0.getOperand(i: 0);
13690 } else if (IsVecReduce(N0.getOperand(i: 1))) {
13691 X = N0.getOperand(i: 0);
13692 N0 = N0.getOperand(i: 1);
13693 } else
13694 return SDValue();
13695 } else if (IsForward && IsVecReduce(N0) && IsVecReduce(N1) &&
13696 IsKnownOrderedLoad(N0.getOperand(i: 0), N1.getOperand(i: 0)) < 0) {
13697 // Note this is backward to how you would expect. We create
13698 // add(reduce(load + 16), reduce(load + 0)) so that the
13699 // add(reduce(load+16), X) is combined into VADDVA(X, load+16)), leaving
13700 // the X as VADDV(load + 0)
13701 return DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1, N2: N0);
13702 } else
13703 return SDValue();
13704
13705 if (!IsVecReduce(N0) || !IsVecReduce(N1))
13706 return SDValue();
13707
13708 if (IsKnownOrderedLoad(N1.getOperand(i: 0), N0.getOperand(i: 0)) >= 0)
13709 return SDValue();
13710
13711 // Switch from add(add(X, N0), N1) to add(add(X, N1), N0)
13712 SDValue Add0 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: X, N2: N1);
13713 return DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Add0, N2: N0);
13714 };
13715 if (SDValue R = DistrubuteVecReduceLoad(N0, N1, true))
13716 return R;
13717 if (SDValue R = DistrubuteVecReduceLoad(N1, N0, false))
13718 return R;
13719 return SDValue();
13720}
13721
13722static SDValue PerformADDVecReduce(SDNode *N, SelectionDAG &DAG,
13723 const ARMSubtarget *Subtarget) {
13724 if (!Subtarget->hasMVEIntegerOps())
13725 return SDValue();
13726
13727 if (SDValue R = TryDistrubutionADDVecReduce(N, DAG))
13728 return R;
13729
13730 EVT VT = N->getValueType(ResNo: 0);
13731 SDValue N0 = N->getOperand(Num: 0);
13732 SDValue N1 = N->getOperand(Num: 1);
13733 SDLoc dl(N);
13734
13735 if (VT != MVT::i64)
13736 return SDValue();
13737
13738 // We are looking for a i64 add of a VADDLVx. Due to these being i64's, this
13739 // will look like:
13740 // t1: i32,i32 = ARMISD::VADDLVs x
13741 // t2: i64 = build_pair t1, t1:1
13742 // t3: i64 = add t2, y
13743 // Otherwise we try to push the add up above VADDLVAx, to potentially allow
13744 // the add to be simplified separately.
13745 // We also need to check for sext / zext and commutitive adds.
13746 auto MakeVecReduce = [&](unsigned Opcode, unsigned OpcodeA, SDValue NA,
13747 SDValue NB) {
13748 if (NB->getOpcode() != ISD::BUILD_PAIR)
13749 return SDValue();
13750 SDValue VecRed = NB->getOperand(Num: 0);
13751 if ((VecRed->getOpcode() != Opcode && VecRed->getOpcode() != OpcodeA) ||
13752 VecRed.getResNo() != 0 ||
13753 NB->getOperand(Num: 1) != SDValue(VecRed.getNode(), 1))
13754 return SDValue();
13755
13756 if (VecRed->getOpcode() == OpcodeA) {
13757 // add(NA, VADDLVA(Inp), Y) -> VADDLVA(add(NA, Inp), Y)
13758 SDValue Inp = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: MVT::i64,
13759 N1: VecRed.getOperand(i: 0), N2: VecRed.getOperand(i: 1));
13760 NA = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::i64, N1: Inp, N2: NA);
13761 }
13762
13763 SmallVector<SDValue, 4> Ops(2);
13764 std::tie(args&: Ops[0], args&: Ops[1]) = DAG.SplitScalar(N: NA, DL: dl, LoVT: MVT::i32, HiVT: MVT::i32);
13765
13766 unsigned S = VecRed->getOpcode() == OpcodeA ? 2 : 0;
13767 for (unsigned I = S, E = VecRed.getNumOperands(); I < E; I++)
13768 Ops.push_back(Elt: VecRed->getOperand(Num: I));
13769 SDValue Red =
13770 DAG.getNode(Opcode: OpcodeA, DL: dl, VTList: DAG.getVTList(VTs: {MVT::i32, MVT::i32}), Ops);
13771 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: MVT::i64, N1: Red,
13772 N2: SDValue(Red.getNode(), 1));
13773 };
13774
13775 if (SDValue M = MakeVecReduce(ARMISD::VADDLVs, ARMISD::VADDLVAs, N0, N1))
13776 return M;
13777 if (SDValue M = MakeVecReduce(ARMISD::VADDLVu, ARMISD::VADDLVAu, N0, N1))
13778 return M;
13779 if (SDValue M = MakeVecReduce(ARMISD::VADDLVs, ARMISD::VADDLVAs, N1, N0))
13780 return M;
13781 if (SDValue M = MakeVecReduce(ARMISD::VADDLVu, ARMISD::VADDLVAu, N1, N0))
13782 return M;
13783 if (SDValue M = MakeVecReduce(ARMISD::VADDLVps, ARMISD::VADDLVAps, N0, N1))
13784 return M;
13785 if (SDValue M = MakeVecReduce(ARMISD::VADDLVpu, ARMISD::VADDLVApu, N0, N1))
13786 return M;
13787 if (SDValue M = MakeVecReduce(ARMISD::VADDLVps, ARMISD::VADDLVAps, N1, N0))
13788 return M;
13789 if (SDValue M = MakeVecReduce(ARMISD::VADDLVpu, ARMISD::VADDLVApu, N1, N0))
13790 return M;
13791 if (SDValue M = MakeVecReduce(ARMISD::VMLALVs, ARMISD::VMLALVAs, N0, N1))
13792 return M;
13793 if (SDValue M = MakeVecReduce(ARMISD::VMLALVu, ARMISD::VMLALVAu, N0, N1))
13794 return M;
13795 if (SDValue M = MakeVecReduce(ARMISD::VMLALVs, ARMISD::VMLALVAs, N1, N0))
13796 return M;
13797 if (SDValue M = MakeVecReduce(ARMISD::VMLALVu, ARMISD::VMLALVAu, N1, N0))
13798 return M;
13799 if (SDValue M = MakeVecReduce(ARMISD::VMLALVps, ARMISD::VMLALVAps, N0, N1))
13800 return M;
13801 if (SDValue M = MakeVecReduce(ARMISD::VMLALVpu, ARMISD::VMLALVApu, N0, N1))
13802 return M;
13803 if (SDValue M = MakeVecReduce(ARMISD::VMLALVps, ARMISD::VMLALVAps, N1, N0))
13804 return M;
13805 if (SDValue M = MakeVecReduce(ARMISD::VMLALVpu, ARMISD::VMLALVApu, N1, N0))
13806 return M;
13807 return SDValue();
13808}
13809
13810bool
13811ARMTargetLowering::isDesirableToCommuteWithShift(const SDNode *N,
13812 CombineLevel Level) const {
13813 assert((N->getOpcode() == ISD::SHL || N->getOpcode() == ISD::SRA ||
13814 N->getOpcode() == ISD::SRL) &&
13815 "Expected shift op");
13816
13817 SDValue ShiftLHS = N->getOperand(Num: 0);
13818 if (!ShiftLHS->hasOneUse())
13819 return false;
13820
13821 if (ShiftLHS.getOpcode() == ISD::SIGN_EXTEND &&
13822 !ShiftLHS.getOperand(i: 0)->hasOneUse())
13823 return false;
13824
13825 if (Level == BeforeLegalizeTypes)
13826 return true;
13827
13828 if (N->getOpcode() != ISD::SHL)
13829 return true;
13830
13831 if (Subtarget->isThumb1Only()) {
13832 // Avoid making expensive immediates by commuting shifts. (This logic
13833 // only applies to Thumb1 because ARM and Thumb2 immediates can be shifted
13834 // for free.)
13835 if (N->getOpcode() != ISD::SHL)
13836 return true;
13837 SDValue N1 = N->getOperand(Num: 0);
13838 if (N1->getOpcode() != ISD::ADD && N1->getOpcode() != ISD::AND &&
13839 N1->getOpcode() != ISD::OR && N1->getOpcode() != ISD::XOR)
13840 return true;
13841 if (auto *Const = dyn_cast<ConstantSDNode>(Val: N1->getOperand(Num: 1))) {
13842 if (Const->getAPIntValue().ult(RHS: 256))
13843 return false;
13844 if (N1->getOpcode() == ISD::ADD && Const->getAPIntValue().slt(RHS: 0) &&
13845 Const->getAPIntValue().sgt(RHS: -256))
13846 return false;
13847 }
13848 return true;
13849 }
13850
13851 // Turn off commute-with-shift transform after legalization, so it doesn't
13852 // conflict with PerformSHLSimplify. (We could try to detect when
13853 // PerformSHLSimplify would trigger more precisely, but it isn't
13854 // really necessary.)
13855 return false;
13856}
13857
13858bool ARMTargetLowering::isDesirableToCommuteXorWithShift(
13859 const SDNode *N) const {
13860 assert(N->getOpcode() == ISD::XOR &&
13861 (N->getOperand(0).getOpcode() == ISD::SHL ||
13862 N->getOperand(0).getOpcode() == ISD::SRL) &&
13863 "Expected XOR(SHIFT) pattern");
13864
13865 // Only commute if the entire NOT mask is a hidden shifted mask.
13866 auto *XorC = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
13867 auto *ShiftC = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 0).getOperand(i: 1));
13868 if (XorC && ShiftC) {
13869 unsigned MaskIdx, MaskLen;
13870 if (XorC->getAPIntValue().isShiftedMask(MaskIdx, MaskLen)) {
13871 unsigned ShiftAmt = ShiftC->getZExtValue();
13872 unsigned BitWidth = N->getValueType(ResNo: 0).getScalarSizeInBits();
13873 if (N->getOperand(Num: 0).getOpcode() == ISD::SHL)
13874 return MaskIdx == ShiftAmt && MaskLen == (BitWidth - ShiftAmt);
13875 return MaskIdx == 0 && MaskLen == (BitWidth - ShiftAmt);
13876 }
13877 }
13878
13879 return false;
13880}
13881
13882bool ARMTargetLowering::shouldFoldConstantShiftPairToMask(
13883 const SDNode *N) const {
13884 assert(((N->getOpcode() == ISD::SHL &&
13885 N->getOperand(0).getOpcode() == ISD::SRL) ||
13886 (N->getOpcode() == ISD::SRL &&
13887 N->getOperand(0).getOpcode() == ISD::SHL)) &&
13888 "Expected shift-shift mask");
13889
13890 if (!Subtarget->isThumb1Only())
13891 return true;
13892
13893 EVT VT = N->getValueType(ResNo: 0);
13894 if (VT.getScalarSizeInBits() > 32)
13895 return true;
13896
13897 return false;
13898}
13899
13900bool ARMTargetLowering::shouldFoldSelectWithIdentityConstant(
13901 unsigned BinOpcode, EVT VT, unsigned SelectOpcode, SDValue X,
13902 SDValue Y) const {
13903 return Subtarget->hasMVEIntegerOps() && isTypeLegal(VT) &&
13904 SelectOpcode == ISD::VSELECT;
13905}
13906
13907bool ARMTargetLowering::preferIncOfAddToSubOfNot(EVT VT) const {
13908 if (!Subtarget->hasNEON() && !Subtarget->hasMVEIntegerOps()) {
13909 if (Subtarget->isThumb1Only())
13910 return VT.getScalarSizeInBits() <= 32;
13911 return true;
13912 }
13913 return VT.isScalarInteger();
13914}
13915
13916bool ARMTargetLowering::shouldConvertFpToSat(unsigned Op, EVT FPVT,
13917 EVT VT) const {
13918 if (!isOperationLegalOrCustom(Op, VT) || !FPVT.isSimple())
13919 return false;
13920
13921 switch (FPVT.getSimpleVT().SimpleTy) {
13922 case MVT::f16:
13923 return Subtarget->hasVFP2Base();
13924 case MVT::f32:
13925 return Subtarget->hasVFP2Base();
13926 case MVT::f64:
13927 return Subtarget->hasFP64();
13928 case MVT::v4f32:
13929 case MVT::v8f16:
13930 return Subtarget->hasMVEFloatOps();
13931 default:
13932 return false;
13933 }
13934}
13935
13936static SDValue PerformSHLSimplify(SDNode *N,
13937 TargetLowering::DAGCombinerInfo &DCI,
13938 const ARMSubtarget *ST) {
13939 // Allow the generic combiner to identify potential bswaps.
13940 if (DCI.isBeforeLegalize())
13941 return SDValue();
13942
13943 // DAG combiner will fold:
13944 // (shl (add x, c1), c2) -> (add (shl x, c2), c1 << c2)
13945 // (shl (or x, c1), c2) -> (or (shl x, c2), c1 << c2
13946 // Other code patterns that can be also be modified have the following form:
13947 // b + ((a << 1) | 510)
13948 // b + ((a << 1) & 510)
13949 // b + ((a << 1) ^ 510)
13950 // b + ((a << 1) + 510)
13951
13952 // Many instructions can perform the shift for free, but it requires both
13953 // the operands to be registers. If c1 << c2 is too large, a mov immediate
13954 // instruction will needed. So, unfold back to the original pattern if:
13955 // - if c1 and c2 are small enough that they don't require mov imms.
13956 // - the user(s) of the node can perform an shl
13957
13958 // No shifted operands for 16-bit instructions.
13959 if (ST->isThumb1Only())
13960 return SDValue();
13961
13962 // Check that all the users could perform the shl themselves.
13963 for (auto *U : N->users()) {
13964 switch(U->getOpcode()) {
13965 default:
13966 return SDValue();
13967 case ISD::SUB:
13968 case ISD::ADD:
13969 case ISD::AND:
13970 case ISD::OR:
13971 case ISD::XOR:
13972 case ISD::SETCC:
13973 case ARMISD::CMP:
13974 // Check that the user isn't already using a constant because there
13975 // aren't any instructions that support an immediate operand and a
13976 // shifted operand.
13977 if (isa<ConstantSDNode>(Val: U->getOperand(Num: 0)) ||
13978 isa<ConstantSDNode>(Val: U->getOperand(Num: 1)))
13979 return SDValue();
13980
13981 // Check that it's not already using a shift.
13982 if (U->getOperand(Num: 0).getOpcode() == ISD::SHL ||
13983 U->getOperand(Num: 1).getOpcode() == ISD::SHL)
13984 return SDValue();
13985 break;
13986 }
13987 }
13988
13989 if (N->getOpcode() != ISD::ADD && N->getOpcode() != ISD::OR &&
13990 N->getOpcode() != ISD::XOR && N->getOpcode() != ISD::AND)
13991 return SDValue();
13992
13993 if (N->getOperand(Num: 0).getOpcode() != ISD::SHL)
13994 return SDValue();
13995
13996 SDValue SHL = N->getOperand(Num: 0);
13997
13998 auto *C1ShlC2 = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
13999 auto *C2 = dyn_cast<ConstantSDNode>(Val: SHL.getOperand(i: 1));
14000 if (!C1ShlC2 || !C2)
14001 return SDValue();
14002
14003 APInt C2Int = C2->getAPIntValue();
14004 APInt C1Int = C1ShlC2->getAPIntValue();
14005 unsigned C2Width = C2Int.getBitWidth();
14006 if (C2Int.uge(RHS: C2Width))
14007 return SDValue();
14008 uint64_t C2Value = C2Int.getZExtValue();
14009
14010 // Check that performing a lshr will not lose any information.
14011 APInt Mask = APInt::getHighBitsSet(numBits: C2Width, hiBitsSet: C2Width - C2Value);
14012 if ((C1Int & Mask) != C1Int)
14013 return SDValue();
14014
14015 // Shift the first constant.
14016 C1Int.lshrInPlace(ShiftAmt: C2Int);
14017
14018 // The immediates are encoded as an 8-bit value that can be rotated.
14019 auto LargeImm = [](const APInt &Imm) {
14020 unsigned Zeros = Imm.countl_zero() + Imm.countr_zero();
14021 return Imm.getBitWidth() - Zeros > 8;
14022 };
14023
14024 if (LargeImm(C1Int) || LargeImm(C2Int))
14025 return SDValue();
14026
14027 SelectionDAG &DAG = DCI.DAG;
14028 SDLoc dl(N);
14029 SDValue X = SHL.getOperand(i: 0);
14030 SDValue BinOp = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: MVT::i32, N1: X,
14031 N2: DAG.getConstant(Val: C1Int, DL: dl, VT: MVT::i32));
14032 // Shift left to compensate for the lshr of C1Int.
14033 SDValue Res = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: MVT::i32, N1: BinOp, N2: SHL.getOperand(i: 1));
14034
14035 LLVM_DEBUG(dbgs() << "Simplify shl use:\n"; SHL.getOperand(0).dump();
14036 SHL.dump(); N->dump());
14037 LLVM_DEBUG(dbgs() << "Into:\n"; X.dump(); BinOp.dump(); Res.dump());
14038 return Res;
14039}
14040
14041
14042/// PerformADDCombine - Target-specific dag combine xforms for ISD::ADD.
14043///
14044static SDValue PerformADDCombine(SDNode *N,
14045 TargetLowering::DAGCombinerInfo &DCI,
14046 const ARMSubtarget *Subtarget) {
14047 SDValue N0 = N->getOperand(Num: 0);
14048 SDValue N1 = N->getOperand(Num: 1);
14049
14050 // Only works one way, because it needs an immediate operand.
14051 if (SDValue Result = PerformSHLSimplify(N, DCI, ST: Subtarget))
14052 return Result;
14053
14054 if (SDValue Result = PerformADDVecReduce(N, DAG&: DCI.DAG, Subtarget))
14055 return Result;
14056
14057 // First try with the default operand order.
14058 if (SDValue Result = PerformADDCombineWithOperands(N, N0, N1, DCI, Subtarget))
14059 return Result;
14060
14061 // If that didn't work, try again with the operands commuted.
14062 return PerformADDCombineWithOperands(N, N0: N1, N1: N0, DCI, Subtarget);
14063}
14064
14065// Combine (sub 0, (csinc X, Y, CC)) -> (csinv -X, Y, CC)
14066// providing -X is as cheap as X (currently, just a constant).
14067static SDValue PerformSubCSINCCombine(SDNode *N, SelectionDAG &DAG) {
14068 if (N->getValueType(ResNo: 0) != MVT::i32 || !isNullConstant(V: N->getOperand(Num: 0)))
14069 return SDValue();
14070 SDValue CSINC = N->getOperand(Num: 1);
14071 if (CSINC.getOpcode() != ARMISD::CSINC || !CSINC.hasOneUse())
14072 return SDValue();
14073
14074 ConstantSDNode *X = dyn_cast<ConstantSDNode>(Val: CSINC.getOperand(i: 0));
14075 if (!X)
14076 return SDValue();
14077
14078 return DAG.getNode(Opcode: ARMISD::CSINV, DL: SDLoc(N), VT: MVT::i32,
14079 N1: DAG.getNode(Opcode: ISD::SUB, DL: SDLoc(N), VT: MVT::i32, N1: N->getOperand(Num: 0),
14080 N2: CSINC.getOperand(i: 0)),
14081 N2: CSINC.getOperand(i: 1), N3: CSINC.getOperand(i: 2),
14082 N4: CSINC.getOperand(i: 3));
14083}
14084
14085static int getNegationCost(SDValue Op) {
14086 // Free to negate.
14087 if (isa<ConstantSDNode>(Val: Op))
14088 return 0;
14089
14090 // Will save one instruction.
14091 if (Op.getOpcode() == ISD::SUB && isNullConstant(V: Op.getOperand(i: 0)))
14092 return -1;
14093
14094 // Can freely negate by converting sra <-> srl.
14095 if (Op.getOpcode() == ISD::SRA || Op.getOpcode() == ISD::SRL) {
14096 ConstantSDNode *ShiftAmt = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1));
14097 if (Op.hasOneUse() && ShiftAmt &&
14098 ShiftAmt->getZExtValue() == Op.getValueType().getScalarSizeInBits() - 1)
14099 return 0;
14100 }
14101
14102 // Will have to create sub.
14103 return 1;
14104}
14105
14106// Try to fold
14107//
14108// (neg (cmov X, Y)) -> (cmov (neg X), (neg Y))
14109//
14110// The folding helps cmov to be matched with csneg without generating
14111// redundant neg instruction.
14112static SDValue performNegCMovCombine(SDNode *N, SelectionDAG &DAG) {
14113 assert(N->getOpcode() == ISD::SUB);
14114 if (!isNullConstant(V: N->getOperand(Num: 0)))
14115 return SDValue();
14116
14117 SDValue CMov = N->getOperand(Num: 1);
14118 if (CMov.getOpcode() != ARMISD::CMOV || !CMov->hasOneUse())
14119 return SDValue();
14120
14121 SDValue N0 = CMov.getOperand(i: 0);
14122 SDValue N1 = CMov.getOperand(i: 1);
14123
14124 // Only perform the fold if we actually save something.
14125 if (getNegationCost(Op: N0) + getNegationCost(Op: N1) > 0)
14126 return SDValue();
14127
14128 SDLoc DL(N);
14129 EVT VT = CMov.getValueType();
14130
14131 SDValue N0N = DAG.getNegative(Val: N0, DL, VT);
14132 SDValue N1N = DAG.getNegative(Val: N1, DL, VT);
14133 return DAG.getNode(Opcode: ARMISD::CMOV, DL, VT, N1: N0N, N2: N1N, N3: CMov.getOperand(i: 2),
14134 N4: CMov.getOperand(i: 3));
14135}
14136
14137/// PerformSUBCombine - Target-specific dag combine xforms for ISD::SUB.
14138///
14139static SDValue PerformSUBCombine(SDNode *N,
14140 TargetLowering::DAGCombinerInfo &DCI,
14141 const ARMSubtarget *Subtarget) {
14142 SDValue N0 = N->getOperand(Num: 0);
14143 SDValue N1 = N->getOperand(Num: 1);
14144
14145 // fold (sub x, (select cc, 0, c)) -> (select cc, x, (sub, x, c))
14146 if (N1.getNode()->hasOneUse())
14147 if (SDValue Result = combineSelectAndUse(N, Slct: N1, OtherOp: N0, DCI))
14148 return Result;
14149
14150 if (SDValue R = PerformSubCSINCCombine(N, DAG&: DCI.DAG))
14151 return R;
14152
14153 if (SDValue Val = performNegCMovCombine(N, DAG&: DCI.DAG))
14154 return Val;
14155
14156 if (!Subtarget->hasMVEIntegerOps() || !N->getValueType(ResNo: 0).isVector())
14157 return SDValue();
14158
14159 // Fold (sub (ARMvmovImm 0), (ARMvdup x)) -> (ARMvdup (sub 0, x))
14160 // so that we can readily pattern match more mve instructions which can use
14161 // a scalar operand.
14162 SDValue VDup = N->getOperand(Num: 1);
14163 if (VDup->getOpcode() != ARMISD::VDUP)
14164 return SDValue();
14165
14166 SDValue VMov = N->getOperand(Num: 0);
14167 if (VMov->getOpcode() == ISD::BITCAST)
14168 VMov = VMov->getOperand(Num: 0);
14169
14170 if (VMov->getOpcode() != ARMISD::VMOVIMM || !isZeroVector(N: VMov))
14171 return SDValue();
14172
14173 SDLoc dl(N);
14174 SDValue Negate = DCI.DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32,
14175 N1: DCI.DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32),
14176 N2: VDup->getOperand(Num: 0));
14177 return DCI.DAG.getNode(Opcode: ARMISD::VDUP, DL: dl, VT: N->getValueType(ResNo: 0), Operand: Negate);
14178}
14179
14180/// PerformVMULCombine
14181/// Distribute (A + B) * C to (A * C) + (B * C) to take advantage of the
14182/// special multiplier accumulator forwarding.
14183/// vmul d3, d0, d2
14184/// vmla d3, d1, d2
14185/// is faster than
14186/// vadd d3, d0, d1
14187/// vmul d3, d3, d2
14188// However, for (A + B) * (A + B),
14189// vadd d2, d0, d1
14190// vmul d3, d0, d2
14191// vmla d3, d1, d2
14192// is slower than
14193// vadd d2, d0, d1
14194// vmul d3, d2, d2
14195static SDValue PerformVMULCombine(SDNode *N,
14196 TargetLowering::DAGCombinerInfo &DCI,
14197 const ARMSubtarget *Subtarget) {
14198 if (!Subtarget->hasVMLxForwarding())
14199 return SDValue();
14200
14201 SelectionDAG &DAG = DCI.DAG;
14202 SDValue N0 = N->getOperand(Num: 0);
14203 SDValue N1 = N->getOperand(Num: 1);
14204 unsigned Opcode = N0.getOpcode();
14205 if (Opcode != ISD::ADD && Opcode != ISD::SUB &&
14206 Opcode != ISD::FADD && Opcode != ISD::FSUB) {
14207 Opcode = N1.getOpcode();
14208 if (Opcode != ISD::ADD && Opcode != ISD::SUB &&
14209 Opcode != ISD::FADD && Opcode != ISD::FSUB)
14210 return SDValue();
14211 std::swap(a&: N0, b&: N1);
14212 }
14213
14214 if (N0 == N1)
14215 return SDValue();
14216
14217 EVT VT = N->getValueType(ResNo: 0);
14218 SDLoc DL(N);
14219 SDValue N00 = N0->getOperand(Num: 0);
14220 SDValue N01 = N0->getOperand(Num: 1);
14221 return DAG.getNode(Opcode, DL, VT,
14222 N1: DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: N00, N2: N1),
14223 N2: DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: N01, N2: N1));
14224}
14225
14226static SDValue PerformMVEVMULLCombine(SDNode *N, SelectionDAG &DAG,
14227 const ARMSubtarget *Subtarget) {
14228 EVT VT = N->getValueType(ResNo: 0);
14229 if (VT != MVT::v2i64)
14230 return SDValue();
14231
14232 SDValue N0 = N->getOperand(Num: 0);
14233 SDValue N1 = N->getOperand(Num: 1);
14234
14235 auto IsSignExt = [&](SDValue Op) {
14236 if (Op->getOpcode() != ISD::SIGN_EXTEND_INREG)
14237 return SDValue();
14238 EVT VT = cast<VTSDNode>(Val: Op->getOperand(Num: 1))->getVT();
14239 if (VT.getScalarSizeInBits() == 32)
14240 return Op->getOperand(Num: 0);
14241 return SDValue();
14242 };
14243 auto IsZeroExt = [&](SDValue Op) {
14244 // Zero extends are a little more awkward. At the point we are matching
14245 // this, we are looking for an AND with a (-1, 0, -1, 0) buildvector mask.
14246 // That might be before of after a bitcast depending on how the and is
14247 // placed. Because this has to look through bitcasts, it is currently only
14248 // supported on LE.
14249 if (!Subtarget->isLittle())
14250 return SDValue();
14251
14252 SDValue And = Op;
14253 if (And->getOpcode() == ISD::BITCAST)
14254 And = And->getOperand(Num: 0);
14255 if (And->getOpcode() != ISD::AND)
14256 return SDValue();
14257 SDValue Mask = And->getOperand(Num: 1);
14258 if (Mask->getOpcode() == ISD::BITCAST)
14259 Mask = Mask->getOperand(Num: 0);
14260
14261 if (Mask->getOpcode() != ISD::BUILD_VECTOR ||
14262 Mask.getValueType() != MVT::v4i32)
14263 return SDValue();
14264 if (isAllOnesConstant(V: Mask->getOperand(Num: 0)) &&
14265 isNullConstant(V: Mask->getOperand(Num: 1)) &&
14266 isAllOnesConstant(V: Mask->getOperand(Num: 2)) &&
14267 isNullConstant(V: Mask->getOperand(Num: 3)))
14268 return And->getOperand(Num: 0);
14269 return SDValue();
14270 };
14271
14272 SDLoc dl(N);
14273 if (SDValue Op0 = IsSignExt(N0)) {
14274 if (SDValue Op1 = IsSignExt(N1)) {
14275 SDValue New0a = DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT: MVT::v4i32, Operand: Op0);
14276 SDValue New1a = DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT: MVT::v4i32, Operand: Op1);
14277 return DAG.getNode(Opcode: ARMISD::VMULLs, DL: dl, VT, N1: New0a, N2: New1a);
14278 }
14279 }
14280 if (SDValue Op0 = IsZeroExt(N0)) {
14281 if (SDValue Op1 = IsZeroExt(N1)) {
14282 SDValue New0a = DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT: MVT::v4i32, Operand: Op0);
14283 SDValue New1a = DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT: MVT::v4i32, Operand: Op1);
14284 return DAG.getNode(Opcode: ARMISD::VMULLu, DL: dl, VT, N1: New0a, N2: New1a);
14285 }
14286 }
14287
14288 return SDValue();
14289}
14290
14291static SDValue PerformMULCombine(SDNode *N,
14292 TargetLowering::DAGCombinerInfo &DCI,
14293 const ARMSubtarget *Subtarget) {
14294 SelectionDAG &DAG = DCI.DAG;
14295
14296 EVT VT = N->getValueType(ResNo: 0);
14297 if (Subtarget->hasMVEIntegerOps() && VT == MVT::v2i64)
14298 return PerformMVEVMULLCombine(N, DAG, Subtarget);
14299
14300 if (Subtarget->isThumb1Only())
14301 return SDValue();
14302
14303 if (DCI.isBeforeLegalize() || DCI.isCalledByLegalizer())
14304 return SDValue();
14305
14306 if (VT.is64BitVector() || VT.is128BitVector())
14307 return PerformVMULCombine(N, DCI, Subtarget);
14308 if (VT != MVT::i32)
14309 return SDValue();
14310
14311 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
14312 if (!C)
14313 return SDValue();
14314
14315 int64_t MulAmt = C->getSExtValue();
14316 unsigned ShiftAmt = llvm::countr_zero<uint64_t>(Val: MulAmt);
14317
14318 ShiftAmt = ShiftAmt & (32 - 1);
14319 SDValue V = N->getOperand(Num: 0);
14320 SDLoc DL(N);
14321
14322 SDValue Res;
14323 MulAmt >>= ShiftAmt;
14324
14325 if (MulAmt >= 0) {
14326 if (llvm::has_single_bit<uint32_t>(Value: MulAmt - 1)) {
14327 // (mul x, 2^N + 1) => (add (shl x, N), x)
14328 Res = DAG.getNode(Opcode: ISD::ADD, DL, VT,
14329 N1: V,
14330 N2: DAG.getNode(Opcode: ISD::SHL, DL, VT,
14331 N1: V,
14332 N2: DAG.getConstant(Val: Log2_32(Value: MulAmt - 1), DL,
14333 VT: MVT::i32)));
14334 } else if (llvm::has_single_bit<uint32_t>(Value: MulAmt + 1)) {
14335 // (mul x, 2^N - 1) => (sub (shl x, N), x)
14336 Res = DAG.getNode(Opcode: ISD::SUB, DL, VT,
14337 N1: DAG.getNode(Opcode: ISD::SHL, DL, VT,
14338 N1: V,
14339 N2: DAG.getConstant(Val: Log2_32(Value: MulAmt + 1), DL,
14340 VT: MVT::i32)),
14341 N2: V);
14342 } else
14343 return SDValue();
14344 } else {
14345 uint64_t MulAmtAbs = -MulAmt;
14346 if (llvm::has_single_bit<uint32_t>(Value: MulAmtAbs + 1)) {
14347 // (mul x, -(2^N - 1)) => (sub x, (shl x, N))
14348 Res = DAG.getNode(Opcode: ISD::SUB, DL, VT,
14349 N1: V,
14350 N2: DAG.getNode(Opcode: ISD::SHL, DL, VT,
14351 N1: V,
14352 N2: DAG.getConstant(Val: Log2_32(Value: MulAmtAbs + 1), DL,
14353 VT: MVT::i32)));
14354 } else if (llvm::has_single_bit<uint32_t>(Value: MulAmtAbs - 1)) {
14355 // (mul x, -(2^N + 1)) => - (add (shl x, N), x)
14356 Res = DAG.getNode(Opcode: ISD::ADD, DL, VT,
14357 N1: V,
14358 N2: DAG.getNode(Opcode: ISD::SHL, DL, VT,
14359 N1: V,
14360 N2: DAG.getConstant(Val: Log2_32(Value: MulAmtAbs - 1), DL,
14361 VT: MVT::i32)));
14362 Res = DAG.getNode(Opcode: ISD::SUB, DL, VT,
14363 N1: DAG.getConstant(Val: 0, DL, VT: MVT::i32), N2: Res);
14364 } else
14365 return SDValue();
14366 }
14367
14368 if (ShiftAmt != 0)
14369 Res = DAG.getNode(Opcode: ISD::SHL, DL, VT,
14370 N1: Res, N2: DAG.getConstant(Val: ShiftAmt, DL, VT: MVT::i32));
14371
14372 // Do not add new nodes to DAG combiner worklist.
14373 DCI.CombineTo(N, Res, AddTo: false);
14374 return SDValue();
14375}
14376
14377static SDValue CombineANDShift(SDNode *N,
14378 TargetLowering::DAGCombinerInfo &DCI,
14379 const ARMSubtarget *Subtarget) {
14380 // Allow DAGCombine to pattern-match before we touch the canonical form.
14381 if (DCI.isBeforeLegalize() || DCI.isCalledByLegalizer())
14382 return SDValue();
14383
14384 if (N->getValueType(ResNo: 0) != MVT::i32)
14385 return SDValue();
14386
14387 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
14388 if (!N1C)
14389 return SDValue();
14390
14391 uint32_t C1 = (uint32_t)N1C->getZExtValue();
14392 // Don't transform uxtb/uxth.
14393 if (C1 == 255 || C1 == 65535)
14394 return SDValue();
14395
14396 SDNode *N0 = N->getOperand(Num: 0).getNode();
14397 if (!N0->hasOneUse())
14398 return SDValue();
14399
14400 if (N0->getOpcode() != ISD::SHL && N0->getOpcode() != ISD::SRL)
14401 return SDValue();
14402
14403 bool LeftShift = N0->getOpcode() == ISD::SHL;
14404
14405 ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(Val: N0->getOperand(Num: 1));
14406 if (!N01C)
14407 return SDValue();
14408
14409 uint32_t C2 = (uint32_t)N01C->getZExtValue();
14410 if (!C2 || C2 >= 32)
14411 return SDValue();
14412
14413 // Clear irrelevant bits in the mask.
14414 if (LeftShift)
14415 C1 &= (-1U << C2);
14416 else
14417 C1 &= (-1U >> C2);
14418
14419 SelectionDAG &DAG = DCI.DAG;
14420 SDLoc DL(N);
14421
14422 // We have a pattern of the form "(and (shl x, c2) c1)" or
14423 // "(and (srl x, c2) c1)", where c1 is a shifted mask. Try to
14424 // transform to a pair of shifts, to save materializing c1.
14425
14426 // First pattern: right shift, then mask off leading bits.
14427 // FIXME: Use demanded bits?
14428 if (!LeftShift && isMask_32(Value: C1)) {
14429 uint32_t C3 = llvm::countl_zero(Val: C1);
14430 if (C2 < C3) {
14431 SDValue SHL = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i32, N1: N0->getOperand(Num: 0),
14432 N2: DAG.getConstant(Val: C3 - C2, DL, VT: MVT::i32));
14433 return DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: SHL,
14434 N2: DAG.getConstant(Val: C3, DL, VT: MVT::i32));
14435 }
14436 }
14437
14438 // First pattern, reversed: left shift, then mask off trailing bits.
14439 if (LeftShift && isMask_32(Value: ~C1)) {
14440 uint32_t C3 = llvm::countr_zero(Val: C1);
14441 if (C2 < C3) {
14442 SDValue SHL = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: N0->getOperand(Num: 0),
14443 N2: DAG.getConstant(Val: C3 - C2, DL, VT: MVT::i32));
14444 return DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i32, N1: SHL,
14445 N2: DAG.getConstant(Val: C3, DL, VT: MVT::i32));
14446 }
14447 }
14448
14449 // Second pattern: left shift, then mask off leading bits.
14450 // FIXME: Use demanded bits?
14451 if (LeftShift && isShiftedMask_32(Value: C1)) {
14452 uint32_t Trailing = llvm::countr_zero(Val: C1);
14453 uint32_t C3 = llvm::countl_zero(Val: C1);
14454 if (Trailing == C2 && C2 + C3 < 32) {
14455 SDValue SHL = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i32, N1: N0->getOperand(Num: 0),
14456 N2: DAG.getConstant(Val: C2 + C3, DL, VT: MVT::i32));
14457 return DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: SHL,
14458 N2: DAG.getConstant(Val: C3, DL, VT: MVT::i32));
14459 }
14460 }
14461
14462 // Second pattern, reversed: right shift, then mask off trailing bits.
14463 // FIXME: Handle other patterns of known/demanded bits.
14464 if (!LeftShift && isShiftedMask_32(Value: C1)) {
14465 uint32_t Leading = llvm::countl_zero(Val: C1);
14466 uint32_t C3 = llvm::countr_zero(Val: C1);
14467 if (Leading == C2 && C2 + C3 < 32) {
14468 SDValue SHL = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: N0->getOperand(Num: 0),
14469 N2: DAG.getConstant(Val: C2 + C3, DL, VT: MVT::i32));
14470 return DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i32, N1: SHL,
14471 N2: DAG.getConstant(Val: C3, DL, VT: MVT::i32));
14472 }
14473 }
14474
14475 // Transform "(and (shl x, c2) c1)" into "(shl (and x, c1>>c2), c2)"
14476 // if "c1 >> c2" is a cheaper immediate than "c1"
14477 if (LeftShift &&
14478 HasLowerConstantMaterializationCost(Val1: C1 >> C2, Val2: C1, Subtarget)) {
14479
14480 SDValue And = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: N0->getOperand(Num: 0),
14481 N2: DAG.getConstant(Val: C1 >> C2, DL, VT: MVT::i32));
14482 return DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i32, N1: And,
14483 N2: DAG.getConstant(Val: C2, DL, VT: MVT::i32));
14484 }
14485
14486 return SDValue();
14487}
14488
14489static SDValue PerformANDCombine(SDNode *N,
14490 TargetLowering::DAGCombinerInfo &DCI,
14491 const ARMSubtarget *Subtarget) {
14492 // Attempt to use immediate-form VBIC
14493 BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(Val: N->getOperand(Num: 1));
14494 SDLoc dl(N);
14495 EVT VT = N->getValueType(ResNo: 0);
14496 SelectionDAG &DAG = DCI.DAG;
14497
14498 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT) || VT == MVT::v2i1 ||
14499 VT == MVT::v4i1 || VT == MVT::v8i1 || VT == MVT::v16i1)
14500 return SDValue();
14501
14502 APInt SplatBits, SplatUndef;
14503 unsigned SplatBitSize;
14504 bool HasAnyUndefs;
14505 if (BVN && (Subtarget->hasNEON() || Subtarget->hasMVEIntegerOps()) &&
14506 BVN->isConstantSplat(SplatValue&: SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
14507 if (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32 ||
14508 SplatBitSize == 64) {
14509 EVT VbicVT;
14510 SDValue Val = isVMOVModifiedImm(SplatBits: (~SplatBits).getZExtValue(),
14511 SplatUndef: SplatUndef.getZExtValue(), SplatBitSize,
14512 DAG, dl, VT&: VbicVT, VectorVT: VT, type: OtherModImm);
14513 if (Val.getNode()) {
14514 SDValue Input =
14515 DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT: VbicVT, Operand: N->getOperand(Num: 0));
14516 SDValue Vbic = DAG.getNode(Opcode: ARMISD::VBICIMM, DL: dl, VT: VbicVT, N1: Input, N2: Val);
14517 return DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT, Operand: Vbic);
14518 }
14519 }
14520 }
14521
14522 if (!Subtarget->isThumb1Only()) {
14523 // fold (and (select cc, -1, c), x) -> (select cc, x, (and, x, c))
14524 if (SDValue Result = combineSelectAndUseCommutative(N, AllOnes: true, DCI))
14525 return Result;
14526
14527 if (SDValue Result = PerformSHLSimplify(N, DCI, ST: Subtarget))
14528 return Result;
14529 }
14530
14531 if (Subtarget->isThumb1Only())
14532 if (SDValue Result = CombineANDShift(N, DCI, Subtarget))
14533 return Result;
14534
14535 return SDValue();
14536}
14537
14538// Try combining OR nodes to SMULWB, SMULWT.
14539static SDValue PerformORCombineToSMULWBT(SDNode *OR,
14540 TargetLowering::DAGCombinerInfo &DCI,
14541 const ARMSubtarget *Subtarget) {
14542 if (!Subtarget->hasV6Ops() ||
14543 (Subtarget->isThumb() &&
14544 (!Subtarget->hasThumb2() || !Subtarget->hasDSP())))
14545 return SDValue();
14546
14547 SDValue SRL = OR->getOperand(Num: 0);
14548 SDValue SHL = OR->getOperand(Num: 1);
14549
14550 if (SRL.getOpcode() != ISD::SRL || SHL.getOpcode() != ISD::SHL) {
14551 SRL = OR->getOperand(Num: 1);
14552 SHL = OR->getOperand(Num: 0);
14553 }
14554 if (!isSRL16(Op: SRL) || !isSHL16(Op: SHL))
14555 return SDValue();
14556
14557 // The first operands to the shifts need to be the two results from the
14558 // same smul_lohi node.
14559 if ((SRL.getOperand(i: 0).getNode() != SHL.getOperand(i: 0).getNode()) ||
14560 SRL.getOperand(i: 0).getOpcode() != ISD::SMUL_LOHI)
14561 return SDValue();
14562
14563 SDNode *SMULLOHI = SRL.getOperand(i: 0).getNode();
14564 if (SRL.getOperand(i: 0) != SDValue(SMULLOHI, 0) ||
14565 SHL.getOperand(i: 0) != SDValue(SMULLOHI, 1))
14566 return SDValue();
14567
14568 // Now we have:
14569 // (or (srl (smul_lohi ?, ?), 16), (shl (smul_lohi ?, ?), 16)))
14570 // For SMUL[B|T] smul_lohi will take a 32-bit and a 16-bit arguments.
14571 // For SMUWB the 16-bit value will signed extended somehow.
14572 // For SMULWT only the SRA is required.
14573 // Check both sides of SMUL_LOHI
14574 SDValue OpS16 = SMULLOHI->getOperand(Num: 0);
14575 SDValue OpS32 = SMULLOHI->getOperand(Num: 1);
14576
14577 SelectionDAG &DAG = DCI.DAG;
14578 if (!isS16(Op: OpS16, DAG) && !isSRA16(Op: OpS16)) {
14579 OpS16 = OpS32;
14580 OpS32 = SMULLOHI->getOperand(Num: 0);
14581 }
14582
14583 SDLoc dl(OR);
14584 unsigned Opcode = 0;
14585 if (isS16(Op: OpS16, DAG))
14586 Opcode = ARMISD::SMULWB;
14587 else if (isSRA16(Op: OpS16)) {
14588 Opcode = ARMISD::SMULWT;
14589 OpS16 = OpS16->getOperand(Num: 0);
14590 }
14591 else
14592 return SDValue();
14593
14594 SDValue Res = DAG.getNode(Opcode, DL: dl, VT: MVT::i32, N1: OpS32, N2: OpS16);
14595 DAG.ReplaceAllUsesOfValueWith(From: SDValue(OR, 0), To: Res);
14596 return SDValue(OR, 0);
14597}
14598
14599static SDValue PerformORCombineToBFI(SDNode *N,
14600 TargetLowering::DAGCombinerInfo &DCI,
14601 const ARMSubtarget *Subtarget) {
14602 // BFI is only available on V6T2+
14603 if (Subtarget->isThumb1Only() || !Subtarget->hasV6T2Ops())
14604 return SDValue();
14605
14606 EVT VT = N->getValueType(ResNo: 0);
14607 SDValue N0 = N->getOperand(Num: 0);
14608 SDValue N1 = N->getOperand(Num: 1);
14609 SelectionDAG &DAG = DCI.DAG;
14610 SDLoc DL(N);
14611 // 1) or (and A, mask), val => ARMbfi A, val, mask
14612 // iff (val & mask) == val
14613 //
14614 // 2) or (and A, mask), (and B, mask2) => ARMbfi A, (lsr B, amt), mask
14615 // 2a) iff isBitFieldInvertedMask(mask) && isBitFieldInvertedMask(~mask2)
14616 // && mask == ~mask2
14617 // 2b) iff isBitFieldInvertedMask(~mask) && isBitFieldInvertedMask(mask2)
14618 // && ~mask == mask2
14619 // (i.e., copy a bitfield value into another bitfield of the same width)
14620
14621 if (VT != MVT::i32)
14622 return SDValue();
14623
14624 SDValue N00 = N0.getOperand(i: 0);
14625
14626 // The value and the mask need to be constants so we can verify this is
14627 // actually a bitfield set. If the mask is 0xffff, we can do better
14628 // via a movt instruction, so don't use BFI in that case.
14629 SDValue MaskOp = N0.getOperand(i: 1);
14630 ConstantSDNode *MaskC = dyn_cast<ConstantSDNode>(Val&: MaskOp);
14631 if (!MaskC)
14632 return SDValue();
14633 unsigned Mask = MaskC->getZExtValue();
14634 if (Mask == 0xffff)
14635 return SDValue();
14636 SDValue Res;
14637 // Case (1): or (and A, mask), val => ARMbfi A, val, mask
14638 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(Val&: N1);
14639 if (N1C) {
14640 unsigned Val = N1C->getZExtValue();
14641 if ((Val & ~Mask) != Val)
14642 return SDValue();
14643
14644 if (ARM::isBitFieldInvertedMask(v: Mask)) {
14645 Val >>= llvm::countr_zero(Val: ~Mask);
14646
14647 Res = DAG.getNode(Opcode: ARMISD::BFI, DL, VT, N1: N00,
14648 N2: DAG.getConstant(Val, DL, VT: MVT::i32),
14649 N3: DAG.getConstant(Val: Mask, DL, VT: MVT::i32));
14650
14651 DCI.CombineTo(N, Res, AddTo: false);
14652 // Return value from the original node to inform the combiner than N is
14653 // now dead.
14654 return SDValue(N, 0);
14655 }
14656 } else if (N1.getOpcode() == ISD::AND) {
14657 // case (2) or (and A, mask), (and B, mask2) => ARMbfi A, (lsr B, amt), mask
14658 ConstantSDNode *N11C = dyn_cast<ConstantSDNode>(Val: N1.getOperand(i: 1));
14659 if (!N11C)
14660 return SDValue();
14661 unsigned Mask2 = N11C->getZExtValue();
14662
14663 // Mask and ~Mask2 (or reverse) must be equivalent for the BFI pattern
14664 // as is to match.
14665 if (ARM::isBitFieldInvertedMask(v: Mask) &&
14666 (Mask == ~Mask2)) {
14667 // The pack halfword instruction works better for masks that fit it,
14668 // so use that when it's available.
14669 if (Subtarget->hasDSP() &&
14670 (Mask == 0xffff || Mask == 0xffff0000))
14671 return SDValue();
14672 // 2a
14673 unsigned amt = llvm::countr_zero(Val: Mask2);
14674 Res = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: N1.getOperand(i: 0),
14675 N2: DAG.getConstant(Val: amt, DL, VT: MVT::i32));
14676 Res = DAG.getNode(Opcode: ARMISD::BFI, DL, VT, N1: N00, N2: Res,
14677 N3: DAG.getConstant(Val: Mask, DL, VT: MVT::i32));
14678 DCI.CombineTo(N, Res, AddTo: false);
14679 // Return value from the original node to inform the combiner than N is
14680 // now dead.
14681 return SDValue(N, 0);
14682 } else if (ARM::isBitFieldInvertedMask(v: ~Mask) &&
14683 (~Mask == Mask2)) {
14684 // The pack halfword instruction works better for masks that fit it,
14685 // so use that when it's available.
14686 if (Subtarget->hasDSP() &&
14687 (Mask2 == 0xffff || Mask2 == 0xffff0000))
14688 return SDValue();
14689 // 2b
14690 unsigned lsb = llvm::countr_zero(Val: Mask);
14691 Res = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: N00,
14692 N2: DAG.getConstant(Val: lsb, DL, VT: MVT::i32));
14693 Res = DAG.getNode(Opcode: ARMISD::BFI, DL, VT, N1: N1.getOperand(i: 0), N2: Res,
14694 N3: DAG.getConstant(Val: Mask2, DL, VT: MVT::i32));
14695 DCI.CombineTo(N, Res, AddTo: false);
14696 // Return value from the original node to inform the combiner than N is
14697 // now dead.
14698 return SDValue(N, 0);
14699 }
14700 }
14701
14702 if (DAG.MaskedValueIsZero(Op: N1, Mask: MaskC->getAPIntValue()) &&
14703 N00.getOpcode() == ISD::SHL && isa<ConstantSDNode>(Val: N00.getOperand(i: 1)) &&
14704 ARM::isBitFieldInvertedMask(v: ~Mask)) {
14705 // Case (3): or (and (shl A, #shamt), mask), B => ARMbfi B, A, ~mask
14706 // where lsb(mask) == #shamt and masked bits of B are known zero.
14707 SDValue ShAmt = N00.getOperand(i: 1);
14708 unsigned ShAmtC = ShAmt->getAsZExtVal();
14709 unsigned LSB = llvm::countr_zero(Val: Mask);
14710 if (ShAmtC != LSB)
14711 return SDValue();
14712
14713 Res = DAG.getNode(Opcode: ARMISD::BFI, DL, VT, N1, N2: N00.getOperand(i: 0),
14714 N3: DAG.getConstant(Val: ~Mask, DL, VT: MVT::i32));
14715
14716 DCI.CombineTo(N, Res, AddTo: false);
14717 // Return value from the original node to inform the combiner than N is
14718 // now dead.
14719 return SDValue(N, 0);
14720 }
14721
14722 return SDValue();
14723}
14724
14725static bool isValidMVECond(unsigned CC, bool IsFloat) {
14726 switch (CC) {
14727 case ARMCC::EQ:
14728 case ARMCC::NE:
14729 case ARMCC::LE:
14730 case ARMCC::GT:
14731 case ARMCC::GE:
14732 case ARMCC::LT:
14733 return true;
14734 case ARMCC::HS:
14735 case ARMCC::HI:
14736 return !IsFloat;
14737 default:
14738 return false;
14739 };
14740}
14741
14742static ARMCC::CondCodes getVCMPCondCode(SDValue N) {
14743 if (N->getOpcode() == ARMISD::VCMP)
14744 return (ARMCC::CondCodes)N->getConstantOperandVal(Num: 2);
14745 else if (N->getOpcode() == ARMISD::VCMPZ)
14746 return (ARMCC::CondCodes)N->getConstantOperandVal(Num: 1);
14747 else
14748 llvm_unreachable("Not a VCMP/VCMPZ!");
14749}
14750
14751static bool CanInvertMVEVCMP(SDValue N) {
14752 ARMCC::CondCodes CC = ARMCC::getOppositeCondition(CC: getVCMPCondCode(N));
14753 return isValidMVECond(CC, IsFloat: N->getOperand(Num: 0).getValueType().isFloatingPoint());
14754}
14755
14756static SDValue PerformORCombine_i1(SDNode *N, SelectionDAG &DAG,
14757 const ARMSubtarget *Subtarget) {
14758 // Try to invert "or A, B" -> "and ~A, ~B", as the "and" is easier to chain
14759 // together with predicates
14760 EVT VT = N->getValueType(ResNo: 0);
14761 SDLoc DL(N);
14762 SDValue N0 = N->getOperand(Num: 0);
14763 SDValue N1 = N->getOperand(Num: 1);
14764
14765 auto IsFreelyInvertable = [&](SDValue V) {
14766 if (V->getOpcode() == ARMISD::VCMP || V->getOpcode() == ARMISD::VCMPZ)
14767 return CanInvertMVEVCMP(N: V);
14768 return false;
14769 };
14770
14771 // At least one operand must be freely invertable.
14772 if (!(IsFreelyInvertable(N0) || IsFreelyInvertable(N1)))
14773 return SDValue();
14774
14775 SDValue NewN0 = DAG.getLogicalNOT(DL, Val: N0, VT);
14776 SDValue NewN1 = DAG.getLogicalNOT(DL, Val: N1, VT);
14777 SDValue And = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: NewN0, N2: NewN1);
14778 return DAG.getLogicalNOT(DL, Val: And, VT);
14779}
14780
14781// Try to form a NEON shift-{right, left}-and-insert (VSRI/VSLI) from:
14782// (or (and X, splat (i32 C1)), (srl Y, splat (i32 C2))) -> VSRI X, Y, #C2
14783// (or (and X, splat (i32 C1)), (shl Y, splat (i32 C2))) -> VSLI X, Y, #C2
14784// where C1 is a mask that preserves the bits not written by the shift/insert,
14785// i.e. `C1 == (1 << C2) - 1`.
14786static SDValue PerformORCombineToShiftInsert(SelectionDAG &DAG, SDValue AndOp,
14787 SDValue ShiftOp, EVT VT,
14788 SDLoc dl) {
14789 // Match (and X, Mask)
14790 if (AndOp.getOpcode() != ISD::AND)
14791 return SDValue();
14792
14793 SDValue X = AndOp.getOperand(i: 0);
14794 SDValue Mask = AndOp.getOperand(i: 1);
14795
14796 ConstantSDNode *MaskC = isConstOrConstSplat(N: Mask, AllowUndefs: false, AllowTruncation: true);
14797 if (!MaskC)
14798 return SDValue();
14799 APInt MaskBits =
14800 MaskC->getAPIntValue().trunc(width: Mask.getScalarValueSizeInBits());
14801
14802 // Match shift (srl/shl Y, CntVec)
14803 int64_t Cnt = 0;
14804 bool IsShiftRight = false;
14805 SDValue Y;
14806
14807 if (ShiftOp.getOpcode() == ARMISD::VSHRuIMM) {
14808 IsShiftRight = true;
14809 Y = ShiftOp.getOperand(i: 0);
14810 Cnt = ShiftOp.getConstantOperandVal(i: 1);
14811 } else if (ShiftOp.getOpcode() == ARMISD::VSHLIMM) {
14812 Y = ShiftOp.getOperand(i: 0);
14813 Cnt = ShiftOp.getConstantOperandVal(i: 1);
14814 } else {
14815 return SDValue();
14816 }
14817
14818 unsigned ElemBits = VT.getScalarSizeInBits();
14819 APInt RequiredMask = IsShiftRight
14820 ? APInt::getHighBitsSet(numBits: ElemBits, hiBitsSet: (unsigned)Cnt)
14821 : APInt::getLowBitsSet(numBits: ElemBits, loBitsSet: (unsigned)Cnt);
14822 if (MaskBits != RequiredMask)
14823 return SDValue();
14824
14825 unsigned Opc = IsShiftRight ? ARMISD::VSRIIMM : ARMISD::VSLIIMM;
14826 return DAG.getNode(Opcode: Opc, DL: dl, VT, N1: X, N2: Y, N3: DAG.getConstant(Val: Cnt, DL: dl, VT: MVT::i32));
14827}
14828
14829/// PerformORCombine - Target-specific dag combine xforms for ISD::OR
14830static SDValue PerformORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
14831 const ARMSubtarget *Subtarget) {
14832 // Attempt to use immediate-form VORR
14833 BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(Val: N->getOperand(Num: 1));
14834 SDLoc dl(N);
14835 EVT VT = N->getValueType(ResNo: 0);
14836 SelectionDAG &DAG = DCI.DAG;
14837
14838 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
14839 return SDValue();
14840
14841 if (Subtarget->hasMVEIntegerOps() && (VT == MVT::v2i1 || VT == MVT::v4i1 ||
14842 VT == MVT::v8i1 || VT == MVT::v16i1))
14843 return PerformORCombine_i1(N, DAG, Subtarget);
14844
14845 APInt SplatBits, SplatUndef;
14846 unsigned SplatBitSize;
14847 bool HasAnyUndefs;
14848 if (BVN && (Subtarget->hasNEON() || Subtarget->hasMVEIntegerOps()) &&
14849 BVN->isConstantSplat(SplatValue&: SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
14850 if (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32 ||
14851 SplatBitSize == 64) {
14852 EVT VorrVT;
14853 SDValue Val =
14854 isVMOVModifiedImm(SplatBits: SplatBits.getZExtValue(), SplatUndef: SplatUndef.getZExtValue(),
14855 SplatBitSize, DAG, dl, VT&: VorrVT, VectorVT: VT, type: OtherModImm);
14856 if (Val.getNode()) {
14857 SDValue Input =
14858 DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT: VorrVT, Operand: N->getOperand(Num: 0));
14859 SDValue Vorr = DAG.getNode(Opcode: ARMISD::VORRIMM, DL: dl, VT: VorrVT, N1: Input, N2: Val);
14860 return DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT, Operand: Vorr);
14861 }
14862 }
14863 }
14864
14865 if (!Subtarget->isThumb1Only()) {
14866 // fold (or (select cc, 0, c), x) -> (select cc, x, (or, x, c))
14867 if (SDValue Result = combineSelectAndUseCommutative(N, AllOnes: false, DCI))
14868 return Result;
14869 if (SDValue Result = PerformORCombineToSMULWBT(OR: N, DCI, Subtarget))
14870 return Result;
14871 }
14872
14873 SDValue N0 = N->getOperand(Num: 0);
14874 SDValue N1 = N->getOperand(Num: 1);
14875
14876 // (or (and X, C1), (srl Y, C2)) -> VSRI X, Y, #C2
14877 // (or (and X, C1), (shl Y, C2)) -> VSLI X, Y, #C2
14878 if (VT.isVector() &&
14879 ((Subtarget->hasNEON() && DAG.getTargetLoweringInfo().isTypeLegal(VT)) ||
14880 (Subtarget->hasMVEIntegerOps() &&
14881 (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32)))) {
14882 if (SDValue ShiftInsert =
14883 PerformORCombineToShiftInsert(DAG, AndOp: N0, ShiftOp: N1, VT, dl))
14884 return ShiftInsert;
14885
14886 if (SDValue ShiftInsert =
14887 PerformORCombineToShiftInsert(DAG, AndOp: N1, ShiftOp: N0, VT, dl))
14888 return ShiftInsert;
14889 }
14890
14891 // (or (and B, A), (and C, ~A)) => (VBSL A, B, C) when A is a constant.
14892 if (Subtarget->hasNEON() && N1.getOpcode() == ISD::AND && VT.isVector() &&
14893 DAG.getTargetLoweringInfo().isTypeLegal(VT)) {
14894
14895 // The code below optimizes (or (and X, Y), Z).
14896 // The AND operand needs to have a single user to make these optimizations
14897 // profitable.
14898 if (N0.getOpcode() != ISD::AND || !N0.hasOneUse())
14899 return SDValue();
14900
14901 APInt SplatUndef;
14902 unsigned SplatBitSize;
14903 bool HasAnyUndefs;
14904
14905 APInt SplatBits0, SplatBits1;
14906 BuildVectorSDNode *BVN0 = dyn_cast<BuildVectorSDNode>(Val: N0->getOperand(Num: 1));
14907 BuildVectorSDNode *BVN1 = dyn_cast<BuildVectorSDNode>(Val: N1->getOperand(Num: 1));
14908 // Ensure that the second operand of both ands are constants
14909 if (BVN0 && BVN0->isConstantSplat(SplatValue&: SplatBits0, SplatUndef, SplatBitSize,
14910 HasAnyUndefs) && !HasAnyUndefs) {
14911 if (BVN1 && BVN1->isConstantSplat(SplatValue&: SplatBits1, SplatUndef, SplatBitSize,
14912 HasAnyUndefs) && !HasAnyUndefs) {
14913 // Ensure that the bit width of the constants are the same and that
14914 // the splat arguments are logical inverses as per the pattern we
14915 // are trying to simplify.
14916 if (SplatBits0.getBitWidth() == SplatBits1.getBitWidth() &&
14917 SplatBits0 == ~SplatBits1) {
14918 // Canonicalize the vector type to make instruction selection
14919 // simpler.
14920 EVT CanonicalVT = VT.is128BitVector() ? MVT::v4i32 : MVT::v2i32;
14921 SDValue Result = DAG.getNode(Opcode: ARMISD::VBSP, DL: dl, VT: CanonicalVT,
14922 N1: N0->getOperand(Num: 1),
14923 N2: N0->getOperand(Num: 0),
14924 N3: N1->getOperand(Num: 0));
14925 return DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT, Operand: Result);
14926 }
14927 }
14928 }
14929 }
14930
14931 // Try to use the ARM/Thumb2 BFI (bitfield insert) instruction when
14932 // reasonable.
14933 if (N0.getOpcode() == ISD::AND && N0.hasOneUse()) {
14934 if (SDValue Res = PerformORCombineToBFI(N, DCI, Subtarget))
14935 return Res;
14936 }
14937
14938 if (SDValue Result = PerformSHLSimplify(N, DCI, ST: Subtarget))
14939 return Result;
14940
14941 // (or x, (csinc 0, 0, cc)) -> (csinc x, 0, cc)
14942 // providing that the x is 0 or 1.
14943 SDValue CSINC = N1;
14944 SDValue Other = N0;
14945 if (CSINC.getOpcode() != ARMISD::CSINC)
14946 std::swap(a&: CSINC, b&: Other);
14947 if (CSINC.getOpcode() == ARMISD::CSINC &&
14948 isNullConstant(V: CSINC.getOperand(i: 0)) &&
14949 isNullConstant(V: CSINC.getOperand(i: 1)) &&
14950 DAG.MaskedValueIsZero(Op: Other, Mask: APInt::getHighBitsSet(numBits: 32, hiBitsSet: 31)))
14951 return DAG.getNode(Opcode: ARMISD::CSINC, DL: dl, VT, N1: Other, N2: CSINC.getOperand(i: 1),
14952 N3: CSINC.getOperand(i: 2), N4: CSINC.getOperand(i: 3));
14953
14954 return SDValue();
14955}
14956
14957static SDValue PerformXORCombine(SDNode *N,
14958 TargetLowering::DAGCombinerInfo &DCI,
14959 const ARMSubtarget *Subtarget) {
14960 EVT VT = N->getValueType(ResNo: 0);
14961 SelectionDAG &DAG = DCI.DAG;
14962
14963 if(!DAG.getTargetLoweringInfo().isTypeLegal(VT))
14964 return SDValue();
14965
14966 if (!Subtarget->isThumb1Only()) {
14967 // fold (xor (select cc, 0, c), x) -> (select cc, x, (xor, x, c))
14968 if (SDValue Result = combineSelectAndUseCommutative(N, AllOnes: false, DCI))
14969 return Result;
14970
14971 if (SDValue Result = PerformSHLSimplify(N, DCI, ST: Subtarget))
14972 return Result;
14973 }
14974
14975 if (Subtarget->hasMVEIntegerOps()) {
14976 // fold (xor(vcmp/z, 1)) into a vcmp with the opposite condition.
14977 SDValue N0 = N->getOperand(Num: 0);
14978 SDValue N1 = N->getOperand(Num: 1);
14979 const TargetLowering *TLI = Subtarget->getTargetLowering();
14980 if (TLI->isConstTrueVal(N: N1) &&
14981 (N0->getOpcode() == ARMISD::VCMP || N0->getOpcode() == ARMISD::VCMPZ)) {
14982 if (CanInvertMVEVCMP(N: N0)) {
14983 SDLoc DL(N0);
14984 ARMCC::CondCodes CC = ARMCC::getOppositeCondition(CC: getVCMPCondCode(N: N0));
14985
14986 SmallVector<SDValue, 4> Ops;
14987 Ops.push_back(Elt: N0->getOperand(Num: 0));
14988 if (N0->getOpcode() == ARMISD::VCMP)
14989 Ops.push_back(Elt: N0->getOperand(Num: 1));
14990 Ops.push_back(Elt: DAG.getConstant(Val: CC, DL, VT: MVT::i32));
14991 return DAG.getNode(Opcode: N0->getOpcode(), DL, VT: N0->getValueType(ResNo: 0), Ops);
14992 }
14993 }
14994 }
14995
14996 return SDValue();
14997}
14998
14999// ParseBFI - given a BFI instruction in N, extract the "from" value (Rn) and return it,
15000// and fill in FromMask and ToMask with (consecutive) bits in "from" to be extracted and
15001// their position in "to" (Rd).
15002static SDValue ParseBFI(SDNode *N, APInt &ToMask, APInt &FromMask) {
15003 assert(N->getOpcode() == ARMISD::BFI);
15004
15005 SDValue From = N->getOperand(Num: 1);
15006 ToMask = ~N->getConstantOperandAPInt(Num: 2);
15007 FromMask = APInt::getLowBitsSet(numBits: ToMask.getBitWidth(), loBitsSet: ToMask.popcount());
15008
15009 // If the Base came from a SHR #C, we can deduce that it is really testing bit
15010 // #C in the base of the SHR.
15011 if (From->getOpcode() == ISD::SRL &&
15012 isa<ConstantSDNode>(Val: From->getOperand(Num: 1))) {
15013 APInt Shift = From->getConstantOperandAPInt(Num: 1);
15014 assert(Shift.getLimitedValue() < 32 && "Shift too large!");
15015 FromMask <<= Shift.getLimitedValue(Limit: 31);
15016 From = From->getOperand(Num: 0);
15017 }
15018
15019 return From;
15020}
15021
15022// If A and B contain one contiguous set of bits, does A | B == A . B?
15023//
15024// Neither A nor B must be zero.
15025static bool BitsProperlyConcatenate(const APInt &A, const APInt &B) {
15026 unsigned LastActiveBitInA = A.countr_zero();
15027 unsigned FirstActiveBitInB = B.getBitWidth() - B.countl_zero() - 1;
15028 return LastActiveBitInA - 1 == FirstActiveBitInB;
15029}
15030
15031static SDValue FindBFIToCombineWith(SDNode *N) {
15032 // We have a BFI in N. Find a BFI it can combine with, if one exists.
15033 APInt ToMask, FromMask;
15034 SDValue From = ParseBFI(N, ToMask, FromMask);
15035 SDValue To = N->getOperand(Num: 0);
15036
15037 SDValue V = To;
15038 if (V.getOpcode() != ARMISD::BFI)
15039 return SDValue();
15040
15041 APInt NewToMask, NewFromMask;
15042 SDValue NewFrom = ParseBFI(N: V.getNode(), ToMask&: NewToMask, FromMask&: NewFromMask);
15043 if (NewFrom != From)
15044 return SDValue();
15045
15046 // Do the written bits conflict with any we've seen so far?
15047 if ((NewToMask & ToMask).getBoolValue())
15048 // Conflicting bits.
15049 return SDValue();
15050
15051 // Are the new bits contiguous when combined with the old bits?
15052 if (BitsProperlyConcatenate(A: ToMask, B: NewToMask) &&
15053 BitsProperlyConcatenate(A: FromMask, B: NewFromMask))
15054 return V;
15055 if (BitsProperlyConcatenate(A: NewToMask, B: ToMask) &&
15056 BitsProperlyConcatenate(A: NewFromMask, B: FromMask))
15057 return V;
15058
15059 return SDValue();
15060}
15061
15062static SDValue PerformBFICombine(SDNode *N, SelectionDAG &DAG) {
15063 SDValue N0 = N->getOperand(Num: 0);
15064 SDValue N1 = N->getOperand(Num: 1);
15065
15066 if (N1.getOpcode() == ISD::AND) {
15067 // (bfi A, (and B, Mask1), Mask2) -> (bfi A, B, Mask2) iff
15068 // the bits being cleared by the AND are not demanded by the BFI.
15069 ConstantSDNode *N11C = dyn_cast<ConstantSDNode>(Val: N1.getOperand(i: 1));
15070 if (!N11C)
15071 return SDValue();
15072 unsigned InvMask = N->getConstantOperandVal(Num: 2);
15073 unsigned LSB = llvm::countr_zero(Val: ~InvMask);
15074 unsigned Width = llvm::bit_width<unsigned>(Value: ~InvMask) - LSB;
15075 assert(Width <
15076 static_cast<unsigned>(std::numeric_limits<unsigned>::digits) &&
15077 "undefined behavior");
15078 unsigned Mask = (1u << Width) - 1;
15079 unsigned Mask2 = N11C->getZExtValue();
15080 if ((Mask & (~Mask2)) == 0)
15081 return DAG.getNode(Opcode: ARMISD::BFI, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
15082 N1: N->getOperand(Num: 0), N2: N1.getOperand(i: 0), N3: N->getOperand(Num: 2));
15083 return SDValue();
15084 }
15085
15086 // Look for another BFI to combine with.
15087 if (SDValue CombineBFI = FindBFIToCombineWith(N)) {
15088 // We've found a BFI.
15089 APInt ToMask1, FromMask1;
15090 SDValue From1 = ParseBFI(N, ToMask&: ToMask1, FromMask&: FromMask1);
15091
15092 APInt ToMask2, FromMask2;
15093 SDValue From2 = ParseBFI(N: CombineBFI.getNode(), ToMask&: ToMask2, FromMask&: FromMask2);
15094 assert(From1 == From2);
15095 (void)From2;
15096
15097 // Create a new BFI, combining the two together.
15098 APInt NewFromMask = FromMask1 | FromMask2;
15099 APInt NewToMask = ToMask1 | ToMask2;
15100
15101 EVT VT = N->getValueType(ResNo: 0);
15102 SDLoc dl(N);
15103
15104 if (NewFromMask[0] == 0)
15105 From1 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: From1,
15106 N2: DAG.getConstant(Val: NewFromMask.countr_zero(), DL: dl, VT));
15107 return DAG.getNode(Opcode: ARMISD::BFI, DL: dl, VT, N1: CombineBFI.getOperand(i: 0), N2: From1,
15108 N3: DAG.getConstant(Val: ~NewToMask, DL: dl, VT));
15109 }
15110
15111 // Reassociate BFI(BFI (A, B, M1), C, M2) to BFI(BFI (A, C, M2), B, M1) so
15112 // that lower bit insertions are performed first, providing that M1 and M2
15113 // do no overlap. This can allow multiple BFI instructions to be combined
15114 // together by the other folds above.
15115 if (N->getOperand(Num: 0).getOpcode() == ARMISD::BFI) {
15116 APInt ToMask1 = ~N->getConstantOperandAPInt(Num: 2);
15117 APInt ToMask2 = ~N0.getConstantOperandAPInt(i: 2);
15118
15119 if (!N0.hasOneUse() || (ToMask1 & ToMask2) != 0 ||
15120 ToMask1.countl_zero() < ToMask2.countl_zero())
15121 return SDValue();
15122
15123 EVT VT = N->getValueType(ResNo: 0);
15124 SDLoc dl(N);
15125 SDValue BFI1 = DAG.getNode(Opcode: ARMISD::BFI, DL: dl, VT, N1: N0.getOperand(i: 0),
15126 N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2));
15127 return DAG.getNode(Opcode: ARMISD::BFI, DL: dl, VT, N1: BFI1, N2: N0.getOperand(i: 1),
15128 N3: N0.getOperand(i: 2));
15129 }
15130
15131 return SDValue();
15132}
15133
15134// Check that N is CMPZ(CSINC(0, 0, CC, X)),
15135// or CMPZ(CMOV(1, 0, CC, X))
15136// return X if valid.
15137static SDValue IsCMPZCSINC(SDNode *Cmp, ARMCC::CondCodes &CC) {
15138 if (Cmp->getOpcode() != ARMISD::CMPZ || !isNullConstant(V: Cmp->getOperand(Num: 1)))
15139 return SDValue();
15140 SDValue CSInc = Cmp->getOperand(Num: 0);
15141
15142 // Ignore any `And 1` nodes that may not yet have been removed. We are
15143 // looking for a value that produces 1/0, so these have no effect on the
15144 // code.
15145 while (CSInc.getOpcode() == ISD::AND &&
15146 isa<ConstantSDNode>(Val: CSInc.getOperand(i: 1)) &&
15147 CSInc.getConstantOperandVal(i: 1) == 1 && CSInc->hasOneUse())
15148 CSInc = CSInc.getOperand(i: 0);
15149
15150 if (CSInc.getOpcode() == ARMISD::CSINC &&
15151 isNullConstant(V: CSInc.getOperand(i: 0)) &&
15152 isNullConstant(V: CSInc.getOperand(i: 1)) && CSInc->hasOneUse()) {
15153 CC = (ARMCC::CondCodes)CSInc.getConstantOperandVal(i: 2);
15154 return CSInc.getOperand(i: 3);
15155 }
15156 if (CSInc.getOpcode() == ARMISD::CMOV && isOneConstant(V: CSInc.getOperand(i: 0)) &&
15157 isNullConstant(V: CSInc.getOperand(i: 1)) && CSInc->hasOneUse()) {
15158 CC = (ARMCC::CondCodes)CSInc.getConstantOperandVal(i: 2);
15159 return CSInc.getOperand(i: 3);
15160 }
15161 if (CSInc.getOpcode() == ARMISD::CMOV && isOneConstant(V: CSInc.getOperand(i: 1)) &&
15162 isNullConstant(V: CSInc.getOperand(i: 0)) && CSInc->hasOneUse()) {
15163 CC = ARMCC::getOppositeCondition(
15164 CC: (ARMCC::CondCodes)CSInc.getConstantOperandVal(i: 2));
15165 return CSInc.getOperand(i: 3);
15166 }
15167 return SDValue();
15168}
15169
15170static SDValue PerformCMPZCombine(SDNode *N, SelectionDAG &DAG) {
15171 // Given CMPZ(CSINC(C, 0, 0, EQ), 0), we can just use C directly. As in
15172 // t92: flags = ARMISD::CMPZ t74, 0
15173 // t93: i32 = ARMISD::CSINC 0, 0, 1, t92
15174 // t96: flags = ARMISD::CMPZ t93, 0
15175 // t114: i32 = ARMISD::CSINV 0, 0, 0, t96
15176 ARMCC::CondCodes Cond;
15177 if (SDValue C = IsCMPZCSINC(Cmp: N, CC&: Cond))
15178 if (Cond == ARMCC::EQ)
15179 return C;
15180 return SDValue();
15181}
15182
15183static SDValue PerformCSETCombine(SDNode *N, SelectionDAG &DAG) {
15184 // Fold away an unnecessary CMPZ/CSINC
15185 // CSXYZ A, B, C1 (CMPZ (CSINC 0, 0, C2, D), 0) ->
15186 // if C1==EQ -> CSXYZ A, B, C2, D
15187 // if C1==NE -> CSXYZ A, B, NOT(C2), D
15188 ARMCC::CondCodes Cond;
15189 if (SDValue C = IsCMPZCSINC(Cmp: N->getOperand(Num: 3).getNode(), CC&: Cond)) {
15190 if (N->getConstantOperandVal(Num: 2) == ARMCC::EQ)
15191 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: MVT::i32, N1: N->getOperand(Num: 0),
15192 N2: N->getOperand(Num: 1),
15193 N3: DAG.getConstant(Val: Cond, DL: SDLoc(N), VT: MVT::i32), N4: C);
15194 if (N->getConstantOperandVal(Num: 2) == ARMCC::NE)
15195 return DAG.getNode(
15196 Opcode: N->getOpcode(), DL: SDLoc(N), VT: MVT::i32, N1: N->getOperand(Num: 0),
15197 N2: N->getOperand(Num: 1),
15198 N3: DAG.getConstant(Val: ARMCC::getOppositeCondition(CC: Cond), DL: SDLoc(N), VT: MVT::i32), N4: C);
15199 }
15200 return SDValue();
15201}
15202
15203/// PerformVMOVRRDCombine - Target-specific dag combine xforms for
15204/// ARMISD::VMOVRRD.
15205static SDValue PerformVMOVRRDCombine(SDNode *N,
15206 TargetLowering::DAGCombinerInfo &DCI,
15207 const ARMSubtarget *Subtarget) {
15208 // vmovrrd(vmovdrr x, y) -> x,y
15209 SDValue InDouble = N->getOperand(Num: 0);
15210 if (InDouble.getOpcode() == ARMISD::VMOVDRR && Subtarget->hasFP64())
15211 return DCI.CombineTo(N, Res0: InDouble.getOperand(i: 0), Res1: InDouble.getOperand(i: 1));
15212
15213 // vmovrrd(load f64) -> (load i32), (load i32)
15214 SDNode *InNode = InDouble.getNode();
15215 if (ISD::isNormalLoad(N: InNode) && InNode->hasOneUse() &&
15216 InNode->getValueType(ResNo: 0) == MVT::f64 &&
15217 InNode->getOperand(Num: 1).getOpcode() == ISD::FrameIndex &&
15218 !cast<LoadSDNode>(Val: InNode)->isVolatile()) {
15219 // TODO: Should this be done for non-FrameIndex operands?
15220 LoadSDNode *LD = cast<LoadSDNode>(Val: InNode);
15221
15222 SelectionDAG &DAG = DCI.DAG;
15223 SDLoc DL(LD);
15224 SDValue BasePtr = LD->getBasePtr();
15225 SDValue NewLD1 =
15226 DAG.getLoad(VT: MVT::i32, dl: DL, Chain: LD->getChain(), Ptr: BasePtr, PtrInfo: LD->getPointerInfo(),
15227 Alignment: LD->getAlign(), MMOFlags: LD->getMemOperand()->getFlags());
15228
15229 SDValue OffsetPtr = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i32, N1: BasePtr,
15230 N2: DAG.getConstant(Val: 4, DL, VT: MVT::i32));
15231
15232 SDValue NewLD2 = DAG.getLoad(VT: MVT::i32, dl: DL, Chain: LD->getChain(), Ptr: OffsetPtr,
15233 PtrInfo: LD->getPointerInfo().getWithOffset(O: 4),
15234 Alignment: commonAlignment(A: LD->getAlign(), Offset: 4),
15235 MMOFlags: LD->getMemOperand()->getFlags());
15236
15237 DAG.ReplaceAllUsesOfValueWith(From: SDValue(LD, 1), To: NewLD2.getValue(R: 1));
15238 if (DCI.DAG.getDataLayout().isBigEndian())
15239 std::swap (a&: NewLD1, b&: NewLD2);
15240 SDValue Result = DCI.CombineTo(N, Res0: NewLD1, Res1: NewLD2);
15241 return Result;
15242 }
15243
15244 // VMOVRRD(extract(..(build_vector(a, b, c, d)))) -> a,b or c,d
15245 // VMOVRRD(extract(insert_vector(insert_vector(.., a, l1), b, l2))) -> a,b
15246 if (InDouble.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
15247 isa<ConstantSDNode>(Val: InDouble.getOperand(i: 1))) {
15248 SDValue BV = InDouble.getOperand(i: 0);
15249 // Look up through any nop bitcasts and vector_reg_casts. bitcasts may
15250 // change lane order under big endian.
15251 bool BVSwap = BV.getOpcode() == ISD::BITCAST;
15252 while (
15253 (BV.getOpcode() == ISD::BITCAST ||
15254 BV.getOpcode() == ARMISD::VECTOR_REG_CAST) &&
15255 (BV.getValueType() == MVT::v2f64 || BV.getValueType() == MVT::v2i64)) {
15256 BVSwap = BV.getOpcode() == ISD::BITCAST;
15257 BV = BV.getOperand(i: 0);
15258 }
15259 if (BV.getValueType() != MVT::v4i32)
15260 return SDValue();
15261
15262 // Handle buildvectors, pulling out the correct lane depending on
15263 // endianness.
15264 unsigned Offset = InDouble.getConstantOperandVal(i: 1) == 1 ? 2 : 0;
15265 if (BV.getOpcode() == ISD::BUILD_VECTOR) {
15266 SDValue Op0 = BV.getOperand(i: Offset);
15267 SDValue Op1 = BV.getOperand(i: Offset + 1);
15268 if (!Subtarget->isLittle() && BVSwap)
15269 std::swap(a&: Op0, b&: Op1);
15270
15271 return DCI.DAG.getMergeValues(Ops: {Op0, Op1}, dl: SDLoc(N));
15272 }
15273
15274 // A chain of insert_vectors, grabbing the correct value of the chain of
15275 // inserts.
15276 SDValue Op0, Op1;
15277 while (BV.getOpcode() == ISD::INSERT_VECTOR_ELT) {
15278 if (isa<ConstantSDNode>(Val: BV.getOperand(i: 2))) {
15279 if (BV.getConstantOperandVal(i: 2) == Offset && !Op0)
15280 Op0 = BV.getOperand(i: 1);
15281 if (BV.getConstantOperandVal(i: 2) == Offset + 1 && !Op1)
15282 Op1 = BV.getOperand(i: 1);
15283 }
15284 BV = BV.getOperand(i: 0);
15285 }
15286 if (!Subtarget->isLittle() && BVSwap)
15287 std::swap(a&: Op0, b&: Op1);
15288 if (Op0 && Op1)
15289 return DCI.DAG.getMergeValues(Ops: {Op0, Op1}, dl: SDLoc(N));
15290 }
15291
15292 return SDValue();
15293}
15294
15295/// PerformVMOVDRRCombine - Target-specific dag combine xforms for
15296/// ARMISD::VMOVDRR. This is also used for BUILD_VECTORs with 2 operands.
15297static SDValue PerformVMOVDRRCombine(SDNode *N, SelectionDAG &DAG) {
15298 // N=vmovrrd(X); vmovdrr(N:0, N:1) -> bit_convert(X)
15299 SDValue Op0 = N->getOperand(Num: 0);
15300 SDValue Op1 = N->getOperand(Num: 1);
15301 if (Op0.getOpcode() == ISD::BITCAST)
15302 Op0 = Op0.getOperand(i: 0);
15303 if (Op1.getOpcode() == ISD::BITCAST)
15304 Op1 = Op1.getOperand(i: 0);
15305 if (Op0.getOpcode() == ARMISD::VMOVRRD &&
15306 Op0.getNode() == Op1.getNode() &&
15307 Op0.getResNo() == 0 && Op1.getResNo() == 1)
15308 return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N),
15309 VT: N->getValueType(ResNo: 0), Operand: Op0.getOperand(i: 0));
15310 return SDValue();
15311}
15312
15313static SDValue PerformVMOVhrCombine(SDNode *N,
15314 TargetLowering::DAGCombinerInfo &DCI) {
15315 SDValue Op0 = N->getOperand(Num: 0);
15316
15317 // VMOVhr (VMOVrh (X)) -> X
15318 if (Op0->getOpcode() == ARMISD::VMOVrh)
15319 return Op0->getOperand(Num: 0);
15320
15321 // FullFP16: half values are passed in S-registers, and we don't
15322 // need any of the bitcast and moves:
15323 //
15324 // t2: f32,ch1,gl1? = CopyFromReg ch, Register:f32 %0, gl?
15325 // t5: i32 = bitcast t2
15326 // t18: f16 = ARMISD::VMOVhr t5
15327 // =>
15328 // tN: f16,ch2,gl2? = CopyFromReg ch, Register::f32 %0, gl?
15329 if (Op0->getOpcode() == ISD::BITCAST) {
15330 SDValue Copy = Op0->getOperand(Num: 0);
15331 if (Copy.getValueType() == MVT::f32 &&
15332 Copy->getOpcode() == ISD::CopyFromReg) {
15333 bool HasGlue = Copy->getNumOperands() == 3;
15334 SDValue Ops[] = {Copy->getOperand(Num: 0), Copy->getOperand(Num: 1),
15335 HasGlue ? Copy->getOperand(Num: 2) : SDValue()};
15336 EVT OutTys[] = {N->getValueType(ResNo: 0), MVT::Other, MVT::Glue};
15337 SDValue NewCopy =
15338 DCI.DAG.getNode(Opcode: ISD::CopyFromReg, DL: SDLoc(N),
15339 VTList: DCI.DAG.getVTList(VTs: ArrayRef(OutTys, HasGlue ? 3 : 2)),
15340 Ops: ArrayRef(Ops, HasGlue ? 3 : 2));
15341
15342 // Update Users, Chains, and Potential Glue.
15343 DCI.DAG.ReplaceAllUsesOfValueWith(From: SDValue(N, 0), To: NewCopy.getValue(R: 0));
15344 DCI.DAG.ReplaceAllUsesOfValueWith(From: Copy.getValue(R: 1), To: NewCopy.getValue(R: 1));
15345 if (HasGlue)
15346 DCI.DAG.ReplaceAllUsesOfValueWith(From: Copy.getValue(R: 2),
15347 To: NewCopy.getValue(R: 2));
15348
15349 return NewCopy;
15350 }
15351 }
15352
15353 // fold (VMOVhr (load x)) -> (load (f16*)x)
15354 if (LoadSDNode *LN0 = dyn_cast<LoadSDNode>(Val&: Op0)) {
15355 if (LN0->hasOneUse() && LN0->isUnindexed() &&
15356 LN0->getMemoryVT() == MVT::i16) {
15357 SDValue Load =
15358 DCI.DAG.getLoad(VT: N->getValueType(ResNo: 0), dl: SDLoc(N), Chain: LN0->getChain(),
15359 Ptr: LN0->getBasePtr(), MMO: LN0->getMemOperand());
15360 DCI.DAG.ReplaceAllUsesOfValueWith(From: SDValue(N, 0), To: Load.getValue(R: 0));
15361 DCI.DAG.ReplaceAllUsesOfValueWith(From: Op0.getValue(R: 1), To: Load.getValue(R: 1));
15362 return Load;
15363 }
15364 }
15365
15366 // Only the bottom 16 bits of the source register are used.
15367 APInt DemandedMask = APInt::getLowBitsSet(numBits: 32, loBitsSet: 16);
15368 const TargetLowering &TLI = DCI.DAG.getTargetLoweringInfo();
15369 if (TLI.SimplifyDemandedBits(Op: Op0, DemandedBits: DemandedMask, DCI))
15370 return SDValue(N, 0);
15371
15372 return SDValue();
15373}
15374
15375static SDValue PerformVMOVrhCombine(SDNode *N, SelectionDAG &DAG) {
15376 SDValue N0 = N->getOperand(Num: 0);
15377 EVT VT = N->getValueType(ResNo: 0);
15378
15379 // fold (VMOVrh (fpconst x)) -> const x
15380 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val&: N0)) {
15381 APFloat V = C->getValueAPF();
15382 return DAG.getConstant(Val: V.bitcastToAPInt().getZExtValue(), DL: SDLoc(N), VT);
15383 }
15384
15385 // fold (VMOVrh (load x)) -> (zextload (i16*)x)
15386 if (ISD::isNormalLoad(N: N0.getNode()) && N0.hasOneUse()) {
15387 LoadSDNode *LN0 = cast<LoadSDNode>(Val&: N0);
15388
15389 SDValue Load =
15390 DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl: SDLoc(N), VT, Chain: LN0->getChain(),
15391 Ptr: LN0->getBasePtr(), MemVT: MVT::i16, MMO: LN0->getMemOperand());
15392 DAG.ReplaceAllUsesOfValueWith(From: SDValue(N, 0), To: Load.getValue(R: 0));
15393 DAG.ReplaceAllUsesOfValueWith(From: N0.getValue(R: 1), To: Load.getValue(R: 1));
15394 return Load;
15395 }
15396
15397 // Fold VMOVrh(extract(x, n)) -> vgetlaneu(x, n)
15398 if (N0->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
15399 isa<ConstantSDNode>(Val: N0->getOperand(Num: 1)))
15400 return DAG.getNode(Opcode: ARMISD::VGETLANEu, DL: SDLoc(N), VT, N1: N0->getOperand(Num: 0),
15401 N2: N0->getOperand(Num: 1));
15402
15403 return SDValue();
15404}
15405
15406/// hasNormalLoadOperand - Check if any of the operands of a BUILD_VECTOR node
15407/// are normal, non-volatile loads. If so, it is profitable to bitcast an
15408/// i64 vector to have f64 elements, since the value can then be loaded
15409/// directly into a VFP register.
15410static bool hasNormalLoadOperand(SDNode *N) {
15411 unsigned NumElts = N->getValueType(ResNo: 0).getVectorNumElements();
15412 for (unsigned i = 0; i < NumElts; ++i) {
15413 SDNode *Elt = N->getOperand(Num: i).getNode();
15414 if (ISD::isNormalLoad(N: Elt) && !cast<LoadSDNode>(Val: Elt)->isVolatile())
15415 return true;
15416 }
15417 return false;
15418}
15419
15420/// PerformBUILD_VECTORCombine - Target-specific dag combine xforms for
15421/// ISD::BUILD_VECTOR.
15422static SDValue PerformBUILD_VECTORCombine(SDNode *N,
15423 TargetLowering::DAGCombinerInfo &DCI,
15424 const ARMSubtarget *Subtarget) {
15425 // build_vector(N=ARMISD::VMOVRRD(X), N:1) -> bit_convert(X):
15426 // VMOVRRD is introduced when legalizing i64 types. It forces the i64 value
15427 // into a pair of GPRs, which is fine when the value is used as a scalar,
15428 // but if the i64 value is converted to a vector, we need to undo the VMOVRRD.
15429 SelectionDAG &DAG = DCI.DAG;
15430 if (N->getNumOperands() == 2)
15431 if (SDValue RV = PerformVMOVDRRCombine(N, DAG))
15432 return RV;
15433
15434 // Load i64 elements as f64 values so that type legalization does not split
15435 // them up into i32 values.
15436 EVT VT = N->getValueType(ResNo: 0);
15437 if (VT.getVectorElementType() != MVT::i64 || !hasNormalLoadOperand(N))
15438 return SDValue();
15439 SDLoc dl(N);
15440 SmallVector<SDValue, 8> Ops;
15441 unsigned NumElts = VT.getVectorNumElements();
15442 for (unsigned i = 0; i < NumElts; ++i) {
15443 SDValue V = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::f64, Operand: N->getOperand(Num: i));
15444 Ops.push_back(Elt: V);
15445 // Make the DAGCombiner fold the bitcast.
15446 DCI.AddToWorklist(N: V.getNode());
15447 }
15448 EVT FloatVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::f64, NumElements: NumElts);
15449 SDValue BV = DAG.getBuildVector(VT: FloatVT, DL: dl, Ops);
15450 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: BV);
15451}
15452
15453/// Target-specific dag combine xforms for ARMISD::BUILD_VECTOR.
15454static SDValue
15455PerformARMBUILD_VECTORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
15456 // ARMISD::BUILD_VECTOR is introduced when legalizing ISD::BUILD_VECTOR.
15457 // At that time, we may have inserted bitcasts from integer to float.
15458 // If these bitcasts have survived DAGCombine, change the lowering of this
15459 // BUILD_VECTOR in something more vector friendly, i.e., that does not
15460 // force to use floating point types.
15461
15462 // Make sure we can change the type of the vector.
15463 // This is possible iff:
15464 // 1. The vector is only used in a bitcast to a integer type. I.e.,
15465 // 1.1. Vector is used only once.
15466 // 1.2. Use is a bit convert to an integer type.
15467 // 2. The size of its operands are 32-bits (64-bits are not legal).
15468 EVT VT = N->getValueType(ResNo: 0);
15469 EVT EltVT = VT.getVectorElementType();
15470
15471 // Check 1.1. and 2.
15472 if (EltVT.getSizeInBits() != 32 || !N->hasOneUse())
15473 return SDValue();
15474
15475 // By construction, the input type must be float.
15476 assert(EltVT == MVT::f32 && "Unexpected type!");
15477
15478 // Check 1.2.
15479 SDNode *Use = *N->user_begin();
15480 if (Use->getOpcode() != ISD::BITCAST ||
15481 Use->getValueType(ResNo: 0).isFloatingPoint())
15482 return SDValue();
15483
15484 // Check profitability.
15485 // Model is, if more than half of the relevant operands are bitcast from
15486 // i32, turn the build_vector into a sequence of insert_vector_elt.
15487 // Relevant operands are everything that is not statically
15488 // (i.e., at compile time) bitcasted.
15489 unsigned NumOfBitCastedElts = 0;
15490 unsigned NumElts = VT.getVectorNumElements();
15491 unsigned NumOfRelevantElts = NumElts;
15492 for (unsigned Idx = 0; Idx < NumElts; ++Idx) {
15493 SDValue Elt = N->getOperand(Num: Idx);
15494 if (Elt->getOpcode() == ISD::BITCAST) {
15495 // Assume only bit cast to i32 will go away.
15496 if (Elt->getOperand(Num: 0).getValueType() == MVT::i32)
15497 ++NumOfBitCastedElts;
15498 } else if (Elt.isUndef() || isa<ConstantSDNode>(Val: Elt))
15499 // Constants are statically casted, thus do not count them as
15500 // relevant operands.
15501 --NumOfRelevantElts;
15502 }
15503
15504 // Check if more than half of the elements require a non-free bitcast.
15505 if (NumOfBitCastedElts <= NumOfRelevantElts / 2)
15506 return SDValue();
15507
15508 SelectionDAG &DAG = DCI.DAG;
15509 // Create the new vector type.
15510 EVT VecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32, NumElements: NumElts);
15511 // Check if the type is legal.
15512 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
15513 if (!TLI.isTypeLegal(VT: VecVT))
15514 return SDValue();
15515
15516 // Combine:
15517 // ARMISD::BUILD_VECTOR E1, E2, ..., EN.
15518 // => BITCAST INSERT_VECTOR_ELT
15519 // (INSERT_VECTOR_ELT (...), (BITCAST EN-1), N-1),
15520 // (BITCAST EN), N.
15521 SDValue Vec = DAG.getUNDEF(VT: VecVT);
15522 SDLoc dl(N);
15523 for (unsigned Idx = 0 ; Idx < NumElts; ++Idx) {
15524 SDValue V = N->getOperand(Num: Idx);
15525 if (V.isUndef())
15526 continue;
15527 if (V.getOpcode() == ISD::BITCAST &&
15528 V->getOperand(Num: 0).getValueType() == MVT::i32)
15529 // Fold obvious case.
15530 V = V.getOperand(i: 0);
15531 else {
15532 V = DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(V), VT: MVT::i32, Operand: V);
15533 // Make the DAGCombiner fold the bitcasts.
15534 DCI.AddToWorklist(N: V.getNode());
15535 }
15536 SDValue LaneIdx = DAG.getConstant(Val: Idx, DL: dl, VT: MVT::i32);
15537 Vec = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: VecVT, N1: Vec, N2: V, N3: LaneIdx);
15538 }
15539 Vec = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: Vec);
15540 // Make the DAGCombiner fold the bitcasts.
15541 DCI.AddToWorklist(N: Vec.getNode());
15542 return Vec;
15543}
15544
15545static SDValue
15546PerformPREDICATE_CASTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
15547 EVT VT = N->getValueType(ResNo: 0);
15548 SDValue Op = N->getOperand(Num: 0);
15549 SDLoc dl(N);
15550
15551 // PREDICATE_CAST(PREDICATE_CAST(x)) == PREDICATE_CAST(x)
15552 if (Op->getOpcode() == ARMISD::PREDICATE_CAST) {
15553 // If the valuetypes are the same, we can remove the cast entirely.
15554 if (Op->getOperand(Num: 0).getValueType() == VT)
15555 return Op->getOperand(Num: 0);
15556 return DCI.DAG.getNode(Opcode: ARMISD::PREDICATE_CAST, DL: dl, VT, Operand: Op->getOperand(Num: 0));
15557 }
15558
15559 // Turn pred_cast(xor x, -1) into xor(pred_cast x, -1), in order to produce
15560 // more VPNOT which might get folded as else predicates.
15561 if (Op.getValueType() == MVT::i32 && isBitwiseNot(V: Op)) {
15562 SDValue X =
15563 DCI.DAG.getNode(Opcode: ARMISD::PREDICATE_CAST, DL: dl, VT, Operand: Op->getOperand(Num: 0));
15564 SDValue C = DCI.DAG.getNode(Opcode: ARMISD::PREDICATE_CAST, DL: dl, VT,
15565 Operand: DCI.DAG.getConstant(Val: 65535, DL: dl, VT: MVT::i32));
15566 return DCI.DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: X, N2: C);
15567 }
15568
15569 // Only the bottom 16 bits of the source register are used.
15570 if (Op.getValueType() == MVT::i32) {
15571 APInt DemandedMask = APInt::getLowBitsSet(numBits: 32, loBitsSet: 16);
15572 const TargetLowering &TLI = DCI.DAG.getTargetLoweringInfo();
15573 if (TLI.SimplifyDemandedBits(Op, DemandedBits: DemandedMask, DCI))
15574 return SDValue(N, 0);
15575 }
15576 return SDValue();
15577}
15578
15579static SDValue PerformVECTOR_REG_CASTCombine(SDNode *N, SelectionDAG &DAG,
15580 const ARMSubtarget *ST) {
15581 EVT VT = N->getValueType(ResNo: 0);
15582 SDValue Op = N->getOperand(Num: 0);
15583 SDLoc dl(N);
15584
15585 // Under Little endian, a VECTOR_REG_CAST is equivalent to a BITCAST
15586 if (ST->isLittle())
15587 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: Op);
15588
15589 // VT VECTOR_REG_CAST (VT Op) -> Op
15590 if (Op.getValueType() == VT)
15591 return Op;
15592 // VECTOR_REG_CAST undef -> undef
15593 if (Op.isUndef())
15594 return DAG.getUNDEF(VT);
15595
15596 // VECTOR_REG_CAST(VECTOR_REG_CAST(x)) == VECTOR_REG_CAST(x)
15597 if (Op->getOpcode() == ARMISD::VECTOR_REG_CAST) {
15598 // If the valuetypes are the same, we can remove the cast entirely.
15599 if (Op->getOperand(Num: 0).getValueType() == VT)
15600 return Op->getOperand(Num: 0);
15601 return DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT, Operand: Op->getOperand(Num: 0));
15602 }
15603
15604 return SDValue();
15605}
15606
15607static SDValue PerformVCMPCombine(SDNode *N, SelectionDAG &DAG,
15608 const ARMSubtarget *Subtarget) {
15609 if (!Subtarget->hasMVEIntegerOps())
15610 return SDValue();
15611
15612 EVT VT = N->getValueType(ResNo: 0);
15613 SDValue Op0 = N->getOperand(Num: 0);
15614 SDValue Op1 = N->getOperand(Num: 1);
15615 ARMCC::CondCodes Cond = (ARMCC::CondCodes)N->getConstantOperandVal(Num: 2);
15616 SDLoc dl(N);
15617
15618 // vcmp X, 0, cc -> vcmpz X, cc
15619 if (isZeroVector(N: Op1))
15620 return DAG.getNode(Opcode: ARMISD::VCMPZ, DL: dl, VT, N1: Op0, N2: N->getOperand(Num: 2));
15621
15622 unsigned SwappedCond = getSwappedCondition(CC: Cond);
15623 if (isValidMVECond(CC: SwappedCond, IsFloat: VT.isFloatingPoint())) {
15624 // vcmp 0, X, cc -> vcmpz X, reversed(cc)
15625 if (isZeroVector(N: Op0))
15626 return DAG.getNode(Opcode: ARMISD::VCMPZ, DL: dl, VT, N1: Op1,
15627 N2: DAG.getConstant(Val: SwappedCond, DL: dl, VT: MVT::i32));
15628 // vcmp vdup(Y), X, cc -> vcmp X, vdup(Y), reversed(cc)
15629 if (Op0->getOpcode() == ARMISD::VDUP && Op1->getOpcode() != ARMISD::VDUP)
15630 return DAG.getNode(Opcode: ARMISD::VCMP, DL: dl, VT, N1: Op1, N2: Op0,
15631 N3: DAG.getConstant(Val: SwappedCond, DL: dl, VT: MVT::i32));
15632 }
15633
15634 return SDValue();
15635}
15636
15637/// PerformInsertEltCombine - Target-specific dag combine xforms for
15638/// ISD::INSERT_VECTOR_ELT.
15639static SDValue PerformInsertEltCombine(SDNode *N,
15640 TargetLowering::DAGCombinerInfo &DCI) {
15641 // Bitcast an i64 load inserted into a vector to f64.
15642 // Otherwise, the i64 value will be legalized to a pair of i32 values.
15643 EVT VT = N->getValueType(ResNo: 0);
15644 SDNode *Elt = N->getOperand(Num: 1).getNode();
15645 if (VT.getVectorElementType() != MVT::i64 ||
15646 !ISD::isNormalLoad(N: Elt) || cast<LoadSDNode>(Val: Elt)->isVolatile())
15647 return SDValue();
15648
15649 SelectionDAG &DAG = DCI.DAG;
15650 SDLoc dl(N);
15651 EVT FloatVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::f64,
15652 NumElements: VT.getVectorNumElements());
15653 SDValue Vec = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: FloatVT, Operand: N->getOperand(Num: 0));
15654 SDValue V = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::f64, Operand: N->getOperand(Num: 1));
15655 // Make the DAGCombiner fold the bitcasts.
15656 DCI.AddToWorklist(N: Vec.getNode());
15657 DCI.AddToWorklist(N: V.getNode());
15658 SDValue InsElt = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: FloatVT,
15659 N1: Vec, N2: V, N3: N->getOperand(Num: 2));
15660 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: InsElt);
15661}
15662
15663// Convert a pair of extracts from the same base vector to a VMOVRRD. Either
15664// directly or bitcast to an integer if the original is a float vector.
15665// extract(x, n); extract(x, n+1) -> VMOVRRD(extract v2f64 x, n/2)
15666// bitcast(extract(x, n)); bitcast(extract(x, n+1)) -> VMOVRRD(extract x, n/2)
15667static SDValue
15668PerformExtractEltToVMOVRRD(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
15669 EVT VT = N->getValueType(ResNo: 0);
15670 SDLoc dl(N);
15671
15672 if (!DCI.isAfterLegalizeDAG() || VT != MVT::i32 ||
15673 !DCI.DAG.getTargetLoweringInfo().isTypeLegal(VT: MVT::f64))
15674 return SDValue();
15675
15676 SDValue Ext = SDValue(N, 0);
15677 if (Ext.getOpcode() == ISD::BITCAST &&
15678 Ext.getOperand(i: 0).getValueType() == MVT::f32)
15679 Ext = Ext.getOperand(i: 0);
15680 if (Ext.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
15681 !isa<ConstantSDNode>(Val: Ext.getOperand(i: 1)) ||
15682 Ext.getConstantOperandVal(i: 1) % 2 != 0)
15683 return SDValue();
15684 if (Ext->hasOneUse() && (Ext->user_begin()->getOpcode() == ISD::SINT_TO_FP ||
15685 Ext->user_begin()->getOpcode() == ISD::UINT_TO_FP))
15686 return SDValue();
15687
15688 SDValue Op0 = Ext.getOperand(i: 0);
15689 EVT VecVT = Op0.getValueType();
15690 unsigned ResNo = Op0.getResNo();
15691 unsigned Lane = Ext.getConstantOperandVal(i: 1);
15692 if (VecVT.getVectorNumElements() != 4)
15693 return SDValue();
15694
15695 // Find another extract, of Lane + 1
15696 auto OtherIt = find_if(Range: Op0->users(), P: [&](SDNode *V) {
15697 return V->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
15698 isa<ConstantSDNode>(Val: V->getOperand(Num: 1)) &&
15699 V->getConstantOperandVal(Num: 1) == Lane + 1 &&
15700 V->getOperand(Num: 0).getResNo() == ResNo;
15701 });
15702 if (OtherIt == Op0->users().end())
15703 return SDValue();
15704
15705 // For float extracts, we need to be converting to a i32 for both vector
15706 // lanes.
15707 SDValue OtherExt(*OtherIt, 0);
15708 if (OtherExt.getValueType() != MVT::i32) {
15709 if (!OtherExt->hasOneUse() ||
15710 OtherExt->user_begin()->getOpcode() != ISD::BITCAST ||
15711 OtherExt->user_begin()->getValueType(ResNo: 0) != MVT::i32)
15712 return SDValue();
15713 OtherExt = SDValue(*OtherExt->user_begin(), 0);
15714 }
15715
15716 // Convert the type to a f64 and extract with a VMOVRRD.
15717 SDValue F64 = DCI.DAG.getNode(
15718 Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: MVT::f64,
15719 N1: DCI.DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: dl, VT: MVT::v2f64, Operand: Op0),
15720 N2: DCI.DAG.getConstant(Val: Ext.getConstantOperandVal(i: 1) / 2, DL: dl, VT: MVT::i32));
15721 SDValue VMOVRRD =
15722 DCI.DAG.getNode(Opcode: ARMISD::VMOVRRD, DL: dl, ResultTys: {MVT::i32, MVT::i32}, Ops: F64);
15723
15724 DCI.CombineTo(N: OtherExt.getNode(), Res: SDValue(VMOVRRD.getNode(), 1));
15725 return VMOVRRD;
15726}
15727
15728static SDValue PerformExtractEltCombine(SDNode *N,
15729 TargetLowering::DAGCombinerInfo &DCI,
15730 const ARMSubtarget *ST) {
15731 SDValue Op0 = N->getOperand(Num: 0);
15732 EVT VT = N->getValueType(ResNo: 0);
15733 SDLoc dl(N);
15734
15735 // extract (vdup x) -> x
15736 if (Op0->getOpcode() == ARMISD::VDUP) {
15737 SDValue X = Op0->getOperand(Num: 0);
15738 if (VT == MVT::f16 && X.getValueType() == MVT::i32)
15739 return DCI.DAG.getNode(Opcode: ARMISD::VMOVhr, DL: dl, VT, Operand: X);
15740 if (VT == MVT::i32 && X.getValueType() == MVT::f16)
15741 return DCI.DAG.getNode(Opcode: ARMISD::VMOVrh, DL: dl, VT, Operand: X);
15742 if (VT == MVT::f32 && X.getValueType() == MVT::i32)
15743 return DCI.DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: X);
15744
15745 while (X.getValueType() != VT && X->getOpcode() == ISD::BITCAST)
15746 X = X->getOperand(Num: 0);
15747 if (X.getValueType() == VT)
15748 return X;
15749 }
15750
15751 // extract ARM_BUILD_VECTOR -> x
15752 if (Op0->getOpcode() == ARMISD::BUILD_VECTOR &&
15753 isa<ConstantSDNode>(Val: N->getOperand(Num: 1)) &&
15754 N->getConstantOperandVal(Num: 1) < Op0.getNumOperands()) {
15755 return Op0.getOperand(i: N->getConstantOperandVal(Num: 1));
15756 }
15757
15758 // extract(bitcast(BUILD_VECTOR(VMOVDRR(a, b), ..))) -> a or b
15759 if (Op0.getValueType() == MVT::v4i32 &&
15760 isa<ConstantSDNode>(Val: N->getOperand(Num: 1)) &&
15761 Op0.getOpcode() == ISD::BITCAST &&
15762 Op0.getOperand(i: 0).getOpcode() == ISD::BUILD_VECTOR &&
15763 Op0.getOperand(i: 0).getValueType() == MVT::v2f64) {
15764 SDValue BV = Op0.getOperand(i: 0);
15765 unsigned Offset = N->getConstantOperandVal(Num: 1);
15766 SDValue MOV = BV.getOperand(i: Offset < 2 ? 0 : 1);
15767 if (MOV.getOpcode() == ARMISD::VMOVDRR)
15768 return MOV.getOperand(i: ST->isLittle() ? Offset % 2 : 1 - Offset % 2);
15769 }
15770
15771 // extract x, n; extract x, n+1 -> VMOVRRD x
15772 if (SDValue R = PerformExtractEltToVMOVRRD(N, DCI))
15773 return R;
15774
15775 // extract (MVETrunc(x)) -> extract x
15776 if (Op0->getOpcode() == ARMISD::MVETRUNC) {
15777 unsigned Idx = N->getConstantOperandVal(Num: 1);
15778 unsigned Vec =
15779 Idx / Op0->getOperand(Num: 0).getValueType().getVectorNumElements();
15780 unsigned SubIdx =
15781 Idx % Op0->getOperand(Num: 0).getValueType().getVectorNumElements();
15782 return DCI.DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT, N1: Op0.getOperand(i: Vec),
15783 N2: DCI.DAG.getConstant(Val: SubIdx, DL: dl, VT: MVT::i32));
15784 }
15785
15786 // extract(bitcast(BUILD_VECTOR(extract(bitcast(a)), ..))) -> extract(a)
15787 if (ST->isLittle() && Op0.getOpcode() == ISD::BITCAST &&
15788 Op0.getOperand(i: 0).getOpcode() == ARMISD::BUILD_VECTOR &&
15789 isa<ConstantSDNode>(Val: N->getOperand(Num: 1)) &&
15790 Op0.getScalarValueSizeInBits() <=
15791 Op0.getOperand(i: 0).getScalarValueSizeInBits()) {
15792 unsigned Lane = N->getConstantOperandVal(Num: 1);
15793 EVT ExtVT = Op0.getValueType();
15794 EVT BVVT = Op0.getOperand(i: 0).getValueType();
15795 unsigned BVLane =
15796 (Lane * BVVT.getVectorNumElements()) / ExtVT.getVectorNumElements();
15797 assert(BVLane < Op0.getOperand(0).getNumOperands());
15798 SDValue Ext = Op0.getOperand(i: 0).getOperand(i: BVLane);
15799 if (Ext.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
15800 Ext.getOperand(i: 0).getOpcode() == ISD::BITCAST &&
15801 isa<ConstantSDNode>(Val: Ext.getOperand(i: 1)) &&
15802 Ext.getOperand(i: 0).getOperand(i: 0).getValueType() == ExtVT) {
15803 unsigned InnerLane = Ext.getConstantOperandVal(i: 1);
15804 unsigned BVSubLane = Lane - (BVLane * ExtVT.getVectorNumElements()) /
15805 BVVT.getVectorNumElements();
15806 unsigned FinalLane = (InnerLane * ExtVT.getVectorNumElements()) /
15807 BVVT.getVectorNumElements() +
15808 BVSubLane;
15809 return DCI.DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT,
15810 N1: Ext.getOperand(i: 0).getOperand(i: 0),
15811 N2: DCI.DAG.getConstant(Val: FinalLane, DL: dl, VT: MVT::i32));
15812 }
15813 }
15814
15815 return SDValue();
15816}
15817
15818static SDValue PerformSignExtendInregCombine(SDNode *N, SelectionDAG &DAG) {
15819 SDValue Op = N->getOperand(Num: 0);
15820 EVT VT = N->getValueType(ResNo: 0);
15821
15822 // sext_inreg(VGETLANEu) -> VGETLANEs
15823 if (Op.getOpcode() == ARMISD::VGETLANEu &&
15824 cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT() ==
15825 Op.getOperand(i: 0).getValueType().getScalarType())
15826 return DAG.getNode(Opcode: ARMISD::VGETLANEs, DL: SDLoc(N), VT, N1: Op.getOperand(i: 0),
15827 N2: Op.getOperand(i: 1));
15828
15829 return SDValue();
15830}
15831
15832static SDValue
15833PerformInsertSubvectorCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
15834 SDValue Vec = N->getOperand(Num: 0);
15835 SDValue SubVec = N->getOperand(Num: 1);
15836 uint64_t IdxVal = N->getConstantOperandVal(Num: 2);
15837 EVT VecVT = Vec.getValueType();
15838 EVT SubVT = SubVec.getValueType();
15839
15840 // Only do this for legal fixed vector types.
15841 if (!VecVT.isFixedLengthVector() ||
15842 !DCI.DAG.getTargetLoweringInfo().isTypeLegal(VT: VecVT) ||
15843 !DCI.DAG.getTargetLoweringInfo().isTypeLegal(VT: SubVT))
15844 return SDValue();
15845
15846 // Ignore widening patterns.
15847 if (IdxVal == 0 && Vec.isUndef())
15848 return SDValue();
15849
15850 // Subvector must be half the width and an "aligned" insertion.
15851 unsigned NumSubElts = SubVT.getVectorNumElements();
15852 if ((SubVT.getSizeInBits() * 2) != VecVT.getSizeInBits() ||
15853 (IdxVal != 0 && IdxVal != NumSubElts))
15854 return SDValue();
15855
15856 // Fold insert_subvector -> concat_vectors
15857 // insert_subvector(Vec,Sub,lo) -> concat_vectors(Sub,extract(Vec,hi))
15858 // insert_subvector(Vec,Sub,hi) -> concat_vectors(extract(Vec,lo),Sub)
15859 SDLoc DL(N);
15860 SDValue Lo, Hi;
15861 if (IdxVal == 0) {
15862 Lo = SubVec;
15863 Hi = DCI.DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: SubVT, N1: Vec,
15864 N2: DCI.DAG.getVectorIdxConstant(Val: NumSubElts, DL));
15865 } else {
15866 Lo = DCI.DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: SubVT, N1: Vec,
15867 N2: DCI.DAG.getVectorIdxConstant(Val: 0, DL));
15868 Hi = SubVec;
15869 }
15870 return DCI.DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: VecVT, N1: Lo, N2: Hi);
15871}
15872
15873// shuffle(MVETrunc(x, y)) -> VMOVN(x, y)
15874static SDValue PerformShuffleVMOVNCombine(ShuffleVectorSDNode *N,
15875 SelectionDAG &DAG) {
15876 SDValue Trunc = N->getOperand(Num: 0);
15877 EVT VT = Trunc.getValueType();
15878 if (Trunc.getOpcode() != ARMISD::MVETRUNC || !N->getOperand(Num: 1).isUndef())
15879 return SDValue();
15880
15881 SDLoc DL(Trunc);
15882 if (isVMOVNTruncMask(M: N->getMask(), ToVT: VT, rev: false))
15883 return DAG.getNode(
15884 Opcode: ARMISD::VMOVN, DL, VT,
15885 N1: DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL, VT, Operand: Trunc.getOperand(i: 0)),
15886 N2: DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL, VT, Operand: Trunc.getOperand(i: 1)),
15887 N3: DAG.getConstant(Val: 1, DL, VT: MVT::i32));
15888 else if (isVMOVNTruncMask(M: N->getMask(), ToVT: VT, rev: true))
15889 return DAG.getNode(
15890 Opcode: ARMISD::VMOVN, DL, VT,
15891 N1: DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL, VT, Operand: Trunc.getOperand(i: 1)),
15892 N2: DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL, VT, Operand: Trunc.getOperand(i: 0)),
15893 N3: DAG.getConstant(Val: 1, DL, VT: MVT::i32));
15894 return SDValue();
15895}
15896
15897/// PerformVECTOR_SHUFFLECombine - Target-specific dag combine xforms for
15898/// ISD::VECTOR_SHUFFLE.
15899static SDValue PerformVECTOR_SHUFFLECombine(SDNode *N, SelectionDAG &DAG) {
15900 if (SDValue R = PerformShuffleVMOVNCombine(N: cast<ShuffleVectorSDNode>(Val: N), DAG))
15901 return R;
15902
15903 // The LLVM shufflevector instruction does not require the shuffle mask
15904 // length to match the operand vector length, but ISD::VECTOR_SHUFFLE does
15905 // have that requirement. When translating to ISD::VECTOR_SHUFFLE, if the
15906 // operands do not match the mask length, they are extended by concatenating
15907 // them with undef vectors. That is probably the right thing for other
15908 // targets, but for NEON it is better to concatenate two double-register
15909 // size vector operands into a single quad-register size vector. Do that
15910 // transformation here:
15911 // shuffle(concat(v1, undef), concat(v2, undef)) ->
15912 // shuffle(concat(v1, v2), undef)
15913 SDValue Op0 = N->getOperand(Num: 0);
15914 SDValue Op1 = N->getOperand(Num: 1);
15915 if (Op0.getOpcode() != ISD::CONCAT_VECTORS ||
15916 Op1.getOpcode() != ISD::CONCAT_VECTORS ||
15917 Op0.getNumOperands() != 2 ||
15918 Op1.getNumOperands() != 2)
15919 return SDValue();
15920 SDValue Concat0Op1 = Op0.getOperand(i: 1);
15921 SDValue Concat1Op1 = Op1.getOperand(i: 1);
15922 if (!Concat0Op1.isUndef() || !Concat1Op1.isUndef())
15923 return SDValue();
15924 // Skip the transformation if any of the types are illegal.
15925 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
15926 EVT VT = N->getValueType(ResNo: 0);
15927 if (!TLI.isTypeLegal(VT) ||
15928 !TLI.isTypeLegal(VT: Concat0Op1.getValueType()) ||
15929 !TLI.isTypeLegal(VT: Concat1Op1.getValueType()))
15930 return SDValue();
15931
15932 SDValue NewConcat = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(N), VT,
15933 N1: Op0.getOperand(i: 0), N2: Op1.getOperand(i: 0));
15934 // Translate the shuffle mask.
15935 SmallVector<int, 16> NewMask;
15936 unsigned NumElts = VT.getVectorNumElements();
15937 unsigned HalfElts = NumElts/2;
15938 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val: N);
15939 for (unsigned n = 0; n < NumElts; ++n) {
15940 int MaskElt = SVN->getMaskElt(Idx: n);
15941 int NewElt = -1;
15942 if (MaskElt < (int)HalfElts)
15943 NewElt = MaskElt;
15944 else if (MaskElt >= (int)NumElts && MaskElt < (int)(NumElts + HalfElts))
15945 NewElt = HalfElts + MaskElt - NumElts;
15946 NewMask.push_back(Elt: NewElt);
15947 }
15948 return DAG.getVectorShuffle(VT, dl: SDLoc(N), N1: NewConcat,
15949 N2: DAG.getUNDEF(VT), Mask: NewMask);
15950}
15951
15952/// Load/store instruction that can be merged with a base address
15953/// update
15954struct BaseUpdateTarget {
15955 SDNode *N;
15956 bool isIntrinsic;
15957 bool isStore;
15958 unsigned AddrOpIdx;
15959};
15960
15961struct BaseUpdateUser {
15962 /// Instruction that updates a pointer
15963 SDNode *N;
15964 /// Pointer increment operand
15965 SDValue Inc;
15966 /// Pointer increment value if it is a constant, or 0 otherwise
15967 unsigned ConstInc;
15968};
15969
15970static bool isValidBaseUpdate(SDNode *N, SDNode *User) {
15971 // Check that the add is independent of the load/store.
15972 // Otherwise, folding it would create a cycle. Search through Addr
15973 // as well, since the User may not be a direct user of Addr and
15974 // only share a base pointer.
15975 SmallPtrSet<const SDNode *, 32> Visited;
15976 SmallVector<const SDNode *, 16> Worklist;
15977 Worklist.push_back(Elt: N);
15978 Worklist.push_back(Elt: User);
15979 const unsigned MaxSteps = 1024;
15980 if (SDNode::hasPredecessorHelper(N, Visited, Worklist, MaxSteps) ||
15981 SDNode::hasPredecessorHelper(N: User, Visited, Worklist, MaxSteps))
15982 return false;
15983 return true;
15984}
15985
15986static bool TryCombineBaseUpdate(struct BaseUpdateTarget &Target,
15987 struct BaseUpdateUser &User,
15988 bool SimpleConstIncOnly,
15989 TargetLowering::DAGCombinerInfo &DCI) {
15990 SelectionDAG &DAG = DCI.DAG;
15991 SDNode *N = Target.N;
15992 MemSDNode *MemN = cast<MemSDNode>(Val: N);
15993 SDLoc dl(N);
15994
15995 // Find the new opcode for the updating load/store.
15996 bool isLoadOp = true;
15997 bool isLaneOp = false;
15998 // Workaround for vst1x and vld1x intrinsics which do not have alignment
15999 // as an operand.
16000 bool hasAlignment = true;
16001 unsigned NewOpc = 0;
16002 unsigned NumVecs = 0;
16003 if (Target.isIntrinsic) {
16004 unsigned IntNo = N->getConstantOperandVal(Num: 1);
16005 switch (IntNo) {
16006 default:
16007 llvm_unreachable("unexpected intrinsic for Neon base update");
16008 case Intrinsic::arm_neon_vld1:
16009 NewOpc = ARMISD::VLD1_UPD;
16010 NumVecs = 1;
16011 break;
16012 case Intrinsic::arm_neon_vld2:
16013 NewOpc = ARMISD::VLD2_UPD;
16014 NumVecs = 2;
16015 break;
16016 case Intrinsic::arm_neon_vld3:
16017 NewOpc = ARMISD::VLD3_UPD;
16018 NumVecs = 3;
16019 break;
16020 case Intrinsic::arm_neon_vld4:
16021 NewOpc = ARMISD::VLD4_UPD;
16022 NumVecs = 4;
16023 break;
16024 case Intrinsic::arm_neon_vld1x2:
16025 NewOpc = ARMISD::VLD1x2_UPD;
16026 NumVecs = 2;
16027 hasAlignment = false;
16028 break;
16029 case Intrinsic::arm_neon_vld1x3:
16030 NewOpc = ARMISD::VLD1x3_UPD;
16031 NumVecs = 3;
16032 hasAlignment = false;
16033 break;
16034 case Intrinsic::arm_neon_vld1x4:
16035 NewOpc = ARMISD::VLD1x4_UPD;
16036 NumVecs = 4;
16037 hasAlignment = false;
16038 break;
16039 case Intrinsic::arm_neon_vld2dup:
16040 NewOpc = ARMISD::VLD2DUP_UPD;
16041 NumVecs = 2;
16042 break;
16043 case Intrinsic::arm_neon_vld3dup:
16044 NewOpc = ARMISD::VLD3DUP_UPD;
16045 NumVecs = 3;
16046 break;
16047 case Intrinsic::arm_neon_vld4dup:
16048 NewOpc = ARMISD::VLD4DUP_UPD;
16049 NumVecs = 4;
16050 break;
16051 case Intrinsic::arm_neon_vld2lane:
16052 NewOpc = ARMISD::VLD2LN_UPD;
16053 NumVecs = 2;
16054 isLaneOp = true;
16055 break;
16056 case Intrinsic::arm_neon_vld3lane:
16057 NewOpc = ARMISD::VLD3LN_UPD;
16058 NumVecs = 3;
16059 isLaneOp = true;
16060 break;
16061 case Intrinsic::arm_neon_vld4lane:
16062 NewOpc = ARMISD::VLD4LN_UPD;
16063 NumVecs = 4;
16064 isLaneOp = true;
16065 break;
16066 case Intrinsic::arm_neon_vst1:
16067 NewOpc = ARMISD::VST1_UPD;
16068 NumVecs = 1;
16069 isLoadOp = false;
16070 break;
16071 case Intrinsic::arm_neon_vst2:
16072 NewOpc = ARMISD::VST2_UPD;
16073 NumVecs = 2;
16074 isLoadOp = false;
16075 break;
16076 case Intrinsic::arm_neon_vst3:
16077 NewOpc = ARMISD::VST3_UPD;
16078 NumVecs = 3;
16079 isLoadOp = false;
16080 break;
16081 case Intrinsic::arm_neon_vst4:
16082 NewOpc = ARMISD::VST4_UPD;
16083 NumVecs = 4;
16084 isLoadOp = false;
16085 break;
16086 case Intrinsic::arm_neon_vst2lane:
16087 NewOpc = ARMISD::VST2LN_UPD;
16088 NumVecs = 2;
16089 isLoadOp = false;
16090 isLaneOp = true;
16091 break;
16092 case Intrinsic::arm_neon_vst3lane:
16093 NewOpc = ARMISD::VST3LN_UPD;
16094 NumVecs = 3;
16095 isLoadOp = false;
16096 isLaneOp = true;
16097 break;
16098 case Intrinsic::arm_neon_vst4lane:
16099 NewOpc = ARMISD::VST4LN_UPD;
16100 NumVecs = 4;
16101 isLoadOp = false;
16102 isLaneOp = true;
16103 break;
16104 case Intrinsic::arm_neon_vst1x2:
16105 NewOpc = ARMISD::VST1x2_UPD;
16106 NumVecs = 2;
16107 isLoadOp = false;
16108 hasAlignment = false;
16109 break;
16110 case Intrinsic::arm_neon_vst1x3:
16111 NewOpc = ARMISD::VST1x3_UPD;
16112 NumVecs = 3;
16113 isLoadOp = false;
16114 hasAlignment = false;
16115 break;
16116 case Intrinsic::arm_neon_vst1x4:
16117 NewOpc = ARMISD::VST1x4_UPD;
16118 NumVecs = 4;
16119 isLoadOp = false;
16120 hasAlignment = false;
16121 break;
16122 }
16123 } else {
16124 isLaneOp = true;
16125 switch (N->getOpcode()) {
16126 default:
16127 llvm_unreachable("unexpected opcode for Neon base update");
16128 case ARMISD::VLD1DUP:
16129 NewOpc = ARMISD::VLD1DUP_UPD;
16130 NumVecs = 1;
16131 break;
16132 case ARMISD::VLD2DUP:
16133 NewOpc = ARMISD::VLD2DUP_UPD;
16134 NumVecs = 2;
16135 break;
16136 case ARMISD::VLD3DUP:
16137 NewOpc = ARMISD::VLD3DUP_UPD;
16138 NumVecs = 3;
16139 break;
16140 case ARMISD::VLD4DUP:
16141 NewOpc = ARMISD::VLD4DUP_UPD;
16142 NumVecs = 4;
16143 break;
16144 case ISD::LOAD:
16145 NewOpc = ARMISD::VLD1_UPD;
16146 NumVecs = 1;
16147 isLaneOp = false;
16148 break;
16149 case ISD::STORE:
16150 NewOpc = ARMISD::VST1_UPD;
16151 NumVecs = 1;
16152 isLaneOp = false;
16153 isLoadOp = false;
16154 break;
16155 }
16156 }
16157
16158 // Find the size of memory referenced by the load/store.
16159 EVT VecTy;
16160 if (isLoadOp) {
16161 VecTy = N->getValueType(ResNo: 0);
16162 } else if (Target.isIntrinsic) {
16163 VecTy = N->getOperand(Num: Target.AddrOpIdx + 1).getValueType();
16164 } else {
16165 assert(Target.isStore &&
16166 "Node has to be a load, a store, or an intrinsic!");
16167 VecTy = N->getOperand(Num: 1).getValueType();
16168 }
16169
16170 bool isVLDDUPOp =
16171 NewOpc == ARMISD::VLD1DUP_UPD || NewOpc == ARMISD::VLD2DUP_UPD ||
16172 NewOpc == ARMISD::VLD3DUP_UPD || NewOpc == ARMISD::VLD4DUP_UPD;
16173
16174 unsigned NumBytes = NumVecs * VecTy.getSizeInBits() / 8;
16175 if (isLaneOp || isVLDDUPOp)
16176 NumBytes /= VecTy.getVectorNumElements();
16177
16178 if (NumBytes >= 3 * 16 && User.ConstInc != NumBytes) {
16179 // VLD3/4 and VST3/4 for 128-bit vectors are implemented with two
16180 // separate instructions that make it harder to use a non-constant update.
16181 return false;
16182 }
16183
16184 if (SimpleConstIncOnly && User.ConstInc != NumBytes)
16185 return false;
16186
16187 if (!isValidBaseUpdate(N, User: User.N))
16188 return false;
16189
16190 // OK, we found an ADD we can fold into the base update.
16191 // Now, create a _UPD node, taking care of not breaking alignment.
16192
16193 EVT AlignedVecTy = VecTy;
16194 Align Alignment = MemN->getAlign();
16195
16196 // If this is a less-than-standard-aligned load/store, change the type to
16197 // match the standard alignment.
16198 // The alignment is overlooked when selecting _UPD variants; and it's
16199 // easier to introduce bitcasts here than fix that.
16200 // There are 3 ways to get to this base-update combine:
16201 // - intrinsics: they are assumed to be properly aligned (to the standard
16202 // alignment of the memory type), so we don't need to do anything.
16203 // - ARMISD::VLDx nodes: they are only generated from the aforementioned
16204 // intrinsics, so, likewise, there's nothing to do.
16205 // - generic load/store instructions: the alignment is specified as an
16206 // explicit operand, rather than implicitly as the standard alignment
16207 // of the memory type (like the intrinsics). We need to change the
16208 // memory type to match the explicit alignment. That way, we don't
16209 // generate non-standard-aligned ARMISD::VLDx nodes.
16210 if (isa<LSBaseSDNode>(Val: N)) {
16211 if (Alignment.value() < VecTy.getScalarSizeInBits() / 8) {
16212 MVT EltTy = MVT::getIntegerVT(BitWidth: Alignment.value() * 8);
16213 assert(NumVecs == 1 && "Unexpected multi-element generic load/store.");
16214 assert(!isLaneOp && "Unexpected generic load/store lane.");
16215 unsigned NumElts = NumBytes / (EltTy.getSizeInBits() / 8);
16216 AlignedVecTy = MVT::getVectorVT(VT: EltTy, NumElements: NumElts);
16217 }
16218 // Don't set an explicit alignment on regular load/stores that we want
16219 // to transform to VLD/VST 1_UPD nodes.
16220 // This matches the behavior of regular load/stores, which only get an
16221 // explicit alignment if the MMO alignment is larger than the standard
16222 // alignment of the memory type.
16223 // Intrinsics, however, always get an explicit alignment, set to the
16224 // alignment of the MMO.
16225 Alignment = Align(1);
16226 }
16227
16228 // Create the new updating load/store node.
16229 // First, create an SDVTList for the new updating node's results.
16230 EVT Tys[6];
16231 unsigned NumResultVecs = (isLoadOp ? NumVecs : 0);
16232 unsigned n;
16233 for (n = 0; n < NumResultVecs; ++n)
16234 Tys[n] = AlignedVecTy;
16235 Tys[n++] = MVT::i32;
16236 Tys[n] = MVT::Other;
16237 SDVTList SDTys = DAG.getVTList(VTs: ArrayRef(Tys, NumResultVecs + 2));
16238
16239 // Then, gather the new node's operands.
16240 SmallVector<SDValue, 8> Ops;
16241 Ops.push_back(Elt: N->getOperand(Num: 0)); // incoming chain
16242 Ops.push_back(Elt: N->getOperand(Num: Target.AddrOpIdx));
16243 Ops.push_back(Elt: User.Inc);
16244
16245 if (StoreSDNode *StN = dyn_cast<StoreSDNode>(Val: N)) {
16246 // Try to match the intrinsic's signature
16247 Ops.push_back(Elt: StN->getValue());
16248 } else {
16249 // Loads (and of course intrinsics) match the intrinsics' signature,
16250 // so just add all but the alignment operand.
16251 unsigned LastOperand =
16252 hasAlignment ? N->getNumOperands() - 1 : N->getNumOperands();
16253 for (unsigned i = Target.AddrOpIdx + 1; i < LastOperand; ++i)
16254 Ops.push_back(Elt: N->getOperand(Num: i));
16255 }
16256
16257 // For all node types, the alignment operand is always the last one.
16258 Ops.push_back(Elt: DAG.getConstant(Val: Alignment.value(), DL: dl, VT: MVT::i32));
16259
16260 // If this is a non-standard-aligned STORE, the penultimate operand is the
16261 // stored value. Bitcast it to the aligned type.
16262 if (AlignedVecTy != VecTy && N->getOpcode() == ISD::STORE) {
16263 SDValue &StVal = Ops[Ops.size() - 2];
16264 StVal = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: AlignedVecTy, Operand: StVal);
16265 }
16266
16267 EVT LoadVT = isLaneOp ? VecTy.getVectorElementType() : AlignedVecTy;
16268 SDValue UpdN = DAG.getMemIntrinsicNode(Opcode: NewOpc, dl, VTList: SDTys, Ops, MemVT: LoadVT,
16269 MMO: MemN->getMemOperand());
16270
16271 // Update the uses.
16272 SmallVector<SDValue, 5> NewResults;
16273 for (unsigned i = 0; i < NumResultVecs; ++i)
16274 NewResults.push_back(Elt: SDValue(UpdN.getNode(), i));
16275
16276 // If this is an non-standard-aligned LOAD, the first result is the loaded
16277 // value. Bitcast it to the expected result type.
16278 if (AlignedVecTy != VecTy && N->getOpcode() == ISD::LOAD) {
16279 SDValue &LdVal = NewResults[0];
16280 LdVal = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: VecTy, Operand: LdVal);
16281 }
16282
16283 NewResults.push_back(Elt: SDValue(UpdN.getNode(), NumResultVecs + 1)); // chain
16284 DCI.CombineTo(N, To: NewResults);
16285 DCI.CombineTo(N: User.N, Res: SDValue(UpdN.getNode(), NumResultVecs));
16286
16287 return true;
16288}
16289
16290// If (opcode ptr inc) is and ADD-like instruction, return the
16291// increment value. Otherwise return 0.
16292static unsigned getPointerConstIncrement(unsigned Opcode, SDValue Ptr,
16293 SDValue Inc, const SelectionDAG &DAG) {
16294 ConstantSDNode *CInc = dyn_cast<ConstantSDNode>(Val: Inc.getNode());
16295 if (!CInc)
16296 return 0;
16297
16298 switch (Opcode) {
16299 case ARMISD::VLD1_UPD:
16300 case ISD::ADD:
16301 return CInc->getZExtValue();
16302 case ISD::OR: {
16303 if (DAG.haveNoCommonBitsSet(A: Ptr, B: Inc)) {
16304 // (OR ptr inc) is the same as (ADD ptr inc)
16305 return CInc->getZExtValue();
16306 }
16307 return 0;
16308 }
16309 default:
16310 return 0;
16311 }
16312}
16313
16314static bool findPointerConstIncrement(SDNode *N, SDValue *Ptr, SDValue *CInc) {
16315 switch (N->getOpcode()) {
16316 case ISD::ADD:
16317 case ISD::OR: {
16318 if (isa<ConstantSDNode>(Val: N->getOperand(Num: 1))) {
16319 *Ptr = N->getOperand(Num: 0);
16320 *CInc = N->getOperand(Num: 1);
16321 return true;
16322 }
16323 return false;
16324 }
16325 case ARMISD::VLD1_UPD: {
16326 if (isa<ConstantSDNode>(Val: N->getOperand(Num: 2))) {
16327 *Ptr = N->getOperand(Num: 1);
16328 *CInc = N->getOperand(Num: 2);
16329 return true;
16330 }
16331 return false;
16332 }
16333 default:
16334 return false;
16335 }
16336}
16337
16338/// CombineBaseUpdate - Target-specific DAG combine function for VLDDUP,
16339/// NEON load/store intrinsics, and generic vector load/stores, to merge
16340/// base address updates.
16341/// For generic load/stores, the memory type is assumed to be a vector.
16342/// The caller is assumed to have checked legality.
16343static SDValue CombineBaseUpdate(SDNode *N,
16344 TargetLowering::DAGCombinerInfo &DCI) {
16345 const bool isIntrinsic = (N->getOpcode() == ISD::INTRINSIC_VOID ||
16346 N->getOpcode() == ISD::INTRINSIC_W_CHAIN);
16347 const bool isStore = N->getOpcode() == ISD::STORE;
16348 const unsigned AddrOpIdx = ((isIntrinsic || isStore) ? 2 : 1);
16349 BaseUpdateTarget Target = {.N: N, .isIntrinsic: isIntrinsic, .isStore: isStore, .AddrOpIdx: AddrOpIdx};
16350
16351 // Limit the number of possible base-updates we look at to prevent degenerate
16352 // cases.
16353 unsigned MaxBaseUpdates = ArmMaxBaseUpdatesToCheck;
16354
16355 SDValue Addr = N->getOperand(Num: AddrOpIdx);
16356
16357 SmallVector<BaseUpdateUser, 8> BaseUpdates;
16358
16359 // Search for a use of the address operand that is an increment.
16360 for (SDUse &Use : Addr->uses()) {
16361 SDNode *User = Use.getUser();
16362 if (Use.getResNo() != Addr.getResNo() || User->getNumOperands() != 2)
16363 continue;
16364
16365 SDValue Inc = User->getOperand(Num: Use.getOperandNo() == 1 ? 0 : 1);
16366 unsigned ConstInc =
16367 getPointerConstIncrement(Opcode: User->getOpcode(), Ptr: Addr, Inc, DAG: DCI.DAG);
16368
16369 if (ConstInc || User->getOpcode() == ISD::ADD) {
16370 BaseUpdates.push_back(Elt: {.N: User, .Inc: Inc, .ConstInc: ConstInc});
16371 if (BaseUpdates.size() >= MaxBaseUpdates)
16372 break;
16373 }
16374 }
16375
16376 // If the address is a constant pointer increment itself, find
16377 // another constant increment that has the same base operand
16378 SDValue Base;
16379 SDValue CInc;
16380 if (findPointerConstIncrement(N: Addr.getNode(), Ptr: &Base, CInc: &CInc)) {
16381 unsigned Offset =
16382 getPointerConstIncrement(Opcode: Addr->getOpcode(), Ptr: Base, Inc: CInc, DAG: DCI.DAG);
16383 if (Offset) {
16384 for (SDUse &Use : Base->uses()) {
16385
16386 SDNode *User = Use.getUser();
16387 if (Use.getResNo() != Base.getResNo() || User == Addr.getNode() ||
16388 User->getNumOperands() != 2)
16389 continue;
16390
16391 SDValue UserInc = User->getOperand(Num: Use.getOperandNo() == 0 ? 1 : 0);
16392 unsigned UserOffset =
16393 getPointerConstIncrement(Opcode: User->getOpcode(), Ptr: Base, Inc: UserInc, DAG: DCI.DAG);
16394
16395 if (!UserOffset || UserOffset <= Offset)
16396 continue;
16397
16398 unsigned NewConstInc = UserOffset - Offset;
16399 SDValue NewInc = DCI.DAG.getConstant(Val: NewConstInc, DL: SDLoc(N), VT: MVT::i32);
16400 BaseUpdates.push_back(Elt: {.N: User, .Inc: NewInc, .ConstInc: NewConstInc});
16401 if (BaseUpdates.size() >= MaxBaseUpdates)
16402 break;
16403 }
16404 }
16405 }
16406
16407 // Try to fold the load/store with an update that matches memory
16408 // access size. This should work well for sequential loads.
16409 unsigned NumValidUpd = BaseUpdates.size();
16410 for (unsigned I = 0; I < NumValidUpd; I++) {
16411 BaseUpdateUser &User = BaseUpdates[I];
16412 if (TryCombineBaseUpdate(Target, User, /*SimpleConstIncOnly=*/true, DCI))
16413 return SDValue();
16414 }
16415
16416 // Try to fold with other users. Non-constant updates are considered
16417 // first, and constant updates are sorted to not break a sequence of
16418 // strided accesses (if there is any).
16419 llvm::stable_sort(Range&: BaseUpdates,
16420 C: [](const BaseUpdateUser &LHS, const BaseUpdateUser &RHS) {
16421 return LHS.ConstInc < RHS.ConstInc;
16422 });
16423 for (BaseUpdateUser &User : BaseUpdates) {
16424 if (TryCombineBaseUpdate(Target, User, /*SimpleConstIncOnly=*/false, DCI))
16425 return SDValue();
16426 }
16427 return SDValue();
16428}
16429
16430static SDValue PerformVLDCombine(SDNode *N,
16431 TargetLowering::DAGCombinerInfo &DCI) {
16432 if (DCI.isBeforeLegalize() || DCI.isCalledByLegalizer())
16433 return SDValue();
16434
16435 return CombineBaseUpdate(N, DCI);
16436}
16437
16438static SDValue PerformMVEVLDCombine(SDNode *N,
16439 TargetLowering::DAGCombinerInfo &DCI) {
16440 if (DCI.isBeforeLegalize() || DCI.isCalledByLegalizer())
16441 return SDValue();
16442
16443 SelectionDAG &DAG = DCI.DAG;
16444 SDValue Addr = N->getOperand(Num: 2);
16445 MemSDNode *MemN = cast<MemSDNode>(Val: N);
16446 SDLoc dl(N);
16447
16448 // For the stores, where there are multiple intrinsics we only actually want
16449 // to post-inc the last of the them.
16450 unsigned IntNo = N->getConstantOperandVal(Num: 1);
16451 if (IntNo == Intrinsic::arm_mve_vst2q && N->getConstantOperandVal(Num: 5) != 1)
16452 return SDValue();
16453 if (IntNo == Intrinsic::arm_mve_vst4q && N->getConstantOperandVal(Num: 7) != 3)
16454 return SDValue();
16455
16456 // Search for a use of the address operand that is an increment.
16457 for (SDUse &Use : Addr->uses()) {
16458 SDNode *User = Use.getUser();
16459 if (User->getOpcode() != ISD::ADD || Use.getResNo() != Addr.getResNo())
16460 continue;
16461
16462 // Check that the add is independent of the load/store. Otherwise, folding
16463 // it would create a cycle. We can avoid searching through Addr as it's a
16464 // predecessor to both.
16465 SmallPtrSet<const SDNode *, 32> Visited;
16466 SmallVector<const SDNode *, 16> Worklist;
16467 Visited.insert(Ptr: Addr.getNode());
16468 Worklist.push_back(Elt: N);
16469 Worklist.push_back(Elt: User);
16470 const unsigned MaxSteps = 1024;
16471 if (SDNode::hasPredecessorHelper(N, Visited, Worklist, MaxSteps) ||
16472 SDNode::hasPredecessorHelper(N: User, Visited, Worklist, MaxSteps))
16473 continue;
16474
16475 // Find the new opcode for the updating load/store.
16476 bool isLoadOp = true;
16477 unsigned NewOpc = 0;
16478 unsigned NumVecs = 0;
16479 switch (IntNo) {
16480 default:
16481 llvm_unreachable("unexpected intrinsic for MVE VLDn combine");
16482 case Intrinsic::arm_mve_vld2q:
16483 NewOpc = ARMISD::VLD2_UPD;
16484 NumVecs = 2;
16485 break;
16486 case Intrinsic::arm_mve_vld4q:
16487 NewOpc = ARMISD::VLD4_UPD;
16488 NumVecs = 4;
16489 break;
16490 case Intrinsic::arm_mve_vst2q:
16491 NewOpc = ARMISD::VST2_UPD;
16492 NumVecs = 2;
16493 isLoadOp = false;
16494 break;
16495 case Intrinsic::arm_mve_vst4q:
16496 NewOpc = ARMISD::VST4_UPD;
16497 NumVecs = 4;
16498 isLoadOp = false;
16499 break;
16500 }
16501
16502 // Find the size of memory referenced by the load/store.
16503 EVT VecTy;
16504 if (isLoadOp) {
16505 VecTy = N->getValueType(ResNo: 0);
16506 } else {
16507 VecTy = N->getOperand(Num: 3).getValueType();
16508 }
16509
16510 unsigned NumBytes = NumVecs * VecTy.getSizeInBits() / 8;
16511
16512 // If the increment is a constant, it must match the memory ref size.
16513 SDValue Inc = User->getOperand(Num: User->getOperand(Num: 0) == Addr ? 1 : 0);
16514 ConstantSDNode *CInc = dyn_cast<ConstantSDNode>(Val: Inc.getNode());
16515 if (!CInc || CInc->getZExtValue() != NumBytes)
16516 continue;
16517
16518 // Create the new updating load/store node.
16519 // First, create an SDVTList for the new updating node's results.
16520 EVT Tys[6];
16521 unsigned NumResultVecs = (isLoadOp ? NumVecs : 0);
16522 unsigned n;
16523 for (n = 0; n < NumResultVecs; ++n)
16524 Tys[n] = VecTy;
16525 Tys[n++] = MVT::i32;
16526 Tys[n] = MVT::Other;
16527 SDVTList SDTys = DAG.getVTList(VTs: ArrayRef(Tys, NumResultVecs + 2));
16528
16529 // Then, gather the new node's operands.
16530 SmallVector<SDValue, 8> Ops;
16531 Ops.push_back(Elt: N->getOperand(Num: 0)); // incoming chain
16532 Ops.push_back(Elt: N->getOperand(Num: 2)); // ptr
16533 Ops.push_back(Elt: Inc);
16534
16535 for (unsigned i = 3; i < N->getNumOperands(); ++i)
16536 Ops.push_back(Elt: N->getOperand(Num: i));
16537
16538 SDValue UpdN = DAG.getMemIntrinsicNode(Opcode: NewOpc, dl, VTList: SDTys, Ops, MemVT: VecTy,
16539 MMO: MemN->getMemOperand());
16540
16541 // Update the uses.
16542 SmallVector<SDValue, 5> NewResults;
16543 for (unsigned i = 0; i < NumResultVecs; ++i)
16544 NewResults.push_back(Elt: SDValue(UpdN.getNode(), i));
16545
16546 NewResults.push_back(Elt: SDValue(UpdN.getNode(), NumResultVecs + 1)); // chain
16547 DCI.CombineTo(N, To: NewResults);
16548 DCI.CombineTo(N: User, Res: SDValue(UpdN.getNode(), NumResultVecs));
16549
16550 break;
16551 }
16552
16553 return SDValue();
16554}
16555
16556/// CombineVLDDUP - For a VDUPLANE node N, check if its source operand is a
16557/// vldN-lane (N > 1) intrinsic, and if all the other uses of that intrinsic
16558/// are also VDUPLANEs. If so, combine them to a vldN-dup operation and
16559/// return true.
16560static bool CombineVLDDUP(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
16561 SelectionDAG &DAG = DCI.DAG;
16562 EVT VT = N->getValueType(ResNo: 0);
16563 // vldN-dup instructions only support 64-bit vectors for N > 1.
16564 if (!VT.is64BitVector())
16565 return false;
16566
16567 // Check if the VDUPLANE operand is a vldN-dup intrinsic.
16568 SDNode *VLD = N->getOperand(Num: 0).getNode();
16569 if (VLD->getOpcode() != ISD::INTRINSIC_W_CHAIN)
16570 return false;
16571 unsigned NumVecs = 0;
16572 unsigned NewOpc = 0;
16573 unsigned IntNo = VLD->getConstantOperandVal(Num: 1);
16574 if (IntNo == Intrinsic::arm_neon_vld2lane) {
16575 NumVecs = 2;
16576 NewOpc = ARMISD::VLD2DUP;
16577 } else if (IntNo == Intrinsic::arm_neon_vld3lane) {
16578 NumVecs = 3;
16579 NewOpc = ARMISD::VLD3DUP;
16580 } else if (IntNo == Intrinsic::arm_neon_vld4lane) {
16581 NumVecs = 4;
16582 NewOpc = ARMISD::VLD4DUP;
16583 } else {
16584 return false;
16585 }
16586
16587 // First check that all the vldN-lane uses are VDUPLANEs and that the lane
16588 // numbers match the load.
16589 unsigned VLDLaneNo = VLD->getConstantOperandVal(Num: NumVecs + 3);
16590 for (SDUse &Use : VLD->uses()) {
16591 // Ignore uses of the chain result.
16592 if (Use.getResNo() == NumVecs)
16593 continue;
16594 SDNode *User = Use.getUser();
16595 if (User->getOpcode() != ARMISD::VDUPLANE ||
16596 VLDLaneNo != User->getConstantOperandVal(Num: 1))
16597 return false;
16598 }
16599
16600 // Create the vldN-dup node.
16601 EVT Tys[5];
16602 unsigned n;
16603 for (n = 0; n < NumVecs; ++n)
16604 Tys[n] = VT;
16605 Tys[n] = MVT::Other;
16606 SDVTList SDTys = DAG.getVTList(VTs: ArrayRef(Tys, NumVecs + 1));
16607 SDValue Ops[] = { VLD->getOperand(Num: 0), VLD->getOperand(Num: 2) };
16608 MemIntrinsicSDNode *VLDMemInt = cast<MemIntrinsicSDNode>(Val: VLD);
16609 SDValue VLDDup = DAG.getMemIntrinsicNode(Opcode: NewOpc, dl: SDLoc(VLD), VTList: SDTys,
16610 Ops, MemVT: VLDMemInt->getMemoryVT(),
16611 MMO: VLDMemInt->getMemOperand());
16612
16613 // Update the uses.
16614 for (SDUse &Use : VLD->uses()) {
16615 unsigned ResNo = Use.getResNo();
16616 // Ignore uses of the chain result.
16617 if (ResNo == NumVecs)
16618 continue;
16619 DCI.CombineTo(N: Use.getUser(), Res: SDValue(VLDDup.getNode(), ResNo));
16620 }
16621
16622 // Now the vldN-lane intrinsic is dead except for its chain result.
16623 // Update uses of the chain.
16624 std::vector<SDValue> VLDDupResults;
16625 for (unsigned n = 0; n < NumVecs; ++n)
16626 VLDDupResults.push_back(x: SDValue(VLDDup.getNode(), n));
16627 VLDDupResults.push_back(x: SDValue(VLDDup.getNode(), NumVecs));
16628 DCI.CombineTo(N: VLD, To: VLDDupResults);
16629
16630 return true;
16631}
16632
16633/// PerformVDUPLANECombine - Target-specific dag combine xforms for
16634/// ARMISD::VDUPLANE.
16635static SDValue PerformVDUPLANECombine(SDNode *N,
16636 TargetLowering::DAGCombinerInfo &DCI,
16637 const ARMSubtarget *Subtarget) {
16638 SDValue Op = N->getOperand(Num: 0);
16639 EVT VT = N->getValueType(ResNo: 0);
16640
16641 // On MVE, we just convert the VDUPLANE to a VDUP with an extract.
16642 if (Subtarget->hasMVEIntegerOps()) {
16643 EVT ExtractVT = VT.getVectorElementType();
16644 // We need to ensure we are creating a legal type.
16645 if (!DCI.DAG.getTargetLoweringInfo().isTypeLegal(VT: ExtractVT))
16646 ExtractVT = MVT::i32;
16647 SDValue Extract = DCI.DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N), VT: ExtractVT,
16648 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1));
16649 return DCI.DAG.getNode(Opcode: ARMISD::VDUP, DL: SDLoc(N), VT, Operand: Extract);
16650 }
16651
16652 // If the source is a vldN-lane (N > 1) intrinsic, and all the other uses
16653 // of that intrinsic are also VDUPLANEs, combine them to a vldN-dup operation.
16654 if (CombineVLDDUP(N, DCI))
16655 return SDValue(N, 0);
16656
16657 // If the source is already a VMOVIMM or VMVNIMM splat, the VDUPLANE is
16658 // redundant. Ignore bit_converts for now; element sizes are checked below.
16659 while (Op.getOpcode() == ISD::BITCAST)
16660 Op = Op.getOperand(i: 0);
16661 if (Op.getOpcode() != ARMISD::VMOVIMM && Op.getOpcode() != ARMISD::VMVNIMM)
16662 return SDValue();
16663
16664 // Make sure the VMOV element size is not bigger than the VDUPLANE elements.
16665 unsigned EltSize = Op.getScalarValueSizeInBits();
16666 // The canonical VMOV for a zero vector uses a 32-bit element size.
16667 unsigned Imm = Op.getConstantOperandVal(i: 0);
16668 unsigned EltBits;
16669 if (ARM_AM::decodeVMOVModImm(ModImm: Imm, EltBits) == 0)
16670 EltSize = 8;
16671 if (EltSize > VT.getScalarSizeInBits())
16672 return SDValue();
16673
16674 return DCI.DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT, Operand: Op);
16675}
16676
16677/// PerformVDUPCombine - Target-specific dag combine xforms for ARMISD::VDUP.
16678static SDValue PerformVDUPCombine(SDNode *N, SelectionDAG &DAG,
16679 const ARMSubtarget *Subtarget) {
16680 SDValue Op = N->getOperand(Num: 0);
16681 SDLoc dl(N);
16682
16683 if (Subtarget->hasMVEIntegerOps()) {
16684 // Convert VDUP f32 -> VDUP BITCAST i32 under MVE, as we know the value will
16685 // need to come from a GPR.
16686 if (Op.getValueType() == MVT::f32)
16687 return DAG.getNode(Opcode: ARMISD::VDUP, DL: dl, VT: N->getValueType(ResNo: 0),
16688 Operand: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: Op));
16689 else if (Op.getValueType() == MVT::f16)
16690 return DAG.getNode(Opcode: ARMISD::VDUP, DL: dl, VT: N->getValueType(ResNo: 0),
16691 Operand: DAG.getNode(Opcode: ARMISD::VMOVrh, DL: dl, VT: MVT::i32, Operand: Op));
16692 }
16693
16694 if (!Subtarget->hasNEON())
16695 return SDValue();
16696
16697 // Match VDUP(LOAD) -> VLD1DUP.
16698 // We match this pattern here rather than waiting for isel because the
16699 // transform is only legal for unindexed loads.
16700 LoadSDNode *LD = dyn_cast<LoadSDNode>(Val: Op.getNode());
16701 if (LD && Op.hasOneUse() && LD->isUnindexed() &&
16702 LD->getMemoryVT() == N->getValueType(ResNo: 0).getVectorElementType()) {
16703 SDValue Ops[] = {LD->getOperand(Num: 0), LD->getOperand(Num: 1),
16704 DAG.getConstant(Val: LD->getAlign().value(), DL: SDLoc(N), VT: MVT::i32)};
16705 SDVTList SDTys = DAG.getVTList(VT1: N->getValueType(ResNo: 0), VT2: MVT::Other);
16706 SDValue VLDDup =
16707 DAG.getMemIntrinsicNode(Opcode: ARMISD::VLD1DUP, dl: SDLoc(N), VTList: SDTys, Ops,
16708 MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand());
16709 DAG.ReplaceAllUsesOfValueWith(From: SDValue(LD, 1), To: VLDDup.getValue(R: 1));
16710 return VLDDup;
16711 }
16712
16713 return SDValue();
16714}
16715
16716static SDValue PerformLOADCombine(SDNode *N,
16717 TargetLowering::DAGCombinerInfo &DCI,
16718 const ARMSubtarget *Subtarget) {
16719 EVT VT = N->getValueType(ResNo: 0);
16720
16721 // If this is a legal vector load, try to combine it into a VLD1_UPD.
16722 if (Subtarget->hasNEON() && ISD::isNormalLoad(N) && VT.isVector() &&
16723 DCI.DAG.getTargetLoweringInfo().isTypeLegal(VT))
16724 return CombineBaseUpdate(N, DCI);
16725
16726 return SDValue();
16727}
16728
16729// Optimize trunc store (of multiple scalars) to shuffle and store. First,
16730// pack all of the elements in one place. Next, store to memory in fewer
16731// chunks.
16732static SDValue PerformTruncatingStoreCombine(StoreSDNode *St,
16733 SelectionDAG &DAG) {
16734 SDValue StVal = St->getValue();
16735 EVT VT = StVal.getValueType();
16736 if (!St->isTruncatingStore() || !VT.isVector())
16737 return SDValue();
16738 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
16739 EVT StVT = St->getMemoryVT();
16740 unsigned NumElems = VT.getVectorNumElements();
16741 assert(StVT != VT && "Cannot truncate to the same type");
16742 unsigned FromEltSz = VT.getScalarSizeInBits();
16743 unsigned ToEltSz = StVT.getScalarSizeInBits();
16744
16745 // From, To sizes and ElemCount must be pow of two
16746 if (!isPowerOf2_32(Value: NumElems * FromEltSz * ToEltSz))
16747 return SDValue();
16748
16749 // We are going to use the original vector elt for storing.
16750 // Accumulated smaller vector elements must be a multiple of the store size.
16751 if (0 != (NumElems * FromEltSz) % ToEltSz)
16752 return SDValue();
16753
16754 unsigned SizeRatio = FromEltSz / ToEltSz;
16755 assert(SizeRatio * NumElems * ToEltSz == VT.getSizeInBits());
16756
16757 // Create a type on which we perform the shuffle.
16758 EVT WideVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: StVT.getScalarType(),
16759 NumElements: NumElems * SizeRatio);
16760 assert(WideVecVT.getSizeInBits() == VT.getSizeInBits());
16761
16762 SDLoc DL(St);
16763 SDValue WideVec = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: WideVecVT, Operand: StVal);
16764 SmallVector<int, 8> ShuffleVec(NumElems * SizeRatio, -1);
16765 for (unsigned i = 0; i < NumElems; ++i)
16766 ShuffleVec[i] = DAG.getDataLayout().isBigEndian() ? (i + 1) * SizeRatio - 1
16767 : i * SizeRatio;
16768
16769 // Can't shuffle using an illegal type.
16770 if (!TLI.isTypeLegal(VT: WideVecVT))
16771 return SDValue();
16772
16773 SDValue Shuff = DAG.getVectorShuffle(
16774 VT: WideVecVT, dl: DL, N1: WideVec, N2: DAG.getUNDEF(VT: WideVec.getValueType()), Mask: ShuffleVec);
16775 // At this point all of the data is stored at the bottom of the
16776 // register. We now need to save it to mem.
16777
16778 // Find the largest store unit
16779 MVT StoreType = MVT::i8;
16780 for (MVT Tp : MVT::integer_valuetypes()) {
16781 if (TLI.isTypeLegal(VT: Tp) && Tp.getSizeInBits() <= NumElems * ToEltSz)
16782 StoreType = Tp;
16783 }
16784 // Didn't find a legal store type.
16785 if (!TLI.isTypeLegal(VT: StoreType))
16786 return SDValue();
16787
16788 // Bitcast the original vector into a vector of store-size units
16789 EVT StoreVecVT =
16790 EVT::getVectorVT(Context&: *DAG.getContext(), VT: StoreType,
16791 NumElements: VT.getSizeInBits() / EVT(StoreType).getSizeInBits());
16792 assert(StoreVecVT.getSizeInBits() == VT.getSizeInBits());
16793 SDValue ShuffWide = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: StoreVecVT, Operand: Shuff);
16794 SmallVector<SDValue, 8> Chains;
16795 SDValue Increment = DAG.getConstant(Val: StoreType.getSizeInBits() / 8, DL,
16796 VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
16797 SDValue BasePtr = St->getBasePtr();
16798
16799 // Perform one or more big stores into memory.
16800 unsigned E = (ToEltSz * NumElems) / StoreType.getSizeInBits();
16801 for (unsigned I = 0; I < E; I++) {
16802 SDValue SubVec = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: StoreType,
16803 N1: ShuffWide, N2: DAG.getIntPtrConstant(Val: I, DL));
16804 SDValue Ch =
16805 DAG.getStore(Chain: St->getChain(), dl: DL, Val: SubVec, Ptr: BasePtr, PtrInfo: St->getPointerInfo(),
16806 Alignment: St->getAlign(), MMOFlags: St->getMemOperand()->getFlags());
16807 BasePtr =
16808 DAG.getNode(Opcode: ISD::ADD, DL, VT: BasePtr.getValueType(), N1: BasePtr, N2: Increment);
16809 Chains.push_back(Elt: Ch);
16810 }
16811 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: Chains);
16812}
16813
16814// Try taking a single vector store from an fpround (which would otherwise turn
16815// into an expensive buildvector) and splitting it into a series of narrowing
16816// stores.
16817static SDValue PerformSplittingToNarrowingStores(StoreSDNode *St,
16818 SelectionDAG &DAG) {
16819 if (!St->isSimple() || St->isTruncatingStore() || !St->isUnindexed())
16820 return SDValue();
16821 SDValue Trunc = St->getValue();
16822 if (Trunc->getOpcode() != ISD::FP_ROUND)
16823 return SDValue();
16824 EVT FromVT = Trunc->getOperand(Num: 0).getValueType();
16825 EVT ToVT = Trunc.getValueType();
16826 if (!ToVT.isVector())
16827 return SDValue();
16828 assert(FromVT.getVectorNumElements() == ToVT.getVectorNumElements());
16829 EVT ToEltVT = ToVT.getVectorElementType();
16830 EVT FromEltVT = FromVT.getVectorElementType();
16831
16832 if (FromEltVT != MVT::f32 || ToEltVT != MVT::f16)
16833 return SDValue();
16834
16835 unsigned NumElements = 4;
16836 if (FromVT.getVectorNumElements() % NumElements != 0)
16837 return SDValue();
16838
16839 // Test if the Trunc will be convertible to a VMOVN with a shuffle, and if so
16840 // use the VMOVN over splitting the store. We are looking for patterns of:
16841 // !rev: 0 N 1 N+1 2 N+2 ...
16842 // rev: N 0 N+1 1 N+2 2 ...
16843 // The shuffle may either be a single source (in which case N = NumElts/2) or
16844 // two inputs extended with concat to the same size (in which case N =
16845 // NumElts).
16846 auto isVMOVNShuffle = [&](ShuffleVectorSDNode *SVN, bool Rev) {
16847 ArrayRef<int> M = SVN->getMask();
16848 unsigned NumElts = ToVT.getVectorNumElements();
16849 if (SVN->getOperand(Num: 1).isUndef())
16850 NumElts /= 2;
16851
16852 unsigned Off0 = Rev ? NumElts : 0;
16853 unsigned Off1 = Rev ? 0 : NumElts;
16854
16855 for (unsigned I = 0; I < NumElts; I += 2) {
16856 if (M[I] >= 0 && M[I] != (int)(Off0 + I / 2))
16857 return false;
16858 if (M[I + 1] >= 0 && M[I + 1] != (int)(Off1 + I / 2))
16859 return false;
16860 }
16861
16862 return true;
16863 };
16864
16865 if (auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Val: Trunc.getOperand(i: 0)))
16866 if (isVMOVNShuffle(Shuffle, false) || isVMOVNShuffle(Shuffle, true))
16867 return SDValue();
16868
16869 LLVMContext &C = *DAG.getContext();
16870 SDLoc DL(St);
16871 // Details about the old store
16872 SDValue Ch = St->getChain();
16873 SDValue BasePtr = St->getBasePtr();
16874 Align Alignment = St->getBaseAlign();
16875 MachineMemOperand::Flags MMOFlags = St->getMemOperand()->getFlags();
16876 AAMDNodes AAInfo = St->getAAInfo();
16877
16878 // We split the store into slices of NumElements. fp16 trunc stores are vcvt
16879 // and then stored as truncating integer stores.
16880 EVT NewFromVT = EVT::getVectorVT(Context&: C, VT: FromEltVT, NumElements);
16881 EVT NewToVT = EVT::getVectorVT(
16882 Context&: C, VT: EVT::getIntegerVT(Context&: C, BitWidth: ToEltVT.getSizeInBits()), NumElements);
16883
16884 SmallVector<SDValue, 4> Stores;
16885 for (unsigned i = 0; i < FromVT.getVectorNumElements() / NumElements; i++) {
16886 unsigned NewOffset = i * NumElements * ToEltVT.getSizeInBits() / 8;
16887 SDValue NewPtr =
16888 DAG.getObjectPtrOffset(SL: DL, Ptr: BasePtr, Offset: TypeSize::getFixed(ExactSize: NewOffset));
16889
16890 SDValue Extract =
16891 DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: NewFromVT, N1: Trunc.getOperand(i: 0),
16892 N2: DAG.getConstant(Val: i * NumElements, DL, VT: MVT::i32));
16893
16894 SDValue FPTrunc =
16895 DAG.getNode(Opcode: ARMISD::VCVTN, DL, VT: MVT::v8f16, N1: DAG.getUNDEF(VT: MVT::v8f16),
16896 N2: Extract, N3: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
16897 Extract = DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL, VT: MVT::v4i32, Operand: FPTrunc);
16898
16899 SDValue Store = DAG.getTruncStore(
16900 Chain: Ch, dl: DL, Val: Extract, Ptr: NewPtr, PtrInfo: St->getPointerInfo().getWithOffset(O: NewOffset),
16901 SVT: NewToVT, Alignment, MMOFlags, Metadata: AAInfo);
16902 Stores.push_back(Elt: Store);
16903 }
16904 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: Stores);
16905}
16906
16907// Try taking a single vector store from an MVETRUNC (which would otherwise turn
16908// into an expensive buildvector) and splitting it into a series of narrowing
16909// stores.
16910static SDValue PerformSplittingMVETruncToNarrowingStores(StoreSDNode *St,
16911 SelectionDAG &DAG) {
16912 if (!St->isSimple() || St->isTruncatingStore() || !St->isUnindexed())
16913 return SDValue();
16914 SDValue Trunc = St->getValue();
16915 if (Trunc->getOpcode() != ARMISD::MVETRUNC)
16916 return SDValue();
16917 EVT FromVT = Trunc->getOperand(Num: 0).getValueType();
16918 EVT ToVT = Trunc.getValueType();
16919
16920 LLVMContext &C = *DAG.getContext();
16921 SDLoc DL(St);
16922 // Details about the old store
16923 SDValue Ch = St->getChain();
16924 SDValue BasePtr = St->getBasePtr();
16925 Align Alignment = St->getBaseAlign();
16926 MachineMemOperand::Flags MMOFlags = St->getMemOperand()->getFlags();
16927 AAMDNodes AAInfo = St->getAAInfo();
16928
16929 EVT NewToVT = EVT::getVectorVT(Context&: C, VT: ToVT.getVectorElementType(),
16930 NumElements: FromVT.getVectorNumElements());
16931
16932 SmallVector<SDValue, 4> Stores;
16933 for (unsigned i = 0; i < Trunc.getNumOperands(); i++) {
16934 unsigned NewOffset =
16935 i * FromVT.getVectorNumElements() * ToVT.getScalarSizeInBits() / 8;
16936 SDValue NewPtr =
16937 DAG.getObjectPtrOffset(SL: DL, Ptr: BasePtr, Offset: TypeSize::getFixed(ExactSize: NewOffset));
16938
16939 SDValue Extract = Trunc.getOperand(i);
16940 SDValue Store = DAG.getTruncStore(
16941 Chain: Ch, dl: DL, Val: Extract, Ptr: NewPtr, PtrInfo: St->getPointerInfo().getWithOffset(O: NewOffset),
16942 SVT: NewToVT, Alignment, MMOFlags, Metadata: AAInfo);
16943 Stores.push_back(Elt: Store);
16944 }
16945 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: Stores);
16946}
16947
16948// Given a floating point store from an extracted vector, with an integer
16949// VGETLANE that already exists, store the existing VGETLANEu directly. This can
16950// help reduce fp register pressure, doesn't require the fp extract and allows
16951// use of more integer post-inc stores not available with vstr.
16952static SDValue PerformExtractFpToIntStores(StoreSDNode *St, SelectionDAG &DAG) {
16953 if (!St->isSimple() || St->isTruncatingStore() || !St->isUnindexed())
16954 return SDValue();
16955 SDValue Extract = St->getValue();
16956 EVT VT = Extract.getValueType();
16957 // For now only uses f16. This may be useful for f32 too, but that will
16958 // be bitcast(extract), not the VGETLANEu we currently check here.
16959 if (VT != MVT::f16 || Extract->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
16960 return SDValue();
16961
16962 SDNode *GetLane =
16963 DAG.getNodeIfExists(Opcode: ARMISD::VGETLANEu, VTList: DAG.getVTList(VT: MVT::i32),
16964 Ops: {Extract.getOperand(i: 0), Extract.getOperand(i: 1)});
16965 if (!GetLane)
16966 return SDValue();
16967
16968 LLVMContext &C = *DAG.getContext();
16969 SDLoc DL(St);
16970 // Create a new integer store to replace the existing floating point version.
16971 SDValue Ch = St->getChain();
16972 SDValue BasePtr = St->getBasePtr();
16973 Align Alignment = St->getBaseAlign();
16974 MachineMemOperand::Flags MMOFlags = St->getMemOperand()->getFlags();
16975 AAMDNodes AAInfo = St->getAAInfo();
16976 EVT NewToVT = EVT::getIntegerVT(Context&: C, BitWidth: VT.getSizeInBits());
16977 SDValue Store = DAG.getTruncStore(Chain: Ch, dl: DL, Val: SDValue(GetLane, 0), Ptr: BasePtr,
16978 PtrInfo: St->getPointerInfo(), SVT: NewToVT, Alignment,
16979 MMOFlags, Metadata: AAInfo);
16980
16981 return Store;
16982}
16983
16984/// PerformSTORECombine - Target-specific dag combine xforms for
16985/// ISD::STORE.
16986static SDValue PerformSTORECombine(SDNode *N,
16987 TargetLowering::DAGCombinerInfo &DCI,
16988 const ARMSubtarget *Subtarget) {
16989 StoreSDNode *St = cast<StoreSDNode>(Val: N);
16990 if (St->isVolatile())
16991 return SDValue();
16992 SDValue StVal = St->getValue();
16993 EVT VT = StVal.getValueType();
16994
16995 if (Subtarget->hasNEON())
16996 if (SDValue Store = PerformTruncatingStoreCombine(St, DAG&: DCI.DAG))
16997 return Store;
16998
16999 if (Subtarget->hasMVEFloatOps())
17000 if (SDValue NewToken = PerformSplittingToNarrowingStores(St, DAG&: DCI.DAG))
17001 return NewToken;
17002
17003 if (Subtarget->hasMVEIntegerOps()) {
17004 if (SDValue NewChain = PerformExtractFpToIntStores(St, DAG&: DCI.DAG))
17005 return NewChain;
17006 if (SDValue NewToken =
17007 PerformSplittingMVETruncToNarrowingStores(St, DAG&: DCI.DAG))
17008 return NewToken;
17009 }
17010
17011 if (!ISD::isNormalStore(N: St))
17012 return SDValue();
17013
17014 // Split a store of a VMOVDRR into two integer stores to avoid mixing NEON and
17015 // ARM stores of arguments in the same cache line.
17016 if (StVal.getOpcode() == ARMISD::VMOVDRR && StVal->hasOneUse()) {
17017 SelectionDAG &DAG = DCI.DAG;
17018 bool isBigEndian = DAG.getDataLayout().isBigEndian();
17019 SDLoc DL(St);
17020 SDValue BasePtr = St->getBasePtr();
17021 SDValue NewST1 =
17022 DAG.getStore(Chain: St->getChain(), dl: DL, Val: StVal.getOperand(i: isBigEndian ? 1 : 0),
17023 Ptr: BasePtr, PtrInfo: St->getPointerInfo(), Alignment: St->getBaseAlign(),
17024 MMOFlags: St->getMemOperand()->getFlags());
17025
17026 SDValue OffsetPtr = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i32, N1: BasePtr,
17027 N2: DAG.getConstant(Val: 4, DL, VT: MVT::i32));
17028 return DAG.getStore(Chain: NewST1.getValue(R: 0), dl: DL,
17029 Val: StVal.getOperand(i: isBigEndian ? 0 : 1), Ptr: OffsetPtr,
17030 PtrInfo: St->getPointerInfo().getWithOffset(O: 4),
17031 Alignment: St->getBaseAlign(), MMOFlags: St->getMemOperand()->getFlags());
17032 }
17033
17034 if (StVal.getValueType() == MVT::i64 &&
17035 StVal.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
17036 // Bitcast an i64 store extracted from a vector to f64.
17037 // Otherwise, the i64 value will be legalized to a pair of i32 values.
17038 SelectionDAG &DAG = DCI.DAG;
17039 SDLoc dl(StVal);
17040 SDValue IntVec = StVal.getOperand(i: 0);
17041 EVT FloatVT =
17042 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::f64,
17043 NumElements: IntVec.getValueType().getVectorNumElements());
17044 SDValue Vec = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: FloatVT, Operand: IntVec);
17045 SDValue ExtElt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: MVT::f64, N1: Vec,
17046 N2: StVal.getOperand(i: 1));
17047 dl = SDLoc(N);
17048 SDValue V = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i64, Operand: ExtElt);
17049 // Make the DAGCombiner fold the bitcasts.
17050 DCI.AddToWorklist(N: Vec.getNode());
17051 DCI.AddToWorklist(N: ExtElt.getNode());
17052 DCI.AddToWorklist(N: V.getNode());
17053 return DAG.getStore(Chain: St->getChain(), dl, Val: V, Ptr: St->getBasePtr(),
17054 PtrInfo: St->getPointerInfo(), Alignment: St->getAlign(),
17055 MMOFlags: St->getMemOperand()->getFlags(), Metadata: St->getAAInfo());
17056 }
17057
17058 // If this is a legal vector store, try to combine it into a VST1_UPD.
17059 if (Subtarget->hasNEON() && ISD::isNormalStore(N) && VT.isVector() &&
17060 DCI.DAG.getTargetLoweringInfo().isTypeLegal(VT))
17061 return CombineBaseUpdate(N, DCI);
17062
17063 return SDValue();
17064}
17065
17066/// PerformVCVTCombine - VCVT (floating-point to fixed-point, Advanced SIMD)
17067/// can replace combinations of VMUL and VCVT (floating-point to integer)
17068/// when the VMUL has a constant operand that is a power of 2.
17069///
17070/// Example (assume d17 = <float 8.000000e+00, float 8.000000e+00>):
17071/// vmul.f32 d16, d17, d16
17072/// vcvt.s32.f32 d16, d16
17073/// becomes:
17074/// vcvt.s32.f32 d16, d16, #3
17075static SDValue PerformVCVTCombine(SDNode *N, SelectionDAG &DAG,
17076 const ARMSubtarget *Subtarget) {
17077 if (!Subtarget->hasNEON())
17078 return SDValue();
17079
17080 SDValue Op = N->getOperand(Num: 0);
17081 if (!Op.getValueType().isVector() || !Op.getValueType().isSimple() ||
17082 Op.getOpcode() != ISD::FMUL)
17083 return SDValue();
17084
17085 SDValue ConstVec = Op->getOperand(Num: 1);
17086 if (!isa<BuildVectorSDNode>(Val: ConstVec))
17087 return SDValue();
17088
17089 MVT FloatTy = Op.getSimpleValueType().getVectorElementType();
17090 uint32_t FloatBits = FloatTy.getSizeInBits();
17091 MVT IntTy = N->getSimpleValueType(ResNo: 0).getVectorElementType();
17092 uint32_t IntBits = IntTy.getSizeInBits();
17093 unsigned NumLanes = Op.getValueType().getVectorNumElements();
17094 if (FloatBits != 32 || IntBits > 32 || (NumLanes != 4 && NumLanes != 2)) {
17095 // These instructions only exist converting from f32 to i32. We can handle
17096 // smaller integers by generating an extra truncate, but larger ones would
17097 // be lossy. We also can't handle anything other than 2 or 4 lanes, since
17098 // these instructions only support v2i32/v4i32 types.
17099 return SDValue();
17100 }
17101
17102 BitVector UndefElements;
17103 BuildVectorSDNode *BV = cast<BuildVectorSDNode>(Val&: ConstVec);
17104 int32_t C = BV->getConstantFPSplatPow2ToLog2Int(UndefElements: &UndefElements, BitWidth: 33);
17105 if (C == -1 || C == 0 || C > 32)
17106 return SDValue();
17107
17108 SDLoc dl(N);
17109 bool isSigned = N->getOpcode() == ISD::FP_TO_SINT;
17110 unsigned IntrinsicOpcode = isSigned ? Intrinsic::arm_neon_vcvtfp2fxs :
17111 Intrinsic::arm_neon_vcvtfp2fxu;
17112 SDValue FixConv = DAG.getNode(
17113 Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: NumLanes == 2 ? MVT::v2i32 : MVT::v4i32,
17114 N1: DAG.getConstant(Val: IntrinsicOpcode, DL: dl, VT: MVT::i32), N2: Op->getOperand(Num: 0),
17115 N3: DAG.getConstant(Val: C, DL: dl, VT: MVT::i32));
17116
17117 if (IntBits < FloatBits)
17118 FixConv = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: N->getValueType(ResNo: 0), Operand: FixConv);
17119
17120 return FixConv;
17121}
17122
17123static SDValue PerformFAddVSelectCombine(SDNode *N, SelectionDAG &DAG,
17124 const ARMSubtarget *Subtarget) {
17125 if (!Subtarget->hasMVEFloatOps())
17126 return SDValue();
17127
17128 // Turn (fadd x, (vselect c, y, -0.0)) into (vselect c, (fadd x, y), x)
17129 // The second form can be more easily turned into a predicated vadd, and
17130 // possibly combined into a fma to become a predicated vfma.
17131 SDValue Op0 = N->getOperand(Num: 0);
17132 SDValue Op1 = N->getOperand(Num: 1);
17133 EVT VT = N->getValueType(ResNo: 0);
17134 SDLoc DL(N);
17135
17136 // The identity element for a fadd is -0.0 or +0.0 when the nsz flag is set,
17137 // which these VMOV's represent.
17138 auto isIdentitySplat = [&](SDValue Op, bool NSZ) {
17139 if (Op.getOpcode() != ISD::BITCAST ||
17140 Op.getOperand(i: 0).getOpcode() != ARMISD::VMOVIMM)
17141 return false;
17142 uint64_t ImmVal = Op.getOperand(i: 0).getConstantOperandVal(i: 0);
17143 if (VT == MVT::v4f32 && (ImmVal == 1664 || (ImmVal == 0 && NSZ)))
17144 return true;
17145 if (VT == MVT::v8f16 && (ImmVal == 2688 || (ImmVal == 0 && NSZ)))
17146 return true;
17147 return false;
17148 };
17149
17150 if (Op0.getOpcode() == ISD::VSELECT && Op1.getOpcode() != ISD::VSELECT)
17151 std::swap(a&: Op0, b&: Op1);
17152
17153 if (Op1.getOpcode() != ISD::VSELECT)
17154 return SDValue();
17155
17156 SDNodeFlags FaddFlags = N->getFlags();
17157 bool NSZ = FaddFlags.hasNoSignedZeros();
17158 if (!isIdentitySplat(Op1.getOperand(i: 2), NSZ))
17159 return SDValue();
17160
17161 SDValue FAdd =
17162 DAG.getNode(Opcode: ISD::FADD, DL, VT, N1: Op0, N2: Op1.getOperand(i: 1), Flags: FaddFlags);
17163 return DAG.getNode(Opcode: ISD::VSELECT, DL, VT, N1: Op1.getOperand(i: 0), N2: FAdd, N3: Op0, Flags: FaddFlags);
17164}
17165
17166static SDValue PerformFADDVCMLACombine(SDNode *N, SelectionDAG &DAG) {
17167 SDValue LHS = N->getOperand(Num: 0);
17168 SDValue RHS = N->getOperand(Num: 1);
17169 EVT VT = N->getValueType(ResNo: 0);
17170 SDLoc DL(N);
17171
17172 if (!N->getFlags().hasAllowReassociation())
17173 return SDValue();
17174
17175 // Combine fadd(a, vcmla(b, c, d)) -> vcmla(fadd(a, b), b, c)
17176 auto ReassocComplex = [&](SDValue A, SDValue B) {
17177 if (A.getOpcode() != ISD::INTRINSIC_WO_CHAIN)
17178 return SDValue();
17179 unsigned Opc = A.getConstantOperandVal(i: 0);
17180 if (Opc != Intrinsic::arm_mve_vcmlaq)
17181 return SDValue();
17182 SDValue VCMLA = DAG.getNode(
17183 Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT, N1: A.getOperand(i: 0), N2: A.getOperand(i: 1),
17184 N3: DAG.getNode(Opcode: ISD::FADD, DL, VT, N1: A.getOperand(i: 2), N2: B, Flags: N->getFlags()),
17185 N4: A.getOperand(i: 3), N5: A.getOperand(i: 4));
17186 VCMLA->setFlags(A->getFlags());
17187 return VCMLA;
17188 };
17189 if (SDValue R = ReassocComplex(LHS, RHS))
17190 return R;
17191 if (SDValue R = ReassocComplex(RHS, LHS))
17192 return R;
17193
17194 return SDValue();
17195}
17196
17197static SDValue PerformFADDCombine(SDNode *N, SelectionDAG &DAG,
17198 const ARMSubtarget *Subtarget) {
17199 if (SDValue S = PerformFAddVSelectCombine(N, DAG, Subtarget))
17200 return S;
17201 if (SDValue S = PerformFADDVCMLACombine(N, DAG))
17202 return S;
17203 return SDValue();
17204}
17205
17206/// PerformVMulVCTPCombine - VCVT (fixed-point to floating-point, Advanced SIMD)
17207/// can replace combinations of VCVT (integer to floating-point) and VMUL
17208/// when the VMUL has a constant operand that is a power of 2.
17209///
17210/// Example (assume d17 = <float 0.125, float 0.125>):
17211/// vcvt.f32.s32 d16, d16
17212/// vmul.f32 d16, d16, d17
17213/// becomes:
17214/// vcvt.f32.s32 d16, d16, #3
17215static SDValue PerformVMulVCTPCombine(SDNode *N, SelectionDAG &DAG,
17216 const ARMSubtarget *Subtarget) {
17217 if (!Subtarget->hasNEON())
17218 return SDValue();
17219
17220 SDValue Op = N->getOperand(Num: 0);
17221 unsigned OpOpcode = Op.getNode()->getOpcode();
17222 if (!N->getValueType(ResNo: 0).isVector() || !N->getValueType(ResNo: 0).isSimple() ||
17223 (OpOpcode != ISD::SINT_TO_FP && OpOpcode != ISD::UINT_TO_FP))
17224 return SDValue();
17225
17226 SDValue ConstVec = N->getOperand(Num: 1);
17227 if (!isa<BuildVectorSDNode>(Val: ConstVec))
17228 return SDValue();
17229
17230 MVT FloatTy = N->getSimpleValueType(ResNo: 0).getVectorElementType();
17231 uint32_t FloatBits = FloatTy.getSizeInBits();
17232 MVT IntTy = Op.getOperand(i: 0).getSimpleValueType().getVectorElementType();
17233 uint32_t IntBits = IntTy.getSizeInBits();
17234 unsigned NumLanes = Op.getValueType().getVectorNumElements();
17235 if (FloatBits != 32 || IntBits > 32 || (NumLanes != 4 && NumLanes != 2)) {
17236 // These instructions only exist converting from i32 to f32. We can handle
17237 // smaller integers by generating an extra extend, but larger ones would
17238 // be lossy. We also can't handle anything other than 2 or 4 lanes, since
17239 // these instructions only support v2i32/v4i32 types.
17240 return SDValue();
17241 }
17242
17243 ConstantFPSDNode *CN = isConstOrConstSplatFP(N: ConstVec, AllowUndefs: true);
17244 APFloat Recip(0.0f);
17245 if (!CN || !CN->getValueAPF().getExactInverse(Inv: &Recip))
17246 return SDValue();
17247
17248 bool IsExact;
17249 APSInt IntVal(33);
17250 if (Recip.convertToInteger(Result&: IntVal, RM: APFloat::rmTowardZero, IsExact: &IsExact) !=
17251 APFloat::opOK ||
17252 !IsExact)
17253 return SDValue();
17254
17255 int32_t C = IntVal.exactLogBase2();
17256 if (C == -1 || C == 0 || C > 32)
17257 return SDValue();
17258
17259 SDLoc DL(N);
17260 bool isSigned = OpOpcode == ISD::SINT_TO_FP;
17261 SDValue ConvInput = Op.getOperand(i: 0);
17262 if (IntBits < FloatBits)
17263 ConvInput = DAG.getNode(Opcode: isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL,
17264 VT: NumLanes == 2 ? MVT::v2i32 : MVT::v4i32, Operand: ConvInput);
17265
17266 unsigned IntrinsicOpcode = isSigned ? Intrinsic::arm_neon_vcvtfxs2fp
17267 : Intrinsic::arm_neon_vcvtfxu2fp;
17268 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: Op.getValueType(),
17269 N1: DAG.getConstant(Val: IntrinsicOpcode, DL, VT: MVT::i32), N2: ConvInput,
17270 N3: DAG.getConstant(Val: C, DL, VT: MVT::i32));
17271}
17272
17273static SDValue PerformVECREDUCE_ADDCombine(SDNode *N, SelectionDAG &DAG,
17274 const ARMSubtarget *ST) {
17275 if (!ST->hasMVEIntegerOps())
17276 return SDValue();
17277
17278 assert(N->getOpcode() == ISD::VECREDUCE_ADD);
17279 EVT ResVT = N->getValueType(ResNo: 0);
17280 SDValue N0 = N->getOperand(Num: 0);
17281 SDLoc dl(N);
17282
17283 // Try to turn vecreduce_add(add(x, y)) into vecreduce(x) + vecreduce(y)
17284 if (ResVT == MVT::i32 && N0.getOpcode() == ISD::ADD &&
17285 (N0.getValueType() == MVT::v4i32 || N0.getValueType() == MVT::v8i16 ||
17286 N0.getValueType() == MVT::v16i8)) {
17287 SDValue Red0 = DAG.getNode(Opcode: ISD::VECREDUCE_ADD, DL: dl, VT: ResVT, Operand: N0.getOperand(i: 0));
17288 SDValue Red1 = DAG.getNode(Opcode: ISD::VECREDUCE_ADD, DL: dl, VT: ResVT, Operand: N0.getOperand(i: 1));
17289 return DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ResVT, N1: Red0, N2: Red1);
17290 }
17291
17292 // We are looking for something that will have illegal types if left alone,
17293 // but that we can convert to a single instruction under MVE. For example
17294 // vecreduce_add(sext(A, v8i32)) => VADDV.s16 A
17295 // or
17296 // vecreduce_add(mul(zext(A, v16i32), zext(B, v16i32))) => VMLADAV.u8 A, B
17297
17298 // The legal cases are:
17299 // VADDV u/s 8/16/32
17300 // VMLAV u/s 8/16/32
17301 // VADDLV u/s 32
17302 // VMLALV u/s 16/32
17303
17304 // If the input vector is smaller than legal (v4i8/v4i16 for example) we can
17305 // extend it and use v4i32 instead.
17306 auto ExtTypeMatches = [](SDValue A, ArrayRef<MVT> ExtTypes) {
17307 EVT AVT = A.getValueType();
17308 return any_of(Range&: ExtTypes, P: [&](MVT Ty) {
17309 return AVT.getVectorNumElements() == Ty.getVectorNumElements() &&
17310 AVT.bitsLE(VT: Ty);
17311 });
17312 };
17313 auto ExtendIfNeeded = [&](SDValue A, unsigned ExtendCode) {
17314 EVT AVT = A.getValueType();
17315 if (!AVT.is128BitVector())
17316 A = DAG.getNode(
17317 Opcode: ExtendCode, DL: dl,
17318 VT: AVT.changeVectorElementType(
17319 Context&: *DAG.getContext(),
17320 EltVT: MVT::getIntegerVT(BitWidth: 128 / AVT.getVectorMinNumElements())),
17321 Operand: A);
17322 return A;
17323 };
17324 auto IsVADDV = [&](MVT RetTy, unsigned ExtendCode, ArrayRef<MVT> ExtTypes) {
17325 if (ResVT != RetTy || N0->getOpcode() != ExtendCode)
17326 return SDValue();
17327 SDValue A = N0->getOperand(Num: 0);
17328 if (ExtTypeMatches(A, ExtTypes))
17329 return ExtendIfNeeded(A, ExtendCode);
17330 return SDValue();
17331 };
17332 auto IsPredVADDV = [&](MVT RetTy, unsigned ExtendCode,
17333 ArrayRef<MVT> ExtTypes, SDValue &Mask) {
17334 if (ResVT != RetTy || N0->getOpcode() != ISD::VSELECT ||
17335 !ISD::isBuildVectorAllZeros(N: N0->getOperand(Num: 2).getNode()))
17336 return SDValue();
17337 Mask = N0->getOperand(Num: 0);
17338 SDValue Ext = N0->getOperand(Num: 1);
17339 if (Ext->getOpcode() != ExtendCode)
17340 return SDValue();
17341 SDValue A = Ext->getOperand(Num: 0);
17342 if (ExtTypeMatches(A, ExtTypes))
17343 return ExtendIfNeeded(A, ExtendCode);
17344 return SDValue();
17345 };
17346 auto IsVMLAV = [&](MVT RetTy, unsigned ExtendCode, ArrayRef<MVT> ExtTypes,
17347 SDValue &A, SDValue &B) {
17348 // For a vmla we are trying to match a larger pattern:
17349 // ExtA = sext/zext A
17350 // ExtB = sext/zext B
17351 // Mul = mul ExtA, ExtB
17352 // vecreduce.add Mul
17353 // There might also be en extra extend between the mul and the addreduce, so
17354 // long as the bitwidth is high enough to make them equivalent (for example
17355 // original v8i16 might be mul at v8i32 and the reduce happens at v8i64).
17356 if (ResVT != RetTy)
17357 return false;
17358 SDValue Mul = N0;
17359 if (Mul->getOpcode() == ExtendCode &&
17360 Mul->getOperand(Num: 0).getScalarValueSizeInBits() * 2 >=
17361 ResVT.getScalarSizeInBits())
17362 Mul = Mul->getOperand(Num: 0);
17363 if (Mul->getOpcode() != ISD::MUL)
17364 return false;
17365 SDValue ExtA = Mul->getOperand(Num: 0);
17366 SDValue ExtB = Mul->getOperand(Num: 1);
17367 if (ExtA->getOpcode() != ExtendCode || ExtB->getOpcode() != ExtendCode)
17368 return false;
17369 A = ExtA->getOperand(Num: 0);
17370 B = ExtB->getOperand(Num: 0);
17371 if (ExtTypeMatches(A, ExtTypes) && ExtTypeMatches(B, ExtTypes)) {
17372 A = ExtendIfNeeded(A, ExtendCode);
17373 B = ExtendIfNeeded(B, ExtendCode);
17374 return true;
17375 }
17376 return false;
17377 };
17378 auto IsPredVMLAV = [&](MVT RetTy, unsigned ExtendCode, ArrayRef<MVT> ExtTypes,
17379 SDValue &A, SDValue &B, SDValue &Mask) {
17380 // Same as the pattern above with a select for the zero predicated lanes
17381 // ExtA = sext/zext A
17382 // ExtB = sext/zext B
17383 // Mul = mul ExtA, ExtB
17384 // N0 = select Mask, Mul, 0
17385 // vecreduce.add N0
17386 if (ResVT != RetTy || N0->getOpcode() != ISD::VSELECT ||
17387 !ISD::isBuildVectorAllZeros(N: N0->getOperand(Num: 2).getNode()))
17388 return false;
17389 Mask = N0->getOperand(Num: 0);
17390 SDValue Mul = N0->getOperand(Num: 1);
17391 if (Mul->getOpcode() == ExtendCode &&
17392 Mul->getOperand(Num: 0).getScalarValueSizeInBits() * 2 >=
17393 ResVT.getScalarSizeInBits())
17394 Mul = Mul->getOperand(Num: 0);
17395 if (Mul->getOpcode() != ISD::MUL)
17396 return false;
17397 SDValue ExtA = Mul->getOperand(Num: 0);
17398 SDValue ExtB = Mul->getOperand(Num: 1);
17399 if (ExtA->getOpcode() != ExtendCode || ExtB->getOpcode() != ExtendCode)
17400 return false;
17401 A = ExtA->getOperand(Num: 0);
17402 B = ExtB->getOperand(Num: 0);
17403 if (ExtTypeMatches(A, ExtTypes) && ExtTypeMatches(B, ExtTypes)) {
17404 A = ExtendIfNeeded(A, ExtendCode);
17405 B = ExtendIfNeeded(B, ExtendCode);
17406 return true;
17407 }
17408 return false;
17409 };
17410 auto Create64bitNode = [&](unsigned Opcode, ArrayRef<SDValue> Ops) {
17411 // Split illegal MVT::v16i8->i64 vector reductions into two legal v8i16->i64
17412 // reductions. The operands are extended with MVEEXT, but as they are
17413 // reductions the lane orders do not matter. MVEEXT may be combined with
17414 // loads to produce two extending loads, or else they will be expanded to
17415 // VREV/VMOVL.
17416 EVT VT = Ops[0].getValueType();
17417 if (VT == MVT::v16i8) {
17418 assert((Opcode == ARMISD::VMLALVs || Opcode == ARMISD::VMLALVu) &&
17419 "Unexpected illegal long reduction opcode");
17420 bool IsUnsigned = Opcode == ARMISD::VMLALVu;
17421
17422 SDValue Ext0 =
17423 DAG.getNode(Opcode: IsUnsigned ? ARMISD::MVEZEXT : ARMISD::MVESEXT, DL: dl,
17424 VTList: DAG.getVTList(VT1: MVT::v8i16, VT2: MVT::v8i16), N: Ops[0]);
17425 SDValue Ext1 =
17426 DAG.getNode(Opcode: IsUnsigned ? ARMISD::MVEZEXT : ARMISD::MVESEXT, DL: dl,
17427 VTList: DAG.getVTList(VT1: MVT::v8i16, VT2: MVT::v8i16), N: Ops[1]);
17428
17429 SDValue MLA0 = DAG.getNode(Opcode, DL: dl, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
17430 N1: Ext0, N2: Ext1);
17431 SDValue MLA1 =
17432 DAG.getNode(Opcode: IsUnsigned ? ARMISD::VMLALVAu : ARMISD::VMLALVAs, DL: dl,
17433 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N1: MLA0, N2: MLA0.getValue(R: 1),
17434 N3: Ext0.getValue(R: 1), N4: Ext1.getValue(R: 1));
17435 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: MVT::i64, N1: MLA1, N2: MLA1.getValue(R: 1));
17436 }
17437 SDValue Node = DAG.getNode(Opcode, DL: dl, ResultTys: {MVT::i32, MVT::i32}, Ops);
17438 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: MVT::i64, N1: Node,
17439 N2: SDValue(Node.getNode(), 1));
17440 };
17441
17442 SDValue A, B;
17443 SDValue Mask;
17444 if (IsVMLAV(MVT::i32, ISD::SIGN_EXTEND, {MVT::v8i16, MVT::v16i8}, A, B))
17445 return DAG.getNode(Opcode: ARMISD::VMLAVs, DL: dl, VT: ResVT, N1: A, N2: B);
17446 if (IsVMLAV(MVT::i32, ISD::ZERO_EXTEND, {MVT::v8i16, MVT::v16i8}, A, B))
17447 return DAG.getNode(Opcode: ARMISD::VMLAVu, DL: dl, VT: ResVT, N1: A, N2: B);
17448 if (IsVMLAV(MVT::i64, ISD::SIGN_EXTEND, {MVT::v16i8, MVT::v8i16, MVT::v4i32},
17449 A, B))
17450 return Create64bitNode(ARMISD::VMLALVs, {A, B});
17451 if (IsVMLAV(MVT::i64, ISD::ZERO_EXTEND, {MVT::v16i8, MVT::v8i16, MVT::v4i32},
17452 A, B))
17453 return Create64bitNode(ARMISD::VMLALVu, {A, B});
17454 if (IsVMLAV(MVT::i16, ISD::SIGN_EXTEND, {MVT::v16i8}, A, B))
17455 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: ResVT,
17456 Operand: DAG.getNode(Opcode: ARMISD::VMLAVs, DL: dl, VT: MVT::i32, N1: A, N2: B));
17457 if (IsVMLAV(MVT::i16, ISD::ZERO_EXTEND, {MVT::v16i8}, A, B))
17458 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: ResVT,
17459 Operand: DAG.getNode(Opcode: ARMISD::VMLAVu, DL: dl, VT: MVT::i32, N1: A, N2: B));
17460
17461 if (IsPredVMLAV(MVT::i32, ISD::SIGN_EXTEND, {MVT::v8i16, MVT::v16i8}, A, B,
17462 Mask))
17463 return DAG.getNode(Opcode: ARMISD::VMLAVps, DL: dl, VT: ResVT, N1: A, N2: B, N3: Mask);
17464 if (IsPredVMLAV(MVT::i32, ISD::ZERO_EXTEND, {MVT::v8i16, MVT::v16i8}, A, B,
17465 Mask))
17466 return DAG.getNode(Opcode: ARMISD::VMLAVpu, DL: dl, VT: ResVT, N1: A, N2: B, N3: Mask);
17467 if (IsPredVMLAV(MVT::i64, ISD::SIGN_EXTEND, {MVT::v8i16, MVT::v4i32}, A, B,
17468 Mask))
17469 return Create64bitNode(ARMISD::VMLALVps, {A, B, Mask});
17470 if (IsPredVMLAV(MVT::i64, ISD::ZERO_EXTEND, {MVT::v8i16, MVT::v4i32}, A, B,
17471 Mask))
17472 return Create64bitNode(ARMISD::VMLALVpu, {A, B, Mask});
17473 if (IsPredVMLAV(MVT::i16, ISD::SIGN_EXTEND, {MVT::v16i8}, A, B, Mask))
17474 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: ResVT,
17475 Operand: DAG.getNode(Opcode: ARMISD::VMLAVps, DL: dl, VT: MVT::i32, N1: A, N2: B, N3: Mask));
17476 if (IsPredVMLAV(MVT::i16, ISD::ZERO_EXTEND, {MVT::v16i8}, A, B, Mask))
17477 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: ResVT,
17478 Operand: DAG.getNode(Opcode: ARMISD::VMLAVpu, DL: dl, VT: MVT::i32, N1: A, N2: B, N3: Mask));
17479
17480 if (SDValue A = IsVADDV(MVT::i32, ISD::SIGN_EXTEND, {MVT::v8i16, MVT::v16i8}))
17481 return DAG.getNode(Opcode: ARMISD::VADDVs, DL: dl, VT: ResVT, Operand: A);
17482 if (SDValue A = IsVADDV(MVT::i32, ISD::ZERO_EXTEND, {MVT::v8i16, MVT::v16i8}))
17483 return DAG.getNode(Opcode: ARMISD::VADDVu, DL: dl, VT: ResVT, Operand: A);
17484 if (SDValue A = IsVADDV(MVT::i64, ISD::SIGN_EXTEND, {MVT::v4i32}))
17485 return Create64bitNode(ARMISD::VADDLVs, {A});
17486 if (SDValue A = IsVADDV(MVT::i64, ISD::ZERO_EXTEND, {MVT::v4i32}))
17487 return Create64bitNode(ARMISD::VADDLVu, {A});
17488 if (SDValue A = IsVADDV(MVT::i16, ISD::SIGN_EXTEND, {MVT::v16i8}))
17489 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: ResVT,
17490 Operand: DAG.getNode(Opcode: ARMISD::VADDVs, DL: dl, VT: MVT::i32, Operand: A));
17491 if (SDValue A = IsVADDV(MVT::i16, ISD::ZERO_EXTEND, {MVT::v16i8}))
17492 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: ResVT,
17493 Operand: DAG.getNode(Opcode: ARMISD::VADDVu, DL: dl, VT: MVT::i32, Operand: A));
17494
17495 if (SDValue A = IsPredVADDV(MVT::i32, ISD::SIGN_EXTEND, {MVT::v8i16, MVT::v16i8}, Mask))
17496 return DAG.getNode(Opcode: ARMISD::VADDVps, DL: dl, VT: ResVT, N1: A, N2: Mask);
17497 if (SDValue A = IsPredVADDV(MVT::i32, ISD::ZERO_EXTEND, {MVT::v8i16, MVT::v16i8}, Mask))
17498 return DAG.getNode(Opcode: ARMISD::VADDVpu, DL: dl, VT: ResVT, N1: A, N2: Mask);
17499 if (SDValue A = IsPredVADDV(MVT::i64, ISD::SIGN_EXTEND, {MVT::v4i32}, Mask))
17500 return Create64bitNode(ARMISD::VADDLVps, {A, Mask});
17501 if (SDValue A = IsPredVADDV(MVT::i64, ISD::ZERO_EXTEND, {MVT::v4i32}, Mask))
17502 return Create64bitNode(ARMISD::VADDLVpu, {A, Mask});
17503 if (SDValue A = IsPredVADDV(MVT::i16, ISD::SIGN_EXTEND, {MVT::v16i8}, Mask))
17504 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: ResVT,
17505 Operand: DAG.getNode(Opcode: ARMISD::VADDVps, DL: dl, VT: MVT::i32, N1: A, N2: Mask));
17506 if (SDValue A = IsPredVADDV(MVT::i16, ISD::ZERO_EXTEND, {MVT::v16i8}, Mask))
17507 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: ResVT,
17508 Operand: DAG.getNode(Opcode: ARMISD::VADDVpu, DL: dl, VT: MVT::i32, N1: A, N2: Mask));
17509
17510 // Some complications. We can get a case where the two inputs of the mul are
17511 // the same, then the output sext will have been helpfully converted to a
17512 // zext. Turn it back.
17513 SDValue Op = N0;
17514 if (Op->getOpcode() == ISD::VSELECT)
17515 Op = Op->getOperand(Num: 1);
17516 if (Op->getOpcode() == ISD::ZERO_EXTEND &&
17517 Op->getOperand(Num: 0)->getOpcode() == ISD::MUL) {
17518 SDValue Mul = Op->getOperand(Num: 0);
17519 if (Mul->getOperand(Num: 0) == Mul->getOperand(Num: 1) &&
17520 Mul->getOperand(Num: 0)->getOpcode() == ISD::SIGN_EXTEND) {
17521 SDValue Ext = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: N0->getValueType(ResNo: 0), Operand: Mul);
17522 if (Op != N0)
17523 Ext = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: N0->getValueType(ResNo: 0),
17524 N1: N0->getOperand(Num: 0), N2: Ext, N3: N0->getOperand(Num: 2));
17525 return DAG.getNode(Opcode: ISD::VECREDUCE_ADD, DL: dl, VT: ResVT, Operand: Ext);
17526 }
17527 }
17528
17529 return SDValue();
17530}
17531
17532// Looks for vaddv(shuffle) or vmlav(shuffle, shuffle), with a shuffle where all
17533// the lanes are used. Due to the reduction being commutative the shuffle can be
17534// removed.
17535static SDValue PerformReduceShuffleCombine(SDNode *N, SelectionDAG &DAG) {
17536 unsigned VecOp = N->getOperand(Num: 0).getValueType().isVector() ? 0 : 2;
17537 auto *Shuf = dyn_cast<ShuffleVectorSDNode>(Val: N->getOperand(Num: VecOp));
17538 if (!Shuf || !Shuf->getOperand(Num: 1).isUndef())
17539 return SDValue();
17540
17541 // Check all elements are used once in the mask.
17542 ArrayRef<int> Mask = Shuf->getMask();
17543 APInt SetElts(Mask.size(), 0);
17544 for (int E : Mask) {
17545 if (E < 0 || E >= (int)Mask.size())
17546 return SDValue();
17547 SetElts.setBit(E);
17548 }
17549 if (!SetElts.isAllOnes())
17550 return SDValue();
17551
17552 if (N->getNumOperands() != VecOp + 1) {
17553 auto *Shuf2 = dyn_cast<ShuffleVectorSDNode>(Val: N->getOperand(Num: VecOp + 1));
17554 if (!Shuf2 || !Shuf2->getOperand(Num: 1).isUndef() || Shuf2->getMask() != Mask)
17555 return SDValue();
17556 }
17557
17558 SmallVector<SDValue> Ops;
17559 for (SDValue Op : N->ops()) {
17560 if (Op.getValueType().isVector())
17561 Ops.push_back(Elt: Op.getOperand(i: 0));
17562 else
17563 Ops.push_back(Elt: Op);
17564 }
17565 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VTList: N->getVTList(), Ops);
17566}
17567
17568static SDValue PerformVMOVNCombine(SDNode *N,
17569 TargetLowering::DAGCombinerInfo &DCI) {
17570 SDValue Op0 = N->getOperand(Num: 0);
17571 SDValue Op1 = N->getOperand(Num: 1);
17572 unsigned IsTop = N->getConstantOperandVal(Num: 2);
17573
17574 // VMOVNT a undef -> a
17575 // VMOVNB a undef -> a
17576 // VMOVNB undef a -> a
17577 if (Op1->isUndef())
17578 return Op0;
17579 if (Op0->isUndef() && !IsTop)
17580 return Op1;
17581
17582 // VMOVNt(c, VQMOVNb(a, b)) => VQMOVNt(c, b)
17583 // VMOVNb(c, VQMOVNb(a, b)) => VQMOVNb(c, b)
17584 if ((Op1->getOpcode() == ARMISD::VQMOVNs ||
17585 Op1->getOpcode() == ARMISD::VQMOVNu) &&
17586 Op1->getConstantOperandVal(Num: 2) == 0)
17587 return DCI.DAG.getNode(Opcode: Op1->getOpcode(), DL: SDLoc(Op1), VT: N->getValueType(ResNo: 0),
17588 N1: Op0, N2: Op1->getOperand(Num: 1), N3: N->getOperand(Num: 2));
17589
17590 // Only the bottom lanes from Qm (Op1) and either the top or bottom lanes from
17591 // Qd (Op0) are demanded from a VMOVN, depending on whether we are inserting
17592 // into the top or bottom lanes.
17593 unsigned NumElts = N->getValueType(ResNo: 0).getVectorNumElements();
17594 APInt Op1DemandedElts = APInt::getSplat(NewLen: NumElts, V: APInt::getLowBitsSet(numBits: 2, loBitsSet: 1));
17595 APInt Op0DemandedElts =
17596 IsTop ? Op1DemandedElts
17597 : APInt::getSplat(NewLen: NumElts, V: APInt::getHighBitsSet(numBits: 2, hiBitsSet: 1));
17598
17599 const TargetLowering &TLI = DCI.DAG.getTargetLoweringInfo();
17600 if (TLI.SimplifyDemandedVectorElts(Op: Op0, DemandedElts: Op0DemandedElts, DCI))
17601 return SDValue(N, 0);
17602 if (TLI.SimplifyDemandedVectorElts(Op: Op1, DemandedElts: Op1DemandedElts, DCI))
17603 return SDValue(N, 0);
17604
17605 return SDValue();
17606}
17607
17608static SDValue PerformVQMOVNCombine(SDNode *N,
17609 TargetLowering::DAGCombinerInfo &DCI) {
17610 SDValue Op0 = N->getOperand(Num: 0);
17611 unsigned IsTop = N->getConstantOperandVal(Num: 2);
17612
17613 unsigned NumElts = N->getValueType(ResNo: 0).getVectorNumElements();
17614 APInt Op0DemandedElts =
17615 APInt::getSplat(NewLen: NumElts, V: IsTop ? APInt::getLowBitsSet(numBits: 2, loBitsSet: 1)
17616 : APInt::getHighBitsSet(numBits: 2, hiBitsSet: 1));
17617
17618 const TargetLowering &TLI = DCI.DAG.getTargetLoweringInfo();
17619 if (TLI.SimplifyDemandedVectorElts(Op: Op0, DemandedElts: Op0DemandedElts, DCI))
17620 return SDValue(N, 0);
17621 return SDValue();
17622}
17623
17624static SDValue PerformVQDMULHCombine(SDNode *N,
17625 TargetLowering::DAGCombinerInfo &DCI) {
17626 EVT VT = N->getValueType(ResNo: 0);
17627 SDValue LHS = N->getOperand(Num: 0);
17628 SDValue RHS = N->getOperand(Num: 1);
17629
17630 auto *Shuf0 = dyn_cast<ShuffleVectorSDNode>(Val&: LHS);
17631 auto *Shuf1 = dyn_cast<ShuffleVectorSDNode>(Val&: RHS);
17632 // Turn VQDMULH(shuffle, shuffle) -> shuffle(VQDMULH)
17633 if (Shuf0 && Shuf1 && Shuf0->getMask().equals(RHS: Shuf1->getMask()) &&
17634 LHS.getOperand(i: 1).isUndef() && RHS.getOperand(i: 1).isUndef() &&
17635 (LHS.hasOneUse() || RHS.hasOneUse() || LHS == RHS)) {
17636 SDLoc DL(N);
17637 SDValue NewBinOp = DCI.DAG.getNode(Opcode: N->getOpcode(), DL, VT,
17638 N1: LHS.getOperand(i: 0), N2: RHS.getOperand(i: 0));
17639 SDValue UndefV = LHS.getOperand(i: 1);
17640 return DCI.DAG.getVectorShuffle(VT, dl: DL, N1: NewBinOp, N2: UndefV, Mask: Shuf0->getMask());
17641 }
17642 return SDValue();
17643}
17644
17645static SDValue PerformLongShiftCombine(SDNode *N, SelectionDAG &DAG) {
17646 SDLoc DL(N);
17647 SDValue Op0 = N->getOperand(Num: 0);
17648 SDValue Op1 = N->getOperand(Num: 1);
17649
17650 // Turn X << -C -> X >> C and viceversa. The negative shifts can come up from
17651 // uses of the intrinsics.
17652 if (auto C = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 2))) {
17653 int ShiftAmt = C->getSExtValue();
17654 if (ShiftAmt == 0) {
17655 SDValue Merge = DAG.getMergeValues(Ops: {Op0, Op1}, dl: DL);
17656 DAG.ReplaceAllUsesWith(From: N, To: Merge.getNode());
17657 return SDValue();
17658 }
17659
17660 if (ShiftAmt >= -32 && ShiftAmt < 0) {
17661 unsigned NewOpcode =
17662 N->getOpcode() == ARMISD::LSLL ? ARMISD::LSRL : ARMISD::LSLL;
17663 SDValue NewShift = DAG.getNode(Opcode: NewOpcode, DL, VTList: N->getVTList(), N1: Op0, N2: Op1,
17664 N3: DAG.getConstant(Val: -ShiftAmt, DL, VT: MVT::i32));
17665 DAG.ReplaceAllUsesWith(From: N, To: NewShift.getNode());
17666 return NewShift;
17667 }
17668 }
17669
17670 return SDValue();
17671}
17672
17673/// PerformIntrinsicCombine - ARM-specific DAG combining for intrinsics.
17674SDValue ARMTargetLowering::PerformIntrinsicCombine(SDNode *N,
17675 DAGCombinerInfo &DCI) const {
17676 SelectionDAG &DAG = DCI.DAG;
17677 unsigned IntNo = N->getConstantOperandVal(Num: 0);
17678 switch (IntNo) {
17679 default:
17680 // Don't do anything for most intrinsics.
17681 break;
17682
17683 // Vector shifts: check for immediate versions and lower them.
17684 // Note: This is done during DAG combining instead of DAG legalizing because
17685 // the build_vectors for 64-bit vector element shift counts are generally
17686 // not legal, and it is hard to see their values after they get legalized to
17687 // loads from a constant pool.
17688 case Intrinsic::arm_neon_vshifts:
17689 case Intrinsic::arm_neon_vshiftu:
17690 case Intrinsic::arm_neon_vrshifts:
17691 case Intrinsic::arm_neon_vrshiftu:
17692 case Intrinsic::arm_neon_vrshiftn:
17693 case Intrinsic::arm_neon_vqshifts:
17694 case Intrinsic::arm_neon_vqshiftu:
17695 case Intrinsic::arm_neon_vqshiftsu:
17696 case Intrinsic::arm_neon_vqshiftns:
17697 case Intrinsic::arm_neon_vqshiftnu:
17698 case Intrinsic::arm_neon_vqshiftnsu:
17699 case Intrinsic::arm_neon_vqrshiftns:
17700 case Intrinsic::arm_neon_vqrshiftnu:
17701 case Intrinsic::arm_neon_vqrshiftnsu: {
17702 EVT VT = N->getOperand(Num: 1).getValueType();
17703 int64_t Cnt;
17704 unsigned VShiftOpc = 0;
17705
17706 switch (IntNo) {
17707 case Intrinsic::arm_neon_vshifts:
17708 case Intrinsic::arm_neon_vshiftu:
17709 if (isVShiftLImm(Op: N->getOperand(Num: 2), VT, isLong: false, Cnt)) {
17710 VShiftOpc = ARMISD::VSHLIMM;
17711 break;
17712 }
17713 if (isVShiftRImm(Op: N->getOperand(Num: 2), VT, isNarrow: false, isIntrinsic: true, Cnt)) {
17714 VShiftOpc = (IntNo == Intrinsic::arm_neon_vshifts ? ARMISD::VSHRsIMM
17715 : ARMISD::VSHRuIMM);
17716 break;
17717 }
17718 return SDValue();
17719
17720 case Intrinsic::arm_neon_vrshifts:
17721 case Intrinsic::arm_neon_vrshiftu:
17722 if (isVShiftRImm(Op: N->getOperand(Num: 2), VT, isNarrow: false, isIntrinsic: true, Cnt))
17723 break;
17724 return SDValue();
17725
17726 case Intrinsic::arm_neon_vqshifts:
17727 case Intrinsic::arm_neon_vqshiftu:
17728 if (isVShiftLImm(Op: N->getOperand(Num: 2), VT, isLong: false, Cnt))
17729 break;
17730 return SDValue();
17731
17732 case Intrinsic::arm_neon_vqshiftsu:
17733 if (isVShiftLImm(Op: N->getOperand(Num: 2), VT, isLong: false, Cnt))
17734 break;
17735 llvm_unreachable("invalid shift count for vqshlu intrinsic");
17736
17737 case Intrinsic::arm_neon_vrshiftn:
17738 case Intrinsic::arm_neon_vqshiftns:
17739 case Intrinsic::arm_neon_vqshiftnu:
17740 case Intrinsic::arm_neon_vqshiftnsu:
17741 case Intrinsic::arm_neon_vqrshiftns:
17742 case Intrinsic::arm_neon_vqrshiftnu:
17743 case Intrinsic::arm_neon_vqrshiftnsu:
17744 // Narrowing shifts require an immediate right shift.
17745 if (isVShiftRImm(Op: N->getOperand(Num: 2), VT, isNarrow: true, isIntrinsic: true, Cnt))
17746 break;
17747 llvm_unreachable("invalid shift count for narrowing vector shift "
17748 "intrinsic");
17749
17750 default:
17751 llvm_unreachable("unhandled vector shift");
17752 }
17753
17754 switch (IntNo) {
17755 case Intrinsic::arm_neon_vshifts:
17756 case Intrinsic::arm_neon_vshiftu:
17757 // Opcode already set above.
17758 break;
17759 case Intrinsic::arm_neon_vrshifts:
17760 VShiftOpc = ARMISD::VRSHRsIMM;
17761 break;
17762 case Intrinsic::arm_neon_vrshiftu:
17763 VShiftOpc = ARMISD::VRSHRuIMM;
17764 break;
17765 case Intrinsic::arm_neon_vrshiftn:
17766 VShiftOpc = ARMISD::VRSHRNIMM;
17767 break;
17768 case Intrinsic::arm_neon_vqshifts:
17769 VShiftOpc = ARMISD::VQSHLsIMM;
17770 break;
17771 case Intrinsic::arm_neon_vqshiftu:
17772 VShiftOpc = ARMISD::VQSHLuIMM;
17773 break;
17774 case Intrinsic::arm_neon_vqshiftsu:
17775 VShiftOpc = ARMISD::VQSHLsuIMM;
17776 break;
17777 case Intrinsic::arm_neon_vqshiftns:
17778 VShiftOpc = ARMISD::VQSHRNsIMM;
17779 break;
17780 case Intrinsic::arm_neon_vqshiftnu:
17781 VShiftOpc = ARMISD::VQSHRNuIMM;
17782 break;
17783 case Intrinsic::arm_neon_vqshiftnsu:
17784 VShiftOpc = ARMISD::VQSHRNsuIMM;
17785 break;
17786 case Intrinsic::arm_neon_vqrshiftns:
17787 VShiftOpc = ARMISD::VQRSHRNsIMM;
17788 break;
17789 case Intrinsic::arm_neon_vqrshiftnu:
17790 VShiftOpc = ARMISD::VQRSHRNuIMM;
17791 break;
17792 case Intrinsic::arm_neon_vqrshiftnsu:
17793 VShiftOpc = ARMISD::VQRSHRNsuIMM;
17794 break;
17795 }
17796
17797 SDLoc dl(N);
17798 return DAG.getNode(Opcode: VShiftOpc, DL: dl, VT: N->getValueType(ResNo: 0),
17799 N1: N->getOperand(Num: 1), N2: DAG.getConstant(Val: Cnt, DL: dl, VT: MVT::i32));
17800 }
17801
17802 case Intrinsic::arm_neon_vshiftins: {
17803 EVT VT = N->getOperand(Num: 1).getValueType();
17804 int64_t Cnt;
17805 unsigned VShiftOpc = 0;
17806
17807 if (isVShiftLImm(Op: N->getOperand(Num: 3), VT, isLong: false, Cnt))
17808 VShiftOpc = ARMISD::VSLIIMM;
17809 else if (isVShiftRImm(Op: N->getOperand(Num: 3), VT, isNarrow: false, isIntrinsic: true, Cnt))
17810 VShiftOpc = ARMISD::VSRIIMM;
17811 else {
17812 llvm_unreachable("invalid shift count for vsli/vsri intrinsic");
17813 }
17814
17815 SDLoc dl(N);
17816 return DAG.getNode(Opcode: VShiftOpc, DL: dl, VT: N->getValueType(ResNo: 0),
17817 N1: N->getOperand(Num: 1), N2: N->getOperand(Num: 2),
17818 N3: DAG.getConstant(Val: Cnt, DL: dl, VT: MVT::i32));
17819 }
17820
17821 case Intrinsic::arm_neon_vqrshifts:
17822 case Intrinsic::arm_neon_vqrshiftu:
17823 // No immediate versions of these to check for.
17824 break;
17825
17826 case Intrinsic::arm_neon_vbsl: {
17827 SDLoc dl(N);
17828 return DAG.getNode(Opcode: ARMISD::VBSP, DL: dl, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
17829 N2: N->getOperand(Num: 2), N3: N->getOperand(Num: 3));
17830 }
17831 case Intrinsic::arm_mve_vqdmlah:
17832 case Intrinsic::arm_mve_vqdmlash:
17833 case Intrinsic::arm_mve_vqrdmlah:
17834 case Intrinsic::arm_mve_vqrdmlash:
17835 case Intrinsic::arm_mve_vmla_n_predicated:
17836 case Intrinsic::arm_mve_vmlas_n_predicated:
17837 case Intrinsic::arm_mve_vqdmlah_predicated:
17838 case Intrinsic::arm_mve_vqdmlash_predicated:
17839 case Intrinsic::arm_mve_vqrdmlah_predicated:
17840 case Intrinsic::arm_mve_vqrdmlash_predicated: {
17841 // These intrinsics all take an i32 scalar operand which is narrowed to the
17842 // size of a single lane of the vector type they return. So we don't need
17843 // any bits of that operand above that point, which allows us to eliminate
17844 // uxth/sxth.
17845 unsigned BitWidth = N->getValueType(ResNo: 0).getScalarSizeInBits();
17846 APInt DemandedMask = APInt::getLowBitsSet(numBits: 32, loBitsSet: BitWidth);
17847 if (SimplifyDemandedBits(Op: N->getOperand(Num: 3), DemandedBits: DemandedMask, DCI))
17848 return SDValue();
17849 break;
17850 }
17851
17852 case Intrinsic::arm_mve_minv:
17853 case Intrinsic::arm_mve_maxv:
17854 case Intrinsic::arm_mve_minav:
17855 case Intrinsic::arm_mve_maxav:
17856 case Intrinsic::arm_mve_minv_predicated:
17857 case Intrinsic::arm_mve_maxv_predicated:
17858 case Intrinsic::arm_mve_minav_predicated:
17859 case Intrinsic::arm_mve_maxav_predicated: {
17860 // These intrinsics all take an i32 scalar operand which is narrowed to the
17861 // size of a single lane of the vector type they take as the other input.
17862 unsigned BitWidth = N->getOperand(Num: 2)->getValueType(ResNo: 0).getScalarSizeInBits();
17863 APInt DemandedMask = APInt::getLowBitsSet(numBits: 32, loBitsSet: BitWidth);
17864 if (SimplifyDemandedBits(Op: N->getOperand(Num: 1), DemandedBits: DemandedMask, DCI))
17865 return SDValue();
17866 break;
17867 }
17868
17869 case Intrinsic::arm_mve_addv: {
17870 // Turn this intrinsic straight into the appropriate ARMISD::VADDV node,
17871 // which allow PerformADDVecReduce to turn it into VADDLV when possible.
17872 bool Unsigned = N->getConstantOperandVal(Num: 2);
17873 unsigned Opc = Unsigned ? ARMISD::VADDVu : ARMISD::VADDVs;
17874 return DAG.getNode(Opcode: Opc, DL: SDLoc(N), VTList: N->getVTList(), N: N->getOperand(Num: 1));
17875 }
17876
17877 case Intrinsic::arm_mve_addlv:
17878 case Intrinsic::arm_mve_addlv_predicated: {
17879 // Same for these, but ARMISD::VADDLV has to be followed by a BUILD_PAIR
17880 // which recombines the two outputs into an i64
17881 bool Unsigned = N->getConstantOperandVal(Num: 2);
17882 unsigned Opc = IntNo == Intrinsic::arm_mve_addlv ?
17883 (Unsigned ? ARMISD::VADDLVu : ARMISD::VADDLVs) :
17884 (Unsigned ? ARMISD::VADDLVpu : ARMISD::VADDLVps);
17885
17886 SmallVector<SDValue, 4> Ops;
17887 for (unsigned i = 1, e = N->getNumOperands(); i < e; i++)
17888 if (i != 2) // skip the unsigned flag
17889 Ops.push_back(Elt: N->getOperand(Num: i));
17890
17891 SDLoc dl(N);
17892 SDValue val = DAG.getNode(Opcode: Opc, DL: dl, ResultTys: {MVT::i32, MVT::i32}, Ops);
17893 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: MVT::i64, N1: val.getValue(R: 0),
17894 N2: val.getValue(R: 1));
17895 }
17896 }
17897
17898 return SDValue();
17899}
17900
17901bool ARMTargetLowering::hasAndNot(SDValue Y) const {
17902 EVT VT = Y.getValueType();
17903 if (!VT.isVector())
17904 return hasAndNotCompare(V: Y);
17905 if (Subtarget->hasMVEIntegerOps())
17906 return VT.is128BitVector();
17907 if (Subtarget->hasNEON())
17908 return VT.is64BitVector() || VT.is128BitVector();
17909 return false;
17910}
17911
17912/// PerformShiftCombine - Checks for immediate versions of vector shifts and
17913/// lowers them. As with the vector shift intrinsics, this is done during DAG
17914/// combining instead of DAG legalizing because the build_vectors for 64-bit
17915/// vector element shift counts are generally not legal, and it is hard to see
17916/// their values after they get legalized to loads from a constant pool.
17917static SDValue PerformShiftCombine(SDNode *N,
17918 TargetLowering::DAGCombinerInfo &DCI,
17919 const ARMSubtarget *ST) {
17920 SelectionDAG &DAG = DCI.DAG;
17921 EVT VT = N->getValueType(ResNo: 0);
17922
17923 if (ST->isThumb1Only() && N->getOpcode() == ISD::SHL && VT == MVT::i32 &&
17924 N->getOperand(Num: 0)->getOpcode() == ISD::AND &&
17925 N->getOperand(Num: 0)->hasOneUse()) {
17926 if (DCI.isBeforeLegalize() || DCI.isCalledByLegalizer())
17927 return SDValue();
17928 // Look for the pattern (shl (and x, AndMask), ShiftAmt). This doesn't
17929 // usually show up because instcombine prefers to canonicalize it to
17930 // (and (shl x, ShiftAmt) (shl AndMask, ShiftAmt)), but the shift can come
17931 // out of GEP lowering in some cases.
17932 SDValue N0 = N->getOperand(Num: 0);
17933 ConstantSDNode *ShiftAmtNode = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
17934 if (!ShiftAmtNode)
17935 return SDValue();
17936 uint32_t ShiftAmt = static_cast<uint32_t>(ShiftAmtNode->getZExtValue());
17937 ConstantSDNode *AndMaskNode = dyn_cast<ConstantSDNode>(Val: N0->getOperand(Num: 1));
17938 if (!AndMaskNode)
17939 return SDValue();
17940 uint32_t AndMask = static_cast<uint32_t>(AndMaskNode->getZExtValue());
17941 // Don't transform uxtb/uxth.
17942 if (AndMask == 255 || AndMask == 65535)
17943 return SDValue();
17944 if (isMask_32(Value: AndMask)) {
17945 uint32_t MaskedBits = llvm::countl_zero(Val: AndMask);
17946 if (MaskedBits > ShiftAmt) {
17947 SDLoc DL(N);
17948 SDValue SHL = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i32, N1: N0->getOperand(Num: 0),
17949 N2: DAG.getConstant(Val: MaskedBits, DL, VT: MVT::i32));
17950 return DAG.getNode(
17951 Opcode: ISD::SRL, DL, VT: MVT::i32, N1: SHL,
17952 N2: DAG.getConstant(Val: MaskedBits - ShiftAmt, DL, VT: MVT::i32));
17953 }
17954 }
17955 }
17956
17957 // Nothing to be done for scalar shifts.
17958 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
17959 if (!VT.isVector() || !TLI.isTypeLegal(VT))
17960 return SDValue();
17961 if (ST->hasMVEIntegerOps())
17962 return SDValue();
17963
17964 int64_t Cnt;
17965
17966 switch (N->getOpcode()) {
17967 default: llvm_unreachable("unexpected shift opcode");
17968
17969 case ISD::SHL:
17970 if (isVShiftLImm(Op: N->getOperand(Num: 1), VT, isLong: false, Cnt)) {
17971 SDLoc dl(N);
17972 return DAG.getNode(Opcode: ARMISD::VSHLIMM, DL: dl, VT, N1: N->getOperand(Num: 0),
17973 N2: DAG.getConstant(Val: Cnt, DL: dl, VT: MVT::i32));
17974 }
17975 break;
17976
17977 case ISD::SRA:
17978 case ISD::SRL:
17979 if (isVShiftRImm(Op: N->getOperand(Num: 1), VT, isNarrow: false, isIntrinsic: false, Cnt)) {
17980 unsigned VShiftOpc =
17981 (N->getOpcode() == ISD::SRA ? ARMISD::VSHRsIMM : ARMISD::VSHRuIMM);
17982 SDLoc dl(N);
17983 return DAG.getNode(Opcode: VShiftOpc, DL: dl, VT, N1: N->getOperand(Num: 0),
17984 N2: DAG.getConstant(Val: Cnt, DL: dl, VT: MVT::i32));
17985 }
17986 }
17987 return SDValue();
17988}
17989
17990// Look for a sign/zero/fpextend extend of a larger than legal load. This can be
17991// split into multiple extending loads, which are simpler to deal with than an
17992// arbitrary extend. For fp extends we use an integer extending load and a VCVTL
17993// to convert the type to an f32.
17994static SDValue PerformSplittingToWideningLoad(SDNode *N, SelectionDAG &DAG) {
17995 SDValue N0 = N->getOperand(Num: 0);
17996 if (N0.getOpcode() != ISD::LOAD)
17997 return SDValue();
17998 LoadSDNode *LD = cast<LoadSDNode>(Val: N0.getNode());
17999 if (!LD->isSimple() || !N0.hasOneUse() || LD->isIndexed() ||
18000 LD->getExtensionType() != ISD::NON_EXTLOAD)
18001 return SDValue();
18002 EVT FromVT = LD->getValueType(ResNo: 0);
18003 EVT ToVT = N->getValueType(ResNo: 0);
18004 if (!ToVT.isVector())
18005 return SDValue();
18006 assert(FromVT.getVectorNumElements() == ToVT.getVectorNumElements());
18007 EVT ToEltVT = ToVT.getVectorElementType();
18008 EVT FromEltVT = FromVT.getVectorElementType();
18009
18010 unsigned NumElements = 0;
18011 if (ToEltVT == MVT::i32 && FromEltVT == MVT::i8)
18012 NumElements = 4;
18013 if (ToEltVT == MVT::f32 && FromEltVT == MVT::f16)
18014 NumElements = 4;
18015 if (NumElements == 0 ||
18016 (FromEltVT != MVT::f16 && FromVT.getVectorNumElements() == NumElements) ||
18017 FromVT.getVectorNumElements() % NumElements != 0 ||
18018 !isPowerOf2_32(Value: NumElements))
18019 return SDValue();
18020
18021 LLVMContext &C = *DAG.getContext();
18022 SDLoc DL(LD);
18023 // Details about the old load
18024 SDValue Ch = LD->getChain();
18025 SDValue BasePtr = LD->getBasePtr();
18026 Align Alignment = LD->getBaseAlign();
18027 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
18028 AAMDNodes AAInfo = LD->getAAInfo();
18029
18030 ISD::LoadExtType NewExtType =
18031 N->getOpcode() == ISD::SIGN_EXTEND ? ISD::SEXTLOAD : ISD::ZEXTLOAD;
18032 SDValue Offset = DAG.getPOISON(VT: BasePtr.getValueType());
18033 EVT NewFromVT = EVT::getVectorVT(
18034 Context&: C, VT: EVT::getIntegerVT(Context&: C, BitWidth: FromEltVT.getScalarSizeInBits()), NumElements);
18035 EVT NewToVT = EVT::getVectorVT(
18036 Context&: C, VT: EVT::getIntegerVT(Context&: C, BitWidth: ToEltVT.getScalarSizeInBits()), NumElements);
18037
18038 SmallVector<SDValue, 4> Loads;
18039 SmallVector<SDValue, 4> Chains;
18040 for (unsigned i = 0; i < FromVT.getVectorNumElements() / NumElements; i++) {
18041 unsigned NewOffset = (i * NewFromVT.getSizeInBits()) / 8;
18042 SDValue NewPtr =
18043 DAG.getObjectPtrOffset(SL: DL, Ptr: BasePtr, Offset: TypeSize::getFixed(ExactSize: NewOffset));
18044
18045 SDValue NewLoad =
18046 DAG.getLoad(AM: ISD::UNINDEXED, ExtType: NewExtType, VT: NewToVT, dl: DL, Chain: Ch, Ptr: NewPtr, Offset,
18047 PtrInfo: LD->getPointerInfo().getWithOffset(O: NewOffset), MemVT: NewFromVT,
18048 Alignment, MMOFlags, Metadata: AAInfo);
18049 Loads.push_back(Elt: NewLoad);
18050 Chains.push_back(Elt: SDValue(NewLoad.getNode(), 1));
18051 }
18052
18053 // Float truncs need to extended with VCVTB's into their floating point types.
18054 if (FromEltVT == MVT::f16) {
18055 SmallVector<SDValue, 4> Extends;
18056
18057 for (unsigned i = 0; i < Loads.size(); i++) {
18058 SDValue LoadBC =
18059 DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL, VT: MVT::v8f16, Operand: Loads[i]);
18060 SDValue FPExt = DAG.getNode(Opcode: ARMISD::VCVTL, DL, VT: MVT::v4f32, N1: LoadBC,
18061 N2: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
18062 Extends.push_back(Elt: FPExt);
18063 }
18064
18065 Loads = Extends;
18066 }
18067
18068 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: Chains);
18069 DAG.ReplaceAllUsesOfValueWith(From: SDValue(LD, 1), To: NewChain);
18070 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: ToVT, Ops: Loads);
18071}
18072
18073/// PerformExtendCombine - Target-specific DAG combining for ISD::SIGN_EXTEND,
18074/// ISD::ZERO_EXTEND, and ISD::ANY_EXTEND.
18075static SDValue PerformExtendCombine(SDNode *N, SelectionDAG &DAG,
18076 const ARMSubtarget *ST) {
18077 SDValue N0 = N->getOperand(Num: 0);
18078 EVT VT = N->getValueType(ResNo: 0);
18079 SDLoc DL(N);
18080
18081 // Check for sign- and zero-extensions of vector extract operations of 8- and
18082 // 16-bit vector elements. NEON and MVE support these directly. They are
18083 // handled during DAG combining because type legalization will promote them
18084 // to 32-bit types and it is messy to recognize the operations after that.
18085 if ((ST->hasNEON() || ST->hasMVEIntegerOps()) &&
18086 N0.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
18087 SDValue Vec = N0.getOperand(i: 0);
18088 SDValue Lane = N0.getOperand(i: 1);
18089 EVT EltVT = N0.getValueType();
18090 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
18091
18092 if (VT == MVT::i32 &&
18093 (EltVT == MVT::i8 || EltVT == MVT::i16) &&
18094 TLI.isTypeLegal(VT: Vec.getValueType()) &&
18095 isa<ConstantSDNode>(Val: Lane)) {
18096
18097 unsigned Opc = 0;
18098 switch (N->getOpcode()) {
18099 default: llvm_unreachable("unexpected opcode");
18100 case ISD::SIGN_EXTEND:
18101 Opc = ARMISD::VGETLANEs;
18102 break;
18103 case ISD::ZERO_EXTEND:
18104 case ISD::ANY_EXTEND:
18105 Opc = ARMISD::VGETLANEu;
18106 break;
18107 }
18108 return DAG.getNode(Opcode: Opc, DL, VT, N1: Vec, N2: Lane);
18109 }
18110 }
18111
18112 if (ST->hasMVEIntegerOps())
18113 if (SDValue NewLoad = PerformSplittingToWideningLoad(N, DAG))
18114 return NewLoad;
18115
18116 // Combine sext(buildvector(..)) to buildvector(sext(..)) to help avoid
18117 // difficult to lower i1 buildvector.
18118 if (ST->hasMVEIntegerOps() && N0.getValueType().getScalarSizeInBits() == 1 &&
18119 N0.getOpcode() == ISD::BUILD_VECTOR && VT.getScalarSizeInBits() <= 32) {
18120 SmallVector<SDValue> Ops;
18121 for (unsigned I = 0; I < N0.getNumOperands(); I++) {
18122 SDValue InReg = N0.getOperand(i: I);
18123 if (N->getOpcode() == ISD::ZERO_EXTEND)
18124 InReg = DAG.getNode(Opcode: ISD::AND, DL, VT: InReg.getValueType(), N1: InReg,
18125 N2: DAG.getConstant(Val: 1, DL, VT: InReg.getValueType()));
18126 else if (N->getOpcode() == ISD::SIGN_EXTEND)
18127 InReg = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: InReg.getValueType(),
18128 N1: InReg, N2: DAG.getValueType(MVT::i1));
18129 SDValue Ext = DAG.getNode(Opcode: N->getOpcode(), DL, VT: MVT::i32, Operand: InReg);
18130 Ops.push_back(Elt: Ext);
18131 }
18132 return DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT, Ops);
18133 }
18134
18135 return SDValue();
18136}
18137
18138static SDValue PerformFPExtendCombine(SDNode *N, SelectionDAG &DAG,
18139 const ARMSubtarget *ST) {
18140 if (ST->hasMVEFloatOps())
18141 if (SDValue NewLoad = PerformSplittingToWideningLoad(N, DAG))
18142 return NewLoad;
18143
18144 return SDValue();
18145}
18146
18147// Lower smin(smax(x, C1), C2) to ssat or usat, if they have saturating
18148// constant bounds.
18149static SDValue PerformMinMaxToSatCombine(SDValue Op, SelectionDAG &DAG,
18150 const ARMSubtarget *Subtarget) {
18151 if ((Subtarget->isThumb() || !Subtarget->hasV6Ops()) &&
18152 !Subtarget->isThumb2())
18153 return SDValue();
18154
18155 EVT VT = Op.getValueType();
18156 SDValue Op0 = Op.getOperand(i: 0);
18157
18158 if (VT != MVT::i32 ||
18159 (Op0.getOpcode() != ISD::SMIN && Op0.getOpcode() != ISD::SMAX) ||
18160 !isa<ConstantSDNode>(Val: Op.getOperand(i: 1)) ||
18161 !isa<ConstantSDNode>(Val: Op0.getOperand(i: 1)))
18162 return SDValue();
18163
18164 SDValue Min = Op;
18165 SDValue Max = Op0;
18166 SDValue Input = Op0.getOperand(i: 0);
18167 if (Min.getOpcode() == ISD::SMAX)
18168 std::swap(a&: Min, b&: Max);
18169
18170 if (Min.getOpcode() != ISD::SMIN || Max.getOpcode() != ISD::SMAX)
18171 return SDValue();
18172
18173 APInt MinC = Min.getConstantOperandAPInt(i: 1);
18174 APInt MaxC = Max.getConstantOperandAPInt(i: 1);
18175 if (MaxC.sgt(RHS: MinC))
18176 return SDValue();
18177
18178 SDLoc DL(Op);
18179
18180 // A clamp whose bounds are already a saturation range maps to a single
18181 // SSAT / USAT.
18182 if ((MinC + 1).isPowerOf2()) {
18183 if (MinC == ~MaxC)
18184 return DAG.getNode(Opcode: ARMISD::SSAT, DL, VT, N1: Input,
18185 N2: DAG.getConstant(Val: MinC.countr_one(), DL, VT));
18186 if (MaxC == 0)
18187 return DAG.getNode(Opcode: ARMISD::USAT, DL, VT, N1: Input,
18188 N2: DAG.getConstant(Val: MinC.countr_one(), DL, VT));
18189 }
18190
18191 // For power-of-two clamp widths, convert the range to be zero-centered,
18192 // apply SSAT, and convert the result back.
18193 //
18194 // Width = Hi - Lo + 1
18195 // Center = Lo + Width / 2
18196 // Result = ssat(X - Center) + Center
18197 //
18198 // The idea is to shift the input so that the clamp range is centered
18199 // around zero, apply ssat, and then shift the result back.
18200 //
18201 // For example clamp(X, -118, 137) -> Width = 256, Center = 10, so it becomes
18202 // ssat(X - 10, 8) + 10
18203
18204 APInt Width = MinC - MaxC + 1;
18205 if (!Width.isPowerOf2() || Width.isOne())
18206 return SDValue();
18207 unsigned SatBit = Width.logBase2() - 1; // ssat to SatBit + 1 signed bits
18208 APInt Center = MaxC + Width.lshr(shiftAmt: 1);
18209
18210 // The rewrite is only valid when X - Center does not overflow;
18211 SDValue NegC = DAG.getConstant(Val: -Center, DL, VT);
18212 if (DAG.computeOverflowForSignedAdd(N0: Input, N1: NegC) != SelectionDAG::OFK_Never)
18213 return SDValue();
18214
18215 SDValue Shifted = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Input, N2: NegC);
18216 SDValue Sat = DAG.getNode(Opcode: ARMISD::SSAT, DL, VT, N1: Shifted,
18217 N2: DAG.getConstant(Val: SatBit, DL, VT));
18218 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Sat, N2: DAG.getConstant(Val: Center, DL, VT));
18219}
18220
18221/// PerformMinMaxCombine - Target-specific DAG combining for creating truncating
18222/// saturates.
18223static SDValue PerformMinMaxCombine(SDNode *N, SelectionDAG &DAG,
18224 const ARMSubtarget *ST) {
18225 EVT VT = N->getValueType(ResNo: 0);
18226 SDValue N0 = N->getOperand(Num: 0);
18227
18228 if (VT == MVT::i32)
18229 return PerformMinMaxToSatCombine(Op: SDValue(N, 0), DAG, Subtarget: ST);
18230
18231 if (!ST->hasMVEIntegerOps())
18232 return SDValue();
18233
18234 if (SDValue V = PerformVQDMULHCombine(N, DAG))
18235 return V;
18236
18237 if (VT != MVT::v4i32 && VT != MVT::v8i16)
18238 return SDValue();
18239
18240 auto IsSignedSaturate = [&](SDNode *Min, SDNode *Max) {
18241 // Check one is a smin and the other is a smax
18242 if (Min->getOpcode() != ISD::SMIN)
18243 std::swap(a&: Min, b&: Max);
18244 if (Min->getOpcode() != ISD::SMIN || Max->getOpcode() != ISD::SMAX)
18245 return false;
18246
18247 APInt SaturateC;
18248 if (VT == MVT::v4i32)
18249 SaturateC = APInt(32, (1 << 15) - 1, true);
18250 else //if (VT == MVT::v8i16)
18251 SaturateC = APInt(16, (1 << 7) - 1, true);
18252
18253 APInt MinC, MaxC;
18254 if (!ISD::isConstantSplatVector(N: Min->getOperand(Num: 1).getNode(), SplatValue&: MinC) ||
18255 MinC != SaturateC)
18256 return false;
18257 if (!ISD::isConstantSplatVector(N: Max->getOperand(Num: 1).getNode(), SplatValue&: MaxC) ||
18258 MaxC != ~SaturateC)
18259 return false;
18260 return true;
18261 };
18262
18263 if (IsSignedSaturate(N, N0.getNode())) {
18264 SDLoc DL(N);
18265 MVT ExtVT, HalfVT;
18266 if (VT == MVT::v4i32) {
18267 HalfVT = MVT::v8i16;
18268 ExtVT = MVT::v4i16;
18269 } else { // if (VT == MVT::v8i16)
18270 HalfVT = MVT::v16i8;
18271 ExtVT = MVT::v8i8;
18272 }
18273
18274 // Create a VQMOVNB with undef top lanes, then signed extended into the top
18275 // half. That extend will hopefully be removed if only the bottom bits are
18276 // demanded (though a truncating store, for example).
18277 SDValue VQMOVN =
18278 DAG.getNode(Opcode: ARMISD::VQMOVNs, DL, VT: HalfVT, N1: DAG.getUNDEF(VT: HalfVT),
18279 N2: N0->getOperand(Num: 0), N3: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
18280 SDValue Bitcast = DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL, VT, Operand: VQMOVN);
18281 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT, N1: Bitcast,
18282 N2: DAG.getValueType(ExtVT));
18283 }
18284
18285 auto IsUnsignedSaturate = [&](SDNode *Min) {
18286 // For unsigned, we just need to check for <= 0xffff
18287 if (Min->getOpcode() != ISD::UMIN)
18288 return false;
18289
18290 APInt SaturateC;
18291 if (VT == MVT::v4i32)
18292 SaturateC = APInt(32, (1 << 16) - 1, true);
18293 else //if (VT == MVT::v8i16)
18294 SaturateC = APInt(16, (1 << 8) - 1, true);
18295
18296 APInt MinC;
18297 if (!ISD::isConstantSplatVector(N: Min->getOperand(Num: 1).getNode(), SplatValue&: MinC) ||
18298 MinC != SaturateC)
18299 return false;
18300 return true;
18301 };
18302
18303 if (IsUnsignedSaturate(N)) {
18304 SDLoc DL(N);
18305 MVT HalfVT;
18306 unsigned ExtConst;
18307 if (VT == MVT::v4i32) {
18308 HalfVT = MVT::v8i16;
18309 ExtConst = 0x0000FFFF;
18310 } else { //if (VT == MVT::v8i16)
18311 HalfVT = MVT::v16i8;
18312 ExtConst = 0x00FF;
18313 }
18314
18315 // Create a VQMOVNB with undef top lanes, then ZExt into the top half with
18316 // an AND. That extend will hopefully be removed if only the bottom bits are
18317 // demanded (though a truncating store, for example).
18318 SDValue VQMOVN =
18319 DAG.getNode(Opcode: ARMISD::VQMOVNu, DL, VT: HalfVT, N1: DAG.getUNDEF(VT: HalfVT), N2: N0,
18320 N3: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
18321 SDValue Bitcast = DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL, VT, Operand: VQMOVN);
18322 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Bitcast,
18323 N2: DAG.getConstant(Val: ExtConst, DL, VT));
18324 }
18325
18326 return SDValue();
18327}
18328
18329static const APInt *isPowerOf2Constant(SDValue V) {
18330 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: V);
18331 if (!C)
18332 return nullptr;
18333 const APInt *CV = &C->getAPIntValue();
18334 return CV->isPowerOf2() ? CV : nullptr;
18335}
18336
18337SDValue ARMTargetLowering::PerformCMOVToBFICombine(SDNode *CMOV, SelectionDAG &DAG) const {
18338 // If we have a CMOV, OR and AND combination such as:
18339 // if (x & CN)
18340 // y |= CM;
18341 //
18342 // And:
18343 // * CN is a single bit;
18344 // * All bits covered by CM are known zero in y
18345 //
18346 // Then we can convert this into a sequence of BFI instructions. This will
18347 // always be a win if CM is a single bit, will always be no worse than the
18348 // TST&OR sequence if CM is two bits, and for thumb will be no worse if CM is
18349 // three bits (due to the extra IT instruction).
18350
18351 SDValue Op0 = CMOV->getOperand(Num: 0);
18352 SDValue Op1 = CMOV->getOperand(Num: 1);
18353 auto CC = CMOV->getConstantOperandAPInt(Num: 2).getLimitedValue();
18354 SDValue CmpZ = CMOV->getOperand(Num: 3);
18355
18356 // The compare must be against zero.
18357 if (!isNullConstant(V: CmpZ->getOperand(Num: 1)))
18358 return SDValue();
18359
18360 assert(CmpZ->getOpcode() == ARMISD::CMPZ);
18361 SDValue And = CmpZ->getOperand(Num: 0);
18362 if (And->getOpcode() != ISD::AND)
18363 return SDValue();
18364 const APInt *AndC = isPowerOf2Constant(V: And->getOperand(Num: 1));
18365 if (!AndC)
18366 return SDValue();
18367 SDValue X = And->getOperand(Num: 0);
18368
18369 if (CC == ARMCC::EQ) {
18370 // We're performing an "equal to zero" compare. Swap the operands so we
18371 // canonicalize on a "not equal to zero" compare.
18372 std::swap(a&: Op0, b&: Op1);
18373 } else {
18374 assert(CC == ARMCC::NE && "How can a CMPZ node not be EQ or NE?");
18375 }
18376
18377 if (Op1->getOpcode() != ISD::OR)
18378 return SDValue();
18379
18380 ConstantSDNode *OrC = dyn_cast<ConstantSDNode>(Val: Op1->getOperand(Num: 1));
18381 if (!OrC)
18382 return SDValue();
18383 SDValue Y = Op1->getOperand(Num: 0);
18384
18385 if (Op0 != Y)
18386 return SDValue();
18387
18388 // Now, is it profitable to continue?
18389 APInt OrCI = OrC->getAPIntValue();
18390 unsigned Heuristic = Subtarget->isThumb() ? 3 : 2;
18391 if (OrCI.popcount() > Heuristic)
18392 return SDValue();
18393
18394 // Lastly, can we determine that the bits defined by OrCI
18395 // are zero in Y?
18396 KnownBits Known = DAG.computeKnownBits(Op: Y);
18397 if ((OrCI & Known.Zero) != OrCI)
18398 return SDValue();
18399
18400 // OK, we can do the combine.
18401 SDValue V = Y;
18402 SDLoc dl(X);
18403 EVT VT = X.getValueType();
18404 unsigned BitInX = AndC->logBase2();
18405
18406 if (BitInX != 0) {
18407 // We must shift X first.
18408 X = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: X,
18409 N2: DAG.getConstant(Val: BitInX, DL: dl, VT));
18410 }
18411
18412 for (unsigned BitInY = 0, NumActiveBits = OrCI.getActiveBits();
18413 BitInY < NumActiveBits; ++BitInY) {
18414 if (OrCI[BitInY] == 0)
18415 continue;
18416 APInt Mask(VT.getSizeInBits(), 0);
18417 Mask.setBit(BitInY);
18418 V = DAG.getNode(Opcode: ARMISD::BFI, DL: dl, VT, N1: V, N2: X,
18419 // Confusingly, the operand is an *inverted* mask.
18420 N3: DAG.getConstant(Val: ~Mask, DL: dl, VT));
18421 }
18422
18423 return V;
18424}
18425
18426// Given N, the value controlling the conditional branch, search for the loop
18427// intrinsic, returning it, along with how the value is used. We need to handle
18428// patterns such as the following:
18429// (brcond (xor (setcc (loop.decrement), 0, ne), 1), exit)
18430// (brcond (setcc (loop.decrement), 0, eq), exit)
18431// (brcond (setcc (loop.decrement), 0, ne), header)
18432static SDValue SearchLoopIntrinsic(SDValue N, ISD::CondCode &CC, int &Imm,
18433 bool &Negate) {
18434 switch (N->getOpcode()) {
18435 default:
18436 break;
18437 case ISD::XOR: {
18438 if (!isa<ConstantSDNode>(Val: N.getOperand(i: 1)))
18439 return SDValue();
18440 if (!cast<ConstantSDNode>(Val: N.getOperand(i: 1))->isOne())
18441 return SDValue();
18442 Negate = !Negate;
18443 return SearchLoopIntrinsic(N: N.getOperand(i: 0), CC, Imm, Negate);
18444 }
18445 case ISD::SETCC: {
18446 auto *Const = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1));
18447 if (!Const)
18448 return SDValue();
18449 if (Const->isZero())
18450 Imm = 0;
18451 else if (Const->isOne())
18452 Imm = 1;
18453 else
18454 return SDValue();
18455 CC = cast<CondCodeSDNode>(Val: N.getOperand(i: 2))->get();
18456 return SearchLoopIntrinsic(N: N->getOperand(Num: 0), CC, Imm, Negate);
18457 }
18458 case ISD::INTRINSIC_W_CHAIN: {
18459 unsigned IntOp = N.getConstantOperandVal(i: 1);
18460 if (IntOp != Intrinsic::test_start_loop_iterations &&
18461 IntOp != Intrinsic::loop_decrement_reg)
18462 return SDValue();
18463 return N;
18464 }
18465 }
18466 return SDValue();
18467}
18468
18469static SDValue PerformHWLoopCombine(SDNode *N,
18470 TargetLowering::DAGCombinerInfo &DCI,
18471 const ARMSubtarget *ST) {
18472
18473 // The hwloop intrinsics that we're interested are used for control-flow,
18474 // either for entering or exiting the loop:
18475 // - test.start.loop.iterations will test whether its operand is zero. If it
18476 // is zero, the proceeding branch should not enter the loop.
18477 // - loop.decrement.reg also tests whether its operand is zero. If it is
18478 // zero, the proceeding branch should not branch back to the beginning of
18479 // the loop.
18480 // So here, we need to check that how the brcond is using the result of each
18481 // of the intrinsics to ensure that we're branching to the right place at the
18482 // right time.
18483
18484 ISD::CondCode CC;
18485 SDValue Cond;
18486 int Imm = 1;
18487 bool Negate = false;
18488 SDValue Chain = N->getOperand(Num: 0);
18489 SDValue Dest;
18490
18491 if (N->getOpcode() == ISD::BRCOND) {
18492 CC = ISD::SETEQ;
18493 Cond = N->getOperand(Num: 1);
18494 Dest = N->getOperand(Num: 2);
18495 } else {
18496 assert(N->getOpcode() == ISD::BR_CC && "Expected BRCOND or BR_CC!");
18497 CC = cast<CondCodeSDNode>(Val: N->getOperand(Num: 1))->get();
18498 Cond = N->getOperand(Num: 2);
18499 Dest = N->getOperand(Num: 4);
18500 if (auto *Const = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 3))) {
18501 if (!Const->isOne() && !Const->isZero())
18502 return SDValue();
18503 Imm = Const->getZExtValue();
18504 } else
18505 return SDValue();
18506 }
18507
18508 SDValue Int = SearchLoopIntrinsic(N: Cond, CC, Imm, Negate);
18509 if (!Int)
18510 return SDValue();
18511
18512 if (Negate)
18513 CC = ISD::getSetCCInverse(Operation: CC, /* Integer inverse */ Type: MVT::i32);
18514
18515 auto IsTrueIfZero = [](ISD::CondCode CC, int Imm) {
18516 return (CC == ISD::SETEQ && Imm == 0) ||
18517 (CC == ISD::SETNE && Imm == 1) ||
18518 (CC == ISD::SETLT && Imm == 1) ||
18519 (CC == ISD::SETULT && Imm == 1);
18520 };
18521
18522 auto IsFalseIfZero = [](ISD::CondCode CC, int Imm) {
18523 return (CC == ISD::SETEQ && Imm == 1) ||
18524 (CC == ISD::SETNE && Imm == 0) ||
18525 (CC == ISD::SETGT && Imm == 0) ||
18526 (CC == ISD::SETUGT && Imm == 0) ||
18527 (CC == ISD::SETGE && Imm == 1) ||
18528 (CC == ISD::SETUGE && Imm == 1);
18529 };
18530
18531 assert((IsTrueIfZero(CC, Imm) || IsFalseIfZero(CC, Imm)) &&
18532 "unsupported condition");
18533
18534 SDLoc dl(Int);
18535 SelectionDAG &DAG = DCI.DAG;
18536 SDValue Elements = Int.getOperand(i: 2);
18537 unsigned IntOp = Int->getConstantOperandVal(Num: 1);
18538 assert((N->hasOneUse() && N->user_begin()->getOpcode() == ISD::BR) &&
18539 "expected single br user");
18540 SDNode *Br = *N->user_begin();
18541 SDValue OtherTarget = Br->getOperand(Num: 1);
18542
18543 // Update the unconditional branch to branch to the given Dest.
18544 auto UpdateUncondBr = [](SDNode *Br, SDValue Dest, SelectionDAG &DAG) {
18545 SDValue NewBrOps[] = { Br->getOperand(Num: 0), Dest };
18546 SDValue NewBr = DAG.getNode(Opcode: ISD::BR, DL: SDLoc(Br), VT: MVT::Other, Ops: NewBrOps);
18547 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Br, 0), To: NewBr);
18548 };
18549
18550 if (IntOp == Intrinsic::test_start_loop_iterations) {
18551 SDValue Res;
18552 SDValue Setup = DAG.getNode(Opcode: ARMISD::WLSSETUP, DL: dl, VT: MVT::i32, Operand: Elements);
18553 // We expect this 'instruction' to branch when the counter is zero.
18554 if (IsTrueIfZero(CC, Imm)) {
18555 SDValue Ops[] = {Chain, Setup, Dest};
18556 Res = DAG.getNode(Opcode: ARMISD::WLS, DL: dl, VT: MVT::Other, Ops);
18557 } else {
18558 // The logic is the reverse of what we need for WLS, so find the other
18559 // basic block target: the target of the proceeding br.
18560 UpdateUncondBr(Br, Dest, DAG);
18561
18562 SDValue Ops[] = {Chain, Setup, OtherTarget};
18563 Res = DAG.getNode(Opcode: ARMISD::WLS, DL: dl, VT: MVT::Other, Ops);
18564 }
18565 // Update LR count to the new value
18566 DAG.ReplaceAllUsesOfValueWith(From: Int.getValue(R: 0), To: Setup);
18567 // Update chain
18568 DAG.ReplaceAllUsesOfValueWith(From: Int.getValue(R: 2), To: Int.getOperand(i: 0));
18569 return Res;
18570 } else {
18571 SDValue Size =
18572 DAG.getTargetConstant(Val: Int.getConstantOperandVal(i: 3), DL: dl, VT: MVT::i32);
18573 SDValue Args[] = { Int.getOperand(i: 0), Elements, Size, };
18574 SDValue LoopDec = DAG.getNode(Opcode: ARMISD::LOOP_DEC, DL: dl,
18575 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other), Ops: Args);
18576 DAG.ReplaceAllUsesWith(From: Int.getNode(), To: LoopDec.getNode());
18577
18578 // We expect this instruction to branch when the count is not zero.
18579 SDValue Target = IsFalseIfZero(CC, Imm) ? Dest : OtherTarget;
18580
18581 // Update the unconditional branch to target the loop preheader if we've
18582 // found the condition has been reversed.
18583 if (Target == OtherTarget)
18584 UpdateUncondBr(Br, Dest, DAG);
18585
18586 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other,
18587 N1: SDValue(LoopDec.getNode(), 1), N2: Chain);
18588
18589 SDValue EndArgs[] = { Chain, SDValue(LoopDec.getNode(), 0), Target };
18590 return DAG.getNode(Opcode: ARMISD::LE, DL: dl, VT: MVT::Other, Ops: EndArgs);
18591 }
18592 return SDValue();
18593}
18594
18595/// PerformBRCONDCombine - Target-specific DAG combining for ARMISD::BRCOND.
18596SDValue
18597ARMTargetLowering::PerformBRCONDCombine(SDNode *N, SelectionDAG &DAG) const {
18598 SDValue Cmp = N->getOperand(Num: 3);
18599 if (Cmp.getOpcode() != ARMISD::CMPZ)
18600 // Only looking at NE cases.
18601 return SDValue();
18602
18603 SDLoc dl(N);
18604 SDValue LHS = Cmp.getOperand(i: 0);
18605 SDValue RHS = Cmp.getOperand(i: 1);
18606 SDValue Chain = N->getOperand(Num: 0);
18607 SDValue BB = N->getOperand(Num: 1);
18608 SDValue ARMcc = N->getOperand(Num: 2);
18609 ARMCC::CondCodes CC = (ARMCC::CondCodes)ARMcc->getAsZExtVal();
18610
18611 // (brcond Chain BB ne (cmpz (and (cmov 0 1 CC Flags) 1) 0))
18612 // -> (brcond Chain BB CC Flags)
18613 if (CC == ARMCC::NE && LHS.getOpcode() == ISD::AND && LHS->hasOneUse() &&
18614 LHS->getOperand(Num: 0)->getOpcode() == ARMISD::CMOV &&
18615 LHS->getOperand(Num: 0)->hasOneUse() &&
18616 isNullConstant(V: LHS->getOperand(Num: 0)->getOperand(Num: 0)) &&
18617 isOneConstant(V: LHS->getOperand(Num: 0)->getOperand(Num: 1)) &&
18618 isOneConstant(V: LHS->getOperand(Num: 1)) && isNullConstant(V: RHS)) {
18619 return DAG.getNode(Opcode: ARMISD::BRCOND, DL: dl, VT: MVT::Other, N1: Chain, N2: BB,
18620 N3: LHS->getOperand(Num: 0)->getOperand(Num: 2),
18621 N4: LHS->getOperand(Num: 0)->getOperand(Num: 3));
18622 }
18623
18624 return SDValue();
18625}
18626
18627/// PerformCMOVCombine - Target-specific DAG combining for ARMISD::CMOV.
18628SDValue
18629ARMTargetLowering::PerformCMOVCombine(SDNode *N, SelectionDAG &DAG) const {
18630 SDLoc dl(N);
18631 EVT VT = N->getValueType(ResNo: 0);
18632 SDValue FalseVal = N->getOperand(Num: 0);
18633 SDValue TrueVal = N->getOperand(Num: 1);
18634 SDValue ARMcc = N->getOperand(Num: 2);
18635 SDValue Cmp = N->getOperand(Num: 3);
18636
18637 // Try to form CSINV etc.
18638 unsigned Opcode;
18639 bool InvertCond;
18640 if (SDValue CSetOp =
18641 matchCSET(Opcode, InvertCond, TrueVal, FalseVal, Subtarget)) {
18642 if (InvertCond) {
18643 ARMCC::CondCodes CondCode =
18644 (ARMCC::CondCodes)cast<const ConstantSDNode>(Val&: ARMcc)->getZExtValue();
18645 CondCode = ARMCC::getOppositeCondition(CC: CondCode);
18646 ARMcc = DAG.getConstant(Val: CondCode, DL: SDLoc(ARMcc), VT: MVT::i32);
18647 }
18648 return DAG.getNode(Opcode, DL: dl, VT, N1: CSetOp, N2: CSetOp, N3: ARMcc, N4: Cmp);
18649 }
18650
18651 if (Cmp.getOpcode() != ARMISD::CMPZ)
18652 // Only looking at EQ and NE cases.
18653 return SDValue();
18654
18655 SDValue LHS = Cmp.getOperand(i: 0);
18656 SDValue RHS = Cmp.getOperand(i: 1);
18657 ARMCC::CondCodes CC = (ARMCC::CondCodes)ARMcc->getAsZExtVal();
18658
18659 // BFI is only available on V6T2+.
18660 if (!Subtarget->isThumb1Only() && Subtarget->hasV6T2Ops()) {
18661 SDValue R = PerformCMOVToBFICombine(CMOV: N, DAG);
18662 if (R)
18663 return R;
18664 }
18665
18666 // Simplify
18667 // mov r1, r0
18668 // cmp r1, x
18669 // mov r0, y
18670 // moveq r0, x
18671 // to
18672 // cmp r0, x
18673 // movne r0, y
18674 //
18675 // mov r1, r0
18676 // cmp r1, x
18677 // mov r0, x
18678 // movne r0, y
18679 // to
18680 // cmp r0, x
18681 // movne r0, y
18682 /// FIXME: Turn this into a target neutral optimization?
18683 SDValue Res;
18684 if (CC == ARMCC::NE && FalseVal == RHS && FalseVal != LHS) {
18685 Res = DAG.getNode(Opcode: ARMISD::CMOV, DL: dl, VT, N1: LHS, N2: TrueVal, N3: ARMcc, N4: Cmp);
18686 } else if (CC == ARMCC::EQ && TrueVal == RHS) {
18687 SDValue ARMcc;
18688 SDValue NewCmp = getARMCmp(LHS, RHS, CC: ISD::SETNE, ARMcc, DAG, dl);
18689 Res = DAG.getNode(Opcode: ARMISD::CMOV, DL: dl, VT, N1: LHS, N2: FalseVal, N3: ARMcc, N4: NewCmp);
18690 }
18691
18692 // (cmov F T ne (cmpz (cmov 0 1 CC Flags) 0))
18693 // -> (cmov F T CC Flags)
18694 if (CC == ARMCC::NE && LHS.getOpcode() == ARMISD::CMOV && LHS->hasOneUse() &&
18695 isNullConstant(V: LHS->getOperand(Num: 0)) && isOneConstant(V: LHS->getOperand(Num: 1)) &&
18696 isNullConstant(V: RHS)) {
18697 return DAG.getNode(Opcode: ARMISD::CMOV, DL: dl, VT, N1: FalseVal, N2: TrueVal,
18698 N3: LHS->getOperand(Num: 2), N4: LHS->getOperand(Num: 3));
18699 }
18700
18701 if (!VT.isInteger())
18702 return SDValue();
18703
18704 // Fold away an unnecessary CMPZ/CMOV
18705 // CMOV A, B, C1, (CMPZ (CMOV 1, 0, C2, D), 0) ->
18706 // if C1==EQ -> CMOV A, B, C2, D
18707 // if C1==NE -> CMOV A, B, NOT(C2), D
18708 if (N->getConstantOperandVal(Num: 2) == ARMCC::EQ ||
18709 N->getConstantOperandVal(Num: 2) == ARMCC::NE) {
18710 ARMCC::CondCodes Cond;
18711 if (SDValue C = IsCMPZCSINC(Cmp: N->getOperand(Num: 3).getNode(), CC&: Cond)) {
18712 if (N->getConstantOperandVal(Num: 2) == ARMCC::NE)
18713 Cond = ARMCC::getOppositeCondition(CC: Cond);
18714 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: MVT::i32, N1: N->getOperand(Num: 0),
18715 N2: N->getOperand(Num: 1),
18716 N3: DAG.getConstant(Val: Cond, DL: SDLoc(N), VT: MVT::i32), N4: C);
18717 }
18718 }
18719
18720 // Materialize a boolean comparison for integers so we can avoid branching.
18721 if (isNullConstant(V: FalseVal)) {
18722 if (CC == ARMCC::EQ && isOneConstant(V: TrueVal)) {
18723 if (!Subtarget->isThumb1Only() && Subtarget->hasV5TOps()) {
18724 // If x == y then x - y == 0 and ARM's CLZ will return 32, shifting it
18725 // right 5 bits will make that 32 be 1, otherwise it will be 0.
18726 // CMOV 0, 1, ==, (CMPZ x, y) -> SRL (CTLZ (SUB x, y)), 5
18727 SDValue Sub = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: LHS, N2: RHS);
18728 Res = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: DAG.getNode(Opcode: ISD::CTLZ, DL: dl, VT, Operand: Sub),
18729 N2: DAG.getConstant(Val: 5, DL: dl, VT: MVT::i32));
18730 } else {
18731 // CMOV 0, 1, ==, (CMPZ x, y) ->
18732 // (UADDO_CARRY (SUB x, y), t:0, t:1)
18733 // where t = (USUBO_CARRY 0, (SUB x, y), 0)
18734 //
18735 // The USUBO_CARRY computes 0 - (x - y) and this will give a borrow when
18736 // x != y. In other words, a carry C == 1 when x == y, C == 0
18737 // otherwise.
18738 // The final UADDO_CARRY computes
18739 // x - y + (0 - (x - y)) + C == C
18740 SDValue Sub = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: LHS, N2: RHS);
18741 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: MVT::i32);
18742 SDValue Neg = DAG.getNode(Opcode: ISD::USUBO, DL: dl, VTList: VTs, N1: FalseVal, N2: Sub);
18743 // ISD::USUBO_CARRY returns a borrow but we want the carry here
18744 // actually.
18745 SDValue Carry =
18746 DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32,
18747 N1: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32), N2: Neg.getValue(R: 1));
18748 Res = DAG.getNode(Opcode: ISD::UADDO_CARRY, DL: dl, VTList: VTs, N1: Sub, N2: Neg, N3: Carry);
18749 }
18750 } else if (CC == ARMCC::NE && !isNullConstant(V: RHS) &&
18751 (!Subtarget->isThumb1Only() || isPowerOf2Constant(V: TrueVal))) {
18752 // This seems pointless but will allow us to combine it further below.
18753 // CMOV 0, z, !=, (CMPZ x, y) -> CMOV (SUBC x, y), z, !=, (SUBC x, y):1
18754 SDValue Sub =
18755 DAG.getNode(Opcode: ARMISD::SUBC, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: MVT::i32), N1: LHS, N2: RHS);
18756 Res = DAG.getNode(Opcode: ARMISD::CMOV, DL: dl, VT, N1: Sub, N2: TrueVal, N3: ARMcc,
18757 N4: Sub.getValue(R: 1));
18758 FalseVal = Sub;
18759 }
18760 } else if (isNullConstant(V: TrueVal)) {
18761 if (CC == ARMCC::EQ && !isNullConstant(V: RHS) &&
18762 (!Subtarget->isThumb1Only() || isPowerOf2Constant(V: FalseVal))) {
18763 // This seems pointless but will allow us to combine it further below
18764 // Note that we change == for != as this is the dual for the case above.
18765 // CMOV z, 0, ==, (CMPZ x, y) -> CMOV (SUBC x, y), z, !=, (SUBC x, y):1
18766 SDValue Sub =
18767 DAG.getNode(Opcode: ARMISD::SUBC, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: MVT::i32), N1: LHS, N2: RHS);
18768 Res = DAG.getNode(Opcode: ARMISD::CMOV, DL: dl, VT, N1: Sub, N2: FalseVal,
18769 N3: DAG.getConstant(Val: ARMCC::NE, DL: dl, VT: MVT::i32),
18770 N4: Sub.getValue(R: 1));
18771 FalseVal = Sub;
18772 }
18773 }
18774
18775 // On Thumb1, the DAG above may be further combined if z is a power of 2
18776 // (z == 2 ^ K).
18777 // CMOV (SUBC x, y), z, !=, (SUBC x, y):1 ->
18778 // t1 = (USUBO (SUB x, y), 1)
18779 // t2 = (USUBO_CARRY (SUB x, y), t1:0, t1:1)
18780 // Result = if K != 0 then (SHL t2:0, K) else t2:0
18781 //
18782 // This also handles the special case of comparing against zero; it's
18783 // essentially, the same pattern, except there's no SUBC:
18784 // CMOV x, z, !=, (CMPZ x, 0) ->
18785 // t1 = (USUBO x, 1)
18786 // t2 = (USUBO_CARRY x, t1:0, t1:1)
18787 // Result = if K != 0 then (SHL t2:0, K) else t2:0
18788 const APInt *TrueConst;
18789 if (Subtarget->isThumb1Only() && CC == ARMCC::NE &&
18790 ((FalseVal.getOpcode() == ARMISD::SUBC && FalseVal.getOperand(i: 0) == LHS &&
18791 FalseVal.getOperand(i: 1) == RHS) ||
18792 (FalseVal == LHS && isNullConstant(V: RHS))) &&
18793 (TrueConst = isPowerOf2Constant(V: TrueVal))) {
18794 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: MVT::i32);
18795 unsigned ShiftAmount = TrueConst->logBase2();
18796 if (ShiftAmount)
18797 TrueVal = DAG.getConstant(Val: 1, DL: dl, VT);
18798 SDValue Subc = DAG.getNode(Opcode: ISD::USUBO, DL: dl, VTList: VTs, N1: FalseVal, N2: TrueVal);
18799 Res = DAG.getNode(Opcode: ISD::USUBO_CARRY, DL: dl, VTList: VTs, N1: FalseVal, N2: Subc,
18800 N3: Subc.getValue(R: 1));
18801
18802 if (ShiftAmount)
18803 Res = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Res,
18804 N2: DAG.getConstant(Val: ShiftAmount, DL: dl, VT: MVT::i32));
18805 }
18806
18807 if (Res.getNode()) {
18808 KnownBits Known = DAG.computeKnownBits(Op: SDValue(N,0));
18809 // Capture demanded bits information that would be otherwise lost.
18810 if (Known.Zero == 0xfffffffe)
18811 Res = DAG.getNode(Opcode: ISD::AssertZext, DL: dl, VT: MVT::i32, N1: Res,
18812 N2: DAG.getValueType(MVT::i1));
18813 else if (Known.Zero == 0xffffff00)
18814 Res = DAG.getNode(Opcode: ISD::AssertZext, DL: dl, VT: MVT::i32, N1: Res,
18815 N2: DAG.getValueType(MVT::i8));
18816 else if (Known.Zero == 0xffff0000)
18817 Res = DAG.getNode(Opcode: ISD::AssertZext, DL: dl, VT: MVT::i32, N1: Res,
18818 N2: DAG.getValueType(MVT::i16));
18819 }
18820
18821 return Res;
18822}
18823
18824static SDValue PerformBITCASTCombine(SDNode *N,
18825 TargetLowering::DAGCombinerInfo &DCI,
18826 const ARMSubtarget *ST) {
18827 SelectionDAG &DAG = DCI.DAG;
18828 SDValue Src = N->getOperand(Num: 0);
18829 EVT DstVT = N->getValueType(ResNo: 0);
18830
18831 // Convert v4f32 bitcast (v4i32 vdup (i32)) -> v4f32 vdup (i32) under MVE.
18832 if (ST->hasMVEIntegerOps() && Src.getOpcode() == ARMISD::VDUP) {
18833 EVT SrcVT = Src.getValueType();
18834 if (SrcVT.getScalarSizeInBits() == DstVT.getScalarSizeInBits())
18835 return DAG.getNode(Opcode: ARMISD::VDUP, DL: SDLoc(N), VT: DstVT, Operand: Src.getOperand(i: 0));
18836 }
18837
18838 // We may have a bitcast of something that has already had this bitcast
18839 // combine performed on it, so skip past any VECTOR_REG_CASTs.
18840 if (Src.getOpcode() == ARMISD::VECTOR_REG_CAST &&
18841 Src.getOperand(i: 0).getValueType().getScalarSizeInBits() <=
18842 Src.getValueType().getScalarSizeInBits())
18843 Src = Src.getOperand(i: 0);
18844
18845 // Bitcast from element-wise VMOV or VMVN doesn't need VREV if the VREV that
18846 // would be generated is at least the width of the element type.
18847 EVT SrcVT = Src.getValueType();
18848 if ((Src.getOpcode() == ARMISD::VMOVIMM ||
18849 Src.getOpcode() == ARMISD::VMVNIMM ||
18850 Src.getOpcode() == ARMISD::VMOVFPIMM) &&
18851 SrcVT.getScalarSizeInBits() <= DstVT.getScalarSizeInBits() &&
18852 DAG.getDataLayout().isBigEndian())
18853 return DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL: SDLoc(N), VT: DstVT, Operand: Src);
18854
18855 // bitcast(extract(x, n)); bitcast(extract(x, n+1)) -> VMOVRRD x
18856 if (SDValue R = PerformExtractEltToVMOVRRD(N, DCI))
18857 return R;
18858
18859 return SDValue();
18860}
18861
18862// Some combines for the MVETrunc truncations legalizer helper. Also lowers the
18863// node into stack operations after legalizeOps.
18864SDValue ARMTargetLowering::PerformMVETruncCombine(
18865 SDNode *N, TargetLowering::DAGCombinerInfo &DCI) const {
18866 SelectionDAG &DAG = DCI.DAG;
18867 EVT VT = N->getValueType(ResNo: 0);
18868 SDLoc DL(N);
18869
18870 // MVETrunc(Undef, Undef) -> Undef
18871 if (all_of(Range: N->ops(), P: [](SDValue Op) { return Op.isUndef(); }))
18872 return DAG.getUNDEF(VT);
18873
18874 // MVETrunc(MVETrunc a b, MVETrunc c, d) -> MVETrunc
18875 if (N->getNumOperands() == 2 &&
18876 N->getOperand(Num: 0).getOpcode() == ARMISD::MVETRUNC &&
18877 N->getOperand(Num: 1).getOpcode() == ARMISD::MVETRUNC)
18878 return DAG.getNode(Opcode: ARMISD::MVETRUNC, DL, VT, N1: N->getOperand(Num: 0).getOperand(i: 0),
18879 N2: N->getOperand(Num: 0).getOperand(i: 1),
18880 N3: N->getOperand(Num: 1).getOperand(i: 0),
18881 N4: N->getOperand(Num: 1).getOperand(i: 1));
18882
18883 // MVETrunc(shuffle, shuffle) -> VMOVN
18884 if (N->getNumOperands() == 2 &&
18885 N->getOperand(Num: 0).getOpcode() == ISD::VECTOR_SHUFFLE &&
18886 N->getOperand(Num: 1).getOpcode() == ISD::VECTOR_SHUFFLE) {
18887 auto *S0 = cast<ShuffleVectorSDNode>(Val: N->getOperand(Num: 0).getNode());
18888 auto *S1 = cast<ShuffleVectorSDNode>(Val: N->getOperand(Num: 1).getNode());
18889
18890 if (S0->getOperand(Num: 0) == S1->getOperand(Num: 0) &&
18891 S0->getOperand(Num: 1) == S1->getOperand(Num: 1)) {
18892 // Construct complete shuffle mask
18893 SmallVector<int, 8> Mask(S0->getMask());
18894 Mask.append(in_start: S1->getMask().begin(), in_end: S1->getMask().end());
18895
18896 if (isVMOVNTruncMask(M: Mask, ToVT: VT, rev: false))
18897 return DAG.getNode(
18898 Opcode: ARMISD::VMOVN, DL, VT,
18899 N1: DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL, VT, Operand: S0->getOperand(Num: 0)),
18900 N2: DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL, VT, Operand: S0->getOperand(Num: 1)),
18901 N3: DAG.getConstant(Val: 1, DL, VT: MVT::i32));
18902 if (isVMOVNTruncMask(M: Mask, ToVT: VT, rev: true))
18903 return DAG.getNode(
18904 Opcode: ARMISD::VMOVN, DL, VT,
18905 N1: DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL, VT, Operand: S0->getOperand(Num: 1)),
18906 N2: DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL, VT, Operand: S0->getOperand(Num: 0)),
18907 N3: DAG.getConstant(Val: 1, DL, VT: MVT::i32));
18908 }
18909 }
18910
18911 // For MVETrunc of a buildvector or shuffle, it can be beneficial to lower the
18912 // truncate to a buildvector to allow the generic optimisations to kick in.
18913 if (all_of(Range: N->ops(), P: [](SDValue Op) {
18914 return Op.getOpcode() == ISD::BUILD_VECTOR ||
18915 Op.getOpcode() == ISD::VECTOR_SHUFFLE ||
18916 (Op.getOpcode() == ISD::BITCAST &&
18917 Op.getOperand(i: 0).getOpcode() == ISD::BUILD_VECTOR);
18918 })) {
18919 SmallVector<SDValue, 8> Extracts;
18920 for (unsigned Op = 0; Op < N->getNumOperands(); Op++) {
18921 SDValue O = N->getOperand(Num: Op);
18922 for (unsigned i = 0; i < O.getValueType().getVectorNumElements(); i++) {
18923 SDValue Ext = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: O,
18924 N2: DAG.getConstant(Val: i, DL, VT: MVT::i32));
18925 Extracts.push_back(Elt: Ext);
18926 }
18927 }
18928 return DAG.getBuildVector(VT, DL, Ops: Extracts);
18929 }
18930
18931 // If we are late in the legalization process and nothing has optimised
18932 // the trunc to anything better, lower it to a stack store and reload,
18933 // performing the truncation whilst keeping the lanes in the correct order:
18934 // VSTRH.32 a, stack; VSTRH.32 b, stack+8; VLDRW.32 stack;
18935 if (!DCI.isAfterLegalizeDAG())
18936 return SDValue();
18937
18938 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: TypeSize::getFixed(ExactSize: 16), Alignment: Align(4));
18939 int SPFI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
18940 int NumIns = N->getNumOperands();
18941 assert((NumIns == 2 || NumIns == 4) &&
18942 "Expected 2 or 4 inputs to an MVETrunc");
18943 EVT StoreVT = VT.getHalfNumVectorElementsVT(Context&: *DAG.getContext());
18944 if (N->getNumOperands() == 4)
18945 StoreVT = StoreVT.getHalfNumVectorElementsVT(Context&: *DAG.getContext());
18946
18947 SmallVector<SDValue> Chains;
18948 for (int I = 0; I < NumIns; I++) {
18949 SDValue Ptr = DAG.getNode(
18950 Opcode: ISD::ADD, DL, VT: StackPtr.getValueType(), N1: StackPtr,
18951 N2: DAG.getConstant(Val: I * 16 / NumIns, DL, VT: StackPtr.getValueType()));
18952 MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(
18953 MF&: DAG.getMachineFunction(), FI: SPFI, Offset: I * 16 / NumIns);
18954 SDValue Ch = DAG.getTruncStore(Chain: DAG.getEntryNode(), dl: DL, Val: N->getOperand(Num: I),
18955 Ptr, PtrInfo: MPI, SVT: StoreVT, Alignment: Align(4));
18956 Chains.push_back(Elt: Ch);
18957 }
18958
18959 SDValue Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: Chains);
18960 MachinePointerInfo MPI =
18961 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI, Offset: 0);
18962 return DAG.getLoad(VT, dl: DL, Chain, Ptr: StackPtr, PtrInfo: MPI, Alignment: Align(4));
18963}
18964
18965// Take a MVEEXT(load x) and split that into (extload x, extload x+8)
18966static SDValue PerformSplittingMVEEXTToWideningLoad(SDNode *N,
18967 SelectionDAG &DAG) {
18968 SDValue N0 = N->getOperand(Num: 0);
18969 LoadSDNode *LD = dyn_cast<LoadSDNode>(Val: N0.getNode());
18970 if (!LD || !LD->isSimple() || !N0.hasOneUse() || LD->isIndexed())
18971 return SDValue();
18972
18973 EVT FromVT = LD->getMemoryVT();
18974 EVT ToVT = N->getValueType(ResNo: 0);
18975 if (!ToVT.isVector())
18976 return SDValue();
18977 assert(FromVT.getVectorNumElements() == ToVT.getVectorNumElements() * 2);
18978 EVT ToEltVT = ToVT.getVectorElementType();
18979 EVT FromEltVT = FromVT.getVectorElementType();
18980
18981 unsigned NumElements = 0;
18982 if (ToEltVT == MVT::i32 && (FromEltVT == MVT::i16 || FromEltVT == MVT::i8))
18983 NumElements = 4;
18984 if (ToEltVT == MVT::i16 && FromEltVT == MVT::i8)
18985 NumElements = 8;
18986 assert(NumElements != 0);
18987
18988 ISD::LoadExtType NewExtType =
18989 N->getOpcode() == ARMISD::MVESEXT ? ISD::SEXTLOAD : ISD::ZEXTLOAD;
18990 if (LD->getExtensionType() != ISD::NON_EXTLOAD &&
18991 LD->getExtensionType() != ISD::EXTLOAD &&
18992 LD->getExtensionType() != NewExtType)
18993 return SDValue();
18994
18995 LLVMContext &C = *DAG.getContext();
18996 SDLoc DL(LD);
18997 // Details about the old load
18998 SDValue Ch = LD->getChain();
18999 SDValue BasePtr = LD->getBasePtr();
19000 Align Alignment = LD->getBaseAlign();
19001 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
19002 AAMDNodes AAInfo = LD->getAAInfo();
19003
19004 SDValue Offset = DAG.getPOISON(VT: BasePtr.getValueType());
19005 EVT NewFromVT = EVT::getVectorVT(
19006 Context&: C, VT: EVT::getIntegerVT(Context&: C, BitWidth: FromEltVT.getScalarSizeInBits()), NumElements);
19007 EVT NewToVT = EVT::getVectorVT(
19008 Context&: C, VT: EVT::getIntegerVT(Context&: C, BitWidth: ToEltVT.getScalarSizeInBits()), NumElements);
19009
19010 SmallVector<SDValue, 4> Loads;
19011 SmallVector<SDValue, 4> Chains;
19012 for (unsigned i = 0; i < FromVT.getVectorNumElements() / NumElements; i++) {
19013 unsigned NewOffset = (i * NewFromVT.getSizeInBits()) / 8;
19014 SDValue NewPtr =
19015 DAG.getObjectPtrOffset(SL: DL, Ptr: BasePtr, Offset: TypeSize::getFixed(ExactSize: NewOffset));
19016
19017 SDValue NewLoad =
19018 DAG.getLoad(AM: ISD::UNINDEXED, ExtType: NewExtType, VT: NewToVT, dl: DL, Chain: Ch, Ptr: NewPtr, Offset,
19019 PtrInfo: LD->getPointerInfo().getWithOffset(O: NewOffset), MemVT: NewFromVT,
19020 Alignment, MMOFlags, Metadata: AAInfo);
19021 Loads.push_back(Elt: NewLoad);
19022 Chains.push_back(Elt: SDValue(NewLoad.getNode(), 1));
19023 }
19024
19025 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: Chains);
19026 DAG.ReplaceAllUsesOfValueWith(From: SDValue(LD, 1), To: NewChain);
19027 return DAG.getMergeValues(Ops: Loads, dl: DL);
19028}
19029
19030// Perform combines for MVEEXT. If it has not be optimized to anything better
19031// before lowering, it gets converted to stack store and extloads performing the
19032// extend whilst still keeping the same lane ordering.
19033SDValue ARMTargetLowering::PerformMVEExtCombine(
19034 SDNode *N, TargetLowering::DAGCombinerInfo &DCI) const {
19035 SelectionDAG &DAG = DCI.DAG;
19036 EVT VT = N->getValueType(ResNo: 0);
19037 SDLoc DL(N);
19038 assert(N->getNumValues() == 2 && "Expected MVEEXT with 2 elements");
19039 assert((VT == MVT::v4i32 || VT == MVT::v8i16) && "Unexpected MVEEXT type");
19040
19041 EVT ExtVT = N->getOperand(Num: 0).getValueType().getHalfNumVectorElementsVT(
19042 Context&: *DAG.getContext());
19043 auto Extend = [&](SDValue V) {
19044 SDValue VVT = DAG.getNode(Opcode: ARMISD::VECTOR_REG_CAST, DL, VT, Operand: V);
19045 return N->getOpcode() == ARMISD::MVESEXT
19046 ? DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT, N1: VVT,
19047 N2: DAG.getValueType(ExtVT))
19048 : DAG.getZeroExtendInReg(Op: VVT, DL, VT: ExtVT);
19049 };
19050
19051 // MVEEXT(VDUP) -> SIGN_EXTEND_INREG(VDUP)
19052 if (N->getOperand(Num: 0).getOpcode() == ARMISD::VDUP) {
19053 SDValue Ext = Extend(N->getOperand(Num: 0));
19054 return DAG.getMergeValues(Ops: {Ext, Ext}, dl: DL);
19055 }
19056
19057 // MVEEXT(shuffle) -> SIGN_EXTEND_INREG/ZERO_EXTEND_INREG
19058 if (auto *SVN = dyn_cast<ShuffleVectorSDNode>(Val: N->getOperand(Num: 0))) {
19059 ArrayRef<int> Mask = SVN->getMask();
19060 assert(Mask.size() == 2 * VT.getVectorNumElements());
19061 assert(Mask.size() == SVN->getValueType(0).getVectorNumElements());
19062 unsigned Rev = VT == MVT::v4i32 ? ARMISD::VREV32 : ARMISD::VREV16;
19063 SDValue Op0 = SVN->getOperand(Num: 0);
19064 SDValue Op1 = SVN->getOperand(Num: 1);
19065
19066 auto CheckInregMask = [&](int Start, int Offset) {
19067 for (int Idx = 0, E = VT.getVectorNumElements(); Idx < E; ++Idx)
19068 if (Mask[Start + Idx] >= 0 && Mask[Start + Idx] != Idx * 2 + Offset)
19069 return false;
19070 return true;
19071 };
19072 SDValue V0 = SDValue(N, 0);
19073 SDValue V1 = SDValue(N, 1);
19074 if (CheckInregMask(0, 0))
19075 V0 = Extend(Op0);
19076 else if (CheckInregMask(0, 1))
19077 V0 = Extend(DAG.getNode(Opcode: Rev, DL, VT: SVN->getValueType(ResNo: 0), Operand: Op0));
19078 else if (CheckInregMask(0, Mask.size()))
19079 V0 = Extend(Op1);
19080 else if (CheckInregMask(0, Mask.size() + 1))
19081 V0 = Extend(DAG.getNode(Opcode: Rev, DL, VT: SVN->getValueType(ResNo: 0), Operand: Op1));
19082
19083 if (CheckInregMask(VT.getVectorNumElements(), Mask.size()))
19084 V1 = Extend(Op1);
19085 else if (CheckInregMask(VT.getVectorNumElements(), Mask.size() + 1))
19086 V1 = Extend(DAG.getNode(Opcode: Rev, DL, VT: SVN->getValueType(ResNo: 0), Operand: Op1));
19087 else if (CheckInregMask(VT.getVectorNumElements(), 0))
19088 V1 = Extend(Op0);
19089 else if (CheckInregMask(VT.getVectorNumElements(), 1))
19090 V1 = Extend(DAG.getNode(Opcode: Rev, DL, VT: SVN->getValueType(ResNo: 0), Operand: Op0));
19091
19092 if (V0.getNode() != N || V1.getNode() != N)
19093 return DAG.getMergeValues(Ops: {V0, V1}, dl: DL);
19094 }
19095
19096 // MVEEXT(load) -> extload, extload
19097 if (N->getOperand(Num: 0)->getOpcode() == ISD::LOAD)
19098 if (SDValue L = PerformSplittingMVEEXTToWideningLoad(N, DAG))
19099 return L;
19100
19101 if (!DCI.isAfterLegalizeDAG())
19102 return SDValue();
19103
19104 // Lower to a stack store and reload:
19105 // VSTRW.32 a, stack; VLDRH.32 stack; VLDRH.32 stack+8;
19106 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: TypeSize::getFixed(ExactSize: 16), Alignment: Align(4));
19107 int SPFI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
19108 int NumOuts = N->getNumValues();
19109 assert((NumOuts == 2 || NumOuts == 4) &&
19110 "Expected 2 or 4 outputs to an MVEEXT");
19111 EVT LoadVT = N->getOperand(Num: 0).getValueType().getHalfNumVectorElementsVT(
19112 Context&: *DAG.getContext());
19113 if (N->getNumOperands() == 4)
19114 LoadVT = LoadVT.getHalfNumVectorElementsVT(Context&: *DAG.getContext());
19115
19116 MachinePointerInfo MPI =
19117 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI, Offset: 0);
19118 SDValue Chain = DAG.getStore(Chain: DAG.getEntryNode(), dl: DL, Val: N->getOperand(Num: 0),
19119 Ptr: StackPtr, PtrInfo: MPI, Alignment: Align(4));
19120
19121 SmallVector<SDValue> Loads;
19122 for (int I = 0; I < NumOuts; I++) {
19123 SDValue Ptr = DAG.getNode(
19124 Opcode: ISD::ADD, DL, VT: StackPtr.getValueType(), N1: StackPtr,
19125 N2: DAG.getConstant(Val: I * 16 / NumOuts, DL, VT: StackPtr.getValueType()));
19126 MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(
19127 MF&: DAG.getMachineFunction(), FI: SPFI, Offset: I * 16 / NumOuts);
19128 SDValue Load = DAG.getExtLoad(
19129 ExtType: N->getOpcode() == ARMISD::MVESEXT ? ISD::SEXTLOAD : ISD::ZEXTLOAD, dl: DL,
19130 VT, Chain, Ptr, PtrInfo: MPI, MemVT: LoadVT, Alignment: Align(4));
19131 Loads.push_back(Elt: Load);
19132 }
19133
19134 return DAG.getMergeValues(Ops: Loads, dl: DL);
19135}
19136
19137SDValue ARMTargetLowering::PerformDAGCombine(SDNode *N,
19138 DAGCombinerInfo &DCI) const {
19139 switch (N->getOpcode()) {
19140 default: break;
19141 case ISD::SELECT_CC:
19142 case ISD::SELECT: return PerformSELECTCombine(N, DCI, Subtarget);
19143 case ISD::VSELECT: return PerformVSELECTCombine(N, DCI, Subtarget);
19144 case ISD::SETCC: return PerformVSetCCToVCTPCombine(N, DCI, Subtarget);
19145 case ARMISD::ADDE: return PerformADDECombine(N, DCI, Subtarget);
19146 case ARMISD::UMLAL: return PerformUMLALCombine(N, DAG&: DCI.DAG, Subtarget);
19147 case ISD::ADD: return PerformADDCombine(N, DCI, Subtarget);
19148 case ISD::SUB: return PerformSUBCombine(N, DCI, Subtarget);
19149 case ISD::MUL: return PerformMULCombine(N, DCI, Subtarget);
19150 case ISD::OR: return PerformORCombine(N, DCI, Subtarget);
19151 case ISD::XOR: return PerformXORCombine(N, DCI, Subtarget);
19152 case ISD::AND: return PerformANDCombine(N, DCI, Subtarget);
19153 case ISD::BRCOND:
19154 case ISD::BR_CC: return PerformHWLoopCombine(N, DCI, ST: Subtarget);
19155 case ARMISD::ADDC:
19156 case ARMISD::SUBC: return PerformAddcSubcCombine(N, DCI, Subtarget);
19157 case ARMISD::SUBE: return PerformAddeSubeCombine(N, DCI, Subtarget);
19158 case ARMISD::BFI: return PerformBFICombine(N, DAG&: DCI.DAG);
19159 case ARMISD::VMOVRRD: return PerformVMOVRRDCombine(N, DCI, Subtarget);
19160 case ARMISD::VMOVDRR: return PerformVMOVDRRCombine(N, DAG&: DCI.DAG);
19161 case ARMISD::VMOVhr: return PerformVMOVhrCombine(N, DCI);
19162 case ARMISD::VMOVrh: return PerformVMOVrhCombine(N, DAG&: DCI.DAG);
19163 case ISD::STORE: return PerformSTORECombine(N, DCI, Subtarget);
19164 case ISD::BUILD_VECTOR: return PerformBUILD_VECTORCombine(N, DCI, Subtarget);
19165 case ISD::INSERT_VECTOR_ELT: return PerformInsertEltCombine(N, DCI);
19166 case ISD::EXTRACT_VECTOR_ELT:
19167 return PerformExtractEltCombine(N, DCI, ST: Subtarget);
19168 case ISD::SIGN_EXTEND_INREG: return PerformSignExtendInregCombine(N, DAG&: DCI.DAG);
19169 case ISD::INSERT_SUBVECTOR: return PerformInsertSubvectorCombine(N, DCI);
19170 case ISD::VECTOR_SHUFFLE: return PerformVECTOR_SHUFFLECombine(N, DAG&: DCI.DAG);
19171 case ARMISD::VDUPLANE: return PerformVDUPLANECombine(N, DCI, Subtarget);
19172 case ARMISD::VDUP: return PerformVDUPCombine(N, DAG&: DCI.DAG, Subtarget);
19173 case ISD::FP_TO_SINT:
19174 case ISD::FP_TO_UINT:
19175 return PerformVCVTCombine(N, DAG&: DCI.DAG, Subtarget);
19176 case ISD::FADD:
19177 return PerformFADDCombine(N, DAG&: DCI.DAG, Subtarget);
19178 case ISD::FMUL:
19179 return PerformVMulVCTPCombine(N, DAG&: DCI.DAG, Subtarget);
19180 case ISD::INTRINSIC_WO_CHAIN:
19181 return PerformIntrinsicCombine(N, DCI);
19182 case ISD::SHL:
19183 case ISD::SRA:
19184 case ISD::SRL:
19185 return PerformShiftCombine(N, DCI, ST: Subtarget);
19186 case ISD::SIGN_EXTEND:
19187 case ISD::ZERO_EXTEND:
19188 case ISD::ANY_EXTEND:
19189 return PerformExtendCombine(N, DAG&: DCI.DAG, ST: Subtarget);
19190 case ISD::FP_EXTEND:
19191 return PerformFPExtendCombine(N, DAG&: DCI.DAG, ST: Subtarget);
19192 case ISD::SMIN:
19193 case ISD::UMIN:
19194 case ISD::SMAX:
19195 case ISD::UMAX:
19196 return PerformMinMaxCombine(N, DAG&: DCI.DAG, ST: Subtarget);
19197 case ARMISD::CMOV:
19198 return PerformCMOVCombine(N, DAG&: DCI.DAG);
19199 case ARMISD::BRCOND:
19200 return PerformBRCONDCombine(N, DAG&: DCI.DAG);
19201 case ARMISD::CMPZ:
19202 return PerformCMPZCombine(N, DAG&: DCI.DAG);
19203 case ARMISD::CSINC:
19204 case ARMISD::CSINV:
19205 case ARMISD::CSNEG:
19206 return PerformCSETCombine(N, DAG&: DCI.DAG);
19207 case ISD::LOAD:
19208 return PerformLOADCombine(N, DCI, Subtarget);
19209 case ARMISD::VLD1DUP:
19210 case ARMISD::VLD2DUP:
19211 case ARMISD::VLD3DUP:
19212 case ARMISD::VLD4DUP:
19213 return PerformVLDCombine(N, DCI);
19214 case ARMISD::BUILD_VECTOR:
19215 return PerformARMBUILD_VECTORCombine(N, DCI);
19216 case ISD::BITCAST:
19217 return PerformBITCASTCombine(N, DCI, ST: Subtarget);
19218 case ARMISD::PREDICATE_CAST:
19219 return PerformPREDICATE_CASTCombine(N, DCI);
19220 case ARMISD::VECTOR_REG_CAST:
19221 return PerformVECTOR_REG_CASTCombine(N, DAG&: DCI.DAG, ST: Subtarget);
19222 case ARMISD::MVETRUNC:
19223 return PerformMVETruncCombine(N, DCI);
19224 case ARMISD::MVESEXT:
19225 case ARMISD::MVEZEXT:
19226 return PerformMVEExtCombine(N, DCI);
19227 case ARMISD::VCMP:
19228 return PerformVCMPCombine(N, DAG&: DCI.DAG, Subtarget);
19229 case ISD::VECREDUCE_ADD:
19230 return PerformVECREDUCE_ADDCombine(N, DAG&: DCI.DAG, ST: Subtarget);
19231 case ARMISD::VADDVs:
19232 case ARMISD::VADDVu:
19233 case ARMISD::VADDLVs:
19234 case ARMISD::VADDLVu:
19235 case ARMISD::VADDLVAs:
19236 case ARMISD::VADDLVAu:
19237 case ARMISD::VMLAVs:
19238 case ARMISD::VMLAVu:
19239 case ARMISD::VMLALVs:
19240 case ARMISD::VMLALVu:
19241 case ARMISD::VMLALVAs:
19242 case ARMISD::VMLALVAu:
19243 return PerformReduceShuffleCombine(N, DAG&: DCI.DAG);
19244 case ARMISD::VMOVN:
19245 return PerformVMOVNCombine(N, DCI);
19246 case ARMISD::VQMOVNs:
19247 case ARMISD::VQMOVNu:
19248 return PerformVQMOVNCombine(N, DCI);
19249 case ARMISD::VQDMULH:
19250 return PerformVQDMULHCombine(N, DCI);
19251 case ARMISD::ASRL:
19252 case ARMISD::LSRL:
19253 case ARMISD::LSLL:
19254 return PerformLongShiftCombine(N, DAG&: DCI.DAG);
19255 case ARMISD::SMULWB: {
19256 unsigned BitWidth = N->getValueType(ResNo: 0).getSizeInBits();
19257 APInt DemandedMask = APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: 16);
19258 if (SimplifyDemandedBits(Op: N->getOperand(Num: 1), DemandedBits: DemandedMask, DCI))
19259 return SDValue();
19260 break;
19261 }
19262 case ARMISD::SMULWT: {
19263 unsigned BitWidth = N->getValueType(ResNo: 0).getSizeInBits();
19264 APInt DemandedMask = APInt::getHighBitsSet(numBits: BitWidth, hiBitsSet: 16);
19265 if (SimplifyDemandedBits(Op: N->getOperand(Num: 1), DemandedBits: DemandedMask, DCI))
19266 return SDValue();
19267 break;
19268 }
19269 case ARMISD::SMLALBB:
19270 case ARMISD::QADD16b:
19271 case ARMISD::QSUB16b:
19272 case ARMISD::UQADD16b:
19273 case ARMISD::UQSUB16b: {
19274 unsigned BitWidth = N->getValueType(ResNo: 0).getSizeInBits();
19275 APInt DemandedMask = APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: 16);
19276 if ((SimplifyDemandedBits(Op: N->getOperand(Num: 0), DemandedBits: DemandedMask, DCI)) ||
19277 (SimplifyDemandedBits(Op: N->getOperand(Num: 1), DemandedBits: DemandedMask, DCI)))
19278 return SDValue();
19279 break;
19280 }
19281 case ARMISD::SMLALBT: {
19282 unsigned LowWidth = N->getOperand(Num: 0).getValueType().getSizeInBits();
19283 APInt LowMask = APInt::getLowBitsSet(numBits: LowWidth, loBitsSet: 16);
19284 unsigned HighWidth = N->getOperand(Num: 1).getValueType().getSizeInBits();
19285 APInt HighMask = APInt::getHighBitsSet(numBits: HighWidth, hiBitsSet: 16);
19286 if ((SimplifyDemandedBits(Op: N->getOperand(Num: 0), DemandedBits: LowMask, DCI)) ||
19287 (SimplifyDemandedBits(Op: N->getOperand(Num: 1), DemandedBits: HighMask, DCI)))
19288 return SDValue();
19289 break;
19290 }
19291 case ARMISD::SMLALTB: {
19292 unsigned HighWidth = N->getOperand(Num: 0).getValueType().getSizeInBits();
19293 APInt HighMask = APInt::getHighBitsSet(numBits: HighWidth, hiBitsSet: 16);
19294 unsigned LowWidth = N->getOperand(Num: 1).getValueType().getSizeInBits();
19295 APInt LowMask = APInt::getLowBitsSet(numBits: LowWidth, loBitsSet: 16);
19296 if ((SimplifyDemandedBits(Op: N->getOperand(Num: 0), DemandedBits: HighMask, DCI)) ||
19297 (SimplifyDemandedBits(Op: N->getOperand(Num: 1), DemandedBits: LowMask, DCI)))
19298 return SDValue();
19299 break;
19300 }
19301 case ARMISD::SMLALTT: {
19302 unsigned BitWidth = N->getValueType(ResNo: 0).getSizeInBits();
19303 APInt DemandedMask = APInt::getHighBitsSet(numBits: BitWidth, hiBitsSet: 16);
19304 if ((SimplifyDemandedBits(Op: N->getOperand(Num: 0), DemandedBits: DemandedMask, DCI)) ||
19305 (SimplifyDemandedBits(Op: N->getOperand(Num: 1), DemandedBits: DemandedMask, DCI)))
19306 return SDValue();
19307 break;
19308 }
19309 case ARMISD::QADD8b:
19310 case ARMISD::QSUB8b:
19311 case ARMISD::UQADD8b:
19312 case ARMISD::UQSUB8b: {
19313 unsigned BitWidth = N->getValueType(ResNo: 0).getSizeInBits();
19314 APInt DemandedMask = APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: 8);
19315 if ((SimplifyDemandedBits(Op: N->getOperand(Num: 0), DemandedBits: DemandedMask, DCI)) ||
19316 (SimplifyDemandedBits(Op: N->getOperand(Num: 1), DemandedBits: DemandedMask, DCI)))
19317 return SDValue();
19318 break;
19319 }
19320 case ARMISD::VBSP:
19321 if (N->getOperand(Num: 1) == N->getOperand(Num: 2))
19322 return N->getOperand(Num: 1);
19323 return SDValue();
19324 case ISD::INTRINSIC_VOID:
19325 case ISD::INTRINSIC_W_CHAIN:
19326 switch (N->getConstantOperandVal(Num: 1)) {
19327 case Intrinsic::arm_neon_vld1:
19328 case Intrinsic::arm_neon_vld1x2:
19329 case Intrinsic::arm_neon_vld1x3:
19330 case Intrinsic::arm_neon_vld1x4:
19331 case Intrinsic::arm_neon_vld2:
19332 case Intrinsic::arm_neon_vld3:
19333 case Intrinsic::arm_neon_vld4:
19334 case Intrinsic::arm_neon_vld2lane:
19335 case Intrinsic::arm_neon_vld3lane:
19336 case Intrinsic::arm_neon_vld4lane:
19337 case Intrinsic::arm_neon_vld2dup:
19338 case Intrinsic::arm_neon_vld3dup:
19339 case Intrinsic::arm_neon_vld4dup:
19340 case Intrinsic::arm_neon_vst1:
19341 case Intrinsic::arm_neon_vst1x2:
19342 case Intrinsic::arm_neon_vst1x3:
19343 case Intrinsic::arm_neon_vst1x4:
19344 case Intrinsic::arm_neon_vst2:
19345 case Intrinsic::arm_neon_vst3:
19346 case Intrinsic::arm_neon_vst4:
19347 case Intrinsic::arm_neon_vst2lane:
19348 case Intrinsic::arm_neon_vst3lane:
19349 case Intrinsic::arm_neon_vst4lane:
19350 return PerformVLDCombine(N, DCI);
19351 case Intrinsic::arm_mve_vld2q:
19352 case Intrinsic::arm_mve_vld4q:
19353 case Intrinsic::arm_mve_vst2q:
19354 case Intrinsic::arm_mve_vst4q:
19355 return PerformMVEVLDCombine(N, DCI);
19356 default: break;
19357 }
19358 break;
19359 }
19360 return SDValue();
19361}
19362
19363bool ARMTargetLowering::isDesirableToTransformToIntegerOp(unsigned Opc,
19364 EVT VT) const {
19365 return (VT == MVT::f32) && (Opc == ISD::LOAD || Opc == ISD::STORE);
19366}
19367
19368bool ARMTargetLowering::allowsMisalignedMemoryAccesses(EVT VT, unsigned,
19369 Align Alignment,
19370 MachineMemOperand::Flags,
19371 unsigned *Fast) const {
19372 // Depends what it gets converted into if the type is weird.
19373 if (!VT.isSimple())
19374 return false;
19375
19376 // The AllowsUnaligned flag models the SCTLR.A setting in ARM cpus
19377 bool AllowsUnaligned = Subtarget->allowsUnalignedMem();
19378 auto Ty = VT.getSimpleVT().SimpleTy;
19379
19380 if (Ty == MVT::i8 || Ty == MVT::i16 || Ty == MVT::i32) {
19381 // Unaligned access can use (for example) LRDB, LRDH, LDR
19382 if (AllowsUnaligned) {
19383 if (Fast)
19384 *Fast = Subtarget->hasV7Ops();
19385 return true;
19386 }
19387 }
19388
19389 if (Ty == MVT::f64 || Ty == MVT::v2f64) {
19390 // For any little-endian targets with neon, we can support unaligned ld/st
19391 // of D and Q (e.g. {D0,D1}) registers by using vld1.i8/vst1.i8.
19392 // A big-endian target may also explicitly support unaligned accesses
19393 if (Subtarget->hasNEON() && (AllowsUnaligned || Subtarget->isLittle())) {
19394 if (Fast)
19395 *Fast = 1;
19396 return true;
19397 }
19398 }
19399
19400 if (!Subtarget->hasMVEIntegerOps())
19401 return false;
19402
19403 // These are for predicates
19404 if ((Ty == MVT::v16i1 || Ty == MVT::v8i1 || Ty == MVT::v4i1 ||
19405 Ty == MVT::v2i1)) {
19406 if (Fast)
19407 *Fast = 1;
19408 return true;
19409 }
19410
19411 // These are for truncated stores/narrowing loads. They are fine so long as
19412 // the alignment is at least the size of the item being loaded
19413 if ((Ty == MVT::v4i8 || Ty == MVT::v8i8 || Ty == MVT::v4i16) &&
19414 Alignment >= VT.getScalarSizeInBits() / 8) {
19415 if (Fast)
19416 *Fast = true;
19417 return true;
19418 }
19419
19420 // In little-endian MVE, the store instructions VSTRB.U8, VSTRH.U16 and
19421 // VSTRW.U32 all store the vector register in exactly the same format, and
19422 // differ only in the range of their immediate offset field and the required
19423 // alignment. So there is always a store that can be used, regardless of
19424 // actual type.
19425 //
19426 // For big endian, that is not the case. But can still emit a (VSTRB.U8;
19427 // VREV64.8) pair and get the same effect. This will likely be better than
19428 // aligning the vector through the stack.
19429 if (Ty == MVT::v16i8 || Ty == MVT::v8i16 || Ty == MVT::v8f16 ||
19430 Ty == MVT::v4i32 || Ty == MVT::v4f32 || Ty == MVT::v2i64 ||
19431 Ty == MVT::v2f64) {
19432 if (Fast)
19433 *Fast = 1;
19434 return true;
19435 }
19436
19437 return false;
19438}
19439
19440EVT ARMTargetLowering::getOptimalMemOpType(
19441 LLVMContext &Context, const MemOp &Op,
19442 const AttributeList &FuncAttributes) const {
19443 // See if we can use NEON instructions for this...
19444 if ((Op.isMemcpyOrMemmove() || Op.isZeroMemset()) && Subtarget->hasNEON() &&
19445 !FuncAttributes.hasFnAttr(Kind: Attribute::NoImplicitFloat)) {
19446 unsigned Fast;
19447 if (Op.size() >= 16 &&
19448 (Op.isAligned(AlignCheck: Align(16)) ||
19449 (allowsMisalignedMemoryAccesses(VT: MVT::v2f64, 0, Alignment: Align(1),
19450 MachineMemOperand::MONone, Fast: &Fast) &&
19451 Fast))) {
19452 return MVT::v2f64;
19453 } else if (Op.size() >= 8 &&
19454 (Op.isAligned(AlignCheck: Align(8)) ||
19455 (allowsMisalignedMemoryAccesses(
19456 VT: MVT::f64, 0, Alignment: Align(1), MachineMemOperand::MONone, Fast: &Fast) &&
19457 Fast))) {
19458 return MVT::f64;
19459 }
19460 }
19461
19462 // Let the target-independent logic figure it out.
19463 return MVT::Other;
19464}
19465
19466// 64-bit integers are split into their high and low parts and held in two
19467// different registers, so the trunc is free since the low register can just
19468// be used.
19469bool ARMTargetLowering::isTruncateFree(Type *SrcTy, Type *DstTy) const {
19470 if (!SrcTy->isIntegerTy() || !DstTy->isIntegerTy())
19471 return false;
19472 unsigned SrcBits = SrcTy->getPrimitiveSizeInBits();
19473 unsigned DestBits = DstTy->getPrimitiveSizeInBits();
19474 return (SrcBits == 64 && DestBits == 32);
19475}
19476
19477bool ARMTargetLowering::isTruncateFree(EVT SrcVT, EVT DstVT) const {
19478 if (SrcVT.isVector() || DstVT.isVector() || !SrcVT.isInteger() ||
19479 !DstVT.isInteger())
19480 return false;
19481 unsigned SrcBits = SrcVT.getSizeInBits();
19482 unsigned DestBits = DstVT.getSizeInBits();
19483 return (SrcBits == 64 && DestBits == 32);
19484}
19485
19486bool ARMTargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
19487 if (Val.getOpcode() != ISD::LOAD)
19488 return false;
19489
19490 EVT VT1 = Val.getValueType();
19491 if (!VT1.isSimple() || !VT1.isInteger() ||
19492 !VT2.isSimple() || !VT2.isInteger())
19493 return false;
19494
19495 switch (VT1.getSimpleVT().SimpleTy) {
19496 default: break;
19497 case MVT::i1:
19498 case MVT::i8:
19499 case MVT::i16:
19500 // 8-bit and 16-bit loads implicitly zero-extend to 32-bits.
19501 return true;
19502 }
19503
19504 return false;
19505}
19506
19507bool ARMTargetLowering::isFNegFree(EVT VT) const {
19508 if (!VT.isSimple())
19509 return false;
19510
19511 // There are quite a few FP16 instructions (e.g. VNMLA, VNMLS, etc.) that
19512 // negate values directly (fneg is free). So, we don't want to let the DAG
19513 // combiner rewrite fneg into xors and some other instructions. For f16 and
19514 // FullFP16 argument passing, some bitcast nodes may be introduced,
19515 // triggering this DAG combine rewrite, so we are avoiding that with this.
19516 switch (VT.getSimpleVT().SimpleTy) {
19517 default: break;
19518 case MVT::f16:
19519 return Subtarget->hasFullFP16();
19520 }
19521
19522 return false;
19523}
19524
19525Type *ARMTargetLowering::shouldConvertSplatType(ShuffleVectorInst *SVI) const {
19526 if (!Subtarget->hasMVEIntegerOps())
19527 return nullptr;
19528 Type *SVIType = SVI->getType();
19529 Type *ScalarType = SVIType->getScalarType();
19530
19531 if (ScalarType->isFloatTy())
19532 return Type::getInt32Ty(C&: SVIType->getContext());
19533 if (ScalarType->isHalfTy())
19534 return Type::getInt16Ty(C&: SVIType->getContext());
19535 return nullptr;
19536}
19537
19538bool ARMTargetLowering::isVectorLoadExtDesirable(SDValue ExtVal) const {
19539 EVT VT = ExtVal.getValueType();
19540
19541 if (!isTypeLegal(VT))
19542 return false;
19543
19544 if (auto *Ld = dyn_cast<MaskedLoadSDNode>(Val: ExtVal.getOperand(i: 0))) {
19545 if (Ld->isExpandingLoad())
19546 return false;
19547 }
19548
19549 if (Subtarget->hasMVEIntegerOps())
19550 return true;
19551
19552 // Don't create a loadext if we can fold the extension into a wide/long
19553 // instruction.
19554 // If there's more than one user instruction, the loadext is desirable no
19555 // matter what. There can be two uses by the same instruction.
19556 if (ExtVal->use_empty() ||
19557 !ExtVal->user_begin()->isOnlyUserOf(N: ExtVal.getNode()))
19558 return true;
19559
19560 SDNode *U = *ExtVal->user_begin();
19561 if ((U->getOpcode() == ISD::ADD || U->getOpcode() == ISD::SUB ||
19562 U->getOpcode() == ISD::SHL || U->getOpcode() == ARMISD::VSHLIMM))
19563 return false;
19564
19565 return true;
19566}
19567
19568bool ARMTargetLowering::allowTruncateForTailCall(Type *Ty1, Type *Ty2) const {
19569 if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy())
19570 return false;
19571
19572 if (!isTypeLegal(VT: EVT::getEVT(Ty: Ty1)))
19573 return false;
19574
19575 assert(Ty1->getPrimitiveSizeInBits() <= 64 && "i128 is probably not a noop");
19576
19577 // Assuming the caller doesn't have a zeroext or signext return parameter,
19578 // truncation all the way down to i1 is valid.
19579 return true;
19580}
19581
19582/// isFMAFasterThanFMulAndFAdd - Return true if an FMA operation is faster
19583/// than a pair of fmul and fadd instructions. fmuladd intrinsics will be
19584/// expanded to FMAs when this method returns true, otherwise fmuladd is
19585/// expanded to fmul + fadd.
19586///
19587/// ARM supports both fused and unfused multiply-add operations; we already
19588/// lower a pair of fmul and fadd to the latter so it's not clear that there
19589/// would be a gain or that the gain would be worthwhile enough to risk
19590/// correctness bugs.
19591///
19592/// For MVE, we set this to true as it helps simplify the need for some
19593/// patterns (and we don't have the non-fused floating point instruction).
19594bool ARMTargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
19595 EVT VT) const {
19596 if (Subtarget->useSoftFloat())
19597 return false;
19598
19599 if (!VT.isSimple())
19600 return false;
19601
19602 switch (VT.getSimpleVT().SimpleTy) {
19603 case MVT::v4f32:
19604 case MVT::v8f16:
19605 return Subtarget->hasMVEFloatOps();
19606 case MVT::f16:
19607 return Subtarget->useFPVFMx16();
19608 case MVT::f32:
19609 return Subtarget->useFPVFMx();
19610 case MVT::f64:
19611 return Subtarget->useFPVFMx64();
19612 default:
19613 break;
19614 }
19615
19616 return false;
19617}
19618
19619static bool isLegalT1AddressImmediate(int64_t V, EVT VT) {
19620 if (V < 0)
19621 return false;
19622
19623 unsigned Scale = 1;
19624 switch (VT.getSimpleVT().SimpleTy) {
19625 case MVT::i1:
19626 case MVT::i8:
19627 // Scale == 1;
19628 break;
19629 case MVT::i16:
19630 // Scale == 2;
19631 Scale = 2;
19632 break;
19633 default:
19634 // On thumb1 we load most things (i32, i64, floats, etc) with a LDR
19635 // Scale == 4;
19636 Scale = 4;
19637 break;
19638 }
19639
19640 if ((V & (Scale - 1)) != 0)
19641 return false;
19642 return isUInt<5>(x: V / Scale);
19643}
19644
19645static bool isLegalT2AddressImmediate(int64_t V, EVT VT,
19646 const ARMSubtarget *Subtarget) {
19647 if (!VT.isInteger() && !VT.isFloatingPoint())
19648 return false;
19649 if (VT.isVector() && Subtarget->hasNEON())
19650 return false;
19651 if (VT.isVector() && VT.isFloatingPoint() && Subtarget->hasMVEIntegerOps() &&
19652 !Subtarget->hasMVEFloatOps())
19653 return false;
19654
19655 bool IsNeg = false;
19656 if (V < 0) {
19657 IsNeg = true;
19658 V = -V;
19659 }
19660
19661 unsigned NumBytes = std::max(a: (unsigned)VT.getSizeInBits() / 8, b: 1U);
19662
19663 // MVE: size * imm7
19664 if (VT.isVector() && Subtarget->hasMVEIntegerOps()) {
19665 switch (VT.getSimpleVT().getVectorElementType().SimpleTy) {
19666 case MVT::i32:
19667 case MVT::f32:
19668 return isShiftedUInt<7,2>(x: V);
19669 case MVT::i16:
19670 case MVT::f16:
19671 return isShiftedUInt<7,1>(x: V);
19672 case MVT::i8:
19673 return isUInt<7>(x: V);
19674 default:
19675 return false;
19676 }
19677 }
19678
19679 // half VLDR: 2 * imm8
19680 if (VT.isFloatingPoint() && NumBytes == 2 && Subtarget->hasFPRegs16())
19681 return isShiftedUInt<8, 1>(x: V);
19682 // VLDR and LDRD: 4 * imm8
19683 if ((VT.isFloatingPoint() && Subtarget->hasVFP2Base()) || NumBytes == 8)
19684 return isShiftedUInt<8, 2>(x: V);
19685
19686 if (NumBytes == 1 || NumBytes == 2 || NumBytes == 4) {
19687 // + imm12 or - imm8
19688 if (IsNeg)
19689 return isUInt<8>(x: V);
19690 return isUInt<12>(x: V);
19691 }
19692
19693 return false;
19694}
19695
19696/// isLegalAddressImmediate - Return true if the integer value can be used
19697/// as the offset of the target addressing mode for load / store of the
19698/// given type.
19699static bool isLegalAddressImmediate(int64_t V, EVT VT,
19700 const ARMSubtarget *Subtarget) {
19701 if (V == 0)
19702 return true;
19703
19704 if (!VT.isSimple())
19705 return false;
19706
19707 if (Subtarget->isThumb1Only())
19708 return isLegalT1AddressImmediate(V, VT);
19709 else if (Subtarget->isThumb2())
19710 return isLegalT2AddressImmediate(V, VT, Subtarget);
19711
19712 // ARM mode.
19713 if (V < 0)
19714 V = - V;
19715 switch (VT.getSimpleVT().SimpleTy) {
19716 default: return false;
19717 case MVT::i1:
19718 case MVT::i8:
19719 case MVT::i32:
19720 // +- imm12
19721 return isUInt<12>(x: V);
19722 case MVT::i16:
19723 // +- imm8
19724 return isUInt<8>(x: V);
19725 case MVT::f32:
19726 case MVT::f64:
19727 if (!Subtarget->hasVFP2Base()) // FIXME: NEON?
19728 return false;
19729 return isShiftedUInt<8, 2>(x: V);
19730 }
19731}
19732
19733bool ARMTargetLowering::isLegalT2ScaledAddressingMode(const AddrMode &AM,
19734 EVT VT) const {
19735 int Scale = AM.Scale;
19736 if (Scale < 0)
19737 return false;
19738
19739 switch (VT.getSimpleVT().SimpleTy) {
19740 default: return false;
19741 case MVT::i1:
19742 case MVT::i8:
19743 case MVT::i16:
19744 case MVT::i32:
19745 if (Scale == 1)
19746 return true;
19747 // r + r << imm
19748 Scale = Scale & ~1;
19749 return Scale == 2 || Scale == 4 || Scale == 8;
19750 case MVT::i64:
19751 // FIXME: What are we trying to model here? ldrd doesn't have an r + r
19752 // version in Thumb mode.
19753 // r + r
19754 if (Scale == 1)
19755 return true;
19756 // r * 2 (this can be lowered to r + r).
19757 if (!AM.HasBaseReg && Scale == 2)
19758 return true;
19759 return false;
19760 case MVT::isVoid:
19761 // Note, we allow "void" uses (basically, uses that aren't loads or
19762 // stores), because arm allows folding a scale into many arithmetic
19763 // operations. This should be made more precise and revisited later.
19764
19765 // Allow r << imm, but the imm has to be a multiple of two.
19766 if (Scale & 1) return false;
19767 return isPowerOf2_32(Value: Scale);
19768 }
19769}
19770
19771bool ARMTargetLowering::isLegalT1ScaledAddressingMode(const AddrMode &AM,
19772 EVT VT) const {
19773 const int Scale = AM.Scale;
19774
19775 // Negative scales are not supported in Thumb1.
19776 if (Scale < 0)
19777 return false;
19778
19779 // Thumb1 addressing modes do not support register scaling excepting the
19780 // following cases:
19781 // 1. Scale == 1 means no scaling.
19782 // 2. Scale == 2 this can be lowered to r + r if there is no base register.
19783 return (Scale == 1) || (!AM.HasBaseReg && Scale == 2);
19784}
19785
19786/// isLegalAddressingMode - Return true if the addressing mode represented
19787/// by AM is legal for this target, for a load/store of the specified type.
19788bool ARMTargetLowering::isLegalAddressingMode(const DataLayout &DL,
19789 const AddrMode &AM, Type *Ty,
19790 unsigned AS, Instruction *I) const {
19791 EVT VT = getValueType(DL, Ty, AllowUnknown: true);
19792 if (!isLegalAddressImmediate(V: AM.BaseOffs, VT, Subtarget))
19793 return false;
19794
19795 // Can never fold addr of global into load/store.
19796 if (AM.BaseGV)
19797 return false;
19798
19799 switch (AM.Scale) {
19800 case 0: // no scale reg, must be "r+i" or "r", or "i".
19801 break;
19802 default:
19803 // ARM doesn't support any R+R*scale+imm addr modes.
19804 if (AM.BaseOffs)
19805 return false;
19806
19807 if (!VT.isSimple())
19808 return false;
19809
19810 if (Subtarget->isThumb1Only())
19811 return isLegalT1ScaledAddressingMode(AM, VT);
19812
19813 if (Subtarget->isThumb2())
19814 return isLegalT2ScaledAddressingMode(AM, VT);
19815
19816 int Scale = AM.Scale;
19817 switch (VT.getSimpleVT().SimpleTy) {
19818 default: return false;
19819 case MVT::i1:
19820 case MVT::i8:
19821 case MVT::i32:
19822 if (Scale < 0) Scale = -Scale;
19823 if (Scale == 1)
19824 return true;
19825 // r + r << imm
19826 return isPowerOf2_32(Value: Scale & ~1);
19827 case MVT::i16:
19828 case MVT::i64:
19829 // r +/- r
19830 if (Scale == 1 || (AM.HasBaseReg && Scale == -1))
19831 return true;
19832 // r * 2 (this can be lowered to r + r).
19833 if (!AM.HasBaseReg && Scale == 2)
19834 return true;
19835 return false;
19836
19837 case MVT::isVoid:
19838 // Note, we allow "void" uses (basically, uses that aren't loads or
19839 // stores), because arm allows folding a scale into many arithmetic
19840 // operations. This should be made more precise and revisited later.
19841
19842 // Allow r << imm, but the imm has to be a multiple of two.
19843 if (Scale & 1) return false;
19844 return isPowerOf2_32(Value: Scale);
19845 }
19846 }
19847 return true;
19848}
19849
19850/// isLegalICmpImmediate - Return true if the specified immediate is legal
19851/// icmp immediate, that is the target has icmp instructions which can compare
19852/// a register against the immediate without having to materialize the
19853/// immediate into a register.
19854bool ARMTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
19855 // Thumb2 and ARM modes can use cmn for negative immediates.
19856 if (!Subtarget->isThumb())
19857 return ARM_AM::getSOImmVal(Arg: (uint32_t)Imm) != -1 ||
19858 ARM_AM::getSOImmVal(Arg: -(uint32_t)Imm) != -1;
19859 if (Subtarget->isThumb2())
19860 return ARM_AM::getT2SOImmVal(Arg: (uint32_t)Imm) != -1 ||
19861 ARM_AM::getT2SOImmVal(Arg: -(uint32_t)Imm) != -1;
19862 // Thumb1 doesn't have cmn, and only 8-bit immediates.
19863 return Imm >= 0 && Imm <= 255;
19864}
19865
19866/// isLegalAddImmediate - Return true if the specified immediate is a legal add
19867/// *or sub* immediate, that is the target has add or sub instructions which can
19868/// add a register with the immediate without having to materialize the
19869/// immediate into a register.
19870bool ARMTargetLowering::isLegalAddImmediate(int64_t Imm) const {
19871 // Same encoding for add/sub, just flip the sign.
19872 uint64_t AbsImm = AbsoluteValue(X: Imm);
19873 if (!Subtarget->isThumb())
19874 return ARM_AM::getSOImmVal(Arg: AbsImm) != -1;
19875 if (Subtarget->isThumb2())
19876 return ARM_AM::getT2SOImmVal(Arg: AbsImm) != -1;
19877 // Thumb1 only has 8-bit unsigned immediate.
19878 return AbsImm <= 255;
19879}
19880
19881// Return false to prevent folding
19882// (mul (add r, c0), c1) -> (add (mul r, c1), c0*c1) in DAGCombine,
19883// if the folding leads to worse code.
19884bool ARMTargetLowering::isMulAddWithConstProfitable(SDValue AddNode,
19885 SDValue ConstNode) const {
19886 // Let the DAGCombiner decide for vector types and large types.
19887 const EVT VT = AddNode.getValueType();
19888 if (VT.isVector() || VT.getScalarSizeInBits() > 32)
19889 return true;
19890
19891 // It is worse if c0 is legal add immediate, while c1*c0 is not
19892 // and has to be composed by at least two instructions.
19893 const ConstantSDNode *C0Node = cast<ConstantSDNode>(Val: AddNode.getOperand(i: 1));
19894 const ConstantSDNode *C1Node = cast<ConstantSDNode>(Val&: ConstNode);
19895 const int64_t C0 = C0Node->getSExtValue();
19896 APInt CA = C0Node->getAPIntValue() * C1Node->getAPIntValue();
19897 if (!isLegalAddImmediate(Imm: C0) || isLegalAddImmediate(Imm: CA.getSExtValue()))
19898 return true;
19899 if (ConstantMaterializationCost(Val: (unsigned)CA.getZExtValue(), Subtarget) > 1)
19900 return false;
19901
19902 // Default to true and let the DAGCombiner decide.
19903 return true;
19904}
19905
19906static bool getARMIndexedAddressParts(SDNode *Ptr, EVT VT,
19907 bool isSEXTLoad, SDValue &Base,
19908 SDValue &Offset, bool &isInc,
19909 SelectionDAG &DAG) {
19910 if (Ptr->getOpcode() != ISD::ADD && Ptr->getOpcode() != ISD::SUB)
19911 return false;
19912
19913 if (VT == MVT::i16 || ((VT == MVT::i8 || VT == MVT::i1) && isSEXTLoad)) {
19914 // AddressingMode 3
19915 Base = Ptr->getOperand(Num: 0);
19916 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Val: Ptr->getOperand(Num: 1))) {
19917 int RHSC = (int)RHS->getZExtValue();
19918 if (RHSC < 0 && RHSC > -256) {
19919 assert(Ptr->getOpcode() == ISD::ADD);
19920 isInc = false;
19921 Offset = DAG.getConstant(Val: -RHSC, DL: SDLoc(Ptr), VT: RHS->getValueType(ResNo: 0));
19922 return true;
19923 }
19924 }
19925 isInc = (Ptr->getOpcode() == ISD::ADD);
19926 Offset = Ptr->getOperand(Num: 1);
19927 return true;
19928 } else if (VT == MVT::i32 || VT == MVT::i8 || VT == MVT::i1) {
19929 // AddressingMode 2
19930 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Val: Ptr->getOperand(Num: 1))) {
19931 int RHSC = (int)RHS->getZExtValue();
19932 if (RHSC < 0 && RHSC > -0x1000) {
19933 assert(Ptr->getOpcode() == ISD::ADD);
19934 isInc = false;
19935 Offset = DAG.getConstant(Val: -RHSC, DL: SDLoc(Ptr), VT: RHS->getValueType(ResNo: 0));
19936 Base = Ptr->getOperand(Num: 0);
19937 return true;
19938 }
19939 }
19940
19941 if (Ptr->getOpcode() == ISD::ADD) {
19942 isInc = true;
19943 ARM_AM::ShiftOpc ShOpcVal=
19944 ARM_AM::getShiftOpcForNode(Opcode: Ptr->getOperand(Num: 0).getOpcode());
19945 if (ShOpcVal != ARM_AM::no_shift) {
19946 Base = Ptr->getOperand(Num: 1);
19947 Offset = Ptr->getOperand(Num: 0);
19948 } else {
19949 Base = Ptr->getOperand(Num: 0);
19950 Offset = Ptr->getOperand(Num: 1);
19951 }
19952 return true;
19953 }
19954
19955 isInc = (Ptr->getOpcode() == ISD::ADD);
19956 Base = Ptr->getOperand(Num: 0);
19957 Offset = Ptr->getOperand(Num: 1);
19958 return true;
19959 }
19960
19961 // FIXME: Use VLDM / VSTM to emulate indexed FP load / store.
19962 return false;
19963}
19964
19965static bool getT2IndexedAddressParts(SDNode *Ptr, EVT VT,
19966 bool isSEXTLoad, SDValue &Base,
19967 SDValue &Offset, bool &isInc,
19968 SelectionDAG &DAG) {
19969 if (Ptr->getOpcode() != ISD::ADD && Ptr->getOpcode() != ISD::SUB)
19970 return false;
19971
19972 Base = Ptr->getOperand(Num: 0);
19973 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Val: Ptr->getOperand(Num: 1))) {
19974 int RHSC = (int)RHS->getZExtValue();
19975 if (RHSC < 0 && RHSC > -0x100) { // 8 bits.
19976 assert(Ptr->getOpcode() == ISD::ADD);
19977 isInc = false;
19978 Offset = DAG.getConstant(Val: -RHSC, DL: SDLoc(Ptr), VT: RHS->getValueType(ResNo: 0));
19979 return true;
19980 } else if (RHSC > 0 && RHSC < 0x100) { // 8 bit, no zero.
19981 isInc = Ptr->getOpcode() == ISD::ADD;
19982 Offset = DAG.getConstant(Val: RHSC, DL: SDLoc(Ptr), VT: RHS->getValueType(ResNo: 0));
19983 return true;
19984 }
19985 }
19986
19987 return false;
19988}
19989
19990static bool getMVEIndexedAddressParts(SDNode *Ptr, EVT VT, Align Alignment,
19991 bool isSEXTLoad, bool IsMasked, bool isLE,
19992 SDValue &Base, SDValue &Offset,
19993 bool &isInc, SelectionDAG &DAG) {
19994 if (Ptr->getOpcode() != ISD::ADD && Ptr->getOpcode() != ISD::SUB)
19995 return false;
19996 if (!isa<ConstantSDNode>(Val: Ptr->getOperand(Num: 1)))
19997 return false;
19998
19999 // We allow LE non-masked loads to change the type (for example use a vldrb.8
20000 // as opposed to a vldrw.32). This can allow extra addressing modes or
20001 // alignments for what is otherwise an equivalent instruction.
20002 bool CanChangeType = isLE && !IsMasked;
20003
20004 ConstantSDNode *RHS = cast<ConstantSDNode>(Val: Ptr->getOperand(Num: 1));
20005 int RHSC = (int)RHS->getZExtValue();
20006
20007 auto IsInRange = [&](int RHSC, int Limit, int Scale) {
20008 if (RHSC < 0 && RHSC > -Limit * Scale && RHSC % Scale == 0) {
20009 assert(Ptr->getOpcode() == ISD::ADD);
20010 isInc = false;
20011 Offset = DAG.getConstant(Val: -RHSC, DL: SDLoc(Ptr), VT: RHS->getValueType(ResNo: 0));
20012 return true;
20013 } else if (RHSC > 0 && RHSC < Limit * Scale && RHSC % Scale == 0) {
20014 isInc = Ptr->getOpcode() == ISD::ADD;
20015 Offset = DAG.getConstant(Val: RHSC, DL: SDLoc(Ptr), VT: RHS->getValueType(ResNo: 0));
20016 return true;
20017 }
20018 return false;
20019 };
20020
20021 // Try to find a matching instruction based on s/zext, Alignment, Offset and
20022 // (in BE/masked) type.
20023 Base = Ptr->getOperand(Num: 0);
20024 if (VT == MVT::v4i16) {
20025 if (Alignment >= 2 && IsInRange(RHSC, 0x80, 2))
20026 return true;
20027 } else if (VT == MVT::v4i8 || VT == MVT::v8i8) {
20028 if (IsInRange(RHSC, 0x80, 1))
20029 return true;
20030 } else if (Alignment >= 4 &&
20031 (CanChangeType || VT == MVT::v4i32 || VT == MVT::v4f32) &&
20032 IsInRange(RHSC, 0x80, 4))
20033 return true;
20034 else if (Alignment >= 2 &&
20035 (CanChangeType || VT == MVT::v8i16 || VT == MVT::v8f16) &&
20036 IsInRange(RHSC, 0x80, 2))
20037 return true;
20038 else if ((CanChangeType || VT == MVT::v16i8) && IsInRange(RHSC, 0x80, 1))
20039 return true;
20040 return false;
20041}
20042
20043/// getPreIndexedAddressParts - returns true by value, base pointer and
20044/// offset pointer and addressing mode by reference if the node's address
20045/// can be legally represented as pre-indexed load / store address.
20046bool
20047ARMTargetLowering::getPreIndexedAddressParts(SDNode *N, SDValue &Base,
20048 SDValue &Offset,
20049 ISD::MemIndexedMode &AM,
20050 SelectionDAG &DAG) const {
20051 if (Subtarget->isThumb1Only())
20052 return false;
20053
20054 EVT VT;
20055 SDValue Ptr;
20056 Align Alignment;
20057 unsigned AS = 0;
20058 bool isSEXTLoad = false;
20059 bool IsMasked = false;
20060 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Val: N)) {
20061 Ptr = LD->getBasePtr();
20062 VT = LD->getMemoryVT();
20063 Alignment = LD->getAlign();
20064 AS = LD->getAddressSpace();
20065 isSEXTLoad = LD->getExtensionType() == ISD::SEXTLOAD;
20066 } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(Val: N)) {
20067 Ptr = ST->getBasePtr();
20068 VT = ST->getMemoryVT();
20069 Alignment = ST->getAlign();
20070 AS = ST->getAddressSpace();
20071 } else if (MaskedLoadSDNode *LD = dyn_cast<MaskedLoadSDNode>(Val: N)) {
20072 Ptr = LD->getBasePtr();
20073 VT = LD->getMemoryVT();
20074 Alignment = LD->getAlign();
20075 AS = LD->getAddressSpace();
20076 isSEXTLoad = LD->getExtensionType() == ISD::SEXTLOAD;
20077 IsMasked = true;
20078 } else if (MaskedStoreSDNode *ST = dyn_cast<MaskedStoreSDNode>(Val: N)) {
20079 Ptr = ST->getBasePtr();
20080 VT = ST->getMemoryVT();
20081 Alignment = ST->getAlign();
20082 AS = ST->getAddressSpace();
20083 IsMasked = true;
20084 } else
20085 return false;
20086
20087 unsigned Fast = 0;
20088 if (!allowsMisalignedMemoryAccesses(VT, AS, Alignment,
20089 MachineMemOperand::MONone, Fast: &Fast)) {
20090 // Only generate post-increment or pre-increment forms when a real
20091 // hardware instruction exists for them. Do not emit postinc/preinc
20092 // if the operation will end up as a libcall.
20093 return false;
20094 }
20095
20096 bool isInc;
20097 bool isLegal = false;
20098 if (VT.isVector())
20099 isLegal = Subtarget->hasMVEIntegerOps() &&
20100 getMVEIndexedAddressParts(
20101 Ptr: Ptr.getNode(), VT, Alignment, isSEXTLoad, IsMasked,
20102 isLE: Subtarget->isLittle(), Base, Offset, isInc, DAG);
20103 else {
20104 if (Subtarget->isThumb2())
20105 isLegal = getT2IndexedAddressParts(Ptr: Ptr.getNode(), VT, isSEXTLoad, Base,
20106 Offset, isInc, DAG);
20107 else
20108 isLegal = getARMIndexedAddressParts(Ptr: Ptr.getNode(), VT, isSEXTLoad, Base,
20109 Offset, isInc, DAG);
20110 }
20111 if (!isLegal)
20112 return false;
20113
20114 AM = isInc ? ISD::PRE_INC : ISD::PRE_DEC;
20115 return true;
20116}
20117
20118/// getPostIndexedAddressParts - returns true by value, base pointer and
20119/// offset pointer and addressing mode by reference if this node can be
20120/// combined with a load / store to form a post-indexed load / store.
20121bool ARMTargetLowering::getPostIndexedAddressParts(SDNode *N, SDNode *Op,
20122 SDValue &Base,
20123 SDValue &Offset,
20124 ISD::MemIndexedMode &AM,
20125 SelectionDAG &DAG) const {
20126 EVT VT;
20127 SDValue Ptr;
20128 Align Alignment;
20129 bool isSEXTLoad = false, isNonExt;
20130 bool IsMasked = false;
20131 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Val: N)) {
20132 VT = LD->getMemoryVT();
20133 Ptr = LD->getBasePtr();
20134 Alignment = LD->getAlign();
20135 isSEXTLoad = LD->getExtensionType() == ISD::SEXTLOAD;
20136 isNonExt = LD->getExtensionType() == ISD::NON_EXTLOAD;
20137 } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(Val: N)) {
20138 VT = ST->getMemoryVT();
20139 Ptr = ST->getBasePtr();
20140 Alignment = ST->getAlign();
20141 isNonExt = !ST->isTruncatingStore();
20142 } else if (MaskedLoadSDNode *LD = dyn_cast<MaskedLoadSDNode>(Val: N)) {
20143 VT = LD->getMemoryVT();
20144 Ptr = LD->getBasePtr();
20145 Alignment = LD->getAlign();
20146 isSEXTLoad = LD->getExtensionType() == ISD::SEXTLOAD;
20147 isNonExt = LD->getExtensionType() == ISD::NON_EXTLOAD;
20148 IsMasked = true;
20149 } else if (MaskedStoreSDNode *ST = dyn_cast<MaskedStoreSDNode>(Val: N)) {
20150 VT = ST->getMemoryVT();
20151 Ptr = ST->getBasePtr();
20152 Alignment = ST->getAlign();
20153 isNonExt = !ST->isTruncatingStore();
20154 IsMasked = true;
20155 } else
20156 return false;
20157
20158 if (Subtarget->isThumb1Only()) {
20159 // Thumb-1 can do a limited post-inc load or store as an updating LDM. It
20160 // must be non-extending/truncating, i32, with an offset of 4.
20161 assert(Op->getValueType(0) == MVT::i32 && "Non-i32 post-inc op?!");
20162 if (Op->getOpcode() != ISD::ADD || !isNonExt)
20163 return false;
20164 auto *RHS = dyn_cast<ConstantSDNode>(Val: Op->getOperand(Num: 1));
20165 if (!RHS || RHS->getZExtValue() != 4)
20166 return false;
20167 if (Alignment < Align(4))
20168 return false;
20169
20170 Offset = Op->getOperand(Num: 1);
20171 Base = Op->getOperand(Num: 0);
20172 AM = ISD::POST_INC;
20173 return true;
20174 }
20175
20176 bool isInc;
20177 bool isLegal = false;
20178 if (VT.isVector())
20179 isLegal = Subtarget->hasMVEIntegerOps() &&
20180 getMVEIndexedAddressParts(Ptr: Op, VT, Alignment, isSEXTLoad, IsMasked,
20181 isLE: Subtarget->isLittle(), Base, Offset,
20182 isInc, DAG);
20183 else {
20184 if (Subtarget->isThumb2())
20185 isLegal = getT2IndexedAddressParts(Ptr: Op, VT, isSEXTLoad, Base, Offset,
20186 isInc, DAG);
20187 else
20188 isLegal = getARMIndexedAddressParts(Ptr: Op, VT, isSEXTLoad, Base, Offset,
20189 isInc, DAG);
20190 }
20191 if (!isLegal)
20192 return false;
20193
20194 if (Ptr != Base) {
20195 // Swap base ptr and offset to catch more post-index load / store when
20196 // it's legal. In Thumb2 mode, offset must be an immediate.
20197 if (Ptr == Offset && Op->getOpcode() == ISD::ADD &&
20198 !Subtarget->isThumb2())
20199 std::swap(a&: Base, b&: Offset);
20200
20201 // Post-indexed load / store update the base pointer.
20202 if (Ptr != Base)
20203 return false;
20204 }
20205
20206 AM = isInc ? ISD::POST_INC : ISD::POST_DEC;
20207 return true;
20208}
20209
20210void ARMTargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
20211 KnownBits &Known,
20212 const APInt &DemandedElts,
20213 const SelectionDAG &DAG,
20214 unsigned Depth) const {
20215 unsigned BitWidth = Known.getBitWidth();
20216 Known.resetAll();
20217 switch (Op.getOpcode()) {
20218 default: break;
20219 case ARMISD::ADDC:
20220 case ARMISD::ADDE:
20221 case ARMISD::SUBC:
20222 case ARMISD::SUBE:
20223 // Special cases when we convert a carry to a boolean.
20224 if (Op.getResNo() == 0) {
20225 SDValue LHS = Op.getOperand(i: 0);
20226 SDValue RHS = Op.getOperand(i: 1);
20227 // (ADDE 0, 0, C) will give us a single bit.
20228 if (Op->getOpcode() == ARMISD::ADDE && isNullConstant(V: LHS) &&
20229 isNullConstant(V: RHS)) {
20230 Known.Zero |= APInt::getHighBitsSet(numBits: BitWidth, hiBitsSet: BitWidth - 1);
20231 return;
20232 }
20233 }
20234 break;
20235 case ARMISD::CMOV: {
20236 // Bits are known zero/one if known on the LHS and RHS.
20237 Known = DAG.computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth+1);
20238 if (Known.isUnknown())
20239 return;
20240
20241 KnownBits KnownRHS = DAG.computeKnownBits(Op: Op.getOperand(i: 1), Depth: Depth+1);
20242 Known = Known.intersectWith(RHS: KnownRHS);
20243 return;
20244 }
20245 case ISD::INTRINSIC_W_CHAIN: {
20246 Intrinsic::ID IntID =
20247 static_cast<Intrinsic::ID>(Op->getConstantOperandVal(Num: 1));
20248 switch (IntID) {
20249 default: return;
20250 case Intrinsic::arm_ldaex:
20251 case Intrinsic::arm_ldrex: {
20252 EVT VT = cast<MemIntrinsicSDNode>(Val: Op)->getMemoryVT();
20253 unsigned MemBits = VT.getScalarSizeInBits();
20254 Known.Zero |= APInt::getHighBitsSet(numBits: BitWidth, hiBitsSet: BitWidth - MemBits);
20255 return;
20256 }
20257 }
20258 }
20259 case ARMISD::BFI: {
20260 // Conservatively, we can recurse down the first operand
20261 // and just mask out all affected bits.
20262 Known = DAG.computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
20263
20264 // The operand to BFI is already a mask suitable for removing the bits it
20265 // sets.
20266 const APInt &Mask = Op.getConstantOperandAPInt(i: 2);
20267 Known.Zero &= Mask;
20268 Known.One &= Mask;
20269 return;
20270 }
20271 case ARMISD::VGETLANEs:
20272 case ARMISD::VGETLANEu: {
20273 const SDValue &SrcSV = Op.getOperand(i: 0);
20274 EVT VecVT = SrcSV.getValueType();
20275 assert(VecVT.isVector() && "VGETLANE expected a vector type");
20276 const unsigned NumSrcElts = VecVT.getVectorNumElements();
20277 ConstantSDNode *Pos = cast<ConstantSDNode>(Val: Op.getOperand(i: 1).getNode());
20278 assert(Pos->getAPIntValue().ult(NumSrcElts) &&
20279 "VGETLANE index out of bounds");
20280 unsigned Idx = Pos->getZExtValue();
20281 APInt DemandedElt = APInt::getOneBitSet(numBits: NumSrcElts, BitNo: Idx);
20282 Known = DAG.computeKnownBits(Op: SrcSV, DemandedElts: DemandedElt, Depth: Depth + 1);
20283
20284 EVT VT = Op.getValueType();
20285 const unsigned DstSz = VT.getScalarSizeInBits();
20286 const unsigned SrcSz = VecVT.getVectorElementType().getSizeInBits();
20287 (void)SrcSz;
20288 assert(SrcSz == Known.getBitWidth());
20289 assert(DstSz > SrcSz);
20290 if (Op.getOpcode() == ARMISD::VGETLANEs)
20291 Known = Known.sext(BitWidth: DstSz);
20292 else {
20293 Known = Known.zext(BitWidth: DstSz);
20294 }
20295 assert(DstSz == Known.getBitWidth());
20296 break;
20297 }
20298 case ARMISD::VMOVrh: {
20299 KnownBits KnownOp = DAG.computeKnownBits(Op: Op->getOperand(Num: 0), Depth: Depth + 1);
20300 assert(KnownOp.getBitWidth() == 16);
20301 Known = KnownOp.zext(BitWidth: 32);
20302 break;
20303 }
20304 case ARMISD::CSINC:
20305 case ARMISD::CSINV:
20306 case ARMISD::CSNEG: {
20307 KnownBits KnownOp0 = DAG.computeKnownBits(Op: Op->getOperand(Num: 0), Depth: Depth + 1);
20308 KnownBits KnownOp1 = DAG.computeKnownBits(Op: Op->getOperand(Num: 1), Depth: Depth + 1);
20309
20310 // The result is either:
20311 // CSINC: KnownOp0 or KnownOp1 + 1
20312 // CSINV: KnownOp0 or ~KnownOp1
20313 // CSNEG: KnownOp0 or KnownOp1 * -1
20314 if (Op.getOpcode() == ARMISD::CSINC)
20315 KnownOp1 =
20316 KnownBits::add(LHS: KnownOp1, RHS: KnownBits::makeConstant(C: APInt(32, 1)));
20317 else if (Op.getOpcode() == ARMISD::CSINV)
20318 std::swap(a&: KnownOp1.Zero, b&: KnownOp1.One);
20319 else if (Op.getOpcode() == ARMISD::CSNEG)
20320 KnownOp1 = KnownBits::mul(LHS: KnownOp1,
20321 RHS: KnownBits::makeConstant(C: APInt::getAllOnes(numBits: 32)));
20322
20323 Known = KnownOp0.intersectWith(RHS: KnownOp1);
20324 break;
20325 }
20326 case ARMISD::VORRIMM:
20327 case ARMISD::VBICIMM: {
20328 unsigned Encoded = Op.getConstantOperandVal(i: 1);
20329 unsigned DecEltBits = 0;
20330 uint64_t DecodedVal = ARM_AM::decodeVMOVModImm(ModImm: Encoded, EltBits&: DecEltBits);
20331
20332 unsigned EltBits = Op.getScalarValueSizeInBits();
20333 if (EltBits != DecEltBits) {
20334 // Be conservative: only update Known when EltBits == DecEltBits.
20335 // This is believed to always be true for VORRIMM/VBICIMM today, but if
20336 // that changes in the future, doing nothing here is safer than risking
20337 // subtle bugs.
20338 break;
20339 }
20340
20341 KnownBits KnownLHS = DAG.computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
20342 bool IsVORR = Op.getOpcode() == ARMISD::VORRIMM;
20343 APInt Imm(DecEltBits, DecodedVal);
20344
20345 Known.One = IsVORR ? (KnownLHS.One | Imm) : (KnownLHS.One & ~Imm);
20346 Known.Zero = IsVORR ? (KnownLHS.Zero & ~Imm) : (KnownLHS.Zero | Imm);
20347 break;
20348 }
20349 }
20350}
20351
20352static bool isLegalLogicalImmediate(unsigned Imm,
20353 const ARMSubtarget *Subtarget) {
20354 if (!Subtarget->isThumb())
20355 return ARM_AM::getSOImmVal(Arg: Imm) != -1;
20356 if (Subtarget->isThumb2())
20357 return ARM_AM::getT2SOImmVal(Arg: Imm) != -1;
20358 // Thumb1 only has 8-bit unsigned immediate.
20359 return Imm <= 255;
20360}
20361
20362/// Refine i32 AND/OR/XOR with a constant RHS using demanded bits: replace the
20363/// immediate with an equivalent constant that ARM/Thumb can encode as a
20364/// logical immediate (or that selects better lowering), without changing the
20365/// computed result on those demanded bits.
20366static bool optimizeLogicalImm(SDValue Op, unsigned Imm,
20367 const APInt &DemandedBits,
20368 const ARMSubtarget *Subtarget,
20369 TargetLowering::TargetLoweringOpt &TLO) {
20370
20371 if (Imm == 0 || Imm == ~0U)
20372 return false;
20373
20374 unsigned Opc = Op.getOpcode();
20375 unsigned Demanded = DemandedBits.getZExtValue();
20376 EVT VT = Op.getValueType();
20377
20378 unsigned ShrunkImm = Imm & Demanded;
20379 unsigned ExpandedImm = Imm | ~Demanded;
20380
20381 auto IsLegalImm = [ShrunkImm, ExpandedImm](unsigned CandidateImm) -> bool {
20382 return (ShrunkImm & CandidateImm) == ShrunkImm &&
20383 (~ExpandedImm & CandidateImm) == 0;
20384 };
20385 auto UseImm = [Imm, Opc, Op, VT, &TLO](unsigned NewImm) -> bool {
20386 if (NewImm == Imm)
20387 return true;
20388 SDLoc DL(Op);
20389 SDValue NewC = TLO.DAG.getConstant(Val: NewImm, DL, VT);
20390 SDValue NewOp =
20391 TLO.DAG.getNode(Opcode: Opc, DL, VT, N1: Op.getOperand(i: 0), N2: NewC, Flags: Op->getFlags());
20392 return TLO.CombineTo(O: Op, N: NewOp);
20393 };
20394
20395 // Shrunk immediate is 0: AND becomes zero; OR/XOR with 0 leaves the other
20396 // operand (still valid on demanded bits).
20397 if (ShrunkImm == 0) {
20398 ++NumOptimizedImms;
20399 return UseImm(ShrunkImm);
20400 }
20401
20402 // If the immediate is all ones: for AND this removes the operation; for
20403 // OR/XOR it remains a transform valid on demanded bits. (Target-independent
20404 // shrink may not fold this, so keep it to avoid obscure combine loops.)
20405 if (ExpandedImm == ~0U) {
20406 ++NumOptimizedImms;
20407 return UseImm(ExpandedImm);
20408 }
20409
20410 // Thumb1: prefer 0xFF / 0xFFFF when they fit the demanded-bit envelope so
20411 // lowering can match uxtb / uxth (AND immediates only; OR/XOR do not use
20412 // that). Run this before strict ShrunkImm: a tight 8-bit ShrunkImm can be
20413 // legal while 0xFF still matches the envelope and yields better isel (uxtb).
20414 if (Opc == ISD::AND && Subtarget->hasV6Ops()) {
20415 if (IsLegalImm(0xFF)) {
20416 ++NumOptimizedImms;
20417 return UseImm(0xFF);
20418 }
20419
20420 if (IsLegalImm(0xFFFF)) {
20421 ++NumOptimizedImms;
20422 return UseImm(0xFFFF);
20423 }
20424 }
20425
20426 // Don't optimize if it is legal.
20427 if (isLegalLogicalImmediate(Imm, Subtarget))
20428 return false;
20429
20430 // FIXME: Check for BIC being legal causes infinite loop due to target
20431 // independent DAG combine undoing this.
20432
20433 // Prefer strict shrink when ShrunkImm encodes for this target, before
20434 // complement expansion.
20435 if (isLegalLogicalImmediate(Imm: ShrunkImm, Subtarget)) {
20436 ++NumOptimizedImms;
20437 return UseImm(ShrunkImm);
20438 }
20439
20440 // Complement expansion: if all undemanded bits are already one, ExpandedImm
20441 // is Imm with every non-demanded bit set. When (~ExpandedImm) < 256, the
20442 // complement fits in an 8-bit unsigned value, i.e. bits 8–31 of ExpandedImm
20443 // are all ones; only the low byte may differ from ~0. Use that expanded
20444 // constant so isel sees a mask shape that fits logical-immediate patterns.
20445 if ((~ExpandedImm) < 256) {
20446 ++NumOptimizedImms;
20447 return UseImm(ExpandedImm);
20448 }
20449
20450 // FIXME: The check for v6 is because this interferes with some ubfx
20451 // optimizations.
20452 if (Opc == ISD::AND && isLegalLogicalImmediate(Imm: ~ExpandedImm, Subtarget) &&
20453 !Subtarget->hasV6Ops()) {
20454 ++NumOptimizedImms;
20455 return UseImm(ExpandedImm);
20456 }
20457
20458 // Potential improvements:
20459 //
20460 // We could try to recognize lsls+lsrs or lsrs+lsls pairs here.
20461 // We could try to prefer Thumb1 immediates which can be lowered to a
20462 // two-instruction sequence.
20463
20464 return false;
20465}
20466
20467bool ARMTargetLowering::targetShrinkDemandedConstant(
20468 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
20469 TargetLoweringOpt &TLO) const {
20470 // Delay this optimization to as late as possible.
20471 if (!TLO.LegalOps)
20472 return false;
20473
20474 EVT VT = Op.getValueType();
20475
20476 // Ignore vectors.
20477 if (VT.isVector())
20478 return false;
20479
20480 unsigned Size = VT.getSizeInBits();
20481
20482 if (Size != 32)
20483 return false;
20484
20485 // Exit early if we demand all bits.
20486 if (DemandedBits.isAllOnes())
20487 return false;
20488
20489 switch (Op.getOpcode()) {
20490 default:
20491 return false;
20492 case ISD::AND:
20493 case ISD::OR:
20494 case ISD::XOR:
20495 break;
20496 }
20497 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1));
20498 if (!C)
20499 return false;
20500 unsigned Imm = C->getZExtValue();
20501 return optimizeLogicalImm(Op, Imm, DemandedBits, Subtarget, TLO);
20502}
20503
20504bool ARMTargetLowering::SimplifyDemandedBitsForTargetNode(
20505 SDValue Op, const APInt &OriginalDemandedBits,
20506 const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO,
20507 unsigned Depth) const {
20508 unsigned Opc = Op.getOpcode();
20509
20510 switch (Opc) {
20511 case ARMISD::ASRL:
20512 case ARMISD::LSRL: {
20513 // If this is result 0 and the other result is unused, see if the demand
20514 // bits allow us to shrink this long shift into a standard small shift in
20515 // the opposite direction.
20516 if (Op.getResNo() == 0 && !Op->hasAnyUseOfValue(Value: 1) &&
20517 isa<ConstantSDNode>(Val: Op->getOperand(Num: 2))) {
20518 unsigned ShAmt = Op->getConstantOperandVal(Num: 2);
20519 if (ShAmt < 32 && OriginalDemandedBits.isSubsetOf(RHS: APInt::getAllOnes(numBits: 32)
20520 << (32 - ShAmt)))
20521 return TLO.CombineTo(
20522 O: Op, N: TLO.DAG.getNode(
20523 Opcode: ISD::SHL, DL: SDLoc(Op), VT: MVT::i32, N1: Op.getOperand(i: 1),
20524 N2: TLO.DAG.getConstant(Val: 32 - ShAmt, DL: SDLoc(Op), VT: MVT::i32)));
20525 }
20526 break;
20527 }
20528 case ARMISD::VBICIMM: {
20529 SDValue Op0 = Op.getOperand(i: 0);
20530 unsigned ModImm = Op.getConstantOperandVal(i: 1);
20531 unsigned EltBits = 0;
20532 uint64_t Mask = ARM_AM::decodeVMOVModImm(ModImm, EltBits);
20533 if ((OriginalDemandedBits & Mask) == 0)
20534 return TLO.CombineTo(O: Op, N: Op0);
20535 }
20536 }
20537
20538 return TargetLowering::SimplifyDemandedBitsForTargetNode(
20539 Op, DemandedBits: OriginalDemandedBits, DemandedElts: OriginalDemandedElts, Known, TLO, Depth);
20540}
20541
20542//===----------------------------------------------------------------------===//
20543// ARM Inline Assembly Support
20544//===----------------------------------------------------------------------===//
20545
20546const char *ARMTargetLowering::LowerXConstraint(EVT ConstraintVT) const {
20547 // At this point, we have to lower this constraint to something else, so we
20548 // lower it to an "r" or "w". However, by doing this we will force the result
20549 // to be in register, while the X constraint is much more permissive.
20550 //
20551 // Although we are correct (we are free to emit anything, without
20552 // constraints), we might break use cases that would expect us to be more
20553 // efficient and emit something else.
20554 if (!Subtarget->hasVFP2Base())
20555 return "r";
20556 if (ConstraintVT.isFloatingPoint())
20557 return "w";
20558 if (ConstraintVT.isVector() && Subtarget->hasNEON() &&
20559 (ConstraintVT.getSizeInBits() == 64 ||
20560 ConstraintVT.getSizeInBits() == 128))
20561 return "w";
20562
20563 return "r";
20564}
20565
20566/// getConstraintType - Given a constraint letter, return the type of
20567/// constraint it is for this target.
20568ARMTargetLowering::ConstraintType
20569ARMTargetLowering::getConstraintType(StringRef Constraint) const {
20570 unsigned S = Constraint.size();
20571 if (S == 1) {
20572 switch (Constraint[0]) {
20573 default: break;
20574 case 'l': return C_RegisterClass;
20575 case 'w': return C_RegisterClass;
20576 case 'h': return C_RegisterClass;
20577 case 'x': return C_RegisterClass;
20578 case 't': return C_RegisterClass;
20579 case 'j': return C_Immediate; // Constant for movw.
20580 // An address with a single base register. Due to the way we
20581 // currently handle addresses it is the same as an 'r' memory constraint.
20582 case 'Q': return C_Memory;
20583 }
20584 } else if (S == 2) {
20585 switch (Constraint[0]) {
20586 default: break;
20587 case 'T': return C_RegisterClass;
20588 // All 'U+' constraints are addresses.
20589 case 'U': return C_Memory;
20590 }
20591 }
20592 return TargetLowering::getConstraintType(Constraint);
20593}
20594
20595/// Examine constraint type and operand type and determine a weight value.
20596/// This object must already have been set up with the operand type
20597/// and the current alternative constraint selected.
20598TargetLowering::ConstraintWeight
20599ARMTargetLowering::getSingleConstraintMatchWeight(
20600 AsmOperandInfo &info, const char *constraint) const {
20601 ConstraintWeight weight = CW_Invalid;
20602 Value *CallOperandVal = info.CallOperandVal;
20603 // If we don't have a value, we can't do a match,
20604 // but allow it at the lowest weight.
20605 if (!CallOperandVal)
20606 return CW_Default;
20607 Type *type = CallOperandVal->getType();
20608 // Look at the constraint type.
20609 switch (*constraint) {
20610 default:
20611 weight = TargetLowering::getSingleConstraintMatchWeight(info, constraint);
20612 break;
20613 case 'l':
20614 if (type->isIntegerTy()) {
20615 if (Subtarget->isThumb())
20616 weight = CW_SpecificReg;
20617 else
20618 weight = CW_Register;
20619 }
20620 break;
20621 case 'w':
20622 if (type->isFloatingPointTy())
20623 weight = CW_Register;
20624 break;
20625 }
20626 return weight;
20627}
20628
20629static bool isIncompatibleReg(const MCPhysReg &PR, MVT VT) {
20630 if (PR == 0 || VT == MVT::Other)
20631 return false;
20632 if (ARM::SPRRegClass.contains(Reg: PR))
20633 return VT != MVT::f32 && VT != MVT::f16 && VT != MVT::i32;
20634 if (ARM::DPRRegClass.contains(Reg: PR))
20635 return VT != MVT::f64 && !VT.is64BitVector();
20636 return false;
20637}
20638
20639using RCPair = std::pair<unsigned, const TargetRegisterClass *>;
20640
20641RCPair ARMTargetLowering::getRegForInlineAsmConstraint(
20642 const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const {
20643 switch (Constraint.size()) {
20644 case 1:
20645 // GCC ARM Constraint Letters
20646 switch (Constraint[0]) {
20647 case 'l': // Low regs or general regs.
20648 if (Subtarget->isThumb())
20649 return RCPair(0U, &ARM::tGPRRegClass);
20650 return RCPair(0U, &ARM::GPRRegClass);
20651 case 'h': // High regs or no regs.
20652 if (Subtarget->isThumb())
20653 return RCPair(0U, &ARM::hGPRRegClass);
20654 break;
20655 case 'r':
20656 if (Subtarget->isThumb1Only())
20657 return RCPair(0U, &ARM::tGPRRegClass);
20658 return RCPair(0U, &ARM::GPRRegClass);
20659 case 'w':
20660 if (VT == MVT::Other)
20661 break;
20662 if (VT == MVT::f32 || VT == MVT::f16 || VT == MVT::bf16)
20663 return RCPair(0U, &ARM::SPRRegClass);
20664 if (VT.getSizeInBits() == 64)
20665 return RCPair(0U, &ARM::DPRRegClass);
20666 if (VT.getSizeInBits() == 128)
20667 return RCPair(0U, &ARM::QPRRegClass);
20668 break;
20669 case 'x':
20670 if (VT == MVT::Other)
20671 break;
20672 if (VT == MVT::f32 || VT == MVT::f16 || VT == MVT::bf16)
20673 return RCPair(0U, &ARM::SPR_8RegClass);
20674 if (VT.getSizeInBits() == 64)
20675 return RCPair(0U, &ARM::DPR_8RegClass);
20676 if (VT.getSizeInBits() == 128)
20677 return RCPair(0U, &ARM::QPR_8RegClass);
20678 break;
20679 case 't':
20680 if (VT == MVT::Other)
20681 break;
20682 if (VT == MVT::f32 || VT == MVT::i32 || VT == MVT::f16 || VT == MVT::bf16)
20683 return RCPair(0U, &ARM::SPRRegClass);
20684 if (VT.getSizeInBits() == 64)
20685 return RCPair(0U, &ARM::DPR_VFP2RegClass);
20686 if (VT.getSizeInBits() == 128)
20687 return RCPair(0U, &ARM::QPR_VFP2RegClass);
20688 break;
20689 }
20690 break;
20691
20692 case 2:
20693 if (Constraint[0] == 'T') {
20694 switch (Constraint[1]) {
20695 default:
20696 break;
20697 case 'e':
20698 return RCPair(0U, &ARM::tGPREvenRegClass);
20699 case 'o':
20700 return RCPair(0U, &ARM::tGPROddRegClass);
20701 }
20702 }
20703 break;
20704
20705 default:
20706 break;
20707 }
20708
20709 if (StringRef("{cc}").equals_insensitive(RHS: Constraint))
20710 return std::make_pair(x: unsigned(ARM::CPSR), y: &ARM::CCRRegClass);
20711
20712 // r14 is an alias of lr.
20713 if (StringRef("{r14}").equals_insensitive(RHS: Constraint))
20714 return std::make_pair(x: unsigned(ARM::LR), y: getRegClassFor(VT: MVT::i32));
20715
20716 auto RCP = TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
20717 if (isIncompatibleReg(PR: RCP.first, VT))
20718 return {0, nullptr};
20719 return RCP;
20720}
20721
20722/// LowerAsmOperandForConstraint - Lower the specified operand into the Ops
20723/// vector. If it is invalid, don't add anything to Ops.
20724void ARMTargetLowering::LowerAsmOperandForConstraint(SDValue Op,
20725 StringRef Constraint,
20726 std::vector<SDValue> &Ops,
20727 SelectionDAG &DAG) const {
20728 SDValue Result;
20729
20730 // Currently only support length 1 constraints.
20731 if (Constraint.size() != 1)
20732 return;
20733
20734 char ConstraintLetter = Constraint[0];
20735 switch (ConstraintLetter) {
20736 default: break;
20737 case 'j':
20738 case 'I': case 'J': case 'K': case 'L':
20739 case 'M': case 'N': case 'O':
20740 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: Op);
20741 if (!C)
20742 return;
20743
20744 int64_t CVal64 = C->getSExtValue();
20745 int CVal = (int) CVal64;
20746 // None of these constraints allow values larger than 32 bits. Check
20747 // that the value fits in an int.
20748 if (CVal != CVal64)
20749 return;
20750
20751 switch (ConstraintLetter) {
20752 case 'j':
20753 // Constant suitable for movw, must be between 0 and
20754 // 65535.
20755 if (Subtarget->hasV6T2Ops() || (Subtarget->hasV8MBaselineOps()))
20756 if (CVal >= 0 && CVal <= 65535)
20757 break;
20758 return;
20759 case 'I':
20760 if (Subtarget->isThumb1Only()) {
20761 // This must be a constant between 0 and 255, for ADD
20762 // immediates.
20763 if (CVal >= 0 && CVal <= 255)
20764 break;
20765 } else if (Subtarget->isThumb2()) {
20766 // A constant that can be used as an immediate value in a
20767 // data-processing instruction.
20768 if (ARM_AM::getT2SOImmVal(Arg: CVal) != -1)
20769 break;
20770 } else {
20771 // A constant that can be used as an immediate value in a
20772 // data-processing instruction.
20773 if (ARM_AM::getSOImmVal(Arg: CVal) != -1)
20774 break;
20775 }
20776 return;
20777
20778 case 'J':
20779 if (Subtarget->isThumb1Only()) {
20780 // This must be a constant between -255 and -1, for negated ADD
20781 // immediates. This can be used in GCC with an "n" modifier that
20782 // prints the negated value, for use with SUB instructions. It is
20783 // not useful otherwise but is implemented for compatibility.
20784 if (CVal >= -255 && CVal <= -1)
20785 break;
20786 } else {
20787 // This must be a constant between -4095 and 4095. This is suitable
20788 // for use as the immediate offset field in LDR and STR instructions
20789 // such as LDR r0,[r1,#offset].
20790 if (CVal >= -4095 && CVal <= 4095)
20791 break;
20792 }
20793 return;
20794
20795 case 'K':
20796 if (Subtarget->isThumb1Only()) {
20797 // A 32-bit value where only one byte has a nonzero value. Exclude
20798 // zero to match GCC. This constraint is used by GCC internally for
20799 // constants that can be loaded with a move/shift combination.
20800 // It is not useful otherwise but is implemented for compatibility.
20801 if (CVal != 0 && ARM_AM::isThumbImmShiftedVal(V: CVal))
20802 break;
20803 } else if (Subtarget->isThumb2()) {
20804 // A constant whose bitwise inverse can be used as an immediate
20805 // value in a data-processing instruction. This can be used in GCC
20806 // with a "B" modifier that prints the inverted value, for use with
20807 // BIC and MVN instructions. It is not useful otherwise but is
20808 // implemented for compatibility.
20809 if (ARM_AM::getT2SOImmVal(Arg: ~CVal) != -1)
20810 break;
20811 } else {
20812 // A constant whose bitwise inverse can be used as an immediate
20813 // value in a data-processing instruction. This can be used in GCC
20814 // with a "B" modifier that prints the inverted value, for use with
20815 // BIC and MVN instructions. It is not useful otherwise but is
20816 // implemented for compatibility.
20817 if (ARM_AM::getSOImmVal(Arg: ~CVal) != -1)
20818 break;
20819 }
20820 return;
20821
20822 case 'L':
20823 if (Subtarget->isThumb1Only()) {
20824 // This must be a constant between -7 and 7,
20825 // for 3-operand ADD/SUB immediate instructions.
20826 if (CVal >= -7 && CVal < 7)
20827 break;
20828 } else if (Subtarget->isThumb2()) {
20829 // A constant whose negation can be used as an immediate value in a
20830 // data-processing instruction. This can be used in GCC with an "n"
20831 // modifier that prints the negated value, for use with SUB
20832 // instructions. It is not useful otherwise but is implemented for
20833 // compatibility.
20834 if (ARM_AM::getT2SOImmVal(Arg: -CVal) != -1)
20835 break;
20836 } else {
20837 // A constant whose negation can be used as an immediate value in a
20838 // data-processing instruction. This can be used in GCC with an "n"
20839 // modifier that prints the negated value, for use with SUB
20840 // instructions. It is not useful otherwise but is implemented for
20841 // compatibility.
20842 if (ARM_AM::getSOImmVal(Arg: -CVal) != -1)
20843 break;
20844 }
20845 return;
20846
20847 case 'M':
20848 if (Subtarget->isThumb1Only()) {
20849 // This must be a multiple of 4 between 0 and 1020, for
20850 // ADD sp + immediate.
20851 if ((CVal >= 0 && CVal <= 1020) && ((CVal & 3) == 0))
20852 break;
20853 } else {
20854 // A power of two or a constant between 0 and 32. This is used in
20855 // GCC for the shift amount on shifted register operands, but it is
20856 // useful in general for any shift amounts.
20857 if ((CVal >= 0 && CVal <= 32) || ((CVal & (CVal - 1)) == 0))
20858 break;
20859 }
20860 return;
20861
20862 case 'N':
20863 if (Subtarget->isThumb1Only()) {
20864 // This must be a constant between 0 and 31, for shift amounts.
20865 if (CVal >= 0 && CVal <= 31)
20866 break;
20867 }
20868 return;
20869
20870 case 'O':
20871 if (Subtarget->isThumb1Only()) {
20872 // This must be a multiple of 4 between -508 and 508, for
20873 // ADD/SUB sp = sp + immediate.
20874 if ((CVal >= -508 && CVal <= 508) && ((CVal & 3) == 0))
20875 break;
20876 }
20877 return;
20878 }
20879 Result = DAG.getSignedTargetConstant(Val: CVal, DL: SDLoc(Op), VT: Op.getValueType());
20880 break;
20881 }
20882
20883 if (Result.getNode()) {
20884 Ops.push_back(x: Result);
20885 return;
20886 }
20887 return TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
20888}
20889
20890static RTLIB::Libcall getDivRemLibcall(
20891 const SDNode *N, MVT::SimpleValueType SVT) {
20892 assert((N->getOpcode() == ISD::SDIVREM || N->getOpcode() == ISD::UDIVREM ||
20893 N->getOpcode() == ISD::SREM || N->getOpcode() == ISD::UREM) &&
20894 "Unhandled Opcode in getDivRemLibcall");
20895 bool isSigned = N->getOpcode() == ISD::SDIVREM ||
20896 N->getOpcode() == ISD::SREM;
20897 RTLIB::Libcall LC;
20898 switch (SVT) {
20899 default: llvm_unreachable("Unexpected request for libcall!");
20900 case MVT::i8: LC = isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break;
20901 case MVT::i16: LC = isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
20902 case MVT::i32: LC = isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
20903 case MVT::i64: LC = isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
20904 }
20905 return LC;
20906}
20907
20908static TargetLowering::ArgListTy getDivRemArgList(
20909 const SDNode *N, LLVMContext *Context, const ARMSubtarget *Subtarget) {
20910 assert((N->getOpcode() == ISD::SDIVREM || N->getOpcode() == ISD::UDIVREM ||
20911 N->getOpcode() == ISD::SREM || N->getOpcode() == ISD::UREM) &&
20912 "Unhandled Opcode in getDivRemArgList");
20913 bool isSigned = N->getOpcode() == ISD::SDIVREM ||
20914 N->getOpcode() == ISD::SREM;
20915 TargetLowering::ArgListTy Args;
20916 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
20917 EVT ArgVT = N->getOperand(Num: i).getValueType();
20918 Type *ArgTy = ArgVT.getTypeForEVT(Context&: *Context);
20919 TargetLowering::ArgListEntry Entry(N->getOperand(Num: i), ArgTy);
20920 Entry.IsSExt = isSigned;
20921 Entry.IsZExt = !isSigned;
20922 Args.push_back(x: Entry);
20923 }
20924 if (Subtarget->getTargetTriple().isOSWindows() && Args.size() >= 2)
20925 std::swap(a&: Args[0], b&: Args[1]);
20926 return Args;
20927}
20928
20929SDValue ARMTargetLowering::LowerDivRem(SDValue Op, SelectionDAG &DAG) const {
20930 assert((Subtarget->isTargetAEABI() || Subtarget->isTargetAndroid() ||
20931 Subtarget->isTargetGNUAEABI() || Subtarget->isTargetMuslAEABI() ||
20932 Subtarget->isTargetFuchsia() || Subtarget->isTargetWindows()) &&
20933 "Register-based DivRem lowering only");
20934 unsigned Opcode = Op->getOpcode();
20935 assert((Opcode == ISD::SDIVREM || Opcode == ISD::UDIVREM) &&
20936 "Invalid opcode for Div/Rem lowering");
20937 bool isSigned = (Opcode == ISD::SDIVREM);
20938 EVT VT = Op->getValueType(ResNo: 0);
20939 SDLoc dl(Op);
20940
20941 if (VT == MVT::i64 && isa<ConstantSDNode>(Val: Op.getOperand(i: 1))) {
20942 SmallVector<SDValue> Result;
20943 if (expandDIVREMByConstant(N: Op.getNode(), Result, HiLoVT: MVT::i32, DAG)) {
20944 SDValue Res0 =
20945 DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT, N1: Result[0], N2: Result[1]);
20946 SDValue Res1 =
20947 DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT, N1: Result[2], N2: Result[3]);
20948 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: dl, VTList: Op->getVTList(),
20949 Ops: {Res0, Res1});
20950 }
20951 }
20952
20953 Type *Ty = VT.getTypeForEVT(Context&: *DAG.getContext());
20954
20955 // If the target has hardware divide, use divide + multiply + subtract:
20956 // div = a / b
20957 // rem = a - b * div
20958 // return {div, rem}
20959 // This should be lowered into UDIV/SDIV + MLS later on.
20960 bool hasDivide = Subtarget->isThumb() ? Subtarget->hasDivideInThumbMode()
20961 : Subtarget->hasDivideInARMMode();
20962 if (hasDivide && Op->getValueType(ResNo: 0).isSimple() &&
20963 Op->getSimpleValueType(ResNo: 0) == MVT::i32) {
20964 unsigned DivOpcode = isSigned ? ISD::SDIV : ISD::UDIV;
20965 const SDValue Dividend = Op->getOperand(Num: 0);
20966 const SDValue Divisor = Op->getOperand(Num: 1);
20967 SDValue Div = DAG.getNode(Opcode: DivOpcode, DL: dl, VT, N1: Dividend, N2: Divisor);
20968 SDValue Mul = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: Div, N2: Divisor);
20969 SDValue Rem = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Dividend, N2: Mul);
20970
20971 SDValue Values[2] = {Div, Rem};
20972 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: VT), Ops: Values);
20973 }
20974
20975 RTLIB::Libcall LC = getDivRemLibcall(N: Op.getNode(),
20976 SVT: VT.getSimpleVT().SimpleTy);
20977 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
20978
20979 SDValue InChain = DAG.getEntryNode();
20980
20981 TargetLowering::ArgListTy Args = getDivRemArgList(N: Op.getNode(),
20982 Context: DAG.getContext(),
20983 Subtarget);
20984
20985 SDValue Callee =
20986 DAG.getExternalSymbol(LCImpl, VT: getPointerTy(DL: DAG.getDataLayout()));
20987
20988 Type *RetTy = StructType::get(elt1: Ty, elts: Ty);
20989
20990 if (getTM().getTargetTriple().isOSWindows())
20991 InChain = WinDBZCheckDenominator(DAG, N: Op.getNode(), InChain);
20992
20993 TargetLowering::CallLoweringInfo CLI(DAG);
20994 CLI.setDebugLoc(dl)
20995 .setChain(InChain)
20996 .setCallee(CC: DAG.getLibcalls().getLibcallImplCallingConv(Call: LCImpl), ResultType: RetTy,
20997 Target: Callee, ArgsList: std::move(Args))
20998 .setInRegister()
20999 .setSExtResult(isSigned)
21000 .setZExtResult(!isSigned);
21001
21002 std::pair<SDValue, SDValue> CallInfo = LowerCallTo(CLI);
21003 return CallInfo.first;
21004}
21005
21006// Lowers REM using divmod helpers
21007// see RTABI section 4.2/4.3
21008SDValue ARMTargetLowering::LowerREM(SDNode *N, SelectionDAG &DAG) const {
21009 EVT VT = N->getValueType(ResNo: 0);
21010
21011 if (VT == MVT::i64 && isa<ConstantSDNode>(Val: N->getOperand(Num: 1))) {
21012 SmallVector<SDValue> Result;
21013 if (expandDIVREMByConstant(N, Result, HiLoVT: MVT::i32, DAG))
21014 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
21015 N1: Result[0], N2: Result[1]);
21016 }
21017
21018 // Build return types (div and rem)
21019 std::vector<Type*> RetTyParams;
21020 Type *RetTyElement;
21021
21022 switch (VT.getSimpleVT().SimpleTy) {
21023 default: llvm_unreachable("Unexpected request for libcall!");
21024 case MVT::i8: RetTyElement = Type::getInt8Ty(C&: *DAG.getContext()); break;
21025 case MVT::i16: RetTyElement = Type::getInt16Ty(C&: *DAG.getContext()); break;
21026 case MVT::i32: RetTyElement = Type::getInt32Ty(C&: *DAG.getContext()); break;
21027 case MVT::i64: RetTyElement = Type::getInt64Ty(C&: *DAG.getContext()); break;
21028 }
21029
21030 RetTyParams.push_back(x: RetTyElement);
21031 RetTyParams.push_back(x: RetTyElement);
21032 ArrayRef<Type*> ret = ArrayRef<Type*>(RetTyParams);
21033 Type *RetTy = StructType::get(Context&: *DAG.getContext(), Elements: ret);
21034
21035 RTLIB::Libcall LC = getDivRemLibcall(N, SVT: N->getValueType(ResNo: 0).getSimpleVT().
21036 SimpleTy);
21037 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
21038 SDValue InChain = DAG.getEntryNode();
21039 TargetLowering::ArgListTy Args = getDivRemArgList(N, Context: DAG.getContext(),
21040 Subtarget);
21041 bool isSigned = N->getOpcode() == ISD::SREM;
21042
21043 SDValue Callee =
21044 DAG.getExternalSymbol(LCImpl, VT: getPointerTy(DL: DAG.getDataLayout()));
21045
21046 if (getTM().getTargetTriple().isOSWindows())
21047 InChain = WinDBZCheckDenominator(DAG, N, InChain);
21048
21049 // Lower call
21050 CallLoweringInfo CLI(DAG);
21051 CLI.setChain(InChain)
21052 .setCallee(CC: DAG.getLibcalls().getLibcallImplCallingConv(Call: LCImpl), ResultType: RetTy,
21053 Target: Callee, ArgsList: std::move(Args))
21054 .setSExtResult(isSigned)
21055 .setZExtResult(!isSigned)
21056 .setDebugLoc(SDLoc(N));
21057 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
21058
21059 // Return second (rem) result operand (first contains div)
21060 SDNode *ResNode = CallResult.first.getNode();
21061 assert(ResNode->getNumOperands() == 2 && "divmod should return two operands");
21062 return ResNode->getOperand(Num: 1);
21063}
21064
21065SDValue
21066ARMTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const {
21067 assert(getTM().getTargetTriple().isOSWindows() &&
21068 "unsupported target platform");
21069 SDLoc DL(Op);
21070
21071 // Get the inputs.
21072 SDValue Chain = Op.getOperand(i: 0);
21073 SDValue Size = Op.getOperand(i: 1);
21074
21075 if (DAG.getMachineFunction().getFunction().hasFnAttribute(
21076 Kind: "no-stack-arg-probe")) {
21077 MaybeAlign Align =
21078 cast<ConstantSDNode>(Val: Op.getOperand(i: 2))->getMaybeAlignValue();
21079 SDValue SP = DAG.getCopyFromReg(Chain, dl: DL, Reg: ARM::SP, VT: MVT::i32);
21080 Chain = SP.getValue(R: 1);
21081 SP = DAG.getNode(Opcode: ISD::SUB, DL, VT: MVT::i32, N1: SP, N2: Size);
21082 if (Align)
21083 SP = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: SP.getValue(R: 0),
21084 N2: DAG.getSignedConstant(Val: -Align->value(), DL, VT: MVT::i32));
21085 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: ARM::SP, N: SP);
21086 SDValue Ops[2] = { SP, Chain };
21087 return DAG.getMergeValues(Ops, dl: DL);
21088 }
21089
21090 SDValue Words = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: Size,
21091 N2: DAG.getConstant(Val: 2, DL, VT: MVT::i32));
21092
21093 SDValue Glue;
21094 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: ARM::R4, N: Words, Glue);
21095 Glue = Chain.getValue(R: 1);
21096
21097 SDVTList NodeTys = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
21098 Chain = DAG.getNode(Opcode: ARMISD::WIN__CHKSTK, DL, VTList: NodeTys, N1: Chain, N2: Glue);
21099
21100 SDValue NewSP = DAG.getCopyFromReg(Chain, dl: DL, Reg: ARM::SP, VT: MVT::i32);
21101 Chain = NewSP.getValue(R: 1);
21102
21103 SDValue Ops[2] = { NewSP, Chain };
21104 return DAG.getMergeValues(Ops, dl: DL);
21105}
21106
21107SDValue ARMTargetLowering::LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const {
21108 bool IsStrict = Op->isStrictFPOpcode();
21109 SDValue SrcVal = Op.getOperand(i: IsStrict ? 1 : 0);
21110 const unsigned DstSz = Op.getValueType().getSizeInBits();
21111 const unsigned SrcSz = SrcVal.getValueType().getSizeInBits();
21112 assert(DstSz > SrcSz && DstSz <= 64 && SrcSz >= 16 &&
21113 "Unexpected type for custom-lowering FP_EXTEND");
21114
21115 assert((!Subtarget->hasFP64() || !Subtarget->hasFPARMv8Base()) &&
21116 "With both FP DP and 16, any FP conversion is legal!");
21117
21118 assert(!(DstSz == 32 && Subtarget->hasFP16()) &&
21119 "With FP16, 16 to 32 conversion is legal!");
21120
21121 // Converting from 32 -> 64 is valid if we have FP64.
21122 if (SrcSz == 32 && DstSz == 64 && Subtarget->hasFP64()) {
21123 // FIXME: Remove this when we have strict fp instruction selection patterns
21124 if (IsStrict) {
21125 SDLoc Loc(Op);
21126 SDValue Result = DAG.getNode(Opcode: ISD::FP_EXTEND,
21127 DL: Loc, VT: Op.getValueType(), Operand: SrcVal);
21128 return DAG.getMergeValues(Ops: {Result, Op.getOperand(i: 0)}, dl: Loc);
21129 }
21130 return Op;
21131 }
21132
21133 // Either we are converting from 16 -> 64, without FP16 and/or
21134 // FP.double-precision or without Armv8-fp. So we must do it in two
21135 // steps.
21136 // Or we are converting from 32 -> 64 without fp.double-precision or 16 -> 32
21137 // without FP16. So we must do a function call.
21138 SDLoc Loc(Op);
21139 RTLIB::Libcall LC;
21140 MakeLibCallOptions CallOptions;
21141 SDValue Chain = IsStrict ? Op.getOperand(i: 0) : SDValue();
21142 for (unsigned Sz = SrcSz; Sz <= 32 && Sz < DstSz; Sz *= 2) {
21143 bool Supported = (Sz == 16 ? Subtarget->hasFP16() : Subtarget->hasFP64());
21144 MVT SrcVT = (Sz == 16 ? MVT::f16 : MVT::f32);
21145 MVT DstVT = (Sz == 16 ? MVT::f32 : MVT::f64);
21146 if (Supported) {
21147 if (IsStrict) {
21148 SrcVal = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: Loc,
21149 ResultTys: {DstVT, MVT::Other}, Ops: {Chain, SrcVal});
21150 Chain = SrcVal.getValue(R: 1);
21151 } else {
21152 SrcVal = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: Loc, VT: DstVT, Operand: SrcVal);
21153 }
21154 } else {
21155 LC = RTLIB::getFPEXT(OpVT: SrcVT, RetVT: DstVT);
21156 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
21157 "Unexpected type for custom-lowering FP_EXTEND");
21158 std::tie(args&: SrcVal, args&: Chain) = makeLibCall(DAG, LC, RetVT: DstVT, Ops: SrcVal, CallOptions,
21159 dl: Loc, Chain);
21160 }
21161 }
21162
21163 return IsStrict ? DAG.getMergeValues(Ops: {SrcVal, Chain}, dl: Loc) : SrcVal;
21164}
21165
21166SDValue ARMTargetLowering::LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const {
21167 bool IsStrict = Op->isStrictFPOpcode();
21168
21169 SDValue SrcVal = Op.getOperand(i: IsStrict ? 1 : 0);
21170 EVT SrcVT = SrcVal.getValueType();
21171 EVT DstVT = Op.getValueType();
21172
21173 if (DstVT == MVT::bf16) {
21174 if (Subtarget->hasBF16() && SrcVT == MVT::f32)
21175 return Op;
21176 return SDValue();
21177 }
21178
21179 const unsigned DstSz = Op.getValueType().getSizeInBits();
21180 const unsigned SrcSz = SrcVT.getSizeInBits();
21181 (void)DstSz;
21182 assert(DstSz < SrcSz && SrcSz <= 64 && DstSz >= 16 &&
21183 "Unexpected type for custom-lowering FP_ROUND");
21184
21185 assert((!Subtarget->hasFP64() || !Subtarget->hasFPARMv8Base()) &&
21186 "With both FP DP and 16, any FP conversion is legal!");
21187
21188 SDLoc Loc(Op);
21189
21190 // Instruction from 32 -> 16 if hasFP16 is valid
21191 if (SrcSz == 32 && Subtarget->hasFP16())
21192 return Op;
21193
21194 // Lib call from 32 -> 16 / 64 -> [32, 16]
21195 RTLIB::Libcall LC = RTLIB::getFPROUND(OpVT: SrcVT, RetVT: DstVT);
21196 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
21197 "Unexpected type for custom-lowering FP_ROUND");
21198 MakeLibCallOptions CallOptions;
21199 SDValue Chain = IsStrict ? Op.getOperand(i: 0) : SDValue();
21200 SDValue Result;
21201 std::tie(args&: Result, args&: Chain) = makeLibCall(DAG, LC, RetVT: DstVT, Ops: SrcVal, CallOptions,
21202 dl: Loc, Chain);
21203 return IsStrict ? DAG.getMergeValues(Ops: {Result, Chain}, dl: Loc) : Result;
21204}
21205
21206bool
21207ARMTargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const {
21208 // The ARM target isn't yet aware of offsets.
21209 return false;
21210}
21211
21212bool ARM::isBitFieldInvertedMask(unsigned v) {
21213 if (v == 0xffffffff)
21214 return false;
21215
21216 // there can be 1's on either or both "outsides", all the "inside"
21217 // bits must be 0's
21218 return isShiftedMask_32(Value: ~v);
21219}
21220
21221/// isFPImmLegal - Returns true if the target can instruction select the
21222/// specified FP immediate natively. If false, the legalizer will
21223/// materialize the FP immediate as a load from a constant pool.
21224bool ARMTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
21225 bool ForCodeSize) const {
21226 if (!Subtarget->hasVFP3Base())
21227 return false;
21228 if (VT == MVT::f16 && Subtarget->hasFullFP16())
21229 return ARM_AM::getFP16Imm(FPImm: Imm) != -1;
21230 if (VT == MVT::f32 && Subtarget->hasFullFP16() &&
21231 ARM_AM::getFP32FP16Imm(FPImm: Imm) != -1)
21232 return true;
21233 if (VT == MVT::f32)
21234 return ARM_AM::getFP32Imm(FPImm: Imm) != -1;
21235 if (VT == MVT::f64 && Subtarget->hasFP64())
21236 return ARM_AM::getFP64Imm(FPImm: Imm) != -1;
21237 return false;
21238}
21239
21240/// getTgtMemIntrinsic - Represent NEON load and store intrinsics as
21241/// MemIntrinsicNodes. The associated MachineMemOperands record the alignment
21242/// specified in the intrinsic calls.
21243void ARMTargetLowering::getTgtMemIntrinsic(
21244 SmallVectorImpl<IntrinsicInfo> &Infos, const CallBase &I,
21245 MachineFunction &MF, unsigned Intrinsic) const {
21246 IntrinsicInfo Info;
21247 switch (Intrinsic) {
21248 case Intrinsic::arm_neon_vld1:
21249 case Intrinsic::arm_neon_vld2:
21250 case Intrinsic::arm_neon_vld3:
21251 case Intrinsic::arm_neon_vld4:
21252 case Intrinsic::arm_neon_vld2lane:
21253 case Intrinsic::arm_neon_vld3lane:
21254 case Intrinsic::arm_neon_vld4lane:
21255 case Intrinsic::arm_neon_vld2dup:
21256 case Intrinsic::arm_neon_vld3dup:
21257 case Intrinsic::arm_neon_vld4dup: {
21258 Info.opc = ISD::INTRINSIC_W_CHAIN;
21259 // Conservatively set memVT to the entire set of vectors loaded.
21260 auto &DL = I.getDataLayout();
21261 uint64_t NumElts = DL.getTypeSizeInBits(Ty: I.getType()) / 64;
21262 Info.memVT = EVT::getVectorVT(Context&: I.getType()->getContext(), VT: MVT::i64, NumElements: NumElts);
21263 Info.ptrVal = I.getArgOperand(i: 0);
21264 Info.offset = 0;
21265 Value *AlignArg = I.getArgOperand(i: I.arg_size() - 1);
21266 Info.align = cast<ConstantInt>(Val: AlignArg)->getMaybeAlignValue();
21267 // volatile loads with NEON intrinsics not supported
21268 Info.flags = MachineMemOperand::MOLoad;
21269 Infos.push_back(Elt: Info);
21270 return;
21271 }
21272 case Intrinsic::arm_neon_vld1x2:
21273 case Intrinsic::arm_neon_vld1x3:
21274 case Intrinsic::arm_neon_vld1x4: {
21275 Info.opc = ISD::INTRINSIC_W_CHAIN;
21276 // Conservatively set memVT to the entire set of vectors loaded.
21277 auto &DL = I.getDataLayout();
21278 uint64_t NumElts = DL.getTypeSizeInBits(Ty: I.getType()) / 64;
21279 Info.memVT = EVT::getVectorVT(Context&: I.getType()->getContext(), VT: MVT::i64, NumElements: NumElts);
21280 Info.ptrVal = I.getArgOperand(i: I.arg_size() - 1);
21281 Info.offset = 0;
21282 Info.align = I.getParamAlign(ArgNo: I.arg_size() - 1).valueOrOne();
21283 // volatile loads with NEON intrinsics not supported
21284 Info.flags = MachineMemOperand::MOLoad;
21285 Infos.push_back(Elt: Info);
21286 return;
21287 }
21288 case Intrinsic::arm_neon_vst1:
21289 case Intrinsic::arm_neon_vst2:
21290 case Intrinsic::arm_neon_vst3:
21291 case Intrinsic::arm_neon_vst4:
21292 case Intrinsic::arm_neon_vst2lane:
21293 case Intrinsic::arm_neon_vst3lane:
21294 case Intrinsic::arm_neon_vst4lane: {
21295 Info.opc = ISD::INTRINSIC_VOID;
21296 // Conservatively set memVT to the entire set of vectors stored.
21297 auto &DL = I.getDataLayout();
21298 unsigned NumElts = 0;
21299 for (unsigned ArgI = 1, ArgE = I.arg_size(); ArgI < ArgE; ++ArgI) {
21300 Type *ArgTy = I.getArgOperand(i: ArgI)->getType();
21301 if (!ArgTy->isVectorTy())
21302 break;
21303 NumElts += DL.getTypeSizeInBits(Ty: ArgTy) / 64;
21304 }
21305 Info.memVT = EVT::getVectorVT(Context&: I.getType()->getContext(), VT: MVT::i64, NumElements: NumElts);
21306 Info.ptrVal = I.getArgOperand(i: 0);
21307 Info.offset = 0;
21308 Value *AlignArg = I.getArgOperand(i: I.arg_size() - 1);
21309 Info.align = cast<ConstantInt>(Val: AlignArg)->getMaybeAlignValue();
21310 // volatile stores with NEON intrinsics not supported
21311 Info.flags = MachineMemOperand::MOStore;
21312 Infos.push_back(Elt: Info);
21313 return;
21314 }
21315 case Intrinsic::arm_neon_vst1x2:
21316 case Intrinsic::arm_neon_vst1x3:
21317 case Intrinsic::arm_neon_vst1x4: {
21318 Info.opc = ISD::INTRINSIC_VOID;
21319 // Conservatively set memVT to the entire set of vectors stored.
21320 auto &DL = I.getDataLayout();
21321 unsigned NumElts = 0;
21322 for (unsigned ArgI = 1, ArgE = I.arg_size(); ArgI < ArgE; ++ArgI) {
21323 Type *ArgTy = I.getArgOperand(i: ArgI)->getType();
21324 if (!ArgTy->isVectorTy())
21325 break;
21326 NumElts += DL.getTypeSizeInBits(Ty: ArgTy) / 64;
21327 }
21328 Info.memVT = EVT::getVectorVT(Context&: I.getType()->getContext(), VT: MVT::i64, NumElements: NumElts);
21329 Info.ptrVal = I.getArgOperand(i: 0);
21330 Info.offset = 0;
21331 Info.align = I.getParamAlign(ArgNo: 0).valueOrOne();
21332 // volatile stores with NEON intrinsics not supported
21333 Info.flags = MachineMemOperand::MOStore;
21334 Infos.push_back(Elt: Info);
21335 return;
21336 }
21337 case Intrinsic::arm_mve_vld2q:
21338 case Intrinsic::arm_mve_vld4q: {
21339 Info.opc = ISD::INTRINSIC_W_CHAIN;
21340 // Conservatively set memVT to the entire set of vectors loaded.
21341 Type *VecTy = cast<StructType>(Val: I.getType())->getElementType(N: 1);
21342 unsigned Factor = Intrinsic == Intrinsic::arm_mve_vld2q ? 2 : 4;
21343 Info.memVT = EVT::getVectorVT(Context&: VecTy->getContext(), VT: MVT::i64, NumElements: Factor * 2);
21344 Info.ptrVal = I.getArgOperand(i: 0);
21345 Info.offset = 0;
21346 Info.align = Align(VecTy->getScalarSizeInBits() / 8);
21347 // volatile loads with MVE intrinsics not supported
21348 Info.flags = MachineMemOperand::MOLoad;
21349 Infos.push_back(Elt: Info);
21350 return;
21351 }
21352 case Intrinsic::arm_mve_vst2q:
21353 case Intrinsic::arm_mve_vst4q: {
21354 Info.opc = ISD::INTRINSIC_VOID;
21355 // Conservatively set memVT to the entire set of vectors stored.
21356 Type *VecTy = I.getArgOperand(i: 1)->getType();
21357 unsigned Factor = Intrinsic == Intrinsic::arm_mve_vst2q ? 2 : 4;
21358 Info.memVT = EVT::getVectorVT(Context&: VecTy->getContext(), VT: MVT::i64, NumElements: Factor * 2);
21359 Info.ptrVal = I.getArgOperand(i: 0);
21360 Info.offset = 0;
21361 Info.align = Align(VecTy->getScalarSizeInBits() / 8);
21362 // volatile stores with MVE intrinsics not supported
21363 Info.flags = MachineMemOperand::MOStore;
21364 Infos.push_back(Elt: Info);
21365 return;
21366 }
21367 case Intrinsic::arm_mve_vldr_gather_base:
21368 case Intrinsic::arm_mve_vldr_gather_base_predicated: {
21369 Info.opc = ISD::INTRINSIC_W_CHAIN;
21370 Info.ptrVal = nullptr;
21371 Info.memVT = MVT::getVT(Ty: I.getType());
21372 Info.align = Align(1);
21373 Info.flags |= MachineMemOperand::MOLoad;
21374 Infos.push_back(Elt: Info);
21375 return;
21376 }
21377 case Intrinsic::arm_mve_vldr_gather_base_wb:
21378 case Intrinsic::arm_mve_vldr_gather_base_wb_predicated: {
21379 Info.opc = ISD::INTRINSIC_W_CHAIN;
21380 Info.ptrVal = nullptr;
21381 Info.memVT = MVT::getVT(Ty: I.getType()->getContainedType(i: 0));
21382 Info.align = Align(1);
21383 Info.flags |= MachineMemOperand::MOLoad;
21384 Infos.push_back(Elt: Info);
21385 return;
21386 }
21387 case Intrinsic::arm_mve_vldr_gather_offset:
21388 case Intrinsic::arm_mve_vldr_gather_offset_predicated: {
21389 Info.opc = ISD::INTRINSIC_W_CHAIN;
21390 Info.ptrVal = nullptr;
21391 MVT DataVT = MVT::getVT(Ty: I.getType());
21392 unsigned MemSize = cast<ConstantInt>(Val: I.getArgOperand(i: 2))->getZExtValue();
21393 Info.memVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: MemSize),
21394 NumElements: DataVT.getVectorNumElements());
21395 Info.align = Align(1);
21396 Info.flags |= MachineMemOperand::MOLoad;
21397 Infos.push_back(Elt: Info);
21398 return;
21399 }
21400 case Intrinsic::arm_mve_vstr_scatter_base:
21401 case Intrinsic::arm_mve_vstr_scatter_base_predicated: {
21402 Info.opc = ISD::INTRINSIC_VOID;
21403 Info.ptrVal = nullptr;
21404 Info.memVT = MVT::getVT(Ty: I.getArgOperand(i: 2)->getType());
21405 Info.align = Align(1);
21406 Info.flags |= MachineMemOperand::MOStore;
21407 Infos.push_back(Elt: Info);
21408 return;
21409 }
21410 case Intrinsic::arm_mve_vstr_scatter_base_wb:
21411 case Intrinsic::arm_mve_vstr_scatter_base_wb_predicated: {
21412 Info.opc = ISD::INTRINSIC_W_CHAIN;
21413 Info.ptrVal = nullptr;
21414 Info.memVT = MVT::getVT(Ty: I.getArgOperand(i: 2)->getType());
21415 Info.align = Align(1);
21416 Info.flags |= MachineMemOperand::MOStore;
21417 Infos.push_back(Elt: Info);
21418 return;
21419 }
21420 case Intrinsic::arm_mve_vstr_scatter_offset:
21421 case Intrinsic::arm_mve_vstr_scatter_offset_predicated: {
21422 Info.opc = ISD::INTRINSIC_VOID;
21423 Info.ptrVal = nullptr;
21424 MVT DataVT = MVT::getVT(Ty: I.getArgOperand(i: 2)->getType());
21425 unsigned MemSize = cast<ConstantInt>(Val: I.getArgOperand(i: 3))->getZExtValue();
21426 Info.memVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: MemSize),
21427 NumElements: DataVT.getVectorNumElements());
21428 Info.align = Align(1);
21429 Info.flags |= MachineMemOperand::MOStore;
21430 Infos.push_back(Elt: Info);
21431 return;
21432 }
21433 case Intrinsic::arm_ldaex:
21434 case Intrinsic::arm_ldrex: {
21435 auto &DL = I.getDataLayout();
21436 Type *ValTy = I.getParamElementType(ArgNo: 0);
21437 Info.opc = ISD::INTRINSIC_W_CHAIN;
21438 Info.memVT = MVT::getVT(Ty: ValTy);
21439 Info.ptrVal = I.getArgOperand(i: 0);
21440 Info.offset = 0;
21441 Info.align = DL.getABITypeAlign(Ty: ValTy);
21442 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile;
21443 Infos.push_back(Elt: Info);
21444 return;
21445 }
21446 case Intrinsic::arm_stlex:
21447 case Intrinsic::arm_strex: {
21448 auto &DL = I.getDataLayout();
21449 Type *ValTy = I.getParamElementType(ArgNo: 1);
21450 Info.opc = ISD::INTRINSIC_W_CHAIN;
21451 Info.memVT = MVT::getVT(Ty: ValTy);
21452 Info.ptrVal = I.getArgOperand(i: 1);
21453 Info.offset = 0;
21454 Info.align = DL.getABITypeAlign(Ty: ValTy);
21455 Info.flags = MachineMemOperand::MOStore | MachineMemOperand::MOVolatile;
21456 Infos.push_back(Elt: Info);
21457 return;
21458 }
21459 case Intrinsic::arm_stlexd:
21460 case Intrinsic::arm_strexd:
21461 Info.opc = ISD::INTRINSIC_W_CHAIN;
21462 Info.memVT = MVT::i64;
21463 Info.ptrVal = I.getArgOperand(i: 2);
21464 Info.offset = 0;
21465 Info.align = Align(8);
21466 Info.flags = MachineMemOperand::MOStore | MachineMemOperand::MOVolatile;
21467 Infos.push_back(Elt: Info);
21468 return;
21469
21470 case Intrinsic::arm_ldaexd:
21471 case Intrinsic::arm_ldrexd:
21472 Info.opc = ISD::INTRINSIC_W_CHAIN;
21473 Info.memVT = MVT::i64;
21474 Info.ptrVal = I.getArgOperand(i: 0);
21475 Info.offset = 0;
21476 Info.align = Align(8);
21477 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile;
21478 Infos.push_back(Elt: Info);
21479 return;
21480
21481 default:
21482 break;
21483 }
21484}
21485
21486/// Returns true if it is beneficial to convert a load of a constant
21487/// to just the constant itself.
21488bool ARMTargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
21489 Type *Ty) const {
21490 assert(Ty->isIntegerTy());
21491
21492 unsigned Bits = Ty->getPrimitiveSizeInBits();
21493 if (Bits == 0 || Bits > 32)
21494 return false;
21495 return true;
21496}
21497
21498TargetLowering::ExtractSubvectorCost
21499ARMTargetLowering::getExtractSubvectorCost(EVT ResVT, EVT SrcVT,
21500 unsigned Index) const {
21501 if (!isOperationLegalOrCustom(Op: ISD::EXTRACT_SUBVECTOR, VT: ResVT))
21502 return ExtractSubvectorCost::Expensive;
21503
21504 if (Index == 0 || Index == ResVT.getVectorNumElements())
21505 return ExtractSubvectorCost::Free;
21506 return ExtractSubvectorCost::Expensive;
21507}
21508
21509Instruction *ARMTargetLowering::makeDMB(IRBuilderBase &Builder,
21510 ARM_MB::MemBOpt Domain) const {
21511 // First, if the target has no DMB, see what fallback we can use.
21512 if (!Subtarget->hasDataBarrier()) {
21513 // Some ARMv6 cpus can support data barriers with an mcr instruction.
21514 // Thumb1 and pre-v6 ARM mode use a libcall instead and should never get
21515 // here.
21516 if (Subtarget->hasV6Ops() && !Subtarget->isThumb()) {
21517 Value* args[6] = {Builder.getInt32(C: 15), Builder.getInt32(C: 0),
21518 Builder.getInt32(C: 0), Builder.getInt32(C: 7),
21519 Builder.getInt32(C: 10), Builder.getInt32(C: 5)};
21520 return Builder.CreateIntrinsicWithoutFolding(ID: Intrinsic::arm_mcr, Args: args);
21521 }
21522 // Instead of using barriers, atomic accesses on these subtargets use
21523 // libcalls.
21524 llvm_unreachable("makeDMB on a target so old that it has no barriers");
21525 } else {
21526 // Only a full system barrier exists in the M-class architectures.
21527 Domain = Subtarget->isMClass() ? ARM_MB::SY : Domain;
21528 Constant *CDomain = Builder.getInt32(C: Domain);
21529 return Builder.CreateIntrinsicWithoutFolding(ID: Intrinsic::arm_dmb, Args: CDomain);
21530 }
21531}
21532
21533// Based on http://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html
21534Instruction *ARMTargetLowering::emitLeadingFence(IRBuilderBase &Builder,
21535 Instruction *Inst,
21536 AtomicOrdering Ord) const {
21537 switch (Ord) {
21538 case AtomicOrdering::NotAtomic:
21539 case AtomicOrdering::Unordered:
21540 llvm_unreachable("Invalid fence: unordered/non-atomic");
21541 case AtomicOrdering::Monotonic:
21542 case AtomicOrdering::Acquire:
21543 return nullptr; // Nothing to do
21544 case AtomicOrdering::SequentiallyConsistent:
21545 if (!Inst->hasAtomicStore())
21546 return nullptr; // Nothing to do
21547 [[fallthrough]];
21548 case AtomicOrdering::Release:
21549 case AtomicOrdering::AcquireRelease:
21550 if (Subtarget->preferISHSTBarriers())
21551 return makeDMB(Builder, Domain: ARM_MB::ISHST);
21552 // FIXME: add a comment with a link to documentation justifying this.
21553 else
21554 return makeDMB(Builder, Domain: ARM_MB::ISH);
21555 }
21556 llvm_unreachable("Unknown fence ordering in emitLeadingFence");
21557}
21558
21559Instruction *ARMTargetLowering::emitTrailingFence(IRBuilderBase &Builder,
21560 Instruction *Inst,
21561 AtomicOrdering Ord) const {
21562 switch (Ord) {
21563 case AtomicOrdering::NotAtomic:
21564 case AtomicOrdering::Unordered:
21565 llvm_unreachable("Invalid fence: unordered/not-atomic");
21566 case AtomicOrdering::Monotonic:
21567 case AtomicOrdering::Release:
21568 return nullptr; // Nothing to do
21569 case AtomicOrdering::Acquire:
21570 case AtomicOrdering::AcquireRelease:
21571 case AtomicOrdering::SequentiallyConsistent:
21572 return makeDMB(Builder, Domain: ARM_MB::ISH);
21573 }
21574 llvm_unreachable("Unknown fence ordering in emitTrailingFence");
21575}
21576
21577// Loads and stores less than 64-bits are already atomic; ones above that
21578// are doomed anyway, so defer to the default libcall and blame the OS when
21579// things go wrong. Cortex M doesn't have ldrexd/strexd though, so don't emit
21580// anything for those.
21581TargetLoweringBase::AtomicExpansionKind
21582ARMTargetLowering::shouldExpandAtomicStoreInIR(StoreInst *SI) const {
21583 bool has64BitAtomicStore;
21584 if (Subtarget->isMClass())
21585 has64BitAtomicStore = false;
21586 else if (Subtarget->isThumb())
21587 has64BitAtomicStore = Subtarget->hasV7Ops();
21588 else
21589 has64BitAtomicStore = Subtarget->hasV6Ops();
21590
21591 unsigned Size = SI->getValueOperand()->getType()->getPrimitiveSizeInBits();
21592 return Size == 64 && has64BitAtomicStore ? AtomicExpansionKind::Expand
21593 : AtomicExpansionKind::None;
21594}
21595
21596// Loads and stores less than 64-bits are already atomic; ones above that
21597// are doomed anyway, so defer to the default libcall and blame the OS when
21598// things go wrong. Cortex M doesn't have ldrexd/strexd though, so don't emit
21599// anything for those.
21600// FIXME: ldrd and strd are atomic if the CPU has LPAE (e.g. A15 has that
21601// guarantee, see DDI0406C ARM architecture reference manual,
21602// sections A8.8.72-74 LDRD)
21603TargetLowering::AtomicExpansionKind
21604ARMTargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const {
21605 bool has64BitAtomicLoad;
21606 if (Subtarget->isMClass())
21607 has64BitAtomicLoad = false;
21608 else if (Subtarget->isThumb())
21609 has64BitAtomicLoad = Subtarget->hasV7Ops();
21610 else
21611 has64BitAtomicLoad = Subtarget->hasV6Ops();
21612
21613 unsigned Size = LI->getType()->getPrimitiveSizeInBits();
21614 return (Size == 64 && has64BitAtomicLoad) ? AtomicExpansionKind::LLOnly
21615 : AtomicExpansionKind::None;
21616}
21617
21618// For the real atomic operations, we have ldrex/strex up to 32 bits,
21619// and up to 64 bits on the non-M profiles
21620TargetLowering::AtomicExpansionKind
21621ARMTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
21622 if (AI->isFloatingPointOperation())
21623 return AtomicExpansionKind::CmpXChg;
21624
21625 unsigned Size = AI->getType()->getPrimitiveSizeInBits();
21626 bool hasAtomicRMW;
21627 if (Subtarget->isMClass())
21628 hasAtomicRMW = Subtarget->hasV8MBaselineOps();
21629 else if (Subtarget->isThumb())
21630 hasAtomicRMW = Subtarget->hasV7Ops();
21631 else
21632 hasAtomicRMW = Subtarget->hasV6Ops();
21633 if (Size <= (Subtarget->isMClass() ? 32U : 64U) && hasAtomicRMW) {
21634 // At -O0, fast-regalloc cannot cope with the live vregs necessary to
21635 // implement atomicrmw without spilling. If the target address is also on
21636 // the stack and close enough to the spill slot, this can lead to a
21637 // situation where the monitor always gets cleared and the atomic operation
21638 // can never succeed. So at -O0 lower this operation to a CAS loop.
21639 if (getTargetMachine().getOptLevel() == CodeGenOptLevel::None)
21640 return AtomicExpansionKind::CmpXChg;
21641 return AtomicExpansionKind::LLSC;
21642 }
21643 return AtomicExpansionKind::None;
21644}
21645
21646// Similar to shouldExpandAtomicRMWInIR, ldrex/strex can be used up to 32
21647// bits, and up to 64 bits on the non-M profiles.
21648TargetLowering::AtomicExpansionKind
21649ARMTargetLowering::shouldExpandAtomicCmpXchgInIR(
21650 const AtomicCmpXchgInst *AI) const {
21651 // At -O0, fast-regalloc cannot cope with the live vregs necessary to
21652 // implement cmpxchg without spilling. If the address being exchanged is also
21653 // on the stack and close enough to the spill slot, this can lead to a
21654 // situation where the monitor always gets cleared and the atomic operation
21655 // can never succeed. So at -O0 we need a late-expanded pseudo-inst instead.
21656 unsigned Size = AI->getOperand(i_nocapture: 1)->getType()->getPrimitiveSizeInBits();
21657 bool HasAtomicCmpXchg;
21658 if (Subtarget->isMClass())
21659 HasAtomicCmpXchg = Subtarget->hasV8MBaselineOps();
21660 else if (Subtarget->isThumb())
21661 HasAtomicCmpXchg = Subtarget->hasV7Ops();
21662 else
21663 HasAtomicCmpXchg = Subtarget->hasV6Ops();
21664 if (getTargetMachine().getOptLevel() != CodeGenOptLevel::None &&
21665 HasAtomicCmpXchg && Size <= (Subtarget->isMClass() ? 32U : 64U))
21666 return AtomicExpansionKind::LLSC;
21667 return AtomicExpansionKind::None;
21668}
21669
21670bool ARMTargetLowering::shouldInsertFencesForAtomic(
21671 const Instruction *I) const {
21672 return InsertFencesForAtomic;
21673}
21674
21675bool ARMTargetLowering::useLoadStackGuardNode(const Module &M) const {
21676 // ROPI/RWPI are not supported currently.
21677 return !Subtarget->isROPI() && !Subtarget->isRWPI();
21678}
21679
21680void ARMTargetLowering::insertSSPDeclarations(
21681 Module &M, const LibcallLoweringInfo &Libcalls) const {
21682 // MSVC CRT provides functionalities for stack protection.
21683 RTLIB::LibcallImpl SecurityCheckCookieLibcall =
21684 Libcalls.getLibcallImpl(Call: RTLIB::SECURITY_CHECK_COOKIE);
21685
21686 RTLIB::LibcallImpl SecurityCookieVar =
21687 Libcalls.getLibcallImpl(Call: RTLIB::STACK_CHECK_GUARD);
21688 if (SecurityCheckCookieLibcall != RTLIB::Unsupported &&
21689 SecurityCookieVar != RTLIB::Unsupported) {
21690 // MSVC CRT has a global variable holding security cookie.
21691 M.getOrInsertGlobal(Name: getLibcallImplName(Call: SecurityCookieVar),
21692 Ty: PointerType::getUnqual(C&: M.getContext()));
21693
21694 // MSVC CRT has a function to validate security cookie.
21695 FunctionCallee SecurityCheckCookie =
21696 M.getOrInsertFunction(Name: getLibcallImplName(Call: SecurityCheckCookieLibcall),
21697 RetTy: Type::getVoidTy(C&: M.getContext()),
21698 Args: PointerType::getUnqual(C&: M.getContext()));
21699 if (Function *F = dyn_cast<Function>(Val: SecurityCheckCookie.getCallee()))
21700 F->addParamAttr(ArgNo: 0, Kind: Attribute::AttrKind::InReg);
21701 }
21702
21703 TargetLowering::insertSSPDeclarations(M, Libcalls);
21704}
21705
21706bool ARMTargetLowering::canCombineStoreAndExtract(Type *VectorTy, Value *Idx,
21707 unsigned &Cost) const {
21708 // If we do not have NEON, vector types are not natively supported.
21709 if (!Subtarget->hasNEON())
21710 return false;
21711
21712 // Floating point values and vector values map to the same register file.
21713 // Therefore, although we could do a store extract of a vector type, this is
21714 // better to leave at float as we have more freedom in the addressing mode for
21715 // those.
21716 if (VectorTy->isFPOrFPVectorTy())
21717 return false;
21718
21719 // If the index is unknown at compile time, this is very expensive to lower
21720 // and it is not possible to combine the store with the extract.
21721 if (!isa<ConstantInt>(Val: Idx))
21722 return false;
21723
21724 assert(VectorTy->isVectorTy() && "VectorTy is not a vector type");
21725 unsigned BitWidth = VectorTy->getPrimitiveSizeInBits().getFixedValue();
21726 // We can do a store + vector extract on any vector that fits perfectly in a D
21727 // or Q register.
21728 if (BitWidth == 64 || BitWidth == 128) {
21729 Cost = 0;
21730 return true;
21731 }
21732 return false;
21733}
21734
21735bool ARMTargetLowering::canCreateUndefOrPoisonForTargetNode(
21736 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
21737 UndefPoisonKind Kind, bool ConsiderFlags, unsigned Depth) const {
21738 unsigned Opcode = Op.getOpcode();
21739 switch (Opcode) {
21740 case ARMISD::VORRIMM:
21741 case ARMISD::VBICIMM:
21742 return false;
21743 }
21744 return TargetLowering::canCreateUndefOrPoisonForTargetNode(
21745 Op, DemandedElts, DAG, Kind, ConsiderFlags, Depth);
21746}
21747
21748bool ARMTargetLowering::isCheapToSpeculateCttz(Type *Ty) const {
21749 return Subtarget->hasV5TOps() && !Subtarget->isThumb1Only();
21750}
21751
21752bool ARMTargetLowering::isCheapToSpeculateCtlz(Type *Ty) const {
21753 return Subtarget->hasV5TOps() && !Subtarget->isThumb1Only();
21754}
21755
21756bool ARMTargetLowering::isMaskAndCmp0FoldingBeneficial(
21757 const Instruction &AndI) const {
21758 if (!Subtarget->hasV7Ops())
21759 return false;
21760
21761 // Sink the `and` instruction only if the mask would fit into a modified
21762 // immediate operand.
21763 ConstantInt *Mask = dyn_cast<ConstantInt>(Val: AndI.getOperand(i: 1));
21764 if (!Mask || Mask->getValue().getBitWidth() > 32u)
21765 return false;
21766 auto MaskVal = unsigned(Mask->getValue().getZExtValue());
21767 return (Subtarget->isThumb2() ? ARM_AM::getT2SOImmVal(Arg: MaskVal)
21768 : ARM_AM::getSOImmVal(Arg: MaskVal)) != -1;
21769}
21770
21771TargetLowering::ShiftLegalizationStrategy
21772ARMTargetLowering::preferredShiftLegalizationStrategy(
21773 SelectionDAG &DAG, SDNode *N, unsigned ExpansionFactor) const {
21774 if (Subtarget->hasMinSize() && !getTM().getTargetTriple().isOSWindows())
21775 return ShiftLegalizationStrategy::LowerToLibcall;
21776 return TargetLowering::preferredShiftLegalizationStrategy(DAG, N,
21777 ExpansionFactor);
21778}
21779
21780Value *ARMTargetLowering::emitLoadLinked(IRBuilderBase &Builder, Type *ValueTy,
21781 Value *Addr,
21782 AtomicOrdering Ord) const {
21783 Module *M = Builder.GetInsertBlock()->getParent()->getParent();
21784 bool IsAcquire = isAcquireOrStronger(AO: Ord);
21785
21786 // Since i64 isn't legal and intrinsics don't get type-lowered, the ldrexd
21787 // intrinsic must return {i32, i32} and we have to recombine them into a
21788 // single i64 here.
21789 if (ValueTy->getPrimitiveSizeInBits() == 64) {
21790 Intrinsic::ID Int =
21791 IsAcquire ? Intrinsic::arm_ldaexd : Intrinsic::arm_ldrexd;
21792
21793 Value *LoHi =
21794 Builder.CreateIntrinsic(ID: Int, Args: Addr, /*FMFSource=*/nullptr, Name: "lohi");
21795
21796 Value *Lo = Builder.CreateExtractValue(Agg: LoHi, Idxs: 0, Name: "lo");
21797 Value *Hi = Builder.CreateExtractValue(Agg: LoHi, Idxs: 1, Name: "hi");
21798 if (!Subtarget->isLittle())
21799 std::swap (a&: Lo, b&: Hi);
21800 Lo = Builder.CreateZExt(V: Lo, DestTy: ValueTy, Name: "lo64");
21801 Hi = Builder.CreateZExt(V: Hi, DestTy: ValueTy, Name: "hi64");
21802 return Builder.CreateOr(
21803 LHS: Lo, RHS: Builder.CreateShl(LHS: Hi, RHS: ConstantInt::get(Ty: ValueTy, V: 32)), Name: "val64");
21804 }
21805
21806 Type *Tys[] = { Addr->getType() };
21807 Intrinsic::ID Int = IsAcquire ? Intrinsic::arm_ldaex : Intrinsic::arm_ldrex;
21808 CallInst *CI = Builder.CreateIntrinsicWithoutFolding(ID: Int, OverloadTypes: Tys, Args: Addr);
21809
21810 CI->addParamAttr(
21811 ArgNo: 0, Attr: Attribute::get(Context&: M->getContext(), Kind: Attribute::ElementType, Ty: ValueTy));
21812 return Builder.CreateTruncOrBitCast(V: CI, DestTy: ValueTy);
21813}
21814
21815void ARMTargetLowering::emitAtomicCmpXchgNoStoreLLBalance(
21816 IRBuilderBase &Builder) const {
21817 if (!Subtarget->hasV7Ops())
21818 return;
21819 Builder.CreateIntrinsic(ID: Intrinsic::arm_clrex, Args: {});
21820}
21821
21822Value *ARMTargetLowering::emitStoreConditional(IRBuilderBase &Builder,
21823 Value *Val, Value *Addr,
21824 AtomicOrdering Ord) const {
21825 Module *M = Builder.GetInsertBlock()->getParent()->getParent();
21826 bool IsRelease = isReleaseOrStronger(AO: Ord);
21827
21828 // Since the intrinsics must have legal type, the i64 intrinsics take two
21829 // parameters: "i32, i32". We must marshal Val into the appropriate form
21830 // before the call.
21831 if (Val->getType()->getPrimitiveSizeInBits() == 64) {
21832 Intrinsic::ID Int =
21833 IsRelease ? Intrinsic::arm_stlexd : Intrinsic::arm_strexd;
21834 Type *Int32Ty = Type::getInt32Ty(C&: M->getContext());
21835
21836 Value *Lo = Builder.CreateTrunc(V: Val, DestTy: Int32Ty, Name: "lo");
21837 Value *Hi = Builder.CreateTrunc(V: Builder.CreateLShr(LHS: Val, RHS: 32), DestTy: Int32Ty, Name: "hi");
21838 if (!Subtarget->isLittle())
21839 std::swap(a&: Lo, b&: Hi);
21840 return Builder.CreateIntrinsic(ID: Int, Args: {Lo, Hi, Addr});
21841 }
21842
21843 Intrinsic::ID Int = IsRelease ? Intrinsic::arm_stlex : Intrinsic::arm_strex;
21844 Type *Tys[] = { Addr->getType() };
21845 Function *Strex = Intrinsic::getOrInsertDeclaration(M, id: Int, OverloadTys: Tys);
21846
21847 CallInst *CI = Builder.CreateCall(
21848 Callee: Strex, Args: {Builder.CreateZExtOrBitCast(
21849 V: Val, DestTy: Strex->getFunctionType()->getParamType(i: 0)),
21850 Addr});
21851 CI->addParamAttr(ArgNo: 1, Attr: Attribute::get(Context&: M->getContext(), Kind: Attribute::ElementType,
21852 Ty: Val->getType()));
21853 return CI;
21854}
21855
21856
21857bool ARMTargetLowering::alignLoopsWithOptSize() const {
21858 return Subtarget->isMClass();
21859}
21860
21861/// A helper function for determining the number of interleaved accesses we
21862/// will generate when lowering accesses of the given type.
21863unsigned
21864ARMTargetLowering::getNumInterleavedAccesses(VectorType *VecTy,
21865 const DataLayout &DL) const {
21866 return (DL.getTypeSizeInBits(Ty: VecTy) + 127) / 128;
21867}
21868
21869bool ARMTargetLowering::isLegalInterleavedAccessType(
21870 unsigned Factor, FixedVectorType *VecTy, Align Alignment,
21871 const DataLayout &DL) const {
21872
21873 unsigned VecSize = DL.getTypeSizeInBits(Ty: VecTy);
21874 unsigned ElSize = DL.getTypeSizeInBits(Ty: VecTy->getElementType());
21875
21876 if (!Subtarget->hasNEON() && !Subtarget->hasMVEIntegerOps())
21877 return false;
21878
21879 // Ensure the vector doesn't have f16 elements. Even though we could do an
21880 // i16 vldN, we can't hold the f16 vectors and will end up converting via
21881 // f32.
21882 if (Subtarget->hasNEON() && VecTy->getElementType()->isHalfTy())
21883 return false;
21884 if (Subtarget->hasMVEIntegerOps() && Factor == 3)
21885 return false;
21886
21887 // Ensure the number of vector elements is greater than 1.
21888 if (VecTy->getNumElements() < 2)
21889 return false;
21890
21891 // Ensure the element type is legal.
21892 if (ElSize != 8 && ElSize != 16 && ElSize != 32)
21893 return false;
21894 // And the alignment if high enough under MVE.
21895 if (Subtarget->hasMVEIntegerOps() && Alignment < ElSize / 8)
21896 return false;
21897
21898 // Ensure the total vector size is 64 or a multiple of 128. Types larger than
21899 // 128 will be split into multiple interleaved accesses.
21900 if (Subtarget->hasNEON() && VecSize == 64)
21901 return true;
21902 return VecSize % 128 == 0;
21903}
21904
21905unsigned ARMTargetLowering::getMaxSupportedInterleaveFactor() const {
21906 if (Subtarget->hasNEON())
21907 return 4;
21908 if (Subtarget->hasMVEIntegerOps())
21909 return MVEMaxSupportedInterleaveFactor;
21910 return TargetLoweringBase::getMaxSupportedInterleaveFactor();
21911}
21912
21913/// Lower an interleaved load into a vldN intrinsic.
21914///
21915/// E.g. Lower an interleaved load (Factor = 2):
21916/// %wide.vec = load <8 x i32>, <8 x i32>* %ptr, align 4
21917/// %v0 = shuffle %wide.vec, undef, <0, 2, 4, 6> ; Extract even elements
21918/// %v1 = shuffle %wide.vec, undef, <1, 3, 5, 7> ; Extract odd elements
21919///
21920/// Into:
21921/// %vld2 = { <4 x i32>, <4 x i32> } call llvm.arm.neon.vld2(%ptr, 4)
21922/// %vec0 = extractelement { <4 x i32>, <4 x i32> } %vld2, i32 0
21923/// %vec1 = extractelement { <4 x i32>, <4 x i32> } %vld2, i32 1
21924bool ARMTargetLowering::lowerInterleavedLoad(
21925 Instruction *Load, Value *Mask, ArrayRef<ShuffleVectorInst *> Shuffles,
21926 ArrayRef<unsigned> Indices, unsigned Factor, const APInt &GapMask) const {
21927 assert(Factor >= 2 && Factor <= getMaxSupportedInterleaveFactor() &&
21928 "Invalid interleave factor");
21929 assert(!Shuffles.empty() && "Empty shufflevector input");
21930 assert(Shuffles.size() == Indices.size() &&
21931 "Unmatched number of shufflevectors and indices");
21932
21933 auto *LI = dyn_cast<LoadInst>(Val: Load);
21934 if (!LI)
21935 return false;
21936 assert(!Mask && GapMask.popcount() == Factor && "Unexpected mask on a load");
21937
21938 auto *VecTy = cast<FixedVectorType>(Val: Shuffles[0]->getType());
21939 Type *EltTy = VecTy->getElementType();
21940
21941 const DataLayout &DL = LI->getDataLayout();
21942 Align Alignment = LI->getAlign();
21943
21944 // Skip if we do not have NEON and skip illegal vector types. We can
21945 // "legalize" wide vector types into multiple interleaved accesses as long as
21946 // the vector types are divisible by 128.
21947 if (!isLegalInterleavedAccessType(Factor, VecTy, Alignment, DL))
21948 return false;
21949
21950 unsigned NumLoads = getNumInterleavedAccesses(VecTy, DL);
21951
21952 // A pointer vector can not be the return type of the ldN intrinsics. Need to
21953 // load integer vectors first and then convert to pointer vectors.
21954 if (EltTy->isPointerTy())
21955 VecTy = FixedVectorType::get(ElementType: DL.getIntPtrType(EltTy), FVTy: VecTy);
21956
21957 IRBuilder<> Builder(LI);
21958
21959 // The base address of the load.
21960 Value *BaseAddr = LI->getPointerOperand();
21961
21962 if (NumLoads > 1) {
21963 // If we're going to generate more than one load, reset the sub-vector type
21964 // to something legal.
21965 VecTy = FixedVectorType::get(ElementType: VecTy->getElementType(),
21966 NumElts: VecTy->getNumElements() / NumLoads);
21967 }
21968
21969 assert(isTypeLegal(EVT::getEVT(VecTy)) && "Illegal vldN vector type!");
21970
21971 auto createLoadIntrinsic = [&](Value *BaseAddr) {
21972 if (Subtarget->hasNEON()) {
21973 Type *PtrTy = Builder.getPtrTy(AddrSpace: LI->getPointerAddressSpace());
21974 Type *Tys[] = {VecTy, PtrTy};
21975 static const Intrinsic::ID LoadInts[3] = {Intrinsic::arm_neon_vld2,
21976 Intrinsic::arm_neon_vld3,
21977 Intrinsic::arm_neon_vld4};
21978
21979 SmallVector<Value *, 2> Ops;
21980 Ops.push_back(Elt: BaseAddr);
21981 Ops.push_back(Elt: Builder.getInt32(C: LI->getAlign().value()));
21982
21983 return Builder.CreateIntrinsic(ID: LoadInts[Factor - 2], OverloadTypes: Tys, Args: Ops,
21984 /*FMFSource=*/nullptr, Name: "vldN");
21985 } else {
21986 assert((Factor == 2 || Factor == 4) &&
21987 "expected interleave factor of 2 or 4 for MVE");
21988 Intrinsic::ID LoadInts =
21989 Factor == 2 ? Intrinsic::arm_mve_vld2q : Intrinsic::arm_mve_vld4q;
21990 Type *PtrTy = Builder.getPtrTy(AddrSpace: LI->getPointerAddressSpace());
21991 Type *Tys[] = {VecTy, PtrTy};
21992
21993 SmallVector<Value *, 2> Ops;
21994 Ops.push_back(Elt: BaseAddr);
21995 return Builder.CreateIntrinsic(ID: LoadInts, OverloadTypes: Tys, Args: Ops, /*FMFSource=*/nullptr,
21996 Name: "vldN");
21997 }
21998 };
21999
22000 // Holds sub-vectors extracted from the load intrinsic return values. The
22001 // sub-vectors are associated with the shufflevector instructions they will
22002 // replace.
22003 DenseMap<ShuffleVectorInst *, SmallVector<Value *, 4>> SubVecs;
22004
22005 for (unsigned LoadCount = 0; LoadCount < NumLoads; ++LoadCount) {
22006 // If we're generating more than one load, compute the base address of
22007 // subsequent loads as an offset from the previous.
22008 if (LoadCount > 0)
22009 BaseAddr = Builder.CreateConstGEP1_32(Ty: VecTy->getElementType(), Ptr: BaseAddr,
22010 Idx0: VecTy->getNumElements() * Factor);
22011
22012 Value *VldN = createLoadIntrinsic(BaseAddr);
22013
22014 // Replace uses of each shufflevector with the corresponding vector loaded
22015 // by ldN.
22016 for (unsigned i = 0; i < Shuffles.size(); i++) {
22017 ShuffleVectorInst *SV = Shuffles[i];
22018 unsigned Index = Indices[i];
22019
22020 Value *SubVec = Builder.CreateExtractValue(Agg: VldN, Idxs: Index);
22021
22022 // Convert the integer vector to pointer vector if the element is pointer.
22023 if (EltTy->isPointerTy())
22024 SubVec = Builder.CreateIntToPtr(
22025 V: SubVec,
22026 DestTy: FixedVectorType::get(ElementType: SV->getType()->getElementType(), FVTy: VecTy));
22027
22028 SubVecs[SV].push_back(Elt: SubVec);
22029 }
22030 }
22031
22032 // Replace uses of the shufflevector instructions with the sub-vectors
22033 // returned by the load intrinsic. If a shufflevector instruction is
22034 // associated with more than one sub-vector, those sub-vectors will be
22035 // concatenated into a single wide vector.
22036 for (ShuffleVectorInst *SVI : Shuffles) {
22037 auto &SubVec = SubVecs[SVI];
22038 auto *WideVec =
22039 SubVec.size() > 1 ? concatenateVectors(Builder, Vecs: SubVec) : SubVec[0];
22040 SVI->replaceAllUsesWith(V: WideVec);
22041 }
22042
22043 return true;
22044}
22045
22046/// Lower an interleaved store into a vstN intrinsic.
22047///
22048/// E.g. Lower an interleaved store (Factor = 3):
22049/// %i.vec = shuffle <8 x i32> %v0, <8 x i32> %v1,
22050/// <0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11>
22051/// store <12 x i32> %i.vec, <12 x i32>* %ptr, align 4
22052///
22053/// Into:
22054/// %sub.v0 = shuffle <8 x i32> %v0, <8 x i32> v1, <0, 1, 2, 3>
22055/// %sub.v1 = shuffle <8 x i32> %v0, <8 x i32> v1, <4, 5, 6, 7>
22056/// %sub.v2 = shuffle <8 x i32> %v0, <8 x i32> v1, <8, 9, 10, 11>
22057/// call void llvm.arm.neon.vst3(%ptr, %sub.v0, %sub.v1, %sub.v2, 4)
22058///
22059/// Note that the new shufflevectors will be removed and we'll only generate one
22060/// vst3 instruction in CodeGen.
22061///
22062/// Example for a more general valid mask (Factor 3). Lower:
22063/// %i.vec = shuffle <32 x i32> %v0, <32 x i32> %v1,
22064/// <4, 32, 16, 5, 33, 17, 6, 34, 18, 7, 35, 19>
22065/// store <12 x i32> %i.vec, <12 x i32>* %ptr
22066///
22067/// Into:
22068/// %sub.v0 = shuffle <32 x i32> %v0, <32 x i32> v1, <4, 5, 6, 7>
22069/// %sub.v1 = shuffle <32 x i32> %v0, <32 x i32> v1, <32, 33, 34, 35>
22070/// %sub.v2 = shuffle <32 x i32> %v0, <32 x i32> v1, <16, 17, 18, 19>
22071/// call void llvm.arm.neon.vst3(%ptr, %sub.v0, %sub.v1, %sub.v2, 4)
22072bool ARMTargetLowering::lowerInterleavedStore(Instruction *Store,
22073 Value *LaneMask,
22074 ShuffleVectorInst *SVI,
22075 unsigned Factor,
22076 const APInt &GapMask) const {
22077 assert(Factor >= 2 && Factor <= getMaxSupportedInterleaveFactor() &&
22078 "Invalid interleave factor");
22079 auto *SI = dyn_cast<StoreInst>(Val: Store);
22080 if (!SI)
22081 return false;
22082 assert(!LaneMask && GapMask.popcount() == Factor &&
22083 "Unexpected mask on store");
22084
22085 auto *VecTy = cast<FixedVectorType>(Val: SVI->getType());
22086 assert(VecTy->getNumElements() % Factor == 0 && "Invalid interleaved store");
22087
22088 unsigned LaneLen = VecTy->getNumElements() / Factor;
22089 Type *EltTy = VecTy->getElementType();
22090 auto *SubVecTy = FixedVectorType::get(ElementType: EltTy, NumElts: LaneLen);
22091
22092 const DataLayout &DL = SI->getDataLayout();
22093 Align Alignment = SI->getAlign();
22094
22095 // Skip if we do not have NEON and skip illegal vector types. We can
22096 // "legalize" wide vector types into multiple interleaved accesses as long as
22097 // the vector types are divisible by 128.
22098 if (!isLegalInterleavedAccessType(Factor, VecTy: SubVecTy, Alignment, DL))
22099 return false;
22100
22101 unsigned NumStores = getNumInterleavedAccesses(VecTy: SubVecTy, DL);
22102
22103 Value *Op0 = SVI->getOperand(i_nocapture: 0);
22104 Value *Op1 = SVI->getOperand(i_nocapture: 1);
22105 IRBuilder<> Builder(SI);
22106
22107 // StN intrinsics don't support pointer vectors as arguments. Convert pointer
22108 // vectors to integer vectors.
22109 if (EltTy->isPointerTy()) {
22110 Type *IntTy = DL.getIntPtrType(EltTy);
22111
22112 // Convert to the corresponding integer vector.
22113 auto *IntVecTy =
22114 FixedVectorType::get(ElementType: IntTy, FVTy: cast<FixedVectorType>(Val: Op0->getType()));
22115 Op0 = Builder.CreatePtrToInt(V: Op0, DestTy: IntVecTy);
22116 Op1 = Builder.CreatePtrToInt(V: Op1, DestTy: IntVecTy);
22117
22118 SubVecTy = FixedVectorType::get(ElementType: IntTy, NumElts: LaneLen);
22119 }
22120
22121 // The base address of the store.
22122 Value *BaseAddr = SI->getPointerOperand();
22123
22124 if (NumStores > 1) {
22125 // If we're going to generate more than one store, reset the lane length
22126 // and sub-vector type to something legal.
22127 LaneLen /= NumStores;
22128 SubVecTy = FixedVectorType::get(ElementType: SubVecTy->getElementType(), NumElts: LaneLen);
22129 }
22130
22131 assert(isTypeLegal(EVT::getEVT(SubVecTy)) && "Illegal vstN vector type!");
22132
22133 auto Mask = SVI->getShuffleMask();
22134
22135 auto createStoreIntrinsic = [&](Value *BaseAddr,
22136 SmallVectorImpl<Value *> &Shuffles) {
22137 if (Subtarget->hasNEON()) {
22138 static const Intrinsic::ID StoreInts[3] = {Intrinsic::arm_neon_vst2,
22139 Intrinsic::arm_neon_vst3,
22140 Intrinsic::arm_neon_vst4};
22141 Type *PtrTy = Builder.getPtrTy(AddrSpace: SI->getPointerAddressSpace());
22142 Type *Tys[] = {PtrTy, SubVecTy};
22143
22144 SmallVector<Value *, 6> Ops;
22145 Ops.push_back(Elt: BaseAddr);
22146 append_range(C&: Ops, R&: Shuffles);
22147 Ops.push_back(Elt: Builder.getInt32(C: SI->getAlign().value()));
22148 Builder.CreateIntrinsic(ID: StoreInts[Factor - 2], OverloadTypes: Tys, Args: Ops);
22149 } else {
22150 assert((Factor == 2 || Factor == 4) &&
22151 "expected interleave factor of 2 or 4 for MVE");
22152 Intrinsic::ID StoreInts =
22153 Factor == 2 ? Intrinsic::arm_mve_vst2q : Intrinsic::arm_mve_vst4q;
22154 Type *PtrTy = Builder.getPtrTy(AddrSpace: SI->getPointerAddressSpace());
22155 Type *Tys[] = {PtrTy, SubVecTy};
22156
22157 SmallVector<Value *, 6> Ops;
22158 Ops.push_back(Elt: BaseAddr);
22159 append_range(C&: Ops, R&: Shuffles);
22160 for (unsigned F = 0; F < Factor; F++) {
22161 Ops.push_back(Elt: Builder.getInt32(C: F));
22162 Builder.CreateIntrinsic(ID: StoreInts, OverloadTypes: Tys, Args: Ops);
22163 Ops.pop_back();
22164 }
22165 }
22166 };
22167
22168 for (unsigned StoreCount = 0; StoreCount < NumStores; ++StoreCount) {
22169 // If we generating more than one store, we compute the base address of
22170 // subsequent stores as an offset from the previous.
22171 if (StoreCount > 0)
22172 BaseAddr = Builder.CreateConstGEP1_32(Ty: SubVecTy->getElementType(),
22173 Ptr: BaseAddr, Idx0: LaneLen * Factor);
22174
22175 SmallVector<Value *, 4> Shuffles;
22176
22177 // Split the shufflevector operands into sub vectors for the new vstN call.
22178 for (unsigned i = 0; i < Factor; i++) {
22179 unsigned IdxI = StoreCount * LaneLen * Factor + i;
22180 if (Mask[IdxI] >= 0) {
22181 Shuffles.push_back(Elt: Builder.CreateShuffleVector(
22182 V1: Op0, V2: Op1, Mask: createSequentialMask(Start: Mask[IdxI], NumInts: LaneLen, NumUndefs: 0)));
22183 } else {
22184 unsigned StartMask = 0;
22185 for (unsigned j = 1; j < LaneLen; j++) {
22186 unsigned IdxJ = StoreCount * LaneLen * Factor + j;
22187 if (Mask[IdxJ * Factor + IdxI] >= 0) {
22188 StartMask = Mask[IdxJ * Factor + IdxI] - IdxJ;
22189 break;
22190 }
22191 }
22192 // Note: If all elements in a chunk are undefs, StartMask=0!
22193 // Note: Filling undef gaps with random elements is ok, since
22194 // those elements were being written anyway (with undefs).
22195 // In the case of all undefs we're defaulting to using elems from 0
22196 // Note: StartMask cannot be negative, it's checked in
22197 // isReInterleaveMask
22198 Shuffles.push_back(Elt: Builder.CreateShuffleVector(
22199 V1: Op0, V2: Op1, Mask: createSequentialMask(Start: StartMask, NumInts: LaneLen, NumUndefs: 0)));
22200 }
22201 }
22202
22203 createStoreIntrinsic(BaseAddr, Shuffles);
22204 }
22205 return true;
22206}
22207
22208enum HABaseType {
22209 HA_UNKNOWN = 0,
22210 HA_FLOAT,
22211 HA_DOUBLE,
22212 HA_VECT64,
22213 HA_VECT128
22214};
22215
22216static bool isHomogeneousAggregate(Type *Ty, HABaseType &Base,
22217 uint64_t &Members) {
22218 if (auto *ST = dyn_cast<StructType>(Val: Ty)) {
22219 for (unsigned i = 0; i < ST->getNumElements(); ++i) {
22220 uint64_t SubMembers = 0;
22221 if (!isHomogeneousAggregate(Ty: ST->getElementType(N: i), Base, Members&: SubMembers))
22222 return false;
22223 Members += SubMembers;
22224 }
22225 } else if (auto *AT = dyn_cast<ArrayType>(Val: Ty)) {
22226 uint64_t SubMembers = 0;
22227 if (!isHomogeneousAggregate(Ty: AT->getElementType(), Base, Members&: SubMembers))
22228 return false;
22229 Members += SubMembers * AT->getNumElements();
22230 } else if (Ty->isFloatTy()) {
22231 if (Base != HA_UNKNOWN && Base != HA_FLOAT)
22232 return false;
22233 Members = 1;
22234 Base = HA_FLOAT;
22235 } else if (Ty->isDoubleTy()) {
22236 if (Base != HA_UNKNOWN && Base != HA_DOUBLE)
22237 return false;
22238 Members = 1;
22239 Base = HA_DOUBLE;
22240 } else if (auto *VT = dyn_cast<VectorType>(Val: Ty)) {
22241 Members = 1;
22242 switch (Base) {
22243 case HA_FLOAT:
22244 case HA_DOUBLE:
22245 return false;
22246 case HA_VECT64:
22247 return VT->getPrimitiveSizeInBits().getFixedValue() == 64;
22248 case HA_VECT128:
22249 return VT->getPrimitiveSizeInBits().getFixedValue() == 128;
22250 case HA_UNKNOWN:
22251 switch (VT->getPrimitiveSizeInBits().getFixedValue()) {
22252 case 64:
22253 Base = HA_VECT64;
22254 return true;
22255 case 128:
22256 Base = HA_VECT128;
22257 return true;
22258 default:
22259 return false;
22260 }
22261 }
22262 }
22263
22264 return (Members > 0 && Members <= 4);
22265}
22266
22267/// Return the correct alignment for the current calling convention.
22268Align ARMTargetLowering::getABIAlignmentForCallingConv(
22269 Type *ArgTy, const DataLayout &DL) const {
22270 const Align ABITypeAlign = DL.getABITypeAlign(Ty: ArgTy);
22271 if (!ArgTy->isVectorTy())
22272 return ABITypeAlign;
22273
22274 // Avoid over-aligning vector parameters. It would require realigning the
22275 // stack and waste space for no real benefit.
22276 MaybeAlign StackAlign = DL.getStackAlignment();
22277 assert(StackAlign && "data layout string is missing stack alignment");
22278 return std::min(a: ABITypeAlign, b: *StackAlign);
22279}
22280
22281/// Return true if a type is an AAPCS-VFP homogeneous aggregate or one of
22282/// [N x i32] or [N x i64]. This allows front-ends to skip emitting padding when
22283/// passing according to AAPCS rules.
22284bool ARMTargetLowering::functionArgumentNeedsConsecutiveRegisters(
22285 Type *Ty, CallingConv::ID CallConv, bool isVarArg,
22286 const DataLayout &DL) const {
22287 if (getEffectiveCallingConv(CC: CallConv, isVarArg) !=
22288 CallingConv::ARM_AAPCS_VFP)
22289 return false;
22290
22291 HABaseType Base = HA_UNKNOWN;
22292 uint64_t Members = 0;
22293 bool IsHA = isHomogeneousAggregate(Ty, Base, Members);
22294 LLVM_DEBUG(dbgs() << "isHA: " << IsHA << " "; Ty->dump());
22295
22296 bool IsIntArray = Ty->isArrayTy() && Ty->getArrayElementType()->isIntegerTy();
22297 return IsHA || IsIntArray;
22298}
22299
22300Register ARMTargetLowering::getExceptionPointerRegister(
22301 ExceptionHandling EH, const Constant *PersonalityFn) const {
22302 // Platforms which do not use SjLj EH may return values in these registers
22303 // via the personality function.
22304 return EH == ExceptionHandling::SjLj ? Register() : ARM::R0;
22305}
22306
22307Register ARMTargetLowering::getExceptionSelectorRegister(
22308 ExceptionHandling EH, const Constant *PersonalityFn) const {
22309 // Platforms which do not use SjLj EH may return values in these registers
22310 // via the personality function.
22311 return EH == ExceptionHandling::SjLj ? Register() : ARM::R1;
22312}
22313
22314void ARMTargetLowering::initializeSplitCSR(MachineBasicBlock *Entry) const {
22315 // Update IsSplitCSR in ARMFunctionInfo.
22316 ARMFunctionInfo *AFI = Entry->getParent()->getInfo<ARMFunctionInfo>();
22317 AFI->setIsSplitCSR(true);
22318}
22319
22320void ARMTargetLowering::insertCopiesSplitCSR(
22321 MachineBasicBlock *Entry,
22322 const SmallVectorImpl<MachineBasicBlock *> &Exits) const {
22323 const ARMBaseRegisterInfo *TRI = Subtarget->getRegisterInfo();
22324 const MCPhysReg *IStart = TRI->getCalleeSavedRegsViaCopy(MF: Entry->getParent());
22325 if (!IStart)
22326 return;
22327
22328 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
22329 MachineRegisterInfo *MRI = &Entry->getParent()->getRegInfo();
22330 MachineBasicBlock::iterator MBBI = Entry->begin();
22331 for (const MCPhysReg *I = IStart; *I; ++I) {
22332 const TargetRegisterClass *RC = nullptr;
22333 if (ARM::GPRRegClass.contains(Reg: *I))
22334 RC = &ARM::GPRRegClass;
22335 else if (ARM::DPRRegClass.contains(Reg: *I))
22336 RC = &ARM::DPRRegClass;
22337 else
22338 llvm_unreachable("Unexpected register class in CSRsViaCopy!");
22339
22340 Register NewVR = MRI->createVirtualRegister(RegClass: RC);
22341 // Create copy from CSR to a virtual register.
22342 // FIXME: this currently does not emit CFI pseudo-instructions, it works
22343 // fine for CXX_FAST_TLS since the C++-style TLS access functions should be
22344 // nounwind. If we want to generalize this later, we may need to emit
22345 // CFI pseudo-instructions.
22346 assert(Entry->getParent()->getFunction().hasFnAttribute(
22347 Attribute::NoUnwind) &&
22348 "Function should be nounwind in insertCopiesSplitCSR!");
22349 Entry->addLiveIn(PhysReg: *I);
22350 BuildMI(BB&: *Entry, I: MBBI, MIMD: DebugLoc(), MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: NewVR)
22351 .addReg(RegNo: *I);
22352
22353 // Insert the copy-back instructions right before the terminator.
22354 for (auto *Exit : Exits)
22355 BuildMI(BB&: *Exit, I: Exit->getFirstTerminator(), MIMD: DebugLoc(),
22356 MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: *I)
22357 .addReg(RegNo: NewVR);
22358 }
22359}
22360
22361void ARMTargetLowering::finalizeLowering(MachineFunction &MF) const {
22362 MF.getFrameInfo().computeMaxCallFrameSize(MF);
22363 TargetLoweringBase::finalizeLowering(MF);
22364}
22365
22366bool ARMTargetLowering::isComplexDeinterleavingSupported() const {
22367 return Subtarget->hasMVEIntegerOps();
22368}
22369
22370bool ARMTargetLowering::isComplexDeinterleavingOperationSupported(
22371 ComplexDeinterleavingOperation Operation, Type *Ty) const {
22372 auto *VTy = dyn_cast<FixedVectorType>(Val: Ty);
22373 if (!VTy)
22374 return false;
22375
22376 auto *ScalarTy = VTy->getScalarType();
22377 unsigned NumElements = VTy->getNumElements();
22378
22379 unsigned VTyWidth = VTy->getScalarSizeInBits() * NumElements;
22380 if (VTyWidth < 128 || !llvm::isPowerOf2_32(Value: VTyWidth))
22381 return false;
22382
22383 // Both VCADD and VCMUL/VCMLA support the same types, F16 and F32
22384 if (ScalarTy->isHalfTy() || ScalarTy->isFloatTy())
22385 return Subtarget->hasMVEFloatOps();
22386
22387 if (Operation != ComplexDeinterleavingOperation::CAdd)
22388 return false;
22389
22390 return Subtarget->hasMVEIntegerOps() &&
22391 (ScalarTy->isIntegerTy(BitWidth: 8) || ScalarTy->isIntegerTy(BitWidth: 16) ||
22392 ScalarTy->isIntegerTy(BitWidth: 32));
22393}
22394
22395ArrayRef<MCPhysReg> ARMTargetLowering::getRoundingControlRegisters() const {
22396 static const MCPhysReg RCRegs[] = {ARM::FPSCR_RM};
22397 return RCRegs;
22398}
22399
22400Value *ARMTargetLowering::createComplexDeinterleavingIR(
22401 IRBuilderBase &B, ComplexDeinterleavingOperation OperationType,
22402 ComplexDeinterleavingRotation Rotation, Value *InputA, Value *InputB,
22403 Value *Accumulator) const {
22404
22405 FixedVectorType *Ty = cast<FixedVectorType>(Val: InputA->getType());
22406
22407 unsigned TyWidth = Ty->getScalarSizeInBits() * Ty->getNumElements();
22408
22409 assert(TyWidth >= 128 && "Width of vector type must be at least 128 bits");
22410
22411 if (TyWidth > 128) {
22412 int Stride = Ty->getNumElements() / 2;
22413 auto SplitSeq = llvm::seq<int>(Begin: 0, End: Ty->getNumElements());
22414 auto SplitSeqVec = llvm::to_vector(Range&: SplitSeq);
22415 ArrayRef<int> LowerSplitMask(&SplitSeqVec[0], Stride);
22416 ArrayRef<int> UpperSplitMask(&SplitSeqVec[Stride], Stride);
22417
22418 auto *LowerSplitA = B.CreateShuffleVector(V: InputA, Mask: LowerSplitMask);
22419 auto *LowerSplitB = B.CreateShuffleVector(V: InputB, Mask: LowerSplitMask);
22420 auto *UpperSplitA = B.CreateShuffleVector(V: InputA, Mask: UpperSplitMask);
22421 auto *UpperSplitB = B.CreateShuffleVector(V: InputB, Mask: UpperSplitMask);
22422 Value *LowerSplitAcc = nullptr;
22423 Value *UpperSplitAcc = nullptr;
22424
22425 if (Accumulator) {
22426 LowerSplitAcc = B.CreateShuffleVector(V: Accumulator, Mask: LowerSplitMask);
22427 UpperSplitAcc = B.CreateShuffleVector(V: Accumulator, Mask: UpperSplitMask);
22428 }
22429
22430 auto *LowerSplitInt = createComplexDeinterleavingIR(
22431 B, OperationType, Rotation, InputA: LowerSplitA, InputB: LowerSplitB, Accumulator: LowerSplitAcc);
22432 auto *UpperSplitInt = createComplexDeinterleavingIR(
22433 B, OperationType, Rotation, InputA: UpperSplitA, InputB: UpperSplitB, Accumulator: UpperSplitAcc);
22434
22435 ArrayRef<int> JoinMask(&SplitSeqVec[0], Ty->getNumElements());
22436 return B.CreateShuffleVector(V1: LowerSplitInt, V2: UpperSplitInt, Mask: JoinMask);
22437 }
22438
22439 auto *IntTy = Type::getInt32Ty(C&: B.getContext());
22440
22441 ConstantInt *ConstRotation = nullptr;
22442 if (OperationType == ComplexDeinterleavingOperation::CMulPartial) {
22443 ConstRotation = ConstantInt::get(Ty: IntTy, V: (int)Rotation);
22444
22445 if (Accumulator)
22446 return B.CreateIntrinsic(ID: Intrinsic::arm_mve_vcmlaq, OverloadTypes: Ty,
22447 Args: {ConstRotation, Accumulator, InputB, InputA});
22448 return B.CreateIntrinsic(ID: Intrinsic::arm_mve_vcmulq, OverloadTypes: Ty,
22449 Args: {ConstRotation, InputB, InputA});
22450 }
22451
22452 if (OperationType == ComplexDeinterleavingOperation::CAdd) {
22453 // 1 means the value is not halved.
22454 auto *ConstHalving = ConstantInt::get(Ty: IntTy, V: 1);
22455
22456 if (Rotation == ComplexDeinterleavingRotation::Rotation_90)
22457 ConstRotation = ConstantInt::get(Ty: IntTy, V: 0);
22458 else if (Rotation == ComplexDeinterleavingRotation::Rotation_270)
22459 ConstRotation = ConstantInt::get(Ty: IntTy, V: 1);
22460
22461 if (!ConstRotation)
22462 return nullptr; // Invalid rotation for arm_mve_vcaddq
22463
22464 return B.CreateIntrinsic(ID: Intrinsic::arm_mve_vcaddq, OverloadTypes: Ty,
22465 Args: {ConstHalving, ConstRotation, InputA, InputB});
22466 }
22467
22468 return nullptr;
22469}
22470