1//===- ARMISelLowering.h - ARM DAG Lowering Interface -----------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the interfaces that ARM uses to lower LLVM code into a
10// selection DAG.
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_LIB_TARGET_ARM_ARMISELLOWERING_H
15#define LLVM_LIB_TARGET_ARM_ARMISELLOWERING_H
16
17#include "MCTargetDesc/ARMBaseInfo.h"
18#include "llvm/ADT/SmallVector.h"
19#include "llvm/ADT/StringRef.h"
20#include "llvm/CodeGen/CallingConvLower.h"
21#include "llvm/CodeGen/ISDOpcodes.h"
22#include "llvm/CodeGen/MachineFunction.h"
23#include "llvm/CodeGen/SelectionDAGNodes.h"
24#include "llvm/CodeGen/TargetLowering.h"
25#include "llvm/CodeGen/ValueTypes.h"
26#include "llvm/CodeGenTypes/MachineValueType.h"
27#include "llvm/IR/Attributes.h"
28#include "llvm/IR/CallingConv.h"
29#include "llvm/IR/Function.h"
30#include "llvm/IR/InlineAsm.h"
31#include "llvm/Support/CodeGen.h"
32#include <optional>
33#include <utility>
34
35namespace llvm {
36
37class ARMSubtarget;
38class DataLayout;
39class FastISel;
40class FunctionLoweringInfo;
41class GlobalValue;
42class InstrItineraryData;
43class Instruction;
44class IRBuilderBase;
45class MachineBasicBlock;
46class MachineInstr;
47class SelectionDAG;
48class TargetLibraryInfo;
49class TargetMachine;
50class TargetRegisterInfo;
51class VectorType;
52
53 namespace ARMISD {
54
55 // ARM Specific DAG Nodes
56 enum NodeType : unsigned {
57 // Start the numbering where the builtin ops and target ops leave off.
58 FIRST_NUMBER = ISD::BUILTIN_OP_END,
59
60 Wrapper, // Wrapper - A wrapper node for TargetConstantPool,
61 // TargetExternalSymbol, and TargetGlobalAddress.
62 WrapperPIC, // WrapperPIC - A wrapper node for TargetGlobalAddress in
63 // PIC mode.
64 WrapperJT, // WrapperJT - A wrapper node for TargetJumpTable
65
66 // Add pseudo op to model memcpy for struct byval.
67 COPY_STRUCT_BYVAL,
68
69 CALL, // Function call.
70 CALL_PRED, // Function call that's predicable.
71 CALL_NOLINK, // Function call with branch not branch-and-link.
72 tSECALL, // CMSE non-secure function call.
73 t2CALL_BTI, // Thumb function call followed by BTI instruction.
74 BRCOND, // Conditional branch.
75 BR_JT, // Jumptable branch.
76 BR2_JT, // Jumptable branch (2 level - jumptable entry is a jump).
77 RET_GLUE, // Return with a flag operand.
78 SERET_GLUE, // CMSE Entry function return with a flag operand.
79 INTRET_GLUE, // Interrupt return with an LR-offset and a flag operand.
80
81 PIC_ADD, // Add with a PC operand and a PIC label.
82
83 ASRL, // MVE long arithmetic shift right.
84 LSRL, // MVE long shift right.
85 LSLL, // MVE long shift left.
86
87 CMP, // ARM compare instructions.
88 CMN, // ARM CMN instructions.
89 CMPZ, // ARM compare that sets only Z flag.
90 CMPFP, // ARM VFP compare instruction, sets FPSCR.
91 CMPFPE, // ARM VFP signalling compare instruction, sets FPSCR.
92 CMPFPw0, // ARM VFP compare against zero instruction, sets FPSCR.
93 CMPFPEw0, // ARM VFP signalling compare against zero instruction, sets
94 // FPSCR.
95 FMSTAT, // ARM fmstat instruction.
96
97 CMOV, // ARM conditional move instructions.
98
99 SSAT, // Signed saturation
100 USAT, // Unsigned saturation
101
102 BCC_i64,
103
104 SRL_GLUE, // V,Flag = srl_flag X -> srl X, 1 + save carry out.
105 SRA_GLUE, // V,Flag = sra_flag X -> sra X, 1 + save carry out.
106 RRX, // V = RRX X, Flag -> srl X, 1 + shift in carry flag.
107
108 ADDC, // Add with carry
109 ADDE, // Add using carry
110 SUBC, // Sub with carry
111 SUBE, // Sub using carry
112 LSLS, // Shift left producing carry
113
114 VMOVRRD, // double to two gprs.
115 VMOVDRR, // Two gprs to double.
116 VMOVSR, // move gpr to single, used for f32 literal constructed in a gpr
117
118 EH_SJLJ_SETJMP, // SjLj exception handling setjmp.
119 EH_SJLJ_LONGJMP, // SjLj exception handling longjmp.
120 EH_SJLJ_SETUP_DISPATCH, // SjLj exception handling setup_dispatch.
121
122 TC_RETURN, // Tail call return pseudo.
123
124 THREAD_POINTER,
125
126 DYN_ALLOC, // Dynamic allocation on the stack.
127
128 MEMBARRIER_MCR, // Memory barrier (MCR)
129
130 PRELOAD, // Preload
131
132 WIN__CHKSTK, // Windows' __chkstk call to do stack probing.
133 WIN__DBZCHK, // Windows' divide by zero check
134
135 WLS, // Low-overhead loops, While Loop Start branch. See t2WhileLoopStart
136 WLSSETUP, // Setup for the iteration count of a WLS. See t2WhileLoopSetup.
137 LOOP_DEC, // Really a part of LE, performs the sub
138 LE, // Low-overhead loops, Loop End
139
140 PREDICATE_CAST, // Predicate cast for MVE i1 types
141 VECTOR_REG_CAST, // Reinterpret the current contents of a vector register
142
143 MVESEXT, // Legalization aids for extending a vector into two/four vectors.
144 MVEZEXT, // or truncating two/four vectors into one. Eventually becomes
145 MVETRUNC, // stack store/load sequence, if not optimized to anything else.
146
147 VCMP, // Vector compare.
148 VCMPZ, // Vector compare to zero.
149 VTST, // Vector test bits.
150
151 // Vector shift by vector
152 VSHLs, // ...left/right by signed
153 VSHLu, // ...left/right by unsigned
154
155 // Vector shift by immediate:
156 VSHLIMM, // ...left
157 VSHRsIMM, // ...right (signed)
158 VSHRuIMM, // ...right (unsigned)
159
160 // Vector rounding shift by immediate:
161 VRSHRsIMM, // ...right (signed)
162 VRSHRuIMM, // ...right (unsigned)
163 VRSHRNIMM, // ...right narrow
164
165 // Vector saturating shift by immediate:
166 VQSHLsIMM, // ...left (signed)
167 VQSHLuIMM, // ...left (unsigned)
168 VQSHLsuIMM, // ...left (signed to unsigned)
169 VQSHRNsIMM, // ...right narrow (signed)
170 VQSHRNuIMM, // ...right narrow (unsigned)
171 VQSHRNsuIMM, // ...right narrow (signed to unsigned)
172
173 // Vector saturating rounding shift by immediate:
174 VQRSHRNsIMM, // ...right narrow (signed)
175 VQRSHRNuIMM, // ...right narrow (unsigned)
176 VQRSHRNsuIMM, // ...right narrow (signed to unsigned)
177
178 // Vector shift and insert:
179 VSLIIMM, // ...left
180 VSRIIMM, // ...right
181
182 // Vector get lane (VMOV scalar to ARM core register)
183 // (These are used for 8- and 16-bit element types only.)
184 VGETLANEu, // zero-extend vector extract element
185 VGETLANEs, // sign-extend vector extract element
186
187 // Vector move immediate and move negated immediate:
188 VMOVIMM,
189 VMVNIMM,
190
191 // Vector move f32 immediate:
192 VMOVFPIMM,
193
194 // Move H <-> R, clearing top 16 bits
195 VMOVrh,
196 VMOVhr,
197
198 // Vector duplicate:
199 VDUP,
200 VDUPLANE,
201
202 // Vector shuffles:
203 VEXT, // extract
204 VREV64, // reverse elements within 64-bit doublewords
205 VREV32, // reverse elements within 32-bit words
206 VREV16, // reverse elements within 16-bit halfwords
207 VZIP, // zip (interleave)
208 VUZP, // unzip (deinterleave)
209 VTRN, // transpose
210 VTBL1, // 1-register shuffle with mask
211 VTBL2, // 2-register shuffle with mask
212 VMOVN, // MVE vmovn
213
214 // MVE Saturating truncates
215 VQMOVNs, // Vector (V) Saturating (Q) Move and Narrow (N), signed (s)
216 VQMOVNu, // Vector (V) Saturating (Q) Move and Narrow (N), unsigned (u)
217
218 // MVE float <> half converts
219 VCVTN, // MVE vcvt f32 -> f16, truncating into either the bottom or top
220 // lanes
221 VCVTL, // MVE vcvt f16 -> f32, extending from either the bottom or top lanes
222
223 // MVE VIDUP instruction, taking a start value and increment.
224 VIDUP,
225
226 // Vector multiply long:
227 VMULLs, // ...signed
228 VMULLu, // ...unsigned
229
230 VQDMULH, // MVE vqdmulh instruction
231
232 // MVE reductions
233 VADDVs, // sign- or zero-extend the elements of a vector to i32,
234 VADDVu, // add them all together, and return an i32 of their sum
235 VADDVps, // Same as VADDV[su] but with a v4i1 predicate mask
236 VADDVpu,
237 VADDLVs, // sign- or zero-extend elements to i64 and sum, returning
238 VADDLVu, // the low and high 32-bit halves of the sum
239 VADDLVAs, // Same as VADDLV[su] but also add an input accumulator
240 VADDLVAu, // provided as low and high halves
241 VADDLVps, // Same as VADDLV[su] but with a v4i1 predicate mask
242 VADDLVpu,
243 VADDLVAps, // Same as VADDLVp[su] but with a v4i1 predicate mask
244 VADDLVApu,
245 VMLAVs, // sign- or zero-extend the elements of two vectors to i32, multiply
246 VMLAVu, // them and add the results together, returning an i32 of the sum
247 VMLAVps, // Same as VMLAV[su] with a v4i1 predicate mask
248 VMLAVpu,
249 VMLALVs, // Same as VMLAV but with i64, returning the low and
250 VMLALVu, // high 32-bit halves of the sum
251 VMLALVps, // Same as VMLALV[su] with a v4i1 predicate mask
252 VMLALVpu,
253 VMLALVAs, // Same as VMLALV but also add an input accumulator
254 VMLALVAu, // provided as low and high halves
255 VMLALVAps, // Same as VMLALVA[su] with a v4i1 predicate mask
256 VMLALVApu,
257 VMINVu, // Find minimum unsigned value of a vector and register
258 VMINVs, // Find minimum signed value of a vector and register
259 VMAXVu, // Find maximum unsigned value of a vector and register
260 VMAXVs, // Find maximum signed value of a vector and register
261
262 SMULWB, // Signed multiply word by half word, bottom
263 SMULWT, // Signed multiply word by half word, top
264 UMLAL, // 64bit Unsigned Accumulate Multiply
265 SMLAL, // 64bit Signed Accumulate Multiply
266 UMAAL, // 64-bit Unsigned Accumulate Accumulate Multiply
267 SMLALBB, // 64-bit signed accumulate multiply bottom, bottom 16
268 SMLALBT, // 64-bit signed accumulate multiply bottom, top 16
269 SMLALTB, // 64-bit signed accumulate multiply top, bottom 16
270 SMLALTT, // 64-bit signed accumulate multiply top, top 16
271 SMLALD, // Signed multiply accumulate long dual
272 SMLALDX, // Signed multiply accumulate long dual exchange
273 SMLSLD, // Signed multiply subtract long dual
274 SMLSLDX, // Signed multiply subtract long dual exchange
275 SMMLAR, // Signed multiply long, round and add
276 SMMLSR, // Signed multiply long, subtract and round
277
278 // Single Lane QADD8 and QADD16. Only the bottom lane. That's what the b
279 // stands for.
280 QADD8b,
281 QSUB8b,
282 QADD16b,
283 QSUB16b,
284 UQADD8b,
285 UQSUB8b,
286 UQADD16b,
287 UQSUB16b,
288
289 // Operands of the standard BUILD_VECTOR node are not legalized, which
290 // is fine if BUILD_VECTORs are always lowered to shuffles or other
291 // operations, but for ARM some BUILD_VECTORs are legal as-is and their
292 // operands need to be legalized. Define an ARM-specific version of
293 // BUILD_VECTOR for this purpose.
294 BUILD_VECTOR,
295
296 // Bit-field insert
297 BFI,
298
299 // Vector OR with immediate
300 VORRIMM,
301 // Vector AND with NOT of immediate
302 VBICIMM,
303
304 // Pseudo vector bitwise select
305 VBSP,
306
307 // Pseudo-instruction representing a memory copy using ldm/stm
308 // instructions.
309 MEMCPY,
310
311 // Pseudo-instruction representing a memory copy using a tail predicated
312 // loop
313 MEMCPYLOOP,
314 // Pseudo-instruction representing a memset using a tail predicated
315 // loop
316 MEMSETLOOP,
317
318 // V8.1MMainline condition select
319 CSINV, // Conditional select invert.
320 CSNEG, // Conditional select negate.
321 CSINC, // Conditional select increment.
322
323 // Vector load N-element structure to all lanes:
324 VLD1DUP = ISD::FIRST_TARGET_MEMORY_OPCODE,
325 VLD2DUP,
326 VLD3DUP,
327 VLD4DUP,
328
329 // NEON loads with post-increment base updates:
330 VLD1_UPD,
331 VLD2_UPD,
332 VLD3_UPD,
333 VLD4_UPD,
334 VLD2LN_UPD,
335 VLD3LN_UPD,
336 VLD4LN_UPD,
337 VLD1DUP_UPD,
338 VLD2DUP_UPD,
339 VLD3DUP_UPD,
340 VLD4DUP_UPD,
341 VLD1x2_UPD,
342 VLD1x3_UPD,
343 VLD1x4_UPD,
344
345 // NEON stores with post-increment base updates:
346 VST1_UPD,
347 VST2_UPD,
348 VST3_UPD,
349 VST4_UPD,
350 VST2LN_UPD,
351 VST3LN_UPD,
352 VST4LN_UPD,
353 VST1x2_UPD,
354 VST1x3_UPD,
355 VST1x4_UPD,
356
357 // Load/Store of dual registers
358 LDRD,
359 STRD
360 };
361
362 } // end namespace ARMISD
363
364 namespace ARM {
365 /// Possible values of current rounding mode, which is specified in bits
366 /// 23:22 of FPSCR.
367 enum Rounding {
368 RN = 0, // Round to Nearest
369 RP = 1, // Round towards Plus infinity
370 RM = 2, // Round towards Minus infinity
371 RZ = 3, // Round towards Zero
372 rmMask = 3 // Bit mask selecting rounding mode
373 };
374
375 // Bit position of rounding mode bits in FPSCR.
376 const unsigned RoundingBitsPos = 22;
377
378 // Bits of floating-point status. These are NZCV flags, QC bit and cumulative
379 // FP exception bits.
380 const unsigned FPStatusBits = 0xf800009f;
381
382 // Some bits in the FPSCR are not yet defined. They must be preserved when
383 // modifying the contents.
384 const unsigned FPReservedBits = 0x00006060;
385 } // namespace ARM
386
387 /// Define some predicates that are used for node matching.
388 namespace ARM {
389
390 bool isBitFieldInvertedMask(unsigned v);
391
392 } // end namespace ARM
393
394 //===--------------------------------------------------------------------===//
395 // ARMTargetLowering - ARM Implementation of the TargetLowering interface
396
397 class ARMTargetLowering : public TargetLowering {
398 public:
399 explicit ARMTargetLowering(const TargetMachine &TM,
400 const ARMSubtarget &STI);
401
402 unsigned getJumpTableEncoding() const override;
403 bool useSoftFloat() const override;
404
405 SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
406
407 /// ReplaceNodeResults - Replace the results of node with an illegal result
408 /// type with new values built out of custom code.
409 void ReplaceNodeResults(SDNode *N, SmallVectorImpl<SDValue>&Results,
410 SelectionDAG &DAG) const override;
411
412 const char *getTargetNodeName(unsigned Opcode) const override;
413
414 bool isSelectSupported(SelectSupportKind Kind) const override {
415 // ARM does not support scalar condition selects on vectors.
416 return (Kind != ScalarCondVectorVal);
417 }
418
419 bool isReadOnly(const GlobalValue *GV) const;
420
421 /// getSetCCResultType - Return the value type to use for ISD::SETCC.
422 EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context,
423 EVT VT) const override;
424
425 MachineBasicBlock *
426 EmitInstrWithCustomInserter(MachineInstr &MI,
427 MachineBasicBlock *MBB) const override;
428
429 void AdjustInstrPostInstrSelection(MachineInstr &MI,
430 SDNode *Node) const override;
431
432 SDValue PerformCMOVCombine(SDNode *N, SelectionDAG &DAG) const;
433 SDValue PerformBRCONDCombine(SDNode *N, SelectionDAG &DAG) const;
434 SDValue PerformCMOVToBFICombine(SDNode *N, SelectionDAG &DAG) const;
435 SDValue PerformIntrinsicCombine(SDNode *N, DAGCombinerInfo &DCI) const;
436 SDValue PerformMVEExtCombine(SDNode *N, DAGCombinerInfo &DCI) const;
437 SDValue PerformMVETruncCombine(SDNode *N, DAGCombinerInfo &DCI) const;
438 SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override;
439
440 bool SimplifyDemandedBitsForTargetNode(SDValue Op,
441 const APInt &OriginalDemandedBits,
442 const APInt &OriginalDemandedElts,
443 KnownBits &Known,
444 TargetLoweringOpt &TLO,
445 unsigned Depth) const override;
446
447 bool isDesirableToTransformToIntegerOp(unsigned Opc, EVT VT) const override;
448
449 /// allowsMisalignedMemoryAccesses - Returns true if the target allows
450 /// unaligned memory accesses of the specified type. Returns whether it
451 /// is "fast" by reference in the second argument.
452 bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AddrSpace,
453 Align Alignment,
454 MachineMemOperand::Flags Flags,
455 unsigned *Fast) const override;
456
457 EVT getOptimalMemOpType(const MemOp &Op,
458 const AttributeList &FuncAttributes) const override;
459
460 bool isTruncateFree(Type *SrcTy, Type *DstTy) const override;
461 bool isTruncateFree(EVT SrcVT, EVT DstVT) const override;
462 bool isZExtFree(SDValue Val, EVT VT2) const override;
463 bool shouldSinkOperands(Instruction *I,
464 SmallVectorImpl<Use *> &Ops) const override;
465 Type* shouldConvertSplatType(ShuffleVectorInst* SVI) const override;
466
467 bool isFNegFree(EVT VT) const override;
468
469 bool isVectorLoadExtDesirable(SDValue ExtVal) const override;
470
471 bool allowTruncateForTailCall(Type *Ty1, Type *Ty2) const override;
472
473
474 /// isLegalAddressingMode - Return true if the addressing mode represented
475 /// by AM is legal for this target, for a load/store of the specified type.
476 bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM,
477 Type *Ty, unsigned AS,
478 Instruction *I = nullptr) const override;
479
480 bool isLegalT2ScaledAddressingMode(const AddrMode &AM, EVT VT) const;
481
482 /// Returns true if the addressing mode representing by AM is legal
483 /// for the Thumb1 target, for a load/store of the specified type.
484 bool isLegalT1ScaledAddressingMode(const AddrMode &AM, EVT VT) const;
485
486 /// isLegalICmpImmediate - Return true if the specified immediate is legal
487 /// icmp immediate, that is the target has icmp instructions which can
488 /// compare a register against the immediate without having to materialize
489 /// the immediate into a register.
490 bool isLegalICmpImmediate(int64_t Imm) const override;
491
492 /// isLegalAddImmediate - Return true if the specified immediate is legal
493 /// add immediate, that is the target has add instructions which can
494 /// add a register and the immediate without having to materialize
495 /// the immediate into a register.
496 bool isLegalAddImmediate(int64_t Imm) const override;
497
498 /// getPreIndexedAddressParts - returns true by value, base pointer and
499 /// offset pointer and addressing mode by reference if the node's address
500 /// can be legally represented as pre-indexed load / store address.
501 bool getPreIndexedAddressParts(SDNode *N, SDValue &Base, SDValue &Offset,
502 ISD::MemIndexedMode &AM,
503 SelectionDAG &DAG) const override;
504
505 /// getPostIndexedAddressParts - returns true by value, base pointer and
506 /// offset pointer and addressing mode by reference if this node can be
507 /// combined with a load / store to form a post-indexed load / store.
508 bool getPostIndexedAddressParts(SDNode *N, SDNode *Op, SDValue &Base,
509 SDValue &Offset, ISD::MemIndexedMode &AM,
510 SelectionDAG &DAG) const override;
511
512 void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known,
513 const APInt &DemandedElts,
514 const SelectionDAG &DAG,
515 unsigned Depth) const override;
516
517 bool targetShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits,
518 const APInt &DemandedElts,
519 TargetLoweringOpt &TLO) const override;
520
521 bool ExpandInlineAsm(CallInst *CI) const override;
522
523 ConstraintType getConstraintType(StringRef Constraint) const override;
524
525 /// Examine constraint string and operand type and determine a weight value.
526 /// The operand object must already have been set up with the operand type.
527 ConstraintWeight getSingleConstraintMatchWeight(
528 AsmOperandInfo &info, const char *constraint) const override;
529
530 std::pair<unsigned, const TargetRegisterClass *>
531 getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
532 StringRef Constraint, MVT VT) const override;
533
534 const char *LowerXConstraint(EVT ConstraintVT) const override;
535
536 /// LowerAsmOperandForConstraint - Lower the specified operand into the Ops
537 /// vector. If it is invalid, don't add anything to Ops. If hasMemory is
538 /// true it means one of the asm constraint of the inline asm instruction
539 /// being processed is 'm'.
540 void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint,
541 std::vector<SDValue> &Ops,
542 SelectionDAG &DAG) const override;
543
544 InlineAsm::ConstraintCode
545 getInlineAsmMemConstraint(StringRef ConstraintCode) const override {
546 if (ConstraintCode == "Q")
547 return InlineAsm::ConstraintCode::Q;
548 if (ConstraintCode.size() == 2) {
549 if (ConstraintCode[0] == 'U') {
550 switch(ConstraintCode[1]) {
551 default:
552 break;
553 case 'm':
554 return InlineAsm::ConstraintCode::Um;
555 case 'n':
556 return InlineAsm::ConstraintCode::Un;
557 case 'q':
558 return InlineAsm::ConstraintCode::Uq;
559 case 's':
560 return InlineAsm::ConstraintCode::Us;
561 case 't':
562 return InlineAsm::ConstraintCode::Ut;
563 case 'v':
564 return InlineAsm::ConstraintCode::Uv;
565 case 'y':
566 return InlineAsm::ConstraintCode::Uy;
567 }
568 }
569 }
570 return TargetLowering::getInlineAsmMemConstraint(ConstraintCode);
571 }
572
573 const ARMSubtarget* getSubtarget() const {
574 return Subtarget;
575 }
576
577 /// getRegClassFor - Return the register class that should be used for the
578 /// specified value type.
579 const TargetRegisterClass *
580 getRegClassFor(MVT VT, bool isDivergent = false) const override;
581
582 bool shouldAlignPointerArgs(CallInst *CI, unsigned &MinSize,
583 Align &PrefAlign) const override;
584
585 /// createFastISel - This method returns a target specific FastISel object,
586 /// or null if the target does not support "fast" ISel.
587 FastISel *createFastISel(FunctionLoweringInfo &funcInfo,
588 const TargetLibraryInfo *libInfo) const override;
589
590 Sched::Preference getSchedulingPreference(SDNode *N) const override;
591
592 bool preferZeroCompareBranch() const override { return true; }
593
594 bool isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const override;
595
596 bool
597 isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const override;
598 bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override;
599
600 /// isFPImmLegal - Returns true if the target can instruction select the
601 /// specified FP immediate natively. If false, the legalizer will
602 /// materialize the FP immediate as a load from a constant pool.
603 bool isFPImmLegal(const APFloat &Imm, EVT VT,
604 bool ForCodeSize = false) const override;
605
606 bool getTgtMemIntrinsic(IntrinsicInfo &Info,
607 const CallInst &I,
608 MachineFunction &MF,
609 unsigned Intrinsic) const override;
610
611 /// Returns true if it is beneficial to convert a load of a constant
612 /// to just the constant itself.
613 bool shouldConvertConstantLoadToIntImm(const APInt &Imm,
614 Type *Ty) const override;
615
616 /// Return true if EXTRACT_SUBVECTOR is cheap for this result type
617 /// with this index.
618 bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT,
619 unsigned Index) const override;
620
621 bool shouldFormOverflowOp(unsigned Opcode, EVT VT,
622 bool MathUsed) const override {
623 // Using overflow ops for overflow checks only should beneficial on ARM.
624 return TargetLowering::shouldFormOverflowOp(Opcode, VT, MathUsed: true);
625 }
626
627 bool shouldReassociateReduction(unsigned Opc, EVT VT) const override {
628 return Opc != ISD::VECREDUCE_ADD;
629 }
630
631 /// Returns true if an argument of type Ty needs to be passed in a
632 /// contiguous block of registers in calling convention CallConv.
633 bool functionArgumentNeedsConsecutiveRegisters(
634 Type *Ty, CallingConv::ID CallConv, bool isVarArg,
635 const DataLayout &DL) const override;
636
637 /// If a physical register, this returns the register that receives the
638 /// exception address on entry to an EH pad.
639 Register
640 getExceptionPointerRegister(const Constant *PersonalityFn) const override;
641
642 /// If a physical register, this returns the register that receives the
643 /// exception typeid on entry to a landing pad.
644 Register
645 getExceptionSelectorRegister(const Constant *PersonalityFn) const override;
646
647 Instruction *makeDMB(IRBuilderBase &Builder, ARM_MB::MemBOpt Domain) const;
648 Value *emitLoadLinked(IRBuilderBase &Builder, Type *ValueTy, Value *Addr,
649 AtomicOrdering Ord) const override;
650 Value *emitStoreConditional(IRBuilderBase &Builder, Value *Val, Value *Addr,
651 AtomicOrdering Ord) const override;
652
653 void
654 emitAtomicCmpXchgNoStoreLLBalance(IRBuilderBase &Builder) const override;
655
656 Instruction *emitLeadingFence(IRBuilderBase &Builder, Instruction *Inst,
657 AtomicOrdering Ord) const override;
658 Instruction *emitTrailingFence(IRBuilderBase &Builder, Instruction *Inst,
659 AtomicOrdering Ord) const override;
660
661 unsigned getMaxSupportedInterleaveFactor() const override;
662
663 bool lowerInterleavedLoad(LoadInst *LI,
664 ArrayRef<ShuffleVectorInst *> Shuffles,
665 ArrayRef<unsigned> Indices,
666 unsigned Factor) const override;
667 bool lowerInterleavedStore(StoreInst *SI, ShuffleVectorInst *SVI,
668 unsigned Factor) const override;
669
670 bool shouldInsertFencesForAtomic(const Instruction *I) const override;
671 TargetLoweringBase::AtomicExpansionKind
672 shouldExpandAtomicLoadInIR(LoadInst *LI) const override;
673 TargetLoweringBase::AtomicExpansionKind
674 shouldExpandAtomicStoreInIR(StoreInst *SI) const override;
675 TargetLoweringBase::AtomicExpansionKind
676 shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override;
677 TargetLoweringBase::AtomicExpansionKind
678 shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const override;
679
680 bool useLoadStackGuardNode() const override;
681
682 void insertSSPDeclarations(Module &M) const override;
683 Value *getSDagStackGuard(const Module &M) const override;
684 Function *getSSPStackGuardCheck(const Module &M) const override;
685
686 bool canCombineStoreAndExtract(Type *VectorTy, Value *Idx,
687 unsigned &Cost) const override;
688
689 bool canMergeStoresTo(unsigned AddressSpace, EVT MemVT,
690 const MachineFunction &MF) const override {
691 // Do not merge to larger than i32.
692 return (MemVT.getSizeInBits() <= 32);
693 }
694
695 bool isCheapToSpeculateCttz(Type *Ty) const override;
696 bool isCheapToSpeculateCtlz(Type *Ty) const override;
697
698 bool convertSetCCLogicToBitwiseLogic(EVT VT) const override {
699 return VT.isScalarInteger();
700 }
701
702 bool supportSwiftError() const override {
703 return true;
704 }
705
706 bool hasStandaloneRem(EVT VT) const override {
707 return HasStandaloneRem;
708 }
709
710 ShiftLegalizationStrategy
711 preferredShiftLegalizationStrategy(SelectionDAG &DAG, SDNode *N,
712 unsigned ExpansionFactor) const override;
713
714 CCAssignFn *CCAssignFnForCall(CallingConv::ID CC, bool isVarArg) const;
715 CCAssignFn *CCAssignFnForReturn(CallingConv::ID CC, bool isVarArg) const;
716
717 /// Returns true if \p VecTy is a legal interleaved access type. This
718 /// function checks the vector element type and the overall width of the
719 /// vector.
720 bool isLegalInterleavedAccessType(unsigned Factor, FixedVectorType *VecTy,
721 Align Alignment,
722 const DataLayout &DL) const;
723
724 bool isMulAddWithConstProfitable(SDValue AddNode,
725 SDValue ConstNode) const override;
726
727 bool alignLoopsWithOptSize() const override;
728
729 /// Returns the number of interleaved accesses that will be generated when
730 /// lowering accesses of the given type.
731 unsigned getNumInterleavedAccesses(VectorType *VecTy,
732 const DataLayout &DL) const;
733
734 void finalizeLowering(MachineFunction &MF) const override;
735
736 /// Return the correct alignment for the current calling convention.
737 Align getABIAlignmentForCallingConv(Type *ArgTy,
738 const DataLayout &DL) const override;
739
740 bool isDesirableToCommuteWithShift(const SDNode *N,
741 CombineLevel Level) const override;
742
743 bool isDesirableToCommuteXorWithShift(const SDNode *N) const override;
744
745 bool shouldFoldConstantShiftPairToMask(const SDNode *N,
746 CombineLevel Level) const override;
747
748 bool shouldFoldSelectWithIdentityConstant(unsigned BinOpcode,
749 EVT VT) const override;
750
751 bool preferIncOfAddToSubOfNot(EVT VT) const override;
752
753 bool shouldConvertFpToSat(unsigned Op, EVT FPVT, EVT VT) const override;
754
755 bool isComplexDeinterleavingSupported() const override;
756 bool isComplexDeinterleavingOperationSupported(
757 ComplexDeinterleavingOperation Operation, Type *Ty) const override;
758
759 Value *createComplexDeinterleavingIR(
760 IRBuilderBase &B, ComplexDeinterleavingOperation OperationType,
761 ComplexDeinterleavingRotation Rotation, Value *InputA, Value *InputB,
762 Value *Accumulator = nullptr) const override;
763
764 bool softPromoteHalfType() const override { return true; }
765
766 bool useFPRegsForHalfType() const override { return true; }
767
768 protected:
769 std::pair<const TargetRegisterClass *, uint8_t>
770 findRepresentativeClass(const TargetRegisterInfo *TRI,
771 MVT VT) const override;
772
773 private:
774 /// Subtarget - Keep a pointer to the ARMSubtarget around so that we can
775 /// make the right decision when generating code for different targets.
776 const ARMSubtarget *Subtarget;
777
778 const TargetRegisterInfo *RegInfo;
779
780 const InstrItineraryData *Itins;
781
782 // TODO: remove this, and have shouldInsertFencesForAtomic do the proper
783 // check.
784 bool InsertFencesForAtomic;
785
786 bool HasStandaloneRem = true;
787
788 void addTypeForNEON(MVT VT, MVT PromotedLdStVT);
789 void addDRTypeForNEON(MVT VT);
790 void addQRTypeForNEON(MVT VT);
791 std::pair<SDValue, SDValue> getARMXALUOOp(SDValue Op, SelectionDAG &DAG, SDValue &ARMcc) const;
792
793 using RegsToPassVector = SmallVector<std::pair<unsigned, SDValue>, 8>;
794
795 void PassF64ArgInRegs(const SDLoc &dl, SelectionDAG &DAG, SDValue Chain,
796 SDValue &Arg, RegsToPassVector &RegsToPass,
797 CCValAssign &VA, CCValAssign &NextVA,
798 SDValue &StackPtr,
799 SmallVectorImpl<SDValue> &MemOpChains,
800 bool IsTailCall,
801 int SPDiff) const;
802 SDValue GetF64FormalArgument(CCValAssign &VA, CCValAssign &NextVA,
803 SDValue &Root, SelectionDAG &DAG,
804 const SDLoc &dl) const;
805
806 CallingConv::ID getEffectiveCallingConv(CallingConv::ID CC,
807 bool isVarArg) const;
808 CCAssignFn *CCAssignFnForNode(CallingConv::ID CC, bool Return,
809 bool isVarArg) const;
810 std::pair<SDValue, MachinePointerInfo>
811 computeAddrForCallArg(const SDLoc &dl, SelectionDAG &DAG,
812 const CCValAssign &VA, SDValue StackPtr,
813 bool IsTailCall, int SPDiff) const;
814 SDValue LowerEH_SJLJ_SETJMP(SDValue Op, SelectionDAG &DAG) const;
815 SDValue LowerEH_SJLJ_LONGJMP(SDValue Op, SelectionDAG &DAG) const;
816 SDValue LowerEH_SJLJ_SETUP_DISPATCH(SDValue Op, SelectionDAG &DAG) const;
817 SDValue LowerINTRINSIC_VOID(SDValue Op, SelectionDAG &DAG,
818 const ARMSubtarget *Subtarget) const;
819 SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG,
820 const ARMSubtarget *Subtarget) const;
821 SDValue LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const;
822 SDValue LowerConstantPool(SDValue Op, SelectionDAG &DAG) const;
823 SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
824 SDValue LowerGlobalAddressDarwin(SDValue Op, SelectionDAG &DAG) const;
825 SDValue LowerGlobalAddressELF(SDValue Op, SelectionDAG &DAG) const;
826 SDValue LowerGlobalAddressWindows(SDValue Op, SelectionDAG &DAG) const;
827 SDValue LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const;
828 SDValue LowerToTLSGeneralDynamicModel(GlobalAddressSDNode *GA,
829 SelectionDAG &DAG) const;
830 SDValue LowerToTLSExecModels(GlobalAddressSDNode *GA,
831 SelectionDAG &DAG,
832 TLSModel::Model model) const;
833 SDValue LowerGlobalTLSAddressDarwin(SDValue Op, SelectionDAG &DAG) const;
834 SDValue LowerGlobalTLSAddressWindows(SDValue Op, SelectionDAG &DAG) const;
835 SDValue LowerBR_JT(SDValue Op, SelectionDAG &DAG) const;
836 SDValue LowerSignedALUO(SDValue Op, SelectionDAG &DAG) const;
837 SDValue LowerUnsignedALUO(SDValue Op, SelectionDAG &DAG) const;
838 SDValue LowerSELECT(SDValue Op, SelectionDAG &DAG) const;
839 SDValue LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const;
840 SDValue LowerBRCOND(SDValue Op, SelectionDAG &DAG) const;
841 SDValue LowerBR_CC(SDValue Op, SelectionDAG &DAG) const;
842 SDValue LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const;
843 SDValue LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
844 SDValue LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const;
845 SDValue LowerShiftRightParts(SDValue Op, SelectionDAG &DAG) const;
846 SDValue LowerShiftLeftParts(SDValue Op, SelectionDAG &DAG) const;
847 SDValue LowerGET_ROUNDING(SDValue Op, SelectionDAG &DAG) const;
848 SDValue LowerSET_ROUNDING(SDValue Op, SelectionDAG &DAG) const;
849 SDValue LowerSET_FPMODE(SDValue Op, SelectionDAG &DAG) const;
850 SDValue LowerRESET_FPMODE(SDValue Op, SelectionDAG &DAG) const;
851 SDValue LowerConstantFP(SDValue Op, SelectionDAG &DAG,
852 const ARMSubtarget *ST) const;
853 SDValue LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
854 const ARMSubtarget *ST) const;
855 SDValue LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
856 SDValue LowerFSINCOS(SDValue Op, SelectionDAG &DAG) const;
857 SDValue LowerDivRem(SDValue Op, SelectionDAG &DAG) const;
858 SDValue LowerDIV_Windows(SDValue Op, SelectionDAG &DAG, bool Signed) const;
859 void ExpandDIV_Windows(SDValue Op, SelectionDAG &DAG, bool Signed,
860 SmallVectorImpl<SDValue> &Results) const;
861 SDValue ExpandBITCAST(SDNode *N, SelectionDAG &DAG,
862 const ARMSubtarget *Subtarget) const;
863 SDValue LowerWindowsDIVLibCall(SDValue Op, SelectionDAG &DAG, bool Signed,
864 SDValue &Chain) const;
865 SDValue LowerREM(SDNode *N, SelectionDAG &DAG) const;
866 SDValue LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const;
867 SDValue LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const;
868 SDValue LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const;
869 SDValue LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const;
870 SDValue LowerINT_TO_FP(SDValue Op, SelectionDAG &DAG) const;
871 SDValue LowerFSETCC(SDValue Op, SelectionDAG &DAG) const;
872 SDValue LowerSPONENTRY(SDValue Op, SelectionDAG &DAG) const;
873 void LowerLOAD(SDNode *N, SmallVectorImpl<SDValue> &Results,
874 SelectionDAG &DAG) const;
875
876 Register getRegisterByName(const char* RegName, LLT VT,
877 const MachineFunction &MF) const override;
878
879 SDValue BuildSDIVPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG,
880 SmallVectorImpl<SDNode *> &Created) const override;
881
882 bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
883 EVT VT) const override;
884
885 SDValue MoveToHPR(const SDLoc &dl, SelectionDAG &DAG, MVT LocVT, MVT ValVT,
886 SDValue Val) const;
887 SDValue MoveFromHPR(const SDLoc &dl, SelectionDAG &DAG, MVT LocVT,
888 MVT ValVT, SDValue Val) const;
889
890 SDValue ReconstructShuffle(SDValue Op, SelectionDAG &DAG) const;
891
892 SDValue LowerCallResult(SDValue Chain, SDValue InGlue,
893 CallingConv::ID CallConv, bool isVarArg,
894 const SmallVectorImpl<ISD::InputArg> &Ins,
895 const SDLoc &dl, SelectionDAG &DAG,
896 SmallVectorImpl<SDValue> &InVals, bool isThisReturn,
897 SDValue ThisVal, bool isCmseNSCall) const;
898
899 bool supportSplitCSR(MachineFunction *MF) const override {
900 return MF->getFunction().getCallingConv() == CallingConv::CXX_FAST_TLS &&
901 MF->getFunction().hasFnAttribute(Kind: Attribute::NoUnwind);
902 }
903
904 void initializeSplitCSR(MachineBasicBlock *Entry) const override;
905 void insertCopiesSplitCSR(
906 MachineBasicBlock *Entry,
907 const SmallVectorImpl<MachineBasicBlock *> &Exits) const override;
908
909 bool splitValueIntoRegisterParts(
910 SelectionDAG & DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
911 unsigned NumParts, MVT PartVT, std::optional<CallingConv::ID> CC)
912 const override;
913
914 SDValue joinRegisterPartsIntoValue(
915 SelectionDAG & DAG, const SDLoc &DL, const SDValue *Parts,
916 unsigned NumParts, MVT PartVT, EVT ValueVT,
917 std::optional<CallingConv::ID> CC) const override;
918
919 SDValue
920 LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
921 const SmallVectorImpl<ISD::InputArg> &Ins,
922 const SDLoc &dl, SelectionDAG &DAG,
923 SmallVectorImpl<SDValue> &InVals) const override;
924
925 int StoreByValRegs(CCState &CCInfo, SelectionDAG &DAG, const SDLoc &dl,
926 SDValue &Chain, const Value *OrigArg,
927 unsigned InRegsParamRecordIdx, int ArgOffset,
928 unsigned ArgSize) const;
929
930 void VarArgStyleRegisters(CCState &CCInfo, SelectionDAG &DAG,
931 const SDLoc &dl, SDValue &Chain,
932 unsigned ArgOffset, unsigned TotalArgRegsSaveSize,
933 bool ForceMutable = false) const;
934
935 SDValue LowerCall(TargetLowering::CallLoweringInfo &CLI,
936 SmallVectorImpl<SDValue> &InVals) const override;
937
938 /// HandleByVal - Target-specific cleanup for ByVal support.
939 void HandleByVal(CCState *, unsigned &, Align) const override;
940
941 /// IsEligibleForTailCallOptimization - Check whether the call is eligible
942 /// for tail call optimization. Targets which want to do tail call
943 /// optimization should implement this function.
944 bool IsEligibleForTailCallOptimization(
945 TargetLowering::CallLoweringInfo &CLI, CCState &CCInfo,
946 SmallVectorImpl<CCValAssign> &ArgLocs, const bool isIndirect) const;
947
948 bool CanLowerReturn(CallingConv::ID CallConv,
949 MachineFunction &MF, bool isVarArg,
950 const SmallVectorImpl<ISD::OutputArg> &Outs,
951 LLVMContext &Context) const override;
952
953 SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
954 const SmallVectorImpl<ISD::OutputArg> &Outs,
955 const SmallVectorImpl<SDValue> &OutVals,
956 const SDLoc &dl, SelectionDAG &DAG) const override;
957
958 bool isUsedByReturnOnly(SDNode *N, SDValue &Chain) const override;
959
960 bool mayBeEmittedAsTailCall(const CallInst *CI) const override;
961
962 bool shouldConsiderGEPOffsetSplit() const override { return true; }
963
964 bool isUnsupportedFloatingType(EVT VT) const;
965
966 SDValue getCMOV(const SDLoc &dl, EVT VT, SDValue FalseVal, SDValue TrueVal,
967 SDValue ARMcc, SDValue CCR, SDValue Cmp,
968 SelectionDAG &DAG) const;
969 SDValue getARMCmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
970 SDValue &ARMcc, SelectionDAG &DAG, const SDLoc &dl) const;
971 SDValue getVFPCmp(SDValue LHS, SDValue RHS, SelectionDAG &DAG,
972 const SDLoc &dl, bool Signaling = false) const;
973 SDValue duplicateCmp(SDValue Cmp, SelectionDAG &DAG) const;
974
975 SDValue OptimizeVFPBrcond(SDValue Op, SelectionDAG &DAG) const;
976
977 void SetupEntryBlockForSjLj(MachineInstr &MI, MachineBasicBlock *MBB,
978 MachineBasicBlock *DispatchBB, int FI) const;
979
980 void EmitSjLjDispatchBlock(MachineInstr &MI, MachineBasicBlock *MBB) const;
981
982 MachineBasicBlock *EmitStructByval(MachineInstr &MI,
983 MachineBasicBlock *MBB) const;
984
985 MachineBasicBlock *EmitLowered__chkstk(MachineInstr &MI,
986 MachineBasicBlock *MBB) const;
987 MachineBasicBlock *EmitLowered__dbzchk(MachineInstr &MI,
988 MachineBasicBlock *MBB) const;
989 void addMVEVectorTypes(bool HasMVEFP);
990 void addAllExtLoads(const MVT From, const MVT To, LegalizeAction Action);
991 void setAllExpand(MVT VT);
992 };
993
994 enum VMOVModImmType {
995 VMOVModImm,
996 VMVNModImm,
997 MVEVMVNModImm,
998 OtherModImm
999 };
1000
1001 namespace ARM {
1002
1003 FastISel *createFastISel(FunctionLoweringInfo &funcInfo,
1004 const TargetLibraryInfo *libInfo);
1005
1006 } // end namespace ARM
1007
1008} // end namespace llvm
1009
1010#endif // LLVM_LIB_TARGET_ARM_ARMISELLOWERING_H
1011