1//===-- HexagonISelLowering.cpp - Hexagon DAG Lowering Implementation -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the interfaces that Hexagon uses to lower LLVM code
10// into a selection DAG.
11//
12//===----------------------------------------------------------------------===//
13
14#include "HexagonISelLowering.h"
15#include "Hexagon.h"
16#include "HexagonMachineFunctionInfo.h"
17#include "HexagonRegisterInfo.h"
18#include "HexagonSubtarget.h"
19#include "HexagonTargetMachine.h"
20#include "HexagonTargetObjectFile.h"
21#include "llvm/ADT/APInt.h"
22#include "llvm/ADT/ArrayRef.h"
23#include "llvm/ADT/SmallVector.h"
24#include "llvm/ADT/StringSwitch.h"
25#include "llvm/CodeGen/CallingConvLower.h"
26#include "llvm/CodeGen/MachineFrameInfo.h"
27#include "llvm/CodeGen/MachineFunction.h"
28#include "llvm/CodeGen/MachineMemOperand.h"
29#include "llvm/CodeGen/MachineRegisterInfo.h"
30#include "llvm/CodeGen/SelectionDAG.h"
31#include "llvm/CodeGen/TargetCallingConv.h"
32#include "llvm/CodeGen/ValueTypes.h"
33#include "llvm/IR/BasicBlock.h"
34#include "llvm/IR/CallingConv.h"
35#include "llvm/IR/DataLayout.h"
36#include "llvm/IR/DerivedTypes.h"
37#include "llvm/IR/DiagnosticInfo.h"
38#include "llvm/IR/DiagnosticPrinter.h"
39#include "llvm/IR/Function.h"
40#include "llvm/IR/GlobalValue.h"
41#include "llvm/IR/IRBuilder.h"
42#include "llvm/IR/InlineAsm.h"
43#include "llvm/IR/Instructions.h"
44#include "llvm/IR/IntrinsicInst.h"
45#include "llvm/IR/Intrinsics.h"
46#include "llvm/IR/IntrinsicsHexagon.h"
47#include "llvm/IR/Module.h"
48#include "llvm/IR/Type.h"
49#include "llvm/IR/Value.h"
50#include "llvm/Support/Casting.h"
51#include "llvm/Support/CodeGen.h"
52#include "llvm/Support/CommandLine.h"
53#include "llvm/Support/Debug.h"
54#include "llvm/Support/ErrorHandling.h"
55#include "llvm/Support/MathExtras.h"
56#include "llvm/Support/raw_ostream.h"
57#include "llvm/Target/TargetMachine.h"
58#include <algorithm>
59#include <cassert>
60#include <cstdint>
61#include <limits>
62#include <utility>
63
64using namespace llvm;
65
66#define DEBUG_TYPE "hexagon-lowering"
67
68static cl::opt<bool> EmitJumpTables("hexagon-emit-jump-tables",
69 cl::init(Val: true), cl::Hidden,
70 cl::desc("Control jump table emission on Hexagon target"));
71
72static cl::opt<bool>
73 EnableHexSDNodeSched("enable-hexagon-sdnode-sched", cl::Hidden,
74 cl::desc("Enable Hexagon SDNode scheduling"));
75
76static cl::opt<int> MinimumJumpTables("minimum-jump-tables", cl::Hidden,
77 cl::init(Val: 5),
78 cl::desc("Set minimum jump tables"));
79
80static cl::opt<bool>
81 ConstantLoadsToImm("constant-loads-to-imm", cl::Hidden, cl::init(Val: true),
82 cl::desc("Convert constant loads to immediate values."));
83
84static cl::opt<bool> AlignLoads("hexagon-align-loads",
85 cl::Hidden, cl::init(Val: false),
86 cl::desc("Rewrite unaligned loads as a pair of aligned loads"));
87
88static cl::opt<bool>
89 DisableArgsMinAlignment("hexagon-disable-args-min-alignment", cl::Hidden,
90 cl::init(Val: false),
91 cl::desc("Disable minimum alignment of 1 for "
92 "arguments passed by value on stack"));
93
94// Implement calling convention for Hexagon.
95
96static bool CC_SkipOdd(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
97 CCValAssign::LocInfo &LocInfo,
98 ISD::ArgFlagsTy &ArgFlags, CCState &State) {
99 static const MCPhysReg ArgRegs[] = {
100 Hexagon::R0, Hexagon::R1, Hexagon::R2,
101 Hexagon::R3, Hexagon::R4, Hexagon::R5
102 };
103 const unsigned NumArgRegs = std::size(ArgRegs);
104 unsigned RegNum = State.getFirstUnallocated(Regs: ArgRegs);
105
106 // RegNum is an index into ArgRegs: skip a register if RegNum is odd.
107 if (RegNum != NumArgRegs && RegNum % 2 == 1)
108 State.AllocateReg(Reg: ArgRegs[RegNum]);
109
110 // Always return false here, as this function only makes sure that the first
111 // unallocated register has an even register number and does not actually
112 // allocate a register for the current argument.
113 return false;
114}
115
116#define GET_CALLING_CONV_IMPL
117#include "HexagonGenCallingConv.inc"
118
119unsigned HexagonTargetLowering::getVectorTypeBreakdownForCallingConv(
120 LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT,
121 unsigned &NumIntermediates, MVT &RegisterVT) const {
122
123 bool isBoolVector = VT.getVectorElementType() == MVT::i1;
124 bool isPowerOf2 = VT.isPow2VectorType();
125 unsigned NumElts = VT.getVectorNumElements();
126
127 // Split vectors of type vXi1 into (X/8) vectors of type v8i1,
128 // where X is divisible by 8.
129 if (isBoolVector && !Subtarget.useHVXOps() && isPowerOf2 && NumElts >= 8) {
130 RegisterVT = MVT::v8i8;
131 IntermediateVT = MVT::v8i1;
132 NumIntermediates = NumElts / 8;
133 return NumIntermediates;
134 }
135
136 // In HVX 64-byte mode, vectors of type vXi1 are split into (X / 64) vectors
137 // of type v64i1, provided that X is divisible by 64.
138 if (isBoolVector && Subtarget.useHVX64BOps() && isPowerOf2 && NumElts >= 64) {
139 RegisterVT = MVT::v64i8;
140 IntermediateVT = MVT::v64i1;
141 NumIntermediates = NumElts / 64;
142 return NumIntermediates;
143 }
144
145 // In HVX 128-byte mode, vectors of type vXi1 are split into (X / 128) vectors
146 // of type v128i1, provided that X is divisible by 128.
147 if (isBoolVector && Subtarget.useHVX128BOps() && isPowerOf2 &&
148 NumElts >= 128) {
149 RegisterVT = MVT::v128i8;
150 IntermediateVT = MVT::v128i1;
151 NumIntermediates = NumElts / 128;
152 return NumIntermediates;
153 }
154
155 return TargetLowering::getVectorTypeBreakdownForCallingConv(
156 Context, CC, VT, IntermediateVT, NumIntermediates, RegisterVT);
157}
158
159std::pair<MVT, unsigned>
160HexagonTargetLowering::handleMaskRegisterForCallingConv(
161 const HexagonSubtarget &Subtarget, EVT VT) const {
162 assert(VT.getVectorElementType() == MVT::i1);
163
164 const unsigned NumElems = VT.getVectorNumElements();
165
166 if (!VT.isPow2VectorType())
167 return {MVT::INVALID_SIMPLE_VALUE_TYPE, 0};
168
169 if (!Subtarget.useHVXOps() && NumElems >= 8)
170 return {MVT::v8i8, NumElems / 8};
171
172 if (Subtarget.useHVX64BOps() && NumElems >= 64)
173 return {MVT::v64i8, NumElems / 64};
174
175 if (Subtarget.useHVX128BOps() && NumElems >= 128)
176 return {MVT::v128i8, NumElems / 128};
177
178 return {MVT::INVALID_SIMPLE_VALUE_TYPE, 0};
179}
180
181MVT HexagonTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
182 CallingConv::ID CC,
183 EVT VT) const {
184
185 if (VT.isVectorOf(EltVT: MVT::i1)) {
186 auto [RegisterVT, NumRegisters] =
187 handleMaskRegisterForCallingConv(Subtarget, VT);
188 if (RegisterVT != MVT::INVALID_SIMPLE_VALUE_TYPE)
189 return RegisterVT;
190 }
191
192 return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
193}
194
195SDValue
196HexagonTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG)
197 const {
198 unsigned IntNo = Op.getConstantOperandVal(i: 0);
199 SDLoc dl(Op);
200 switch (IntNo) {
201 default:
202 return SDValue(); // Don't custom lower most intrinsics.
203 case Intrinsic::thread_pointer: {
204 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
205 return DAG.getNode(Opcode: HexagonISD::THREAD_POINTER, DL: dl, VT: PtrVT);
206 }
207 }
208}
209
210/// CreateCopyOfByValArgument - Make a copy of an aggregate at address specified
211/// by "Src" to address "Dst" of size "Size". Alignment information is
212/// specified by the specific parameter attribute. The copy will be passed as
213/// a byval function parameter. Sometimes what we are copying is the end of a
214/// larger object, the part that does not fit in registers.
215static SDValue CreateCopyOfByValArgument(SDValue Src, SDValue Dst,
216 SDValue Chain, ISD::ArgFlagsTy Flags,
217 SelectionDAG &DAG, const SDLoc &dl) {
218 SDValue SizeNode = DAG.getConstant(Val: Flags.getByValSize(), DL: dl, VT: MVT::i32);
219 Align Alignment = Flags.getNonZeroByValAlign();
220 return DAG.getMemcpy(Chain, dl, Dst, Src, Size: SizeNode, DstAlign: Alignment, SrcAlign: Alignment,
221 /*isVolatile=*/isVol: false, /*AlwaysInline=*/false,
222 /*CI=*/nullptr, OverrideTailCall: std::nullopt, DstPtrInfo: MachinePointerInfo(),
223 SrcPtrInfo: MachinePointerInfo());
224}
225
226bool
227HexagonTargetLowering::CanLowerReturn(
228 CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg,
229 const SmallVectorImpl<ISD::OutputArg> &Outs,
230 LLVMContext &Context, const Type *RetTy) const {
231 SmallVector<CCValAssign, 16> RVLocs;
232 CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
233
234 if (MF.getSubtarget<HexagonSubtarget>().useHVXOps())
235 return CCInfo.CheckReturn(Outs, Fn: RetCC_Hexagon_HVX);
236 return CCInfo.CheckReturn(Outs, Fn: RetCC_Hexagon);
237}
238
239// LowerReturn - Lower ISD::RET. If a struct is larger than 8 bytes and is
240// passed by value, the function prototype is modified to return void and
241// the value is stored in memory pointed by a pointer passed by caller.
242SDValue
243HexagonTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
244 bool IsVarArg,
245 const SmallVectorImpl<ISD::OutputArg> &Outs,
246 const SmallVectorImpl<SDValue> &OutVals,
247 const SDLoc &dl, SelectionDAG &DAG) const {
248 // CCValAssign - represent the assignment of the return value to locations.
249 SmallVector<CCValAssign, 16> RVLocs;
250
251 // CCState - Info about the registers and stack slot.
252 CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
253 *DAG.getContext());
254
255 // Analyze return values of ISD::RET
256 if (Subtarget.useHVXOps())
257 CCInfo.AnalyzeReturn(Outs, Fn: RetCC_Hexagon_HVX);
258 else
259 CCInfo.AnalyzeReturn(Outs, Fn: RetCC_Hexagon);
260
261 SDValue Glue;
262 SmallVector<SDValue, 4> RetOps(1, Chain);
263
264 // Copy the result values into the output registers.
265 for (unsigned i = 0; i != RVLocs.size(); ++i) {
266 CCValAssign &VA = RVLocs[i];
267 SDValue Val = OutVals[i];
268
269 switch (VA.getLocInfo()) {
270 default:
271 // Loc info must be one of Full, BCvt, SExt, ZExt, or AExt.
272 llvm_unreachable("Unknown loc info!");
273 case CCValAssign::Full:
274 break;
275 case CCValAssign::BCvt:
276 Val = DAG.getBitcast(VT: VA.getLocVT(), V: Val);
277 break;
278 case CCValAssign::SExt:
279 Val = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: Val);
280 break;
281 case CCValAssign::ZExt:
282 Val = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: Val);
283 break;
284 case CCValAssign::AExt:
285 Val = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: Val);
286 break;
287 }
288
289 Chain = DAG.getCopyToReg(Chain, dl, Reg: VA.getLocReg(), N: Val, Glue);
290
291 // Guarantee that all emitted copies are stuck together with flags.
292 Glue = Chain.getValue(R: 1);
293 RetOps.push_back(Elt: DAG.getRegister(Reg: VA.getLocReg(), VT: VA.getLocVT()));
294 }
295
296 RetOps[0] = Chain; // Update chain.
297
298 // Add the glue if we have it.
299 if (Glue.getNode())
300 RetOps.push_back(Elt: Glue);
301
302 return DAG.getNode(Opcode: HexagonISD::RET_GLUE, DL: dl, VT: MVT::Other, Ops: RetOps);
303}
304
305bool HexagonTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
306 // If either no tail call or told not to tail call at all, don't.
307 return CI->isTailCall();
308}
309
310Register HexagonTargetLowering::getRegisterByName(
311 const char* RegName, LLT VT, const MachineFunction &) const {
312 // Just support r19, the linux kernel uses it.
313 Register Reg = StringSwitch<Register>(RegName)
314 .Case(S: "r0", Value: Hexagon::R0)
315 .Case(S: "r1", Value: Hexagon::R1)
316 .Case(S: "r2", Value: Hexagon::R2)
317 .Case(S: "r3", Value: Hexagon::R3)
318 .Case(S: "r4", Value: Hexagon::R4)
319 .Case(S: "r5", Value: Hexagon::R5)
320 .Case(S: "r6", Value: Hexagon::R6)
321 .Case(S: "r7", Value: Hexagon::R7)
322 .Case(S: "r8", Value: Hexagon::R8)
323 .Case(S: "r9", Value: Hexagon::R9)
324 .Case(S: "r10", Value: Hexagon::R10)
325 .Case(S: "r11", Value: Hexagon::R11)
326 .Case(S: "r12", Value: Hexagon::R12)
327 .Case(S: "r13", Value: Hexagon::R13)
328 .Case(S: "r14", Value: Hexagon::R14)
329 .Case(S: "r15", Value: Hexagon::R15)
330 .Case(S: "r16", Value: Hexagon::R16)
331 .Case(S: "r17", Value: Hexagon::R17)
332 .Case(S: "r18", Value: Hexagon::R18)
333 .Case(S: "r19", Value: Hexagon::R19)
334 .Case(S: "r20", Value: Hexagon::R20)
335 .Case(S: "r21", Value: Hexagon::R21)
336 .Case(S: "r22", Value: Hexagon::R22)
337 .Case(S: "r23", Value: Hexagon::R23)
338 .Case(S: "r24", Value: Hexagon::R24)
339 .Case(S: "r25", Value: Hexagon::R25)
340 .Case(S: "r26", Value: Hexagon::R26)
341 .Case(S: "r27", Value: Hexagon::R27)
342 .Case(S: "r28", Value: Hexagon::R28)
343 .Case(S: "r29", Value: Hexagon::R29)
344 .Case(S: "r30", Value: Hexagon::R30)
345 .Case(S: "r31", Value: Hexagon::R31)
346 .Case(S: "r1:0", Value: Hexagon::D0)
347 .Case(S: "r3:2", Value: Hexagon::D1)
348 .Case(S: "r5:4", Value: Hexagon::D2)
349 .Case(S: "r7:6", Value: Hexagon::D3)
350 .Case(S: "r9:8", Value: Hexagon::D4)
351 .Case(S: "r11:10", Value: Hexagon::D5)
352 .Case(S: "r13:12", Value: Hexagon::D6)
353 .Case(S: "r15:14", Value: Hexagon::D7)
354 .Case(S: "r17:16", Value: Hexagon::D8)
355 .Case(S: "r19:18", Value: Hexagon::D9)
356 .Case(S: "r21:20", Value: Hexagon::D10)
357 .Case(S: "r23:22", Value: Hexagon::D11)
358 .Case(S: "r25:24", Value: Hexagon::D12)
359 .Case(S: "r27:26", Value: Hexagon::D13)
360 .Case(S: "r29:28", Value: Hexagon::D14)
361 .Case(S: "r31:30", Value: Hexagon::D15)
362 .Case(S: "sp", Value: Hexagon::R29)
363 .Case(S: "fp", Value: Hexagon::R30)
364 .Case(S: "lr", Value: Hexagon::R31)
365 .Case(S: "p0", Value: Hexagon::P0)
366 .Case(S: "p1", Value: Hexagon::P1)
367 .Case(S: "p2", Value: Hexagon::P2)
368 .Case(S: "p3", Value: Hexagon::P3)
369 .Case(S: "sa0", Value: Hexagon::SA0)
370 .Case(S: "lc0", Value: Hexagon::LC0)
371 .Case(S: "sa1", Value: Hexagon::SA1)
372 .Case(S: "lc1", Value: Hexagon::LC1)
373 .Case(S: "m0", Value: Hexagon::M0)
374 .Case(S: "m1", Value: Hexagon::M1)
375 .Case(S: "usr", Value: Hexagon::USR)
376 .Case(S: "ugp", Value: Hexagon::UGP)
377 .Case(S: "cs0", Value: Hexagon::CS0)
378 .Case(S: "cs1", Value: Hexagon::CS1)
379 .Default(Value: Register());
380 return Reg;
381}
382
383/// LowerCallResult - Lower the result values of an ISD::CALL into the
384/// appropriate copies out of appropriate physical registers. This assumes that
385/// Chain/Glue are the input chain/glue to use, and that TheCall is the call
386/// being lowered. Returns a SDNode with the same number of values as the
387/// ISD::CALL.
388SDValue HexagonTargetLowering::LowerCallResult(
389 SDValue Chain, SDValue Glue, CallingConv::ID CallConv, bool IsVarArg,
390 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
391 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals,
392 const SmallVectorImpl<SDValue> &OutVals, SDValue Callee) const {
393 // Assign locations to each value returned by this call.
394 SmallVector<CCValAssign, 16> RVLocs;
395
396 CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
397 *DAG.getContext());
398
399 if (Subtarget.useHVXOps())
400 CCInfo.AnalyzeCallResult(Ins, Fn: RetCC_Hexagon_HVX);
401 else
402 CCInfo.AnalyzeCallResult(Ins, Fn: RetCC_Hexagon);
403
404 // Copy all of the result registers out of their specified physreg.
405 for (unsigned i = 0; i != RVLocs.size(); ++i) {
406 SDValue RetVal;
407 if (RVLocs[i].getValVT() == MVT::i1) {
408 // Return values of type MVT::i1 require special handling. The reason
409 // is that MVT::i1 is associated with the PredRegs register class, but
410 // values of that type are still returned in R0. Generate an explicit
411 // copy into a predicate register from R0, and treat the value of the
412 // predicate register as the call result.
413 auto &MRI = DAG.getMachineFunction().getRegInfo();
414 SDValue FR0 = DAG.getCopyFromReg(Chain, dl, Reg: RVLocs[i].getLocReg(),
415 VT: MVT::i32, Glue);
416 // FR0 = (Value, Chain, Glue)
417 Register PredR = MRI.createVirtualRegister(RegClass: &Hexagon::PredRegsRegClass);
418 SDValue TPR = DAG.getCopyToReg(Chain: FR0.getValue(R: 1), dl, Reg: PredR,
419 N: FR0.getValue(R: 0), Glue: FR0.getValue(R: 2));
420 // TPR = (Chain, Glue)
421 // Don't glue this CopyFromReg, because it copies from a virtual
422 // register. If it is glued to the call, InstrEmitter will add it
423 // as an implicit def to the call (EmitMachineNode).
424 RetVal = DAG.getCopyFromReg(Chain: TPR.getValue(R: 0), dl, Reg: PredR, VT: MVT::i1);
425 Glue = TPR.getValue(R: 1);
426 Chain = TPR.getValue(R: 0);
427 } else {
428 RetVal = DAG.getCopyFromReg(Chain, dl, Reg: RVLocs[i].getLocReg(),
429 VT: RVLocs[i].getValVT(), Glue);
430 Glue = RetVal.getValue(R: 2);
431 Chain = RetVal.getValue(R: 1);
432 }
433 InVals.push_back(Elt: RetVal.getValue(R: 0));
434 }
435
436 return Chain;
437}
438
439/// LowerCall - Functions arguments are copied from virtual regs to
440/// (physical regs)/(stack frame), CALLSEQ_START and CALLSEQ_END are emitted.
441SDValue
442HexagonTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
443 SmallVectorImpl<SDValue> &InVals) const {
444 SelectionDAG &DAG = CLI.DAG;
445 SDLoc &dl = CLI.DL;
446 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
447 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
448 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
449 SDValue Chain = CLI.Chain;
450 SDValue Callee = CLI.Callee;
451 CallingConv::ID CallConv = CLI.CallConv;
452 bool IsVarArg = CLI.IsVarArg;
453 bool DoesNotReturn = CLI.DoesNotReturn;
454
455 bool IsStructRet = Outs.empty() ? false : Outs[0].Flags.isSRet();
456 MachineFunction &MF = DAG.getMachineFunction();
457 MachineFrameInfo &MFI = MF.getFrameInfo();
458 auto PtrVT = getPointerTy(DL: MF.getDataLayout());
459
460 if (GlobalAddressSDNode *GAN = dyn_cast<GlobalAddressSDNode>(Val&: Callee))
461 Callee = DAG.getTargetGlobalAddress(GV: GAN->getGlobal(), DL: dl, VT: MVT::i32);
462
463 // Linux ABI treats var-arg calls the same way as regular ones.
464 bool TreatAsVarArg = !Subtarget.isEnvironmentMusl() && IsVarArg;
465
466 // Analyze operands of the call, assigning locations to each operand.
467 SmallVector<CCValAssign, 16> ArgLocs;
468 CCState CCInfo(CallConv, TreatAsVarArg, MF, ArgLocs, *DAG.getContext());
469
470 if (Subtarget.useHVXOps())
471 CCInfo.AnalyzeCallOperands(Outs, Fn: CC_Hexagon_HVX);
472 else if (DisableArgsMinAlignment)
473 CCInfo.AnalyzeCallOperands(Outs, Fn: CC_Hexagon_Legacy);
474 else
475 CCInfo.AnalyzeCallOperands(Outs, Fn: CC_Hexagon);
476
477 if (CLI.IsTailCall) {
478 bool StructAttrFlag = MF.getFunction().hasStructRetAttr();
479 CLI.IsTailCall = IsEligibleForTailCallOptimization(Callee, CalleeCC: CallConv,
480 isVarArg: IsVarArg, isCalleeStructRet: IsStructRet, isCallerStructRet: StructAttrFlag, Outs,
481 OutVals, Ins, DAG);
482 for (const CCValAssign &VA : ArgLocs) {
483 if (VA.isMemLoc()) {
484 CLI.IsTailCall = false;
485 break;
486 }
487 }
488 LLVM_DEBUG(dbgs() << (CLI.IsTailCall ? "Eligible for Tail Call\n"
489 : "Argument must be passed on stack. "
490 "Not eligible for Tail Call\n"));
491 }
492 // Get a count of how many bytes are to be pushed on the stack.
493 unsigned NumBytes = CCInfo.getStackSize();
494 SmallVector<std::pair<unsigned, SDValue>, 16> RegsToPass;
495 SmallVector<SDValue, 8> MemOpChains;
496
497 const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo();
498 SDValue StackPtr =
499 DAG.getCopyFromReg(Chain, dl, Reg: HRI.getStackRegister(), VT: PtrVT);
500
501 bool NeedsArgAlign = false;
502 Align LargestAlignSeen;
503 // Walk the register/memloc assignments, inserting copies/loads.
504 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
505 CCValAssign &VA = ArgLocs[i];
506 SDValue Arg = OutVals[i];
507 ISD::ArgFlagsTy Flags = Outs[i].Flags;
508 // Record if we need > 8 byte alignment on an argument.
509 bool ArgAlign = Subtarget.isHVXVectorType(VecTy: VA.getValVT());
510 NeedsArgAlign |= ArgAlign;
511
512 // Promote the value if needed.
513 switch (VA.getLocInfo()) {
514 default:
515 // Loc info must be one of Full, BCvt, SExt, ZExt, or AExt.
516 llvm_unreachable("Unknown loc info!");
517 case CCValAssign::Full:
518 break;
519 case CCValAssign::BCvt:
520 Arg = DAG.getBitcast(VT: VA.getLocVT(), V: Arg);
521 break;
522 case CCValAssign::SExt:
523 Arg = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: Arg);
524 break;
525 case CCValAssign::ZExt:
526 Arg = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: Arg);
527 break;
528 case CCValAssign::AExt:
529 Arg = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: Arg);
530 break;
531 }
532
533 if (VA.isMemLoc()) {
534 unsigned LocMemOffset = VA.getLocMemOffset();
535 SDValue MemAddr = DAG.getConstant(Val: LocMemOffset, DL: dl,
536 VT: StackPtr.getValueType());
537 MemAddr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::i32, N1: StackPtr, N2: MemAddr);
538 if (ArgAlign)
539 LargestAlignSeen = std::max(
540 a: LargestAlignSeen, b: Align(VA.getLocVT().getStoreSizeInBits() / 8));
541 if (Flags.isByVal()) {
542 // The argument is a struct passed by value. According to LLVM, "Arg"
543 // is a pointer.
544 MemOpChains.push_back(Elt: CreateCopyOfByValArgument(Src: Arg, Dst: MemAddr, Chain,
545 Flags, DAG, dl));
546 } else {
547 MachinePointerInfo LocPI = MachinePointerInfo::getStack(
548 MF&: DAG.getMachineFunction(), Offset: LocMemOffset);
549 SDValue S = DAG.getStore(Chain, dl, Val: Arg, Ptr: MemAddr, PtrInfo: LocPI);
550 MemOpChains.push_back(Elt: S);
551 }
552 continue;
553 }
554
555 // Arguments that can be passed on register must be kept at RegsToPass
556 // vector.
557 if (VA.isRegLoc())
558 RegsToPass.push_back(Elt: std::make_pair(x: VA.getLocReg(), y&: Arg));
559 }
560
561 if (NeedsArgAlign && Subtarget.hasV60Ops()) {
562 LLVM_DEBUG(dbgs() << "Function needs byte stack align due to call args\n");
563 Align VecAlign = HRI.getSpillAlign(RC: Hexagon::HvxVRRegClass);
564 LargestAlignSeen = std::max(a: LargestAlignSeen, b: VecAlign);
565 MFI.ensureMaxAlignment(Alignment: LargestAlignSeen);
566 }
567 // Transform all store nodes into one single node because all store
568 // nodes are independent of each other.
569 if (!MemOpChains.empty())
570 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: MemOpChains);
571
572 SDValue Glue;
573 if (!CLI.IsTailCall) {
574 Chain = DAG.getCALLSEQ_START(Chain, InSize: NumBytes, OutSize: 0, DL: dl);
575 Glue = Chain.getValue(R: 1);
576 }
577
578 // Build a sequence of copy-to-reg nodes chained together with token
579 // chain and flag operands which copy the outgoing args into registers.
580 // The Glue is necessary since all emitted instructions must be
581 // stuck together.
582 if (!CLI.IsTailCall) {
583 for (const auto &R : RegsToPass) {
584 Chain = DAG.getCopyToReg(Chain, dl, Reg: R.first, N: R.second, Glue);
585 Glue = Chain.getValue(R: 1);
586 }
587 } else {
588 // For tail calls lower the arguments to the 'real' stack slot.
589 //
590 // Force all the incoming stack arguments to be loaded from the stack
591 // before any new outgoing arguments are stored to the stack, because the
592 // outgoing stack slots may alias the incoming argument stack slots, and
593 // the alias isn't otherwise explicit. This is slightly more conservative
594 // than necessary, because it means that each store effectively depends
595 // on every argument instead of just those arguments it would clobber.
596 //
597 // Do not flag preceding copytoreg stuff together with the following stuff.
598 Glue = SDValue();
599 for (const auto &R : RegsToPass) {
600 Chain = DAG.getCopyToReg(Chain, dl, Reg: R.first, N: R.second, Glue);
601 Glue = Chain.getValue(R: 1);
602 }
603 Glue = SDValue();
604 }
605
606 bool LongCalls = MF.getSubtarget<HexagonSubtarget>().useLongCalls();
607 unsigned Flags = LongCalls ? HexagonII::HMOTF_ConstExtended : 0;
608
609 // If the callee is a GlobalAddress/ExternalSymbol node (quite common, every
610 // direct call is) turn it into a TargetGlobalAddress/TargetExternalSymbol
611 // node so that legalize doesn't hack it.
612 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Val&: Callee)) {
613 Callee = DAG.getTargetGlobalAddress(GV: G->getGlobal(), DL: dl, VT: PtrVT, offset: 0, TargetFlags: Flags);
614 } else if (ExternalSymbolSDNode *S =
615 dyn_cast<ExternalSymbolSDNode>(Val&: Callee)) {
616 Callee = DAG.getTargetExternalSymbol(Sym: S->getSymbol(), VT: PtrVT, TargetFlags: Flags);
617 }
618
619 // Returns a chain & a flag for retval copy to use.
620 SmallVector<SDValue, 8> Ops;
621 Ops.push_back(Elt: Chain);
622 Ops.push_back(Elt: Callee);
623
624 // Add argument registers to the end of the list so that they are
625 // known live into the call.
626 for (const auto &R : RegsToPass)
627 Ops.push_back(Elt: DAG.getRegister(Reg: R.first, VT: R.second.getValueType()));
628
629 const uint32_t *Mask = HRI.getCallPreservedMask(MF, CallConv);
630 assert(Mask && "Missing call preserved mask for calling convention");
631 Ops.push_back(Elt: DAG.getRegisterMask(RegMask: Mask));
632
633 if (Glue.getNode())
634 Ops.push_back(Elt: Glue);
635
636 if (CLI.IsTailCall) {
637 MFI.setHasTailCall();
638 return DAG.getNode(Opcode: HexagonISD::TC_RETURN, DL: dl, VT: MVT::Other, Ops);
639 }
640
641 // Set this here because we need to know this for "hasFP" in frame lowering.
642 // The target-independent code calls getFrameRegister before setting it, and
643 // getFrameRegister uses hasFP to determine whether the function has FP.
644 MFI.setHasCalls(true);
645
646 unsigned OpCode = DoesNotReturn ? HexagonISD::CALLnr : HexagonISD::CALL;
647 Chain = DAG.getNode(Opcode: OpCode, DL: dl, ResultTys: {MVT::Other, MVT::Glue}, Ops);
648 if (CLI.CFIType)
649 Chain.getNode()->setCFIType(CLI.CFIType->getZExtValue());
650 Glue = Chain.getValue(R: 1);
651
652 // Create the CALLSEQ_END node.
653 Chain = DAG.getCALLSEQ_END(Chain, Size1: NumBytes, Size2: 0, Glue, DL: dl);
654 Glue = Chain.getValue(R: 1);
655
656 // Handle result values, copying them out of physregs into vregs that we
657 // return.
658 return LowerCallResult(Chain, Glue, CallConv, IsVarArg, Ins, dl, DAG,
659 InVals, OutVals, Callee);
660}
661
662/// Returns true by value, base pointer and offset pointer and addressing
663/// mode by reference if this node can be combined with a load / store to
664/// form a post-indexed load / store.
665bool HexagonTargetLowering::getPostIndexedAddressParts(SDNode *N, SDNode *Op,
666 SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM,
667 SelectionDAG &DAG) const {
668 LSBaseSDNode *LSN = dyn_cast<LSBaseSDNode>(Val: N);
669 if (!LSN)
670 return false;
671 EVT VT = LSN->getMemoryVT();
672 if (!VT.isSimple())
673 return false;
674 bool IsLegalType = VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32 ||
675 VT == MVT::i64 || VT == MVT::f32 || VT == MVT::f64 ||
676 VT == MVT::v2i16 || VT == MVT::v2i32 || VT == MVT::v4i8 ||
677 VT == MVT::v4i16 || VT == MVT::v8i8 ||
678 Subtarget.isHVXVectorType(VecTy: VT.getSimpleVT());
679 if (!IsLegalType)
680 return false;
681
682 if (Op->getOpcode() != ISD::ADD)
683 return false;
684 Base = Op->getOperand(Num: 0);
685 Offset = Op->getOperand(Num: 1);
686 if (!isa<ConstantSDNode>(Val: Offset.getNode()))
687 return false;
688 AM = ISD::POST_INC;
689
690 int32_t V = cast<ConstantSDNode>(Val: Offset.getNode())->getSExtValue();
691 return Subtarget.getInstrInfo()->isValidAutoIncImm(VT, Offset: V);
692}
693
694SDValue HexagonTargetLowering::LowerFDIV(SDValue Op, SelectionDAG &DAG) const {
695 if (DAG.getMachineFunction().getFunction().hasOptSize())
696 return SDValue();
697 else
698 return Op;
699}
700
701SDValue
702HexagonTargetLowering::LowerINLINEASM(SDValue Op, SelectionDAG &DAG) const {
703 MachineFunction &MF = DAG.getMachineFunction();
704 auto &HMFI = *MF.getInfo<HexagonMachineFunctionInfo>();
705 const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo();
706 unsigned LR = HRI.getRARegister();
707
708 if ((Op.getOpcode() != ISD::INLINEASM &&
709 Op.getOpcode() != ISD::INLINEASM_BR) || HMFI.hasClobberLR())
710 return Op;
711
712 unsigned NumOps = Op.getNumOperands();
713 if (Op.getOperand(i: NumOps-1).getValueType() == MVT::Glue)
714 --NumOps; // Ignore the flag operand.
715
716 for (unsigned i = InlineAsm::Op_FirstOperand; i != NumOps;) {
717 const InlineAsm::Flag Flags(Op.getConstantOperandVal(i));
718 unsigned NumVals = Flags.getNumOperandRegisters();
719 ++i; // Skip the ID value.
720
721 switch (Flags.getKind()) {
722 default:
723 llvm_unreachable("Bad flags!");
724 case InlineAsm::Kind::RegUse:
725 case InlineAsm::Kind::Imm:
726 case InlineAsm::Kind::Mem:
727 i += NumVals;
728 break;
729 case InlineAsm::Kind::Clobber:
730 case InlineAsm::Kind::RegDef:
731 case InlineAsm::Kind::RegDefEarlyClobber: {
732 for (; NumVals; --NumVals, ++i) {
733 Register Reg = cast<RegisterSDNode>(Val: Op.getOperand(i))->getReg();
734 if (Reg != LR)
735 continue;
736 HMFI.setHasClobberLR(true);
737 return Op;
738 }
739 break;
740 }
741 }
742 }
743
744 return Op;
745}
746
747// Need to transform ISD::PREFETCH into something that doesn't inherit
748// all of the properties of ISD::PREFETCH, specifically SDNPMayLoad and
749// SDNPMayStore.
750SDValue HexagonTargetLowering::LowerPREFETCH(SDValue Op,
751 SelectionDAG &DAG) const {
752 SDValue Chain = Op.getOperand(i: 0);
753 SDValue Addr = Op.getOperand(i: 1);
754 // Lower it to DCFETCH($reg, #0). A "pat" will try to merge the offset in,
755 // if the "reg" is fed by an "add".
756 SDLoc DL(Op);
757 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
758 return DAG.getNode(Opcode: HexagonISD::DCFETCH, DL, VT: MVT::Other, N1: Chain, N2: Addr, N3: Zero);
759}
760
761SDValue HexagonTargetLowering::LowerINTRINSIC_VOID(SDValue Op,
762 SelectionDAG &DAG) const {
763 SDValue Chain = Op.getOperand(i: 0);
764 unsigned IntNo = Op.getConstantOperandVal(i: 1);
765 // Lower the hexagon_prefetch builtin to DCFETCH, as above.
766 if (IntNo == Intrinsic::hexagon_prefetch) {
767 SDValue Addr = Op.getOperand(i: 2);
768 SDLoc DL(Op);
769 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
770 return DAG.getNode(Opcode: HexagonISD::DCFETCH, DL, VT: MVT::Other, N1: Chain, N2: Addr, N3: Zero);
771 }
772 return SDValue();
773}
774
775SDValue
776HexagonTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
777 SelectionDAG &DAG) const {
778 SDValue Chain = Op.getOperand(i: 0);
779 SDValue Size = Op.getOperand(i: 1);
780 SDValue Align = Op.getOperand(i: 2);
781 SDLoc dl(Op);
782
783 ConstantSDNode *AlignConst = dyn_cast<ConstantSDNode>(Val&: Align);
784 assert(AlignConst && "Non-constant Align in LowerDYNAMIC_STACKALLOC");
785
786 unsigned A = AlignConst->getSExtValue();
787 auto &HFI = *Subtarget.getFrameLowering();
788 // "Zero" means natural stack alignment.
789 if (A == 0)
790 A = HFI.getStackAlign().value();
791
792 LLVM_DEBUG({
793 dbgs () << __func__ << " Align: " << A << " Size: ";
794 Size.getNode()->dump(&DAG);
795 dbgs() << "\n";
796 });
797
798 SDValue AC = DAG.getConstant(Val: A, DL: dl, VT: MVT::i32);
799 SDVTList VTs = DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other);
800 SDValue AA = DAG.getNode(Opcode: HexagonISD::ALLOCA, DL: dl, VTList: VTs, N1: Chain, N2: Size, N3: AC);
801
802 DAG.ReplaceAllUsesOfValueWith(From: Op, To: AA);
803 return AA;
804}
805
806SDValue HexagonTargetLowering::LowerFormalArguments(
807 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
808 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
809 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
810 MachineFunction &MF = DAG.getMachineFunction();
811 MachineFrameInfo &MFI = MF.getFrameInfo();
812 MachineRegisterInfo &MRI = MF.getRegInfo();
813
814 // Linux ABI treats var-arg calls the same way as regular ones.
815 bool TreatAsVarArg = !Subtarget.isEnvironmentMusl() && IsVarArg;
816
817 // Assign locations to all of the incoming arguments.
818 SmallVector<CCValAssign, 16> ArgLocs;
819 CCState CCInfo(CallConv, TreatAsVarArg, MF, ArgLocs, *DAG.getContext());
820
821 if (Subtarget.useHVXOps())
822 CCInfo.AnalyzeFormalArguments(Ins, Fn: CC_Hexagon_HVX);
823 else if (DisableArgsMinAlignment)
824 CCInfo.AnalyzeFormalArguments(Ins, Fn: CC_Hexagon_Legacy);
825 else
826 CCInfo.AnalyzeFormalArguments(Ins, Fn: CC_Hexagon);
827
828 // For LLVM, in the case when returning a struct by value (>8byte),
829 // the first argument is a pointer that points to the location on caller's
830 // stack where the return value will be stored. For Hexagon, the location on
831 // caller's stack is passed only when the struct size is smaller than (and
832 // equal to) 8 bytes. If not, no address will be passed into callee and
833 // callee return the result directly through R0/R1.
834 auto NextSingleReg = [] (const TargetRegisterClass &RC, unsigned Reg) {
835 switch (RC.getID()) {
836 case Hexagon::IntRegsRegClassID:
837 return Reg - Hexagon::R0 + 1;
838 case Hexagon::DoubleRegsRegClassID:
839 return (Reg - Hexagon::D0 + 1) * 2;
840 case Hexagon::HvxVRRegClassID:
841 return Reg - Hexagon::V0 + 1;
842 case Hexagon::HvxWRRegClassID:
843 return (Reg - Hexagon::W0 + 1) * 2;
844 }
845 llvm_unreachable("Unexpected register class");
846 };
847
848 auto &HFL = const_cast<HexagonFrameLowering&>(*Subtarget.getFrameLowering());
849 auto &HMFI = *MF.getInfo<HexagonMachineFunctionInfo>();
850 HFL.FirstVarArgSavedReg = 0;
851 HMFI.setFirstNamedArgFrameIndex(-int(MFI.getNumFixedObjects()));
852
853 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
854 CCValAssign &VA = ArgLocs[i];
855 ISD::ArgFlagsTy Flags = Ins[i].Flags;
856 bool ByVal = Flags.isByVal();
857
858 // Arguments passed in registers:
859 // 1. 32- and 64-bit values and HVX vectors are passed directly,
860 // 2. Large structs are passed via an address, and the address is
861 // passed in a register.
862 if (VA.isRegLoc() && ByVal && Flags.getByValSize() <= 8)
863 llvm_unreachable("ByValSize must be bigger than 8 bytes");
864
865 bool InReg = VA.isRegLoc() &&
866 (!ByVal || (ByVal && Flags.getByValSize() > 8));
867
868 if (InReg) {
869 MVT RegVT = VA.getLocVT();
870 if (VA.getLocInfo() == CCValAssign::BCvt)
871 RegVT = VA.getValVT();
872
873 const TargetRegisterClass *RC = getRegClassFor(VT: RegVT);
874 Register VReg = MRI.createVirtualRegister(RegClass: RC);
875 SDValue Copy = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: RegVT);
876
877 // Treat values of type MVT::i1 specially: they are passed in
878 // registers of type i32, but they need to remain as values of
879 // type i1 for consistency of the argument lowering.
880 if (VA.getValVT() == MVT::i1) {
881 assert(RegVT.getSizeInBits() <= 32);
882 SDValue T = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: RegVT,
883 N1: Copy, N2: DAG.getConstant(Val: 1, DL: dl, VT: RegVT));
884 Copy = DAG.getSetCC(DL: dl, VT: MVT::i1, LHS: T, RHS: DAG.getConstant(Val: 0, DL: dl, VT: RegVT),
885 Cond: ISD::SETNE);
886 } else {
887#ifndef NDEBUG
888 unsigned RegSize = RegVT.getSizeInBits();
889 assert(RegSize == 32 || RegSize == 64 ||
890 Subtarget.isHVXVectorType(RegVT));
891#endif
892 }
893 InVals.push_back(Elt: Copy);
894 MRI.addLiveIn(Reg: VA.getLocReg(), vreg: VReg);
895 HFL.FirstVarArgSavedReg = NextSingleReg(*RC, VA.getLocReg());
896 } else {
897 assert(VA.isMemLoc() && "Argument should be passed in memory");
898
899 // If it's a byval parameter, then we need to compute the
900 // "real" size, not the size of the pointer.
901 unsigned ObjSize = Flags.isByVal()
902 ? Flags.getByValSize()
903 : VA.getLocVT().getStoreSizeInBits() / 8;
904
905 // Create the frame index object for this incoming parameter.
906 int Offset = HEXAGON_LRFP_SIZE + VA.getLocMemOffset();
907 int FI = MFI.CreateFixedObject(Size: ObjSize, SPOffset: Offset, IsImmutable: true);
908 SDValue FIN = DAG.getFrameIndex(FI, VT: MVT::i32);
909
910 if (Flags.isByVal()) {
911 // If it's a pass-by-value aggregate, then do not dereference the stack
912 // location. Instead, we should generate a reference to the stack
913 // location.
914 InVals.push_back(Elt: FIN);
915 } else {
916 SDValue L = DAG.getLoad(VT: VA.getValVT(), dl, Chain, Ptr: FIN,
917 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI, Offset: 0));
918 InVals.push_back(Elt: L);
919 }
920 }
921 }
922
923 if (IsVarArg && Subtarget.isEnvironmentMusl()) {
924 for (int i = HFL.FirstVarArgSavedReg; i < 6; i++)
925 MRI.addLiveIn(Reg: Hexagon::R0+i);
926 }
927
928 if (IsVarArg && Subtarget.isEnvironmentMusl()) {
929 HMFI.setFirstNamedArgFrameIndex(HMFI.getFirstNamedArgFrameIndex() - 1);
930 HMFI.setLastNamedArgFrameIndex(-int(MFI.getNumFixedObjects()));
931
932 // Create Frame index for the start of register saved area.
933 int NumVarArgRegs = 6 - HFL.FirstVarArgSavedReg;
934 bool RequiresPadding = (NumVarArgRegs & 1);
935 int RegSaveAreaSizePlusPadding = RequiresPadding
936 ? (NumVarArgRegs + 1) * 4
937 : NumVarArgRegs * 4;
938
939 if (RegSaveAreaSizePlusPadding > 0) {
940 // The offset to saved register area should be 8 byte aligned.
941 int RegAreaStart = HEXAGON_LRFP_SIZE + CCInfo.getStackSize();
942 if (!(RegAreaStart % 8))
943 RegAreaStart = (RegAreaStart + 7) & -8;
944
945 int RegSaveAreaFrameIndex =
946 MFI.CreateFixedObject(Size: RegSaveAreaSizePlusPadding, SPOffset: RegAreaStart, IsImmutable: true);
947 HMFI.setRegSavedAreaStartFrameIndex(RegSaveAreaFrameIndex);
948
949 // This will point to the next argument passed via stack.
950 int Offset = RegAreaStart + RegSaveAreaSizePlusPadding;
951 int FI = MFI.CreateFixedObject(Hexagon_PointerSize, SPOffset: Offset, IsImmutable: true);
952 HMFI.setVarArgsFrameIndex(FI);
953 } else {
954 // This will point to the next argument passed via stack, when
955 // there is no saved register area.
956 int Offset = HEXAGON_LRFP_SIZE + CCInfo.getStackSize();
957 int FI = MFI.CreateFixedObject(Hexagon_PointerSize, SPOffset: Offset, IsImmutable: true);
958 HMFI.setRegSavedAreaStartFrameIndex(FI);
959 HMFI.setVarArgsFrameIndex(FI);
960 }
961 }
962
963
964 if (IsVarArg && !Subtarget.isEnvironmentMusl()) {
965 // This will point to the next argument passed via stack.
966 int Offset = HEXAGON_LRFP_SIZE + CCInfo.getStackSize();
967 int FI = MFI.CreateFixedObject(Hexagon_PointerSize, SPOffset: Offset, IsImmutable: true);
968 HMFI.setVarArgsFrameIndex(FI);
969 }
970
971 return Chain;
972}
973
974SDValue
975HexagonTargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {
976 // VASTART stores the address of the VarArgsFrameIndex slot into the
977 // memory location argument.
978 MachineFunction &MF = DAG.getMachineFunction();
979 HexagonMachineFunctionInfo *QFI = MF.getInfo<HexagonMachineFunctionInfo>();
980 SDValue Addr = DAG.getFrameIndex(FI: QFI->getVarArgsFrameIndex(), VT: MVT::i32);
981 const Value *SV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 2))->getValue();
982
983 if (!Subtarget.isEnvironmentMusl()) {
984 return DAG.getStore(Chain: Op.getOperand(i: 0), dl: SDLoc(Op), Val: Addr, Ptr: Op.getOperand(i: 1),
985 PtrInfo: MachinePointerInfo(SV));
986 }
987 auto &FuncInfo = *MF.getInfo<HexagonMachineFunctionInfo>();
988 auto &HFL = *Subtarget.getFrameLowering();
989 SDLoc DL(Op);
990 SmallVector<SDValue, 8> MemOps;
991
992 // Get frame index of va_list.
993 SDValue FIN = Op.getOperand(i: 1);
994
995 // If first Vararg register is odd, add 4 bytes to start of
996 // saved register area to point to the first register location.
997 // This is because the saved register area has to be 8 byte aligned.
998 // In case of an odd start register, there will be 4 bytes of padding in
999 // the beginning of saved register area. If all registers area used up,
1000 // the following condition will handle it correctly.
1001 SDValue SavedRegAreaStartFrameIndex =
1002 DAG.getFrameIndex(FI: FuncInfo.getRegSavedAreaStartFrameIndex(), VT: MVT::i32);
1003
1004 auto PtrVT = getPointerTy(DL: DAG.getDataLayout());
1005
1006 if (HFL.FirstVarArgSavedReg & 1)
1007 SavedRegAreaStartFrameIndex =
1008 DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT,
1009 N1: DAG.getFrameIndex(FI: FuncInfo.getRegSavedAreaStartFrameIndex(),
1010 VT: MVT::i32),
1011 N2: DAG.getIntPtrConstant(Val: 4, DL));
1012
1013 // Store the saved register area start pointer.
1014 SDValue Store =
1015 DAG.getStore(Chain: Op.getOperand(i: 0), dl: DL,
1016 Val: SavedRegAreaStartFrameIndex,
1017 Ptr: FIN, PtrInfo: MachinePointerInfo(SV));
1018 MemOps.push_back(Elt: Store);
1019
1020 // Store saved register area end pointer.
1021 FIN = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT,
1022 N1: FIN, N2: DAG.getIntPtrConstant(Val: 4, DL));
1023 Store = DAG.getStore(Chain: Op.getOperand(i: 0), dl: DL,
1024 Val: DAG.getFrameIndex(FI: FuncInfo.getVarArgsFrameIndex(),
1025 VT: PtrVT),
1026 Ptr: FIN, PtrInfo: MachinePointerInfo(SV, 4));
1027 MemOps.push_back(Elt: Store);
1028
1029 // Store overflow area pointer.
1030 FIN = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT,
1031 N1: FIN, N2: DAG.getIntPtrConstant(Val: 4, DL));
1032 Store = DAG.getStore(Chain: Op.getOperand(i: 0), dl: DL,
1033 Val: DAG.getFrameIndex(FI: FuncInfo.getVarArgsFrameIndex(),
1034 VT: PtrVT),
1035 Ptr: FIN, PtrInfo: MachinePointerInfo(SV, 8));
1036 MemOps.push_back(Elt: Store);
1037
1038 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: MemOps);
1039}
1040
1041SDValue
1042HexagonTargetLowering::LowerVACOPY(SDValue Op, SelectionDAG &DAG) const {
1043 // Assert that the linux ABI is enabled for the current compilation.
1044 assert(Subtarget.isEnvironmentMusl() && "Linux ABI should be enabled");
1045 SDValue Chain = Op.getOperand(i: 0);
1046 SDValue DestPtr = Op.getOperand(i: 1);
1047 SDValue SrcPtr = Op.getOperand(i: 2);
1048 const Value *DestSV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 3))->getValue();
1049 const Value *SrcSV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 4))->getValue();
1050 SDLoc DL(Op);
1051 // Size of the va_list is 12 bytes as it has 3 pointers. Therefore,
1052 // we need to memcopy 12 bytes from va_list to another similar list.
1053 return DAG.getMemcpy(Chain, dl: DL, Dst: DestPtr, Src: SrcPtr,
1054 Size: DAG.getIntPtrConstant(Val: 12, DL), DstAlign: Align(4), SrcAlign: Align(4),
1055 /*isVolatile*/ isVol: false, AlwaysInline: false, /*CI=*/nullptr,
1056 OverrideTailCall: std::nullopt, DstPtrInfo: MachinePointerInfo(DestSV),
1057 SrcPtrInfo: MachinePointerInfo(SrcSV));
1058}
1059
1060SDValue HexagonTargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
1061 const SDLoc &dl(Op);
1062 SDValue LHS = Op.getOperand(i: 0);
1063 SDValue RHS = Op.getOperand(i: 1);
1064 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 2))->get();
1065 MVT ResTy = ty(Op);
1066 MVT OpTy = ty(Op: LHS);
1067
1068 if (OpTy == MVT::v2i16 || OpTy == MVT::v4i8) {
1069 assert(OpTy.getVectorElementType().isScalarInteger());
1070 MVT WideTy = OpTy.widenIntegerElementType();
1071 return DAG.getSetCC(DL: dl, VT: ResTy,
1072 LHS: DAG.getSExtOrTrunc(Op: LHS, DL: SDLoc(LHS), VT: WideTy),
1073 RHS: DAG.getSExtOrTrunc(Op: RHS, DL: SDLoc(RHS), VT: WideTy), Cond: CC);
1074 }
1075
1076 // Treat all other vector types as legal.
1077 if (ResTy.isVector())
1078 return Op;
1079
1080 // Comparisons of short integers should use sign-extend, not zero-extend,
1081 // since we can represent small negative values in the compare instructions.
1082 // The LLVM default is to use zero-extend arbitrarily in these cases.
1083 auto isSExtFree = [this](SDValue N) {
1084 switch (N.getOpcode()) {
1085 case ISD::TRUNCATE: {
1086 // A sign-extend of a truncate of a sign-extend is free.
1087 SDValue Op = N.getOperand(i: 0);
1088 if (Op.getOpcode() != ISD::AssertSext)
1089 return false;
1090 EVT OrigTy = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
1091 unsigned ThisBW = ty(Op: N).getSizeInBits();
1092 unsigned OrigBW = OrigTy.getSizeInBits();
1093 // The type that was sign-extended to get the AssertSext must be
1094 // narrower than the type of N (so that N has still the same value
1095 // as the original).
1096 return ThisBW >= OrigBW;
1097 }
1098 case ISD::LOAD:
1099 // We have sign-extended loads.
1100 return true;
1101 }
1102 return false;
1103 };
1104
1105 if (OpTy == MVT::i8 || OpTy == MVT::i16) {
1106 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: RHS);
1107 bool IsNegative = C && C->getAPIntValue().isNegative();
1108 if (IsNegative || isSExtFree(LHS) || isSExtFree(RHS))
1109 return DAG.getSetCC(DL: dl, VT: ResTy,
1110 LHS: DAG.getSExtOrTrunc(Op: LHS, DL: SDLoc(LHS), VT: MVT::i32),
1111 RHS: DAG.getSExtOrTrunc(Op: RHS, DL: SDLoc(RHS), VT: MVT::i32), Cond: CC);
1112 }
1113
1114 return SDValue();
1115}
1116
1117SDValue
1118HexagonTargetLowering::LowerVSELECT(SDValue Op, SelectionDAG &DAG) const {
1119 SDValue PredOp = Op.getOperand(i: 0);
1120 SDValue Op1 = Op.getOperand(i: 1), Op2 = Op.getOperand(i: 2);
1121 MVT OpTy = ty(Op: Op1);
1122 const SDLoc &dl(Op);
1123
1124 if (OpTy == MVT::v2i16 || OpTy == MVT::v4i8) {
1125 assert(OpTy.getVectorElementType().isScalarInteger());
1126 MVT WideTy = OpTy.widenIntegerElementType();
1127 // Generate (trunc (select (_, sext, sext))).
1128 return DAG.getSExtOrTrunc(
1129 Op: DAG.getSelect(DL: dl, VT: WideTy, Cond: PredOp,
1130 LHS: DAG.getSExtOrTrunc(Op: Op1, DL: dl, VT: WideTy),
1131 RHS: DAG.getSExtOrTrunc(Op: Op2, DL: dl, VT: WideTy)),
1132 DL: dl, VT: OpTy);
1133 }
1134
1135 return SDValue();
1136}
1137
1138SDValue
1139HexagonTargetLowering::LowerConstantPool(SDValue Op, SelectionDAG &DAG) const {
1140 EVT ValTy = Op.getValueType();
1141 ConstantPoolSDNode *CPN = cast<ConstantPoolSDNode>(Val&: Op);
1142 Constant *CVal = nullptr;
1143 bool isVTi1Type = false;
1144 if (auto *CV = dyn_cast<ConstantVector>(Val: CPN->getConstVal())) {
1145 if (cast<VectorType>(Val: CV->getType())->getElementType()->isIntegerTy(BitWidth: 1)) {
1146 IRBuilder<> IRB(CV->getContext());
1147 SmallVector<Constant*, 128> NewConst;
1148 unsigned VecLen = CV->getNumOperands();
1149 assert(isPowerOf2_32(VecLen) &&
1150 "conversion only supported for pow2 VectorSize");
1151 for (unsigned i = 0; i < VecLen; ++i)
1152 NewConst.push_back(Elt: IRB.getInt8(C: CV->getOperand(i_nocapture: i)->isNullValue()));
1153
1154 CVal = ConstantVector::get(V: NewConst);
1155 isVTi1Type = true;
1156 }
1157 }
1158 Align Alignment = CPN->getAlign();
1159 bool IsPositionIndependent = isPositionIndependent();
1160 unsigned char TF = IsPositionIndependent ? HexagonII::MO_PCREL : 0;
1161
1162 unsigned Offset = 0;
1163 SDValue T;
1164 if (CPN->isMachineConstantPoolEntry())
1165 T = DAG.getTargetConstantPool(C: CPN->getMachineCPVal(), VT: ValTy, Align: Alignment,
1166 Offset, TargetFlags: TF);
1167 else if (isVTi1Type)
1168 T = DAG.getTargetConstantPool(C: CVal, VT: ValTy, Align: Alignment, Offset, TargetFlags: TF);
1169 else
1170 T = DAG.getTargetConstantPool(C: CPN->getConstVal(), VT: ValTy, Align: Alignment, Offset,
1171 TargetFlags: TF);
1172
1173 assert(cast<ConstantPoolSDNode>(T)->getTargetFlags() == TF &&
1174 "Inconsistent target flag encountered");
1175
1176 if (IsPositionIndependent)
1177 return DAG.getNode(Opcode: HexagonISD::AT_PCREL, DL: SDLoc(Op), VT: ValTy, Operand: T);
1178 return DAG.getNode(Opcode: HexagonISD::CP, DL: SDLoc(Op), VT: ValTy, Operand: T);
1179}
1180
1181SDValue
1182HexagonTargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const {
1183 EVT VT = Op.getValueType();
1184 int Idx = cast<JumpTableSDNode>(Val&: Op)->getIndex();
1185 if (isPositionIndependent()) {
1186 SDValue T = DAG.getTargetJumpTable(JTI: Idx, VT, TargetFlags: HexagonII::MO_PCREL);
1187 return DAG.getNode(Opcode: HexagonISD::AT_PCREL, DL: SDLoc(Op), VT, Operand: T);
1188 }
1189
1190 SDValue T = DAG.getTargetJumpTable(JTI: Idx, VT);
1191 return DAG.getNode(Opcode: HexagonISD::JT, DL: SDLoc(Op), VT, Operand: T);
1192}
1193
1194SDValue
1195HexagonTargetLowering::LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const {
1196 const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo();
1197 MachineFunction &MF = DAG.getMachineFunction();
1198 MachineFrameInfo &MFI = MF.getFrameInfo();
1199 MFI.setReturnAddressIsTaken(true);
1200
1201 EVT VT = Op.getValueType();
1202 SDLoc dl(Op);
1203 unsigned Depth = Op.getConstantOperandVal(i: 0);
1204 if (Depth) {
1205 SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
1206 SDValue Offset = DAG.getConstant(Val: 4, DL: dl, VT: MVT::i32);
1207 return DAG.getLoad(VT, dl, Chain: DAG.getEntryNode(),
1208 Ptr: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: FrameAddr, N2: Offset),
1209 PtrInfo: MachinePointerInfo());
1210 }
1211
1212 // Return LR, which contains the return address. Mark it an implicit live-in.
1213 Register Reg = MF.addLiveIn(PReg: HRI.getRARegister(), RC: getRegClassFor(VT: MVT::i32));
1214 return DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl, Reg, VT);
1215}
1216
1217SDValue
1218HexagonTargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const {
1219 const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo();
1220 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
1221 MFI.setFrameAddressIsTaken(true);
1222
1223 EVT VT = Op.getValueType();
1224 SDLoc dl(Op);
1225 unsigned Depth = Op.getConstantOperandVal(i: 0);
1226 SDValue FrameAddr = DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl,
1227 Reg: HRI.getFrameRegister(), VT);
1228 while (Depth--)
1229 FrameAddr = DAG.getLoad(VT, dl, Chain: DAG.getEntryNode(), Ptr: FrameAddr,
1230 PtrInfo: MachinePointerInfo());
1231 return FrameAddr;
1232}
1233
1234SDValue
1235HexagonTargetLowering::LowerATOMIC_FENCE(SDValue Op, SelectionDAG& DAG) const {
1236 SDLoc dl(Op);
1237 return DAG.getNode(Opcode: HexagonISD::BARRIER, DL: dl, VT: MVT::Other, Operand: Op.getOperand(i: 0));
1238}
1239
1240SDValue
1241HexagonTargetLowering::LowerGLOBALADDRESS(SDValue Op, SelectionDAG &DAG) const {
1242 SDLoc dl(Op);
1243 auto *GAN = cast<GlobalAddressSDNode>(Val&: Op);
1244 auto PtrVT = getPointerTy(DL: DAG.getDataLayout());
1245 auto *GV = GAN->getGlobal();
1246 int64_t Offset = GAN->getOffset();
1247
1248 auto &HLOF = *HTM.getObjFileLowering();
1249 Reloc::Model RM = HTM.getRelocationModel();
1250
1251 if (RM == Reloc::Static) {
1252 SDValue GA = DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: Offset);
1253 const GlobalObject *GO = GV->getAliaseeObject();
1254 if (GO && Subtarget.useSmallData() && HLOF.isGlobalInSmallSection(GO, TM: HTM))
1255 return DAG.getNode(Opcode: HexagonISD::CONST32_GP, DL: dl, VT: PtrVT, Operand: GA);
1256 return DAG.getNode(Opcode: HexagonISD::CONST32, DL: dl, VT: PtrVT, Operand: GA);
1257 }
1258
1259 bool UsePCRel = getTargetMachine().shouldAssumeDSOLocal(GV);
1260 if (UsePCRel) {
1261 SDValue GA = DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: Offset,
1262 TargetFlags: HexagonII::MO_PCREL);
1263 return DAG.getNode(Opcode: HexagonISD::AT_PCREL, DL: dl, VT: PtrVT, Operand: GA);
1264 }
1265
1266 // Use GOT index.
1267 SDValue GOT = DAG.getGLOBAL_OFFSET_TABLE(VT: PtrVT);
1268 SDValue GA = DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: 0, TargetFlags: HexagonII::MO_GOT);
1269 SDValue Off = DAG.getConstant(Val: Offset, DL: dl, VT: MVT::i32);
1270 return DAG.getNode(Opcode: HexagonISD::AT_GOT, DL: dl, VT: PtrVT, N1: GOT, N2: GA, N3: Off);
1271}
1272
1273// Specifies that for loads and stores VT can be promoted to PromotedLdStVT.
1274SDValue
1275HexagonTargetLowering::LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const {
1276 const BlockAddress *BA = cast<BlockAddressSDNode>(Val&: Op)->getBlockAddress();
1277 SDLoc dl(Op);
1278 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
1279
1280 Reloc::Model RM = HTM.getRelocationModel();
1281 if (RM == Reloc::Static) {
1282 SDValue A = DAG.getTargetBlockAddress(BA, VT: PtrVT);
1283 return DAG.getNode(Opcode: HexagonISD::CONST32_GP, DL: dl, VT: PtrVT, Operand: A);
1284 }
1285
1286 SDValue A = DAG.getTargetBlockAddress(BA, VT: PtrVT, Offset: 0, TargetFlags: HexagonII::MO_PCREL);
1287 return DAG.getNode(Opcode: HexagonISD::AT_PCREL, DL: dl, VT: PtrVT, Operand: A);
1288}
1289
1290SDValue
1291HexagonTargetLowering::LowerGLOBAL_OFFSET_TABLE(SDValue Op, SelectionDAG &DAG)
1292 const {
1293 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
1294 SDValue GOTSym = DAG.getTargetExternalSymbol(HEXAGON_GOT_SYM_NAME, VT: PtrVT,
1295 TargetFlags: HexagonII::MO_PCREL);
1296 return DAG.getNode(Opcode: HexagonISD::AT_PCREL, DL: SDLoc(Op), VT: PtrVT, Operand: GOTSym);
1297}
1298
1299SDValue
1300HexagonTargetLowering::GetDynamicTLSAddr(SelectionDAG &DAG, SDValue Chain,
1301 GlobalAddressSDNode *GA, SDValue Glue, EVT PtrVT, unsigned ReturnReg,
1302 unsigned char OperandFlags) const {
1303 MachineFunction &MF = DAG.getMachineFunction();
1304 MachineFrameInfo &MFI = MF.getFrameInfo();
1305 SDVTList NodeTys = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
1306 SDLoc dl(GA);
1307 SDValue TGA = DAG.getTargetGlobalAddress(GV: GA->getGlobal(), DL: dl,
1308 VT: GA->getValueType(ResNo: 0),
1309 offset: GA->getOffset(),
1310 TargetFlags: OperandFlags);
1311 // Create Operands for the call.The Operands should have the following:
1312 // 1. Chain SDValue
1313 // 2. Callee which in this case is the Global address value.
1314 // 3. Registers live into the call.In this case its R0, as we
1315 // have just one argument to be passed.
1316 // 4. Glue.
1317 // Note: The order is important.
1318
1319 const auto &HRI = *Subtarget.getRegisterInfo();
1320 const uint32_t *Mask = HRI.getCallPreservedMask(MF, CallingConv::C);
1321 assert(Mask && "Missing call preserved mask for calling convention");
1322 SDValue Ops[] = { Chain, TGA, DAG.getRegister(Reg: Hexagon::R0, VT: PtrVT),
1323 DAG.getRegisterMask(RegMask: Mask), Glue };
1324 Chain = DAG.getNode(Opcode: HexagonISD::CALL, DL: dl, VTList: NodeTys, Ops);
1325
1326 // Inform MFI that function has calls.
1327 MFI.setAdjustsStack(true);
1328
1329 Glue = Chain.getValue(R: 1);
1330 return DAG.getCopyFromReg(Chain, dl, Reg: ReturnReg, VT: PtrVT, Glue);
1331}
1332
1333//
1334// Lower using the initial executable model for TLS addresses
1335//
1336SDValue
1337HexagonTargetLowering::LowerToTLSInitialExecModel(GlobalAddressSDNode *GA,
1338 SelectionDAG &DAG) const {
1339 SDLoc dl(GA);
1340 int64_t Offset = GA->getOffset();
1341 auto PtrVT = getPointerTy(DL: DAG.getDataLayout());
1342
1343 // Get the thread pointer.
1344 SDValue TP = DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl, Reg: Hexagon::UGP, VT: PtrVT);
1345
1346 bool IsPositionIndependent = isPositionIndependent();
1347 unsigned char TF =
1348 IsPositionIndependent ? HexagonII::MO_IEGOT : HexagonII::MO_IE;
1349
1350 // First generate the TLS symbol address
1351 SDValue TGA = DAG.getTargetGlobalAddress(GV: GA->getGlobal(), DL: dl, VT: PtrVT,
1352 offset: Offset, TargetFlags: TF);
1353
1354 SDValue Sym = DAG.getNode(Opcode: HexagonISD::CONST32, DL: dl, VT: PtrVT, Operand: TGA);
1355
1356 if (IsPositionIndependent) {
1357 // Generate the GOT pointer in case of position independent code
1358 SDValue GOT = LowerGLOBAL_OFFSET_TABLE(Op: Sym, DAG);
1359
1360 // Add the TLS Symbol address to GOT pointer.This gives
1361 // GOT relative relocation for the symbol.
1362 Sym = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: GOT, N2: Sym);
1363 }
1364
1365 // Load the offset value for TLS symbol.This offset is relative to
1366 // thread pointer.
1367 SDValue LoadOffset =
1368 DAG.getLoad(VT: PtrVT, dl, Chain: DAG.getEntryNode(), Ptr: Sym, PtrInfo: MachinePointerInfo());
1369
1370 // Address of the thread local variable is the add of thread
1371 // pointer and the offset of the variable.
1372 return DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: TP, N2: LoadOffset);
1373}
1374
1375//
1376// Lower using the local executable model for TLS addresses
1377//
1378SDValue
1379HexagonTargetLowering::LowerToTLSLocalExecModel(GlobalAddressSDNode *GA,
1380 SelectionDAG &DAG) const {
1381 SDLoc dl(GA);
1382 int64_t Offset = GA->getOffset();
1383 auto PtrVT = getPointerTy(DL: DAG.getDataLayout());
1384
1385 // Get the thread pointer.
1386 SDValue TP = DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl, Reg: Hexagon::UGP, VT: PtrVT);
1387 // Generate the TLS symbol address
1388 SDValue TGA = DAG.getTargetGlobalAddress(GV: GA->getGlobal(), DL: dl, VT: PtrVT, offset: Offset,
1389 TargetFlags: HexagonII::MO_TPREL);
1390 SDValue Sym = DAG.getNode(Opcode: HexagonISD::CONST32, DL: dl, VT: PtrVT, Operand: TGA);
1391
1392 // Address of the thread local variable is the add of thread
1393 // pointer and the offset of the variable.
1394 return DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: TP, N2: Sym);
1395}
1396
1397//
1398// Lower using the general dynamic model for TLS addresses
1399//
1400SDValue
1401HexagonTargetLowering::LowerToTLSGeneralDynamicModel(GlobalAddressSDNode *GA,
1402 SelectionDAG &DAG) const {
1403 SDLoc dl(GA);
1404 int64_t Offset = GA->getOffset();
1405 auto PtrVT = getPointerTy(DL: DAG.getDataLayout());
1406
1407 // First generate the TLS symbol address
1408 SDValue TGA = DAG.getTargetGlobalAddress(GV: GA->getGlobal(), DL: dl, VT: PtrVT, offset: Offset,
1409 TargetFlags: HexagonII::MO_GDGOT);
1410
1411 // Then, generate the GOT pointer
1412 SDValue GOT = LowerGLOBAL_OFFSET_TABLE(Op: TGA, DAG);
1413
1414 // Add the TLS symbol and the GOT pointer
1415 SDValue Sym = DAG.getNode(Opcode: HexagonISD::CONST32, DL: dl, VT: PtrVT, Operand: TGA);
1416 SDValue Chain = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: GOT, N2: Sym);
1417
1418 // Copy over the argument to R0
1419 SDValue InGlue;
1420 Chain = DAG.getCopyToReg(Chain: DAG.getEntryNode(), dl, Reg: Hexagon::R0, N: Chain, Glue: InGlue);
1421 InGlue = Chain.getValue(R: 1);
1422
1423 unsigned Flags = DAG.getSubtarget<HexagonSubtarget>().useLongCalls()
1424 ? HexagonII::MO_GDPLT | HexagonII::HMOTF_ConstExtended
1425 : HexagonII::MO_GDPLT;
1426
1427 return GetDynamicTLSAddr(DAG, Chain, GA, Glue: InGlue, PtrVT,
1428 ReturnReg: Hexagon::R0, OperandFlags: Flags);
1429}
1430
1431//
1432// Lower TLS addresses.
1433//
1434// For now for dynamic models, we only support the general dynamic model.
1435//
1436SDValue
1437HexagonTargetLowering::LowerGlobalTLSAddress(SDValue Op,
1438 SelectionDAG &DAG) const {
1439 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Val&: Op);
1440
1441 switch (HTM.getTLSModel(GV: GA->getGlobal())) {
1442 case TLSModel::GeneralDynamic:
1443 case TLSModel::LocalDynamic:
1444 return LowerToTLSGeneralDynamicModel(GA, DAG);
1445 case TLSModel::InitialExec:
1446 return LowerToTLSInitialExecModel(GA, DAG);
1447 case TLSModel::LocalExec:
1448 return LowerToTLSLocalExecModel(GA, DAG);
1449 }
1450 llvm_unreachable("Bogus TLS model");
1451}
1452
1453//===----------------------------------------------------------------------===//
1454// TargetLowering Implementation
1455//===----------------------------------------------------------------------===//
1456
1457HexagonTargetLowering::HexagonTargetLowering(const TargetMachine &TM,
1458 const HexagonSubtarget &ST)
1459 : TargetLowering(TM, ST),
1460 HTM(static_cast<const HexagonTargetMachine &>(TM)), Subtarget(ST) {
1461 auto &HRI = *Subtarget.getRegisterInfo();
1462
1463 setPrefLoopAlignment(Align(16));
1464 setMinFunctionAlignment(Align(4));
1465 setPrefFunctionAlignment(Align(16));
1466 setStackPointerRegisterToSaveRestore(HRI.getStackRegister());
1467 setBooleanContents(TargetLoweringBase::UndefinedBooleanContent);
1468 setBooleanVectorContents(TargetLoweringBase::UndefinedBooleanContent);
1469
1470 setMaxAtomicSizeInBitsSupported(64);
1471 setMinCmpXchgSizeInBits(32);
1472
1473 if (EnableHexSDNodeSched)
1474 setSchedulingPreference(Sched::VLIW);
1475 else
1476 setSchedulingPreference(Sched::Source);
1477
1478 // Limits for inline expansion of memcpy/memmove
1479 MaxStoresPerMemcpy = 6;
1480 MaxStoresPerMemcpyOptSize = 4;
1481 MaxStoresPerMemmove = 6;
1482 MaxStoresPerMemmoveOptSize = 4;
1483 MaxStoresPerMemset = 8;
1484 MaxStoresPerMemsetOptSize = 4;
1485
1486 setTargetDAGCombine(ISD::VECREDUCE_ADD);
1487
1488 //
1489 // Set up register classes.
1490 //
1491
1492 addRegisterClass(VT: MVT::i1, RC: &Hexagon::PredRegsRegClass);
1493 addRegisterClass(VT: MVT::v2i1, RC: &Hexagon::PredRegsRegClass); // bbbbaaaa
1494 addRegisterClass(VT: MVT::v4i1, RC: &Hexagon::PredRegsRegClass); // ddccbbaa
1495 addRegisterClass(VT: MVT::v8i1, RC: &Hexagon::PredRegsRegClass); // hgfedcba
1496 addRegisterClass(VT: MVT::i32, RC: &Hexagon::IntRegsRegClass);
1497 addRegisterClass(VT: MVT::v2i16, RC: &Hexagon::IntRegsRegClass);
1498 addRegisterClass(VT: MVT::v4i8, RC: &Hexagon::IntRegsRegClass);
1499 addRegisterClass(VT: MVT::i64, RC: &Hexagon::DoubleRegsRegClass);
1500 addRegisterClass(VT: MVT::v8i8, RC: &Hexagon::DoubleRegsRegClass);
1501 addRegisterClass(VT: MVT::v4i16, RC: &Hexagon::DoubleRegsRegClass);
1502 addRegisterClass(VT: MVT::v2i32, RC: &Hexagon::DoubleRegsRegClass);
1503
1504 addRegisterClass(VT: MVT::f32, RC: &Hexagon::IntRegsRegClass);
1505 addRegisterClass(VT: MVT::f64, RC: &Hexagon::DoubleRegsRegClass);
1506
1507 //
1508 // Handling of scalar operations.
1509 //
1510 // All operations default to "legal", except:
1511 // - indexed loads and stores (pre-/post-incremented),
1512 // - ANY_EXTEND_VECTOR_INREG, ATOMIC_CMP_SWAP_WITH_SUCCESS, CONCAT_VECTORS,
1513 // ConstantFP, FCEIL, FCOPYSIGN, FEXP, FEXP2, FFLOOR, FGETSIGN,
1514 // FLOG, FLOG2, FLOG10, FMAXIMUMNUM, FMINIMUMNUM, FNEARBYINT, FRINT, FROUND,
1515 // TRAP, FTRUNC, PREFETCH, SIGN_EXTEND_VECTOR_INREG,
1516 // ZERO_EXTEND_VECTOR_INREG,
1517 // which default to "expand" for at least one type.
1518
1519 // Misc operations.
1520 setOperationAction(Op: ISD::ConstantFP, VT: MVT::f32, Action: Legal);
1521 setOperationAction(Op: ISD::ConstantFP, VT: MVT::f64, Action: Legal);
1522 setOperationAction(Op: ISD::TRAP, VT: MVT::Other, Action: Legal);
1523 setOperationAction(Op: ISD::DEBUGTRAP, VT: MVT::Other, Action: Legal);
1524 setOperationAction(Op: ISD::ConstantPool, VT: MVT::i32, Action: Custom);
1525 setOperationAction(Op: ISD::JumpTable, VT: MVT::i32, Action: Custom);
1526 setOperationAction(Op: ISD::BUILD_PAIR, VT: MVT::i64, Action: Expand);
1527 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i1, Action: Expand);
1528 setOperationAction(Op: ISD::INLINEASM, VT: MVT::Other, Action: Custom);
1529 setOperationAction(Op: ISD::INLINEASM_BR, VT: MVT::Other, Action: Custom);
1530 setOperationAction(Op: ISD::PREFETCH, VT: MVT::Other, Action: Custom);
1531 setOperationAction(Op: ISD::READCYCLECOUNTER, VT: MVT::i64, Action: Legal);
1532 setOperationAction(Op: ISD::READSTEADYCOUNTER, VT: MVT::i64, Action: Legal);
1533 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::Other, Action: Custom);
1534 setOperationAction(Op: ISD::INTRINSIC_VOID, VT: MVT::Other, Action: Custom);
1535 setOperationAction(Op: ISD::EH_RETURN, VT: MVT::Other, Action: Custom);
1536 setOperationAction(Op: ISD::GLOBAL_OFFSET_TABLE, VT: MVT::i32, Action: Custom);
1537 setOperationAction(Op: ISD::GlobalTLSAddress, VT: MVT::i32, Action: Custom);
1538 setOperationAction(Op: ISD::ATOMIC_FENCE, VT: MVT::Other, Action: Custom);
1539
1540 // Custom legalize GlobalAddress nodes into CONST32.
1541 setOperationAction(Op: ISD::GlobalAddress, VT: MVT::i32, Action: Custom);
1542 setOperationAction(Op: ISD::GlobalAddress, VT: MVT::i8, Action: Custom);
1543 setOperationAction(Op: ISD::BlockAddress, VT: MVT::i32, Action: Custom);
1544
1545 // Hexagon needs to optimize cases with negative constants.
1546 setOperationAction(Op: ISD::SETCC, VT: MVT::i8, Action: Custom);
1547 setOperationAction(Op: ISD::SETCC, VT: MVT::i16, Action: Custom);
1548 setOperationAction(Op: ISD::SETCC, VT: MVT::v4i8, Action: Custom);
1549 setOperationAction(Op: ISD::SETCC, VT: MVT::v2i16, Action: Custom);
1550
1551 // VASTART needs to be custom lowered to use the VarArgsFrameIndex.
1552 setOperationAction(Op: ISD::VASTART, VT: MVT::Other, Action: Custom);
1553 setOperationAction(Op: ISD::VAEND, VT: MVT::Other, Action: Expand);
1554 setOperationAction(Op: ISD::VAARG, VT: MVT::Other, Action: Expand);
1555 if (Subtarget.isEnvironmentMusl())
1556 setOperationAction(Op: ISD::VACOPY, VT: MVT::Other, Action: Custom);
1557 else
1558 setOperationAction(Op: ISD::VACOPY, VT: MVT::Other, Action: Expand);
1559
1560 setOperationAction(Op: ISD::STACKSAVE, VT: MVT::Other, Action: Expand);
1561 setOperationAction(Op: ISD::STACKRESTORE, VT: MVT::Other, Action: Expand);
1562 setOperationAction(Op: ISD::DYNAMIC_STACKALLOC, VT: MVT::i32, Action: Custom);
1563
1564 if (EmitJumpTables)
1565 setMinimumJumpTableEntries(MinimumJumpTables);
1566 else
1567 setMinimumJumpTableEntries(std::numeric_limits<unsigned>::max());
1568 setOperationAction(Op: ISD::BR_JT, VT: MVT::Other, Action: Expand);
1569
1570 for (unsigned LegalIntOp :
1571 {ISD::ABS, ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX}) {
1572 setOperationAction(Op: LegalIntOp, VT: MVT::i32, Action: Legal);
1573 setOperationAction(Op: LegalIntOp, VT: MVT::i64, Action: Legal);
1574 }
1575
1576 // Hexagon has A4_addp_c and A4_subp_c that take and generate a carry bit,
1577 // but they only operate on i64.
1578 for (MVT VT : MVT::integer_valuetypes()) {
1579 setOperationAction(Op: ISD::UADDO, VT, Action: Custom);
1580 setOperationAction(Op: ISD::USUBO, VT, Action: Custom);
1581 setOperationAction(Op: ISD::SADDO, VT, Action: Expand);
1582 setOperationAction(Op: ISD::SSUBO, VT, Action: Expand);
1583 setOperationAction(Op: ISD::UADDO_CARRY, VT, Action: Expand);
1584 setOperationAction(Op: ISD::USUBO_CARRY, VT, Action: Expand);
1585 }
1586 setOperationAction(Op: ISD::UADDO_CARRY, VT: MVT::i64, Action: Custom);
1587 setOperationAction(Op: ISD::USUBO_CARRY, VT: MVT::i64, Action: Custom);
1588
1589 setOperationAction(Op: ISD::CTLZ, VT: MVT::i8, Action: Promote);
1590 setOperationAction(Op: ISD::CTLZ, VT: MVT::i16, Action: Promote);
1591 setOperationAction(Op: ISD::CTTZ, VT: MVT::i8, Action: Promote);
1592 setOperationAction(Op: ISD::CTTZ, VT: MVT::i16, Action: Promote);
1593
1594 // Popcount can count # of 1s in i64 but returns i32.
1595 setOperationAction(Op: ISD::CTPOP, VT: MVT::i8, Action: Promote);
1596 setOperationAction(Op: ISD::CTPOP, VT: MVT::i16, Action: Promote);
1597 setOperationAction(Op: ISD::CTPOP, VT: MVT::i32, Action: Promote);
1598 setOperationAction(Op: ISD::CTPOP, VT: MVT::i64, Action: Legal);
1599
1600 setOperationAction(Op: ISD::BITREVERSE, VT: MVT::i32, Action: Legal);
1601 setOperationAction(Op: ISD::BITREVERSE, VT: MVT::i64, Action: Legal);
1602 setOperationAction(Op: ISD::BSWAP, VT: MVT::i32, Action: Legal);
1603 setOperationAction(Op: ISD::BSWAP, VT: MVT::i64, Action: Legal);
1604
1605 setOperationAction(Op: ISD::FSHL, VT: MVT::i32, Action: Legal);
1606 setOperationAction(Op: ISD::FSHL, VT: MVT::i64, Action: Legal);
1607 setOperationAction(Op: ISD::FSHR, VT: MVT::i32, Action: Legal);
1608 setOperationAction(Op: ISD::FSHR, VT: MVT::i64, Action: Legal);
1609
1610 for (unsigned IntExpOp :
1611 {ISD::SDIV, ISD::UDIV, ISD::SREM, ISD::UREM,
1612 ISD::SDIVREM, ISD::UDIVREM, ISD::ROTL, ISD::ROTR,
1613 ISD::SHL_PARTS, ISD::SRA_PARTS, ISD::SRL_PARTS,
1614 ISD::SMUL_LOHI, ISD::UMUL_LOHI}) {
1615 for (MVT VT : MVT::integer_valuetypes())
1616 setOperationAction(Op: IntExpOp, VT, Action: Expand);
1617 }
1618 for (MVT VT : MVT::fp_valuetypes()) {
1619 for (unsigned FPExpOp : {ISD::FDIV, ISD::FSQRT, ISD::FSIN, ISD::FCOS,
1620 ISD::FSINCOS, ISD::FPOW, ISD::FCOPYSIGN})
1621 setOperationAction(Op: FPExpOp, VT, Action: Expand);
1622
1623 setOperationAction(Op: ISD::FREM, VT, Action: LibCall);
1624 }
1625
1626 // No extending loads from i32.
1627 for (MVT VT : MVT::integer_valuetypes()) {
1628 setLoadExtAction(ExtType: ISD::ZEXTLOAD, ValVT: VT, MemVT: MVT::i32, Action: Expand);
1629 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: VT, MemVT: MVT::i32, Action: Expand);
1630 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: MVT::i32, Action: Expand);
1631 }
1632 // Turn FP truncstore into trunc + store.
1633 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f32, Action: Expand);
1634 setTruncStoreAction(ValVT: MVT::f32, MemVT: MVT::bf16, Action: Expand);
1635 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::bf16, Action: Expand);
1636 // Turn FP extload into load/fpextend.
1637 for (MVT VT : MVT::fp_valuetypes())
1638 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: MVT::f32, Action: Expand);
1639
1640 // Expand BR_CC and SELECT_CC for all integer and fp types.
1641 for (MVT VT : MVT::integer_valuetypes()) {
1642 setOperationAction(Op: ISD::BR_CC, VT, Action: Expand);
1643 setOperationAction(Op: ISD::SELECT_CC, VT, Action: Expand);
1644 }
1645 for (MVT VT : MVT::fp_valuetypes()) {
1646 setOperationAction(Op: ISD::BR_CC, VT, Action: Expand);
1647 setOperationAction(Op: ISD::SELECT_CC, VT, Action: Expand);
1648 }
1649 setOperationAction(Op: ISD::BR_CC, VT: MVT::Other, Action: Expand);
1650
1651 //
1652 // Handling of vector operations.
1653 //
1654
1655 // Set the action for vector operations to "expand", then override it with
1656 // either "custom" or "legal" for specific cases.
1657 // clang-format off
1658 static const unsigned VectExpOps[] = {
1659 // Integer arithmetic:
1660 ISD::ADD, ISD::SUB, ISD::MUL, ISD::SDIV, ISD::UDIV,
1661 ISD::SREM, ISD::UREM, ISD::SDIVREM, ISD::UDIVREM, ISD::SADDO,
1662 ISD::UADDO, ISD::SSUBO, ISD::USUBO, ISD::SMUL_LOHI, ISD::UMUL_LOHI,
1663 // Logical/bit:
1664 ISD::AND, ISD::OR, ISD::XOR, ISD::ROTL, ISD::ROTR,
1665 ISD::CTPOP, ISD::CTLZ, ISD::CTTZ, ISD::BSWAP, ISD::BITREVERSE,
1666 // Floating point arithmetic/math functions:
1667 ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FMA, ISD::FDIV,
1668 ISD::FREM, ISD::FNEG, ISD::FABS, ISD::FSQRT, ISD::FSIN,
1669 ISD::FCOS, ISD::FPOW, ISD::FLOG, ISD::FLOG2,
1670 ISD::FLOG10, ISD::FEXP, ISD::FEXP2, ISD::FCEIL, ISD::FTRUNC,
1671 ISD::FRINT, ISD::FNEARBYINT, ISD::FROUND, ISD::FFLOOR,
1672 ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM,
1673 ISD::FSINCOS, ISD::FLDEXP,
1674 // Misc:
1675 ISD::BR_CC, ISD::SELECT_CC, ISD::ConstantPool,
1676 // Vector:
1677 ISD::BUILD_VECTOR, ISD::SCALAR_TO_VECTOR,
1678 ISD::EXTRACT_VECTOR_ELT, ISD::INSERT_VECTOR_ELT,
1679 ISD::EXTRACT_SUBVECTOR, ISD::INSERT_SUBVECTOR,
1680 ISD::CONCAT_VECTORS, ISD::VECTOR_SHUFFLE,
1681 ISD::SPLAT_VECTOR,
1682 };
1683 // clang-format on
1684
1685 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
1686 for (unsigned VectExpOp : VectExpOps)
1687 setOperationAction(Op: VectExpOp, VT, Action: Expand);
1688
1689 // Expand all extending loads and truncating stores:
1690 for (MVT TargetVT : MVT::fixedlen_vector_valuetypes()) {
1691 if (TargetVT == VT)
1692 continue;
1693 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: TargetVT, MemVT: VT, Action: Expand);
1694 setLoadExtAction(ExtType: ISD::ZEXTLOAD, ValVT: TargetVT, MemVT: VT, Action: Expand);
1695 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: TargetVT, MemVT: VT, Action: Expand);
1696 setTruncStoreAction(ValVT: VT, MemVT: TargetVT, Action: Expand);
1697 }
1698
1699 // Normalize all inputs to SELECT to be vectors of i32.
1700 if (VT.getVectorElementType() != MVT::i32) {
1701 MVT VT32 = MVT::getVectorVT(VT: MVT::i32, NumElements: VT.getSizeInBits()/32);
1702 setOperationAction(Op: ISD::SELECT, VT, Action: Promote);
1703 AddPromotedToType(Opc: ISD::SELECT, OrigVT: VT, DestVT: VT32);
1704 }
1705 setOperationAction(Op: ISD::SRA, VT, Action: Custom);
1706 setOperationAction(Op: ISD::SHL, VT, Action: Custom);
1707 setOperationAction(Op: ISD::SRL, VT, Action: Custom);
1708 }
1709
1710 setOperationAction(Op: ISD::SADDSAT, VT: MVT::i32, Action: Legal);
1711 setOperationAction(Op: ISD::SADDSAT, VT: MVT::i64, Action: Legal);
1712
1713 // Extending loads from (native) vectors of i8 into (native) vectors of i16
1714 // are legal.
1715 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v2i16, MemVT: MVT::v2i8, Action: Legal);
1716 setLoadExtAction(ExtType: ISD::ZEXTLOAD, ValVT: MVT::v2i16, MemVT: MVT::v2i8, Action: Legal);
1717 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: MVT::v2i16, MemVT: MVT::v2i8, Action: Legal);
1718 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v4i16, MemVT: MVT::v4i8, Action: Legal);
1719 setLoadExtAction(ExtType: ISD::ZEXTLOAD, ValVT: MVT::v4i16, MemVT: MVT::v4i8, Action: Legal);
1720 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: MVT::v4i16, MemVT: MVT::v4i8, Action: Legal);
1721
1722 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::v2i8, Action: Legal);
1723 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::v2i16, Action: Legal);
1724 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::v2i32, Action: Legal);
1725
1726 // Types natively supported:
1727 for (MVT NativeVT : {MVT::v8i1, MVT::v4i1, MVT::v2i1, MVT::v4i8,
1728 MVT::v8i8, MVT::v2i16, MVT::v4i16, MVT::v2i32}) {
1729 setOperationAction(Op: ISD::BUILD_VECTOR, VT: NativeVT, Action: Custom);
1730 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: NativeVT, Action: Custom);
1731 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: NativeVT, Action: Custom);
1732 setOperationAction(Op: ISD::EXTRACT_SUBVECTOR, VT: NativeVT, Action: Custom);
1733 setOperationAction(Op: ISD::INSERT_SUBVECTOR, VT: NativeVT, Action: Custom);
1734 setOperationAction(Op: ISD::CONCAT_VECTORS, VT: NativeVT, Action: Custom);
1735
1736 setOperationAction(Op: ISD::ADD, VT: NativeVT, Action: Legal);
1737 setOperationAction(Op: ISD::SUB, VT: NativeVT, Action: Legal);
1738 setOperationAction(Op: ISD::MUL, VT: NativeVT, Action: Legal);
1739 setOperationAction(Op: ISD::AND, VT: NativeVT, Action: Legal);
1740 setOperationAction(Op: ISD::OR, VT: NativeVT, Action: Legal);
1741 setOperationAction(Op: ISD::XOR, VT: NativeVT, Action: Legal);
1742
1743 if (NativeVT.getVectorElementType() != MVT::i1) {
1744 setOperationAction(Op: ISD::SPLAT_VECTOR, VT: NativeVT, Action: Legal);
1745 setOperationAction(Op: ISD::BSWAP, VT: NativeVT, Action: Legal);
1746 setOperationAction(Op: ISD::BITREVERSE, VT: NativeVT, Action: Legal);
1747 }
1748 }
1749
1750 for (MVT VT : {MVT::v8i8, MVT::v4i16, MVT::v2i32}) {
1751 setOperationAction(Op: ISD::SMIN, VT, Action: Legal);
1752 setOperationAction(Op: ISD::SMAX, VT, Action: Legal);
1753 setOperationAction(Op: ISD::UMIN, VT, Action: Legal);
1754 setOperationAction(Op: ISD::UMAX, VT, Action: Legal);
1755 }
1756
1757 // Custom lower unaligned loads.
1758 // Also, for both loads and stores, verify the alignment of the address
1759 // in case it is a compile-time constant. This is a usability feature to
1760 // provide a meaningful error message to users.
1761 for (MVT VT : {MVT::i16, MVT::i32, MVT::v4i8, MVT::i64, MVT::v8i8,
1762 MVT::v2i16, MVT::v4i16, MVT::v2i32}) {
1763 setOperationAction(Op: ISD::LOAD, VT, Action: Custom);
1764 setOperationAction(Op: ISD::STORE, VT, Action: Custom);
1765 }
1766
1767 // Custom-lower load/stores of boolean vectors.
1768 for (MVT VT : {MVT::v2i1, MVT::v4i1, MVT::v8i1}) {
1769 setOperationAction(Op: ISD::LOAD, VT, Action: Custom);
1770 setOperationAction(Op: ISD::STORE, VT, Action: Custom);
1771 }
1772
1773 // Normalize integer compares to EQ/GT/UGT
1774 for (MVT VT : {MVT::v2i16, MVT::v4i8, MVT::v8i8, MVT::v2i32, MVT::v4i16,
1775 MVT::v2i32}) {
1776 setCondCodeAction(CCs: ISD::SETNE, VT, Action: Expand);
1777 setCondCodeAction(CCs: ISD::SETLE, VT, Action: Expand);
1778 setCondCodeAction(CCs: ISD::SETGE, VT, Action: Expand);
1779 setCondCodeAction(CCs: ISD::SETLT, VT, Action: Expand);
1780 setCondCodeAction(CCs: ISD::SETULE, VT, Action: Expand);
1781 setCondCodeAction(CCs: ISD::SETUGE, VT, Action: Expand);
1782 setCondCodeAction(CCs: ISD::SETULT, VT, Action: Expand);
1783 }
1784
1785 // Normalize boolean compares to [U]LE/[U]LT
1786 for (MVT VT : {MVT::i1, MVT::v2i1, MVT::v4i1, MVT::v8i1}) {
1787 setCondCodeAction(CCs: ISD::SETGE, VT, Action: Expand);
1788 setCondCodeAction(CCs: ISD::SETGT, VT, Action: Expand);
1789 setCondCodeAction(CCs: ISD::SETUGE, VT, Action: Expand);
1790 setCondCodeAction(CCs: ISD::SETUGT, VT, Action: Expand);
1791 }
1792
1793 // Custom-lower bitcasts from i8 to v8i1.
1794 setOperationAction(Op: ISD::BITCAST, VT: MVT::i8, Action: Custom);
1795 setOperationAction(Op: ISD::SETCC, VT: MVT::v2i16, Action: Custom);
1796 setOperationAction(Op: ISD::VSELECT, VT: MVT::v4i8, Action: Custom);
1797 setOperationAction(Op: ISD::VSELECT, VT: MVT::v2i16, Action: Custom);
1798 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT: MVT::v4i8, Action: Custom);
1799 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT: MVT::v4i16, Action: Custom);
1800 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT: MVT::v8i8, Action: Custom);
1801
1802 // V5+.
1803 setOperationAction(Op: ISD::FMA, VT: MVT::f64, Action: Expand);
1804 setOperationAction(Op: ISD::FADD, VT: MVT::f64, Action: Expand);
1805 setOperationAction(Op: ISD::FSUB, VT: MVT::f64, Action: Expand);
1806 setOperationAction(Op: ISD::FMUL, VT: MVT::f64, Action: Expand);
1807 setOperationAction(Op: ISD::FDIV, VT: MVT::f32, Action: Custom);
1808
1809 setOperationAction(Op: ISD::FMINIMUMNUM, VT: MVT::f32, Action: Legal);
1810 setOperationAction(Op: ISD::FMAXIMUMNUM, VT: MVT::f32, Action: Legal);
1811 setOperationAction(Op: ISD::FMINNUM, VT: MVT::f32, Action: Legal);
1812 setOperationAction(Op: ISD::FMAXNUM, VT: MVT::f32, Action: Legal);
1813
1814 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i1, Action: Promote);
1815 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i8, Action: Promote);
1816 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i16, Action: Promote);
1817 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::i1, Action: Promote);
1818 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::i8, Action: Promote);
1819 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::i16, Action: Promote);
1820 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::i1, Action: Promote);
1821 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::i8, Action: Promote);
1822 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::i16, Action: Promote);
1823 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i1, Action: Promote);
1824 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i8, Action: Promote);
1825 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i16, Action: Promote);
1826
1827 // Special handling for half-precision floating point conversions.
1828 // Lower half float conversions into library calls.
1829 setOperationAction(Op: ISD::FP16_TO_FP, VT: MVT::f32, Action: Expand);
1830 setOperationAction(Op: ISD::FP16_TO_FP, VT: MVT::f64, Action: Expand);
1831 setOperationAction(Op: ISD::FP_TO_FP16, VT: MVT::f32, Action: Expand);
1832 setOperationAction(Op: ISD::FP_TO_FP16, VT: MVT::f64, Action: Expand);
1833 setOperationAction(Op: ISD::BF16_TO_FP, VT: MVT::f32, Action: Expand);
1834 setOperationAction(Op: ISD::BF16_TO_FP, VT: MVT::f64, Action: Expand);
1835 setOperationAction(Op: ISD::FP_TO_BF16, VT: MVT::f64, Action: Expand);
1836
1837 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f32, MemVT: MVT::f16, Action: Expand);
1838 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::f16, Action: Expand);
1839 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f32, MemVT: MVT::bf16, Action: Expand);
1840 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::bf16, Action: Expand);
1841
1842 setTruncStoreAction(ValVT: MVT::f32, MemVT: MVT::f16, Action: Expand);
1843 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f16, Action: Expand);
1844
1845 // Handling of indexed loads/stores: default is "expand".
1846 //
1847 for (MVT VT : {MVT::i8, MVT::i16, MVT::i32, MVT::i64, MVT::f32, MVT::f64,
1848 MVT::v2i16, MVT::v2i32, MVT::v4i8, MVT::v4i16, MVT::v8i8}) {
1849 setIndexedLoadAction(IdxModes: ISD::POST_INC, VT, Action: Legal);
1850 setIndexedStoreAction(IdxModes: ISD::POST_INC, VT, Action: Legal);
1851 }
1852
1853 // Subtarget-specific operation actions.
1854 //
1855 if (Subtarget.hasV60Ops()) {
1856 setOperationAction(Op: ISD::ROTL, VT: MVT::i32, Action: Legal);
1857 setOperationAction(Op: ISD::ROTL, VT: MVT::i64, Action: Legal);
1858 setOperationAction(Op: ISD::ROTR, VT: MVT::i32, Action: Legal);
1859 setOperationAction(Op: ISD::ROTR, VT: MVT::i64, Action: Legal);
1860 }
1861 if (Subtarget.hasV66Ops()) {
1862 setOperationAction(Op: ISD::FADD, VT: MVT::f64, Action: Legal);
1863 setOperationAction(Op: ISD::FSUB, VT: MVT::f64, Action: Legal);
1864 }
1865 if (Subtarget.hasV67Ops()) {
1866 setOperationAction(Op: ISD::FMINIMUMNUM, VT: MVT::f64, Action: Legal);
1867 setOperationAction(Op: ISD::FMAXIMUMNUM, VT: MVT::f64, Action: Legal);
1868 setOperationAction(Op: ISD::FMINNUM, VT: MVT::f64, Action: Legal);
1869 setOperationAction(Op: ISD::FMAXNUM, VT: MVT::f64, Action: Legal);
1870 setOperationAction(Op: ISD::FMUL, VT: MVT::f64, Action: Legal);
1871 }
1872
1873 setTargetDAGCombine(ISD::OR);
1874 setTargetDAGCombine(ISD::TRUNCATE);
1875 setTargetDAGCombine(ISD::VSELECT);
1876
1877 if (Subtarget.useHVXOps())
1878 initializeHVXLowering();
1879
1880 computeRegisterProperties(TRI: &HRI);
1881}
1882
1883bool
1884HexagonTargetLowering::validateConstPtrAlignment(SDValue Ptr, Align NeedAlign,
1885 const SDLoc &dl, SelectionDAG &DAG) const {
1886 auto *CA = dyn_cast<ConstantSDNode>(Val&: Ptr);
1887 if (!CA)
1888 return true;
1889 unsigned Addr = CA->getZExtValue();
1890 Align HaveAlign =
1891 Addr != 0 ? Align(1ull << llvm::countr_zero(Val: Addr)) : NeedAlign;
1892 if (HaveAlign >= NeedAlign)
1893 return true;
1894
1895 static int DK_MisalignedTrap = llvm::getNextAvailablePluginDiagnosticKind();
1896
1897 struct DiagnosticInfoMisalignedTrap : public DiagnosticInfo {
1898 DiagnosticInfoMisalignedTrap(StringRef M)
1899 : DiagnosticInfo(DK_MisalignedTrap, DS_Remark), Msg(M) {}
1900 void print(DiagnosticPrinter &DP) const override {
1901 DP << Msg;
1902 }
1903 static bool classof(const DiagnosticInfo *DI) {
1904 return DI->getKind() == DK_MisalignedTrap;
1905 }
1906 StringRef Msg;
1907 };
1908
1909 std::string ErrMsg;
1910 raw_string_ostream O(ErrMsg);
1911 O << "Misaligned constant address: " << format_hex(N: Addr, Width: 10)
1912 << " has alignment " << HaveAlign.value()
1913 << ", but the memory access requires " << NeedAlign.value();
1914 if (DebugLoc DL = dl.getDebugLoc())
1915 DL.print(OS&: O << ", at ");
1916 O << ". The instruction has been replaced with a trap.";
1917
1918 DAG.getContext()->diagnose(DI: DiagnosticInfoMisalignedTrap(O.str()));
1919 return false;
1920}
1921
1922SDValue
1923HexagonTargetLowering::replaceMemWithUndef(SDValue Op, SelectionDAG &DAG)
1924 const {
1925 const SDLoc &dl(Op);
1926 auto *LS = cast<LSBaseSDNode>(Val: Op.getNode());
1927 assert(!LS->isIndexed() && "Not expecting indexed ops on constant address");
1928
1929 SDValue Chain = LS->getChain();
1930 SDValue Trap = DAG.getNode(Opcode: ISD::TRAP, DL: dl, VT: MVT::Other, Operand: Chain);
1931 if (LS->getOpcode() == ISD::LOAD)
1932 return DAG.getMergeValues(Ops: {DAG.getUNDEF(VT: ty(Op)), Trap}, dl);
1933 return Trap;
1934}
1935
1936// Bit-reverse Load Intrinsic: Check if the instruction is a bit reverse load
1937// intrinsic.
1938static bool isBrevLdIntrinsic(const Value *Inst) {
1939 unsigned ID = cast<IntrinsicInst>(Val: Inst)->getIntrinsicID();
1940 return (ID == Intrinsic::hexagon_L2_loadrd_pbr ||
1941 ID == Intrinsic::hexagon_L2_loadri_pbr ||
1942 ID == Intrinsic::hexagon_L2_loadrh_pbr ||
1943 ID == Intrinsic::hexagon_L2_loadruh_pbr ||
1944 ID == Intrinsic::hexagon_L2_loadrb_pbr ||
1945 ID == Intrinsic::hexagon_L2_loadrub_pbr);
1946}
1947
1948// Bit-reverse Load Intrinsic :Crawl up and figure out the object from previous
1949// instruction. So far we only handle bitcast, extract value and bit reverse
1950// load intrinsic instructions. Should we handle CGEP ?
1951static Value *getBrevLdObject(Value *V) {
1952 if (Operator::getOpcode(V) == Instruction::ExtractValue ||
1953 Operator::getOpcode(V) == Instruction::BitCast)
1954 V = cast<Operator>(Val: V)->getOperand(i: 0);
1955 else if (isa<IntrinsicInst>(Val: V) && isBrevLdIntrinsic(Inst: V))
1956 V = cast<Instruction>(Val: V)->getOperand(i: 0);
1957 return V;
1958}
1959
1960// Bit-reverse Load Intrinsic: For a PHI Node return either an incoming edge or
1961// a back edge. If the back edge comes from the intrinsic itself, the incoming
1962// edge is returned.
1963static Value *returnEdge(const PHINode *PN, Value *IntrBaseVal) {
1964 const BasicBlock *Parent = PN->getParent();
1965 int Idx = -1;
1966 for (unsigned i = 0, e = PN->getNumIncomingValues(); i < e; ++i) {
1967 BasicBlock *Blk = PN->getIncomingBlock(i);
1968 // Determine if the back edge is originated from intrinsic.
1969 if (Blk == Parent) {
1970 Value *BackEdgeVal = PN->getIncomingValue(i);
1971 Value *BaseVal;
1972 // Loop over till we return the same Value or we hit the IntrBaseVal.
1973 do {
1974 BaseVal = BackEdgeVal;
1975 BackEdgeVal = getBrevLdObject(V: BackEdgeVal);
1976 } while ((BaseVal != BackEdgeVal) && (IntrBaseVal != BackEdgeVal));
1977 // If the getBrevLdObject returns IntrBaseVal, we should return the
1978 // incoming edge.
1979 if (IntrBaseVal == BackEdgeVal)
1980 continue;
1981 Idx = i;
1982 break;
1983 } else // Set the node to incoming edge.
1984 Idx = i;
1985 }
1986 assert(Idx >= 0 && "Unexpected index to incoming argument in PHI");
1987 return PN->getIncomingValue(i: Idx);
1988}
1989
1990// Bit-reverse Load Intrinsic: Figure out the underlying object the base
1991// pointer points to, for the bit-reverse load intrinsic. Setting this to
1992// memoperand might help alias analysis to figure out the dependencies.
1993static Value *getUnderLyingObjectForBrevLdIntr(Value *V) {
1994 Value *IntrBaseVal = V;
1995 Value *BaseVal;
1996 // Loop over till we return the same Value, implies we either figure out
1997 // the object or we hit a PHI
1998 do {
1999 BaseVal = V;
2000 V = getBrevLdObject(V);
2001 } while (BaseVal != V);
2002
2003 // Identify the object from PHINode.
2004 if (const PHINode *PN = dyn_cast<PHINode>(Val: V))
2005 return returnEdge(PN, IntrBaseVal);
2006 // For non PHI nodes, the object is the last value returned by getBrevLdObject
2007 else
2008 return V;
2009}
2010
2011/// Given an intrinsic, checks if on the target the intrinsic will need to map
2012/// to a MemIntrinsicNode (touches memory). If this is the case, it stores
2013/// the intrinsic information into the Infos vector.
2014void HexagonTargetLowering::getTgtMemIntrinsic(
2015 SmallVectorImpl<IntrinsicInfo> &Infos, const CallBase &I,
2016 MachineFunction &MF, unsigned Intrinsic) const {
2017 IntrinsicInfo Info;
2018 switch (Intrinsic) {
2019 case Intrinsic::hexagon_L2_loadrd_pbr:
2020 case Intrinsic::hexagon_L2_loadri_pbr:
2021 case Intrinsic::hexagon_L2_loadrh_pbr:
2022 case Intrinsic::hexagon_L2_loadruh_pbr:
2023 case Intrinsic::hexagon_L2_loadrb_pbr:
2024 case Intrinsic::hexagon_L2_loadrub_pbr: {
2025 Info.opc = ISD::INTRINSIC_W_CHAIN;
2026 auto &DL = I.getDataLayout();
2027 auto &Cont = I.getCalledFunction()->getParent()->getContext();
2028 // The intrinsic function call is of the form { ElTy, i8* }
2029 // @llvm.hexagon.L2.loadXX.pbr(i8*, i32). The pointer and memory access type
2030 // should be derived from ElTy.
2031 Type *ElTy = I.getCalledFunction()->getReturnType()->getStructElementType(N: 0);
2032 Info.memVT = MVT::getVT(Ty: ElTy);
2033 llvm::Value *BasePtrVal = I.getOperand(i_nocapture: 0);
2034 Info.ptrVal = getUnderLyingObjectForBrevLdIntr(V: BasePtrVal);
2035 // The offset value comes through Modifier register. For now, assume the
2036 // offset is 0.
2037 Info.offset = 0;
2038 Info.align = DL.getABITypeAlign(Ty: Info.memVT.getTypeForEVT(Context&: Cont));
2039 Info.flags = MachineMemOperand::MOLoad;
2040 Infos.push_back(Elt: Info);
2041 return;
2042 }
2043 case Intrinsic::hexagon_V6_vgathermw:
2044 case Intrinsic::hexagon_V6_vgathermw_128B:
2045 case Intrinsic::hexagon_V6_vgathermh:
2046 case Intrinsic::hexagon_V6_vgathermh_128B:
2047 case Intrinsic::hexagon_V6_vgathermhw:
2048 case Intrinsic::hexagon_V6_vgathermhw_128B:
2049 case Intrinsic::hexagon_V6_vgathermwq:
2050 case Intrinsic::hexagon_V6_vgathermwq_128B:
2051 case Intrinsic::hexagon_V6_vgathermhq:
2052 case Intrinsic::hexagon_V6_vgathermhq_128B:
2053 case Intrinsic::hexagon_V6_vgathermhwq:
2054 case Intrinsic::hexagon_V6_vgathermhwq_128B:
2055 case Intrinsic::hexagon_V6_vgather_vscattermh:
2056 case Intrinsic::hexagon_V6_vgather_vscattermh_128B: {
2057 const Module &M = *I.getParent()->getParent()->getParent();
2058 Info.opc = ISD::INTRINSIC_W_CHAIN;
2059 Type *VecTy = I.getArgOperand(i: I.arg_size() - 1)->getType();
2060 assert(VecTy->isVectorTy() && "Expected vector operand for vgather");
2061 Info.memVT = MVT::getVT(Ty: VecTy);
2062 Info.ptrVal = I.getArgOperand(i: 0);
2063 Info.offset = 0;
2064 Info.align =
2065 MaybeAlign(M.getDataLayout().getTypeAllocSizeInBits(Ty: VecTy) / 8);
2066 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
2067 MachineMemOperand::MOVolatile;
2068 Infos.push_back(Elt: Info);
2069 return;
2070 }
2071 default:
2072 break;
2073 }
2074}
2075
2076bool HexagonTargetLowering::hasBitTest(SDValue X, SDValue Y) const {
2077 return X.getValueType().isScalarInteger(); // 'tstbit'
2078}
2079
2080bool HexagonTargetLowering::isTruncateFree(Type *Ty1, Type *Ty2) const {
2081 return isTruncateFree(VT1: EVT::getEVT(Ty: Ty1), VT2: EVT::getEVT(Ty: Ty2));
2082}
2083
2084bool HexagonTargetLowering::isTruncateFree(EVT VT1, EVT VT2) const {
2085 if (!VT1.isSimple() || !VT2.isSimple())
2086 return false;
2087 return VT1.getSimpleVT() == MVT::i64 && VT2.getSimpleVT() == MVT::i32;
2088}
2089
2090bool HexagonTargetLowering::isFMAFasterThanFMulAndFAdd(
2091 const MachineFunction &MF, EVT VT) const {
2092 return isOperationLegalOrCustom(Op: ISD::FMA, VT);
2093}
2094
2095// Should we expand the build vector with shuffles?
2096bool HexagonTargetLowering::shouldExpandBuildVectorWithShuffles(EVT VT,
2097 unsigned DefinedValues) const {
2098 return false;
2099}
2100
2101bool HexagonTargetLowering::isExtractSubvectorCheap(EVT ResVT, EVT SrcVT,
2102 unsigned Index) const {
2103 assert(ResVT.getVectorElementType() == SrcVT.getVectorElementType());
2104 if (!ResVT.isSimple() || !SrcVT.isSimple())
2105 return false;
2106
2107 MVT ResTy = ResVT.getSimpleVT(), SrcTy = SrcVT.getSimpleVT();
2108 if (ResTy.getVectorElementType() != MVT::i1)
2109 return true;
2110
2111 // Non-HVX bool vectors are relatively cheap.
2112 return SrcTy.getVectorNumElements() <= 8;
2113}
2114
2115bool HexagonTargetLowering::isTargetCanonicalConstantNode(SDValue Op) const {
2116 return Op.getOpcode() == ISD::CONCAT_VECTORS ||
2117 TargetLowering::isTargetCanonicalConstantNode(Op);
2118}
2119
2120bool HexagonTargetLowering::isShuffleMaskLegal(ArrayRef<int> Mask,
2121 EVT VT) const {
2122 return true;
2123}
2124
2125TargetLoweringBase::LegalizeTypeAction
2126HexagonTargetLowering::getPreferredVectorAction(MVT VT) const {
2127 unsigned VecLen = VT.getVectorMinNumElements();
2128 MVT ElemTy = VT.getVectorElementType();
2129
2130 if (VecLen == 1 || VT.isScalableVector())
2131 return TargetLoweringBase::TypeScalarizeVector;
2132
2133 if (Subtarget.useHVXOps()) {
2134 unsigned Action = getPreferredHvxVectorAction(VecTy: VT);
2135 if (Action != ~0u)
2136 return static_cast<TargetLoweringBase::LegalizeTypeAction>(Action);
2137 }
2138
2139 // Always widen (remaining) vectors of i1.
2140 if (ElemTy == MVT::i1)
2141 return TargetLoweringBase::TypeWidenVector;
2142 // Widen non-power-of-2 vectors. Such types cannot be split right now,
2143 // and computeRegisterProperties will override "split" with "widen",
2144 // which can cause other issues.
2145 if (!isPowerOf2_32(Value: VecLen))
2146 return TargetLoweringBase::TypeWidenVector;
2147
2148 return TargetLoweringBase::TypeSplitVector;
2149}
2150
2151TargetLoweringBase::LegalizeAction
2152HexagonTargetLowering::getCustomOperationAction(SDNode &Op) const {
2153 if (Subtarget.useHVXOps()) {
2154 unsigned Action = getCustomHvxOperationAction(Op);
2155 if (Action != ~0u)
2156 return static_cast<TargetLoweringBase::LegalizeAction>(Action);
2157 }
2158 return TargetLoweringBase::Legal;
2159}
2160
2161std::pair<SDValue, int>
2162HexagonTargetLowering::getBaseAndOffset(SDValue Addr) const {
2163 if (Addr.getOpcode() == ISD::ADD) {
2164 SDValue Op1 = Addr.getOperand(i: 1);
2165 if (auto *CN = dyn_cast<const ConstantSDNode>(Val: Op1.getNode()))
2166 return { Addr.getOperand(i: 0), CN->getSExtValue() };
2167 }
2168 return { Addr, 0 };
2169}
2170
2171// Lower a vector shuffle (V1, V2, V3). V1 and V2 are the two vectors
2172// to select data from, V3 is the permutation.
2173SDValue
2174HexagonTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG)
2175 const {
2176 const auto *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
2177 ArrayRef<int> AM = SVN->getMask();
2178 assert(AM.size() <= 8 && "Unexpected shuffle mask");
2179 unsigned VecLen = AM.size();
2180
2181 MVT VecTy = ty(Op);
2182 assert(!Subtarget.isHVXVectorType(VecTy, true) &&
2183 "HVX shuffles should be legal");
2184 assert(VecTy.getSizeInBits() <= 64 && "Unexpected vector length");
2185
2186 SDValue Op0 = Op.getOperand(i: 0);
2187 SDValue Op1 = Op.getOperand(i: 1);
2188 const SDLoc &dl(Op);
2189
2190 // If the inputs are not the same as the output, bail. This is not an
2191 // error situation, but complicates the handling and the default expansion
2192 // (into BUILD_VECTOR) should be adequate.
2193 if (ty(Op: Op0) != VecTy || ty(Op: Op1) != VecTy)
2194 return SDValue();
2195
2196 // Normalize the mask so that the first non-negative index comes from
2197 // the first operand.
2198 SmallVector<int, 8> Mask(AM);
2199 unsigned F = llvm::find_if(Range&: AM, P: [](int M) { return M >= 0; }) - AM.data();
2200 if (F == AM.size())
2201 return DAG.getUNDEF(VT: VecTy);
2202 if (AM[F] >= int(VecLen)) {
2203 ShuffleVectorSDNode::commuteMask(Mask);
2204 std::swap(a&: Op0, b&: Op1);
2205 }
2206
2207 // Express the shuffle mask in terms of bytes.
2208 SmallVector<int,8> ByteMask;
2209 unsigned ElemBytes = VecTy.getVectorElementType().getSizeInBits() / 8;
2210 for (int M : Mask) {
2211 if (M < 0) {
2212 for (unsigned j = 0; j != ElemBytes; ++j)
2213 ByteMask.push_back(Elt: -1);
2214 } else {
2215 for (unsigned j = 0; j != ElemBytes; ++j)
2216 ByteMask.push_back(Elt: M*ElemBytes + j);
2217 }
2218 }
2219 assert(ByteMask.size() <= 8);
2220
2221 // All non-undef (non-negative) indexes are well within [0..127], so they
2222 // fit in a single byte. Build two 64-bit words:
2223 // - MaskIdx where each byte is the corresponding index (for non-negative
2224 // indexes), and 0xFF for negative indexes, and
2225 // - MaskUnd that has 0xFF for each negative index.
2226 uint64_t MaskIdx = 0;
2227 uint64_t MaskUnd = 0;
2228 for (unsigned i = 0, e = ByteMask.size(); i != e; ++i) {
2229 unsigned S = 8*i;
2230 uint64_t M = ByteMask[i] & 0xFF;
2231 if (M == 0xFF)
2232 MaskUnd |= M << S;
2233 MaskIdx |= M << S;
2234 }
2235
2236 if (ByteMask.size() == 4) {
2237 // Identity.
2238 if (MaskIdx == (0x03020100 | MaskUnd))
2239 return Op0;
2240 // Byte swap.
2241 if (MaskIdx == (0x00010203 | MaskUnd)) {
2242 SDValue T0 = DAG.getBitcast(VT: MVT::i32, V: Op0);
2243 SDValue T1 = DAG.getNode(Opcode: ISD::BSWAP, DL: dl, VT: MVT::i32, Operand: T0);
2244 return DAG.getBitcast(VT: VecTy, V: T1);
2245 }
2246
2247 // Byte packs.
2248 SDValue Concat10 =
2249 getCombine(Hi: Op1, Lo: Op0, dl, ResTy: typeJoin(Tys: {ty(Op: Op1), ty(Op: Op0)}), DAG);
2250 if (MaskIdx == (0x06040200 | MaskUnd))
2251 return getInstr(MachineOpc: Hexagon::S2_vtrunehb, dl, Ty: VecTy, Ops: {Concat10}, DAG);
2252 if (MaskIdx == (0x07050301 | MaskUnd))
2253 return getInstr(MachineOpc: Hexagon::S2_vtrunohb, dl, Ty: VecTy, Ops: {Concat10}, DAG);
2254
2255 SDValue Concat01 =
2256 getCombine(Hi: Op0, Lo: Op1, dl, ResTy: typeJoin(Tys: {ty(Op: Op0), ty(Op: Op1)}), DAG);
2257 if (MaskIdx == (0x02000604 | MaskUnd))
2258 return getInstr(MachineOpc: Hexagon::S2_vtrunehb, dl, Ty: VecTy, Ops: {Concat01}, DAG);
2259 if (MaskIdx == (0x03010705 | MaskUnd))
2260 return getInstr(MachineOpc: Hexagon::S2_vtrunohb, dl, Ty: VecTy, Ops: {Concat01}, DAG);
2261 }
2262
2263 if (ByteMask.size() == 8) {
2264 // Identity.
2265 if (MaskIdx == (0x0706050403020100ull | MaskUnd))
2266 return Op0;
2267 // Byte swap.
2268 if (MaskIdx == (0x0001020304050607ull | MaskUnd)) {
2269 SDValue T0 = DAG.getBitcast(VT: MVT::i64, V: Op0);
2270 SDValue T1 = DAG.getNode(Opcode: ISD::BSWAP, DL: dl, VT: MVT::i64, Operand: T0);
2271 return DAG.getBitcast(VT: VecTy, V: T1);
2272 }
2273
2274 // Halfword picks.
2275 if (MaskIdx == (0x0d0c050409080100ull | MaskUnd))
2276 return getInstr(MachineOpc: Hexagon::S2_shuffeh, dl, Ty: VecTy, Ops: {Op1, Op0}, DAG);
2277 if (MaskIdx == (0x0f0e07060b0a0302ull | MaskUnd))
2278 return getInstr(MachineOpc: Hexagon::S2_shuffoh, dl, Ty: VecTy, Ops: {Op1, Op0}, DAG);
2279 if (MaskIdx == (0x0d0c090805040100ull | MaskUnd))
2280 return getInstr(MachineOpc: Hexagon::S2_vtrunewh, dl, Ty: VecTy, Ops: {Op1, Op0}, DAG);
2281 if (MaskIdx == (0x0f0e0b0a07060302ull | MaskUnd))
2282 return getInstr(MachineOpc: Hexagon::S2_vtrunowh, dl, Ty: VecTy, Ops: {Op1, Op0}, DAG);
2283 if (MaskIdx == (0x0706030205040100ull | MaskUnd)) {
2284 VectorPair P = opSplit(Vec: Op0, dl, DAG);
2285 return getInstr(MachineOpc: Hexagon::S2_packhl, dl, Ty: VecTy, Ops: {P.second, P.first}, DAG);
2286 }
2287
2288 // Byte packs.
2289 if (MaskIdx == (0x0e060c040a020800ull | MaskUnd))
2290 return getInstr(MachineOpc: Hexagon::S2_shuffeb, dl, Ty: VecTy, Ops: {Op1, Op0}, DAG);
2291 if (MaskIdx == (0x0f070d050b030901ull | MaskUnd))
2292 return getInstr(MachineOpc: Hexagon::S2_shuffob, dl, Ty: VecTy, Ops: {Op1, Op0}, DAG);
2293 }
2294
2295 return SDValue();
2296}
2297
2298SDValue
2299HexagonTargetLowering::getSplatValue(SDValue Op, SelectionDAG &DAG) const {
2300 switch (Op.getOpcode()) {
2301 case ISD::BUILD_VECTOR:
2302 if (SDValue S = cast<BuildVectorSDNode>(Val&: Op)->getSplatValue())
2303 return S;
2304 break;
2305 case ISD::SPLAT_VECTOR:
2306 return Op.getOperand(i: 0);
2307 }
2308 return SDValue();
2309}
2310
2311// Create a Hexagon-specific node for shifting a vector by an integer.
2312SDValue
2313HexagonTargetLowering::getVectorShiftByInt(SDValue Op, SelectionDAG &DAG)
2314 const {
2315 unsigned NewOpc;
2316 switch (Op.getOpcode()) {
2317 case ISD::SHL:
2318 NewOpc = HexagonISD::VASL;
2319 break;
2320 case ISD::SRA:
2321 NewOpc = HexagonISD::VASR;
2322 break;
2323 case ISD::SRL:
2324 NewOpc = HexagonISD::VLSR;
2325 break;
2326 default:
2327 llvm_unreachable("Unexpected shift opcode");
2328 }
2329 if (SDValue Sp = getSplatValue(Op: Op.getOperand(i: 1), DAG)) {
2330 const SDLoc dl(Op);
2331 // Canonicalize shift amount to i32 as required.
2332 SDValue Sh = Sp;
2333 if (Sh.getValueType() != MVT::i32)
2334 Sh = DAG.getZExtOrTrunc(Op: Sh, DL: dl, VT: MVT::i32);
2335
2336 assert(Sh.getValueType() == MVT::i32 &&
2337 "Hexagon vector shift-by-int must use i32 shift operand");
2338 return DAG.getNode(Opcode: NewOpc, DL: dl, VT: ty(Op), N1: Op.getOperand(i: 0), N2: Sh);
2339 }
2340
2341 return SDValue();
2342}
2343
2344SDValue
2345HexagonTargetLowering::LowerVECTOR_SHIFT(SDValue Op, SelectionDAG &DAG) const {
2346 const SDLoc &dl(Op);
2347
2348 // First try to convert the shift (by vector) to a shift by a scalar.
2349 // If we first split the shift, the shift amount will become 'extract
2350 // subvector', and will no longer be recognized as scalar.
2351 SDValue Res = Op;
2352 if (SDValue S = getVectorShiftByInt(Op, DAG))
2353 Res = S;
2354
2355 unsigned Opc = Res.getOpcode();
2356 switch (Opc) {
2357 case HexagonISD::VASR:
2358 case HexagonISD::VLSR:
2359 case HexagonISD::VASL:
2360 break;
2361 default:
2362 // No instructions for shifts by non-scalars.
2363 return SDValue();
2364 }
2365
2366 MVT ResTy = ty(Op: Res);
2367 if (ResTy.getVectorElementType() != MVT::i8)
2368 return Res;
2369
2370 // For shifts of i8, extend the inputs to i16, then truncate back to i8.
2371 assert(ResTy.getVectorElementType() == MVT::i8);
2372 SDValue Val = Res.getOperand(i: 0), Amt = Res.getOperand(i: 1);
2373
2374 auto ShiftPartI8 = [&dl, &DAG, this](unsigned Opc, SDValue V, SDValue A) {
2375 MVT Ty = ty(Op: V);
2376 MVT ExtTy = MVT::getVectorVT(VT: MVT::i16, NumElements: Ty.getVectorNumElements());
2377 SDValue ExtV = Opc == HexagonISD::VASR ? DAG.getSExtOrTrunc(Op: V, DL: dl, VT: ExtTy)
2378 : DAG.getZExtOrTrunc(Op: V, DL: dl, VT: ExtTy);
2379 SDValue ExtS = DAG.getNode(Opcode: Opc, DL: dl, VT: ExtTy, Ops: {ExtV, A});
2380 return DAG.getZExtOrTrunc(Op: ExtS, DL: dl, VT: Ty);
2381 };
2382
2383 if (ResTy.getSizeInBits() == 32)
2384 return ShiftPartI8(Opc, Val, Amt);
2385
2386 auto [LoV, HiV] = opSplit(Vec: Val, dl, DAG);
2387 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResTy,
2388 Ops: {ShiftPartI8(Opc, LoV, Amt), ShiftPartI8(Opc, HiV, Amt)});
2389}
2390
2391SDValue
2392HexagonTargetLowering::LowerROTL(SDValue Op, SelectionDAG &DAG) const {
2393 if (isa<ConstantSDNode>(Val: Op.getOperand(i: 1).getNode()))
2394 return Op;
2395 return SDValue();
2396}
2397
2398SDValue
2399HexagonTargetLowering::LowerBITCAST(SDValue Op, SelectionDAG &DAG) const {
2400 MVT ResTy = ty(Op);
2401 SDValue InpV = Op.getOperand(i: 0);
2402 MVT InpTy = ty(Op: InpV);
2403 assert(ResTy.getSizeInBits() == InpTy.getSizeInBits());
2404 const SDLoc &dl(Op);
2405
2406 // Handle conversion from i8 to v8i1.
2407 if (InpTy == MVT::i8) {
2408 if (ResTy == MVT::v8i1) {
2409 SDValue Sc = DAG.getBitcast(VT: tyScalar(Ty: InpTy), V: InpV);
2410 SDValue Ext = DAG.getZExtOrTrunc(Op: Sc, DL: dl, VT: MVT::i32);
2411 return getInstr(MachineOpc: Hexagon::C2_tfrrp, dl, Ty: ResTy, Ops: Ext, DAG);
2412 }
2413 return SDValue();
2414 }
2415
2416 return Op;
2417}
2418
2419bool
2420HexagonTargetLowering::getBuildVectorConstInts(ArrayRef<SDValue> Values,
2421 MVT VecTy, SelectionDAG &DAG,
2422 MutableArrayRef<ConstantInt*> Consts) const {
2423 MVT ElemTy = VecTy.getVectorElementType();
2424 unsigned ElemWidth = ElemTy.getSizeInBits();
2425 IntegerType *IntTy = IntegerType::get(C&: *DAG.getContext(), NumBits: ElemWidth);
2426 bool AllConst = true;
2427
2428 for (unsigned i = 0, e = Values.size(); i != e; ++i) {
2429 SDValue V = Values[i];
2430 if (V.isUndef()) {
2431 Consts[i] = ConstantInt::get(Ty: IntTy, V: 0);
2432 continue;
2433 }
2434 // Make sure to always cast to IntTy.
2435 if (auto *CN = dyn_cast<ConstantSDNode>(Val: V.getNode())) {
2436 const ConstantInt *CI = CN->getConstantIntValue();
2437 Consts[i] = cast<ConstantInt>(
2438 Val: ConstantInt::get(Ty: IntTy, V: CI->getValue().trunc(width: ElemWidth)));
2439 } else if (auto *CN = dyn_cast<ConstantFPSDNode>(Val: V.getNode())) {
2440 const ConstantFP *CF = CN->getConstantFPValue();
2441 APInt A = CF->getValueAPF().bitcastToAPInt();
2442 Consts[i] = ConstantInt::get(Ty: IntTy, V: A.getZExtValue());
2443 } else {
2444 AllConst = false;
2445 }
2446 }
2447 return AllConst;
2448}
2449
2450SDValue
2451HexagonTargetLowering::buildVector32(ArrayRef<SDValue> Elem, const SDLoc &dl,
2452 MVT VecTy, SelectionDAG &DAG) const {
2453 MVT ElemTy = VecTy.getVectorElementType();
2454 assert(VecTy.getVectorNumElements() == Elem.size());
2455
2456 SmallVector<ConstantInt*,4> Consts(Elem.size());
2457 bool AllConst = getBuildVectorConstInts(Values: Elem, VecTy, DAG, Consts);
2458
2459 unsigned First, Num = Elem.size();
2460 for (First = 0; First != Num; ++First) {
2461 if (!isUndef(Op: Elem[First]))
2462 break;
2463 }
2464 if (First == Num)
2465 return DAG.getUNDEF(VT: VecTy);
2466
2467 if (AllConst &&
2468 llvm::all_of(Range&: Consts, P: [](ConstantInt *CI) { return CI->isZero(); }))
2469 return getZero(dl, Ty: VecTy, DAG);
2470
2471 if (ElemTy == MVT::i16 || ElemTy == MVT::f16) {
2472 assert(Elem.size() == 2);
2473 if (AllConst) {
2474 // The 'Consts' array will have all values as integers regardless
2475 // of the vector element type.
2476 uint32_t V = (Consts[0]->getZExtValue() & 0xFFFF) |
2477 Consts[1]->getZExtValue() << 16;
2478 return DAG.getBitcast(VT: VecTy, V: DAG.getConstant(Val: V, DL: dl, VT: MVT::i32));
2479 }
2480 SDValue E0, E1;
2481 if (ElemTy == MVT::f16) {
2482 E0 = DAG.getZExtOrTrunc(Op: DAG.getBitcast(VT: MVT::i16, V: Elem[0]), DL: dl, VT: MVT::i32);
2483 E1 = DAG.getZExtOrTrunc(Op: DAG.getBitcast(VT: MVT::i16, V: Elem[1]), DL: dl, VT: MVT::i32);
2484 } else {
2485 E0 = Elem[0];
2486 E1 = Elem[1];
2487 }
2488 SDValue N = getInstr(MachineOpc: Hexagon::A2_combine_ll, dl, Ty: MVT::i32, Ops: {E1, E0}, DAG);
2489 return DAG.getBitcast(VT: VecTy, V: N);
2490 }
2491
2492 if (ElemTy == MVT::i8) {
2493 // First try generating a constant.
2494 if (AllConst) {
2495 uint32_t V = (Consts[0]->getZExtValue() & 0xFF) |
2496 (Consts[1]->getZExtValue() & 0xFF) << 8 |
2497 (Consts[2]->getZExtValue() & 0xFF) << 16 |
2498 Consts[3]->getZExtValue() << 24;
2499 return DAG.getBitcast(VT: MVT::v4i8, V: DAG.getConstant(Val: V, DL: dl, VT: MVT::i32));
2500 }
2501
2502 // Then try splat.
2503 bool IsSplat = true;
2504 for (unsigned i = First+1; i != Num; ++i) {
2505 if (Elem[i] == Elem[First] || isUndef(Op: Elem[i]))
2506 continue;
2507 IsSplat = false;
2508 break;
2509 }
2510 if (IsSplat) {
2511 // Legalize the operand of SPLAT_VECTOR.
2512 SDValue Ext = DAG.getZExtOrTrunc(Op: Elem[First], DL: dl, VT: MVT::i32);
2513 return DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: VecTy, Operand: Ext);
2514 }
2515
2516 // Generate
2517 // (zxtb(Elem[0]) | (zxtb(Elem[1]) << 8)) |
2518 // (zxtb(Elem[2]) | (zxtb(Elem[3]) << 8)) << 16
2519 assert(Elem.size() == 4);
2520 SDValue Vs[4];
2521 for (unsigned i = 0; i != 4; ++i) {
2522 Vs[i] = DAG.getZExtOrTrunc(Op: Elem[i], DL: dl, VT: MVT::i32);
2523 Vs[i] = DAG.getZeroExtendInReg(Op: Vs[i], DL: dl, VT: MVT::i8);
2524 }
2525 SDValue S8 = DAG.getConstant(Val: 8, DL: dl, VT: MVT::i32);
2526 SDValue T0 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: MVT::i32, Ops: {Vs[1], S8});
2527 SDValue T1 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: MVT::i32, Ops: {Vs[3], S8});
2528 SDValue B0 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: MVT::i32, Ops: {Vs[0], T0});
2529 SDValue B1 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: MVT::i32, Ops: {Vs[2], T1});
2530
2531 SDValue R = getInstr(MachineOpc: Hexagon::A2_combine_ll, dl, Ty: MVT::i32, Ops: {B1, B0}, DAG);
2532 return DAG.getBitcast(VT: MVT::v4i8, V: R);
2533 }
2534
2535#ifndef NDEBUG
2536 dbgs() << "VecTy: " << VecTy << '\n';
2537#endif
2538 llvm_unreachable("Unexpected vector element type");
2539}
2540
2541SDValue
2542HexagonTargetLowering::buildVector64(ArrayRef<SDValue> Elem, const SDLoc &dl,
2543 MVT VecTy, SelectionDAG &DAG) const {
2544 MVT ElemTy = VecTy.getVectorElementType();
2545 assert(VecTy.getVectorNumElements() == Elem.size());
2546
2547 SmallVector<ConstantInt*,8> Consts(Elem.size());
2548 bool AllConst = getBuildVectorConstInts(Values: Elem, VecTy, DAG, Consts);
2549
2550 unsigned First, Num = Elem.size();
2551 for (First = 0; First != Num; ++First) {
2552 if (!isUndef(Op: Elem[First]))
2553 break;
2554 }
2555 if (First == Num)
2556 return DAG.getUNDEF(VT: VecTy);
2557
2558 if (AllConst &&
2559 llvm::all_of(Range&: Consts, P: [](ConstantInt *CI) { return CI->isZero(); }))
2560 return getZero(dl, Ty: VecTy, DAG);
2561
2562 // First try splat if possible.
2563 if (ElemTy == MVT::i16 || ElemTy == MVT::f16) {
2564 bool IsSplat = true;
2565 for (unsigned i = First+1; i != Num; ++i) {
2566 if (Elem[i] == Elem[First] || isUndef(Op: Elem[i]))
2567 continue;
2568 IsSplat = false;
2569 break;
2570 }
2571 if (IsSplat) {
2572 // Legalize the operand of SPLAT_VECTOR
2573 SDValue S = ElemTy == MVT::f16 ? DAG.getBitcast(VT: MVT::i16, V: Elem[First])
2574 : Elem[First];
2575 SDValue Ext = DAG.getZExtOrTrunc(Op: S, DL: dl, VT: MVT::i32);
2576 return DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: VecTy, Operand: Ext);
2577 }
2578 }
2579
2580 // Then try constant.
2581 if (AllConst) {
2582 uint64_t Val = 0;
2583 unsigned W = ElemTy.getSizeInBits();
2584 uint64_t Mask = (1ull << W) - 1;
2585 for (unsigned i = 0; i != Num; ++i)
2586 Val = (Val << W) | (Consts[Num-1-i]->getZExtValue() & Mask);
2587 SDValue V0 = DAG.getConstant(Val, DL: dl, VT: MVT::i64);
2588 return DAG.getBitcast(VT: VecTy, V: V0);
2589 }
2590
2591 // Build two 32-bit vectors and concatenate.
2592 MVT HalfTy = MVT::getVectorVT(VT: ElemTy, NumElements: Num/2);
2593 SDValue L = (ElemTy == MVT::i32)
2594 ? Elem[0]
2595 : buildVector32(Elem: Elem.take_front(N: Num/2), dl, VecTy: HalfTy, DAG);
2596 SDValue H = (ElemTy == MVT::i32)
2597 ? Elem[1]
2598 : buildVector32(Elem: Elem.drop_front(N: Num/2), dl, VecTy: HalfTy, DAG);
2599 return getCombine(Hi: H, Lo: L, dl, ResTy: VecTy, DAG);
2600}
2601
2602SDValue
2603HexagonTargetLowering::extractVector(SDValue VecV, SDValue IdxV,
2604 const SDLoc &dl, MVT ValTy, MVT ResTy,
2605 SelectionDAG &DAG) const {
2606 MVT VecTy = ty(Op: VecV);
2607 assert(!ValTy.isVector() ||
2608 VecTy.getVectorElementType() == ValTy.getVectorElementType());
2609 if (VecTy.getVectorElementType() == MVT::i1)
2610 return extractVectorPred(VecV, IdxV, dl, ValTy, ResTy, DAG);
2611
2612 unsigned VecWidth = VecTy.getSizeInBits();
2613 unsigned ValWidth = ValTy.getSizeInBits();
2614 unsigned ElemWidth = VecTy.getVectorElementType().getSizeInBits();
2615 assert((VecWidth % ElemWidth) == 0);
2616 assert(VecWidth == 32 || VecWidth == 64);
2617
2618 // Cast everything to scalar integer types.
2619 MVT ScalarTy = tyScalar(Ty: VecTy);
2620 VecV = DAG.getBitcast(VT: ScalarTy, V: VecV);
2621
2622 SDValue WidthV = DAG.getConstant(Val: ValWidth, DL: dl, VT: MVT::i32);
2623 SDValue ExtV;
2624
2625 if (auto *IdxN = dyn_cast<ConstantSDNode>(Val&: IdxV)) {
2626 unsigned Off = IdxN->getZExtValue() * ElemWidth;
2627 if (VecWidth == 64 && ValWidth == 32) {
2628 assert(Off == 0 || Off == 32);
2629 ExtV = Off == 0 ? LoHalf(V: VecV, DAG) : HiHalf(V: VecV, DAG);
2630 } else if (Off == 0 && (ValWidth % 8) == 0) {
2631 ExtV = DAG.getZeroExtendInReg(Op: VecV, DL: dl, VT: tyScalar(Ty: ValTy));
2632 } else {
2633 SDValue OffV = DAG.getConstant(Val: Off, DL: dl, VT: MVT::i32);
2634 // The return type of EXTRACTU must be the same as the type of the
2635 // input vector.
2636 ExtV = DAG.getNode(Opcode: HexagonISD::EXTRACTU, DL: dl, VT: ScalarTy,
2637 Ops: {VecV, WidthV, OffV});
2638 }
2639 } else {
2640 if (ty(Op: IdxV) != MVT::i32)
2641 IdxV = DAG.getZExtOrTrunc(Op: IdxV, DL: dl, VT: MVT::i32);
2642 SDValue OffV = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MVT::i32, N1: IdxV,
2643 N2: DAG.getConstant(Val: ElemWidth, DL: dl, VT: MVT::i32));
2644 ExtV = DAG.getNode(Opcode: HexagonISD::EXTRACTU, DL: dl, VT: ScalarTy,
2645 Ops: {VecV, WidthV, OffV});
2646 }
2647
2648 // Cast ExtV to the requested result type.
2649 ExtV = DAG.getZExtOrTrunc(Op: ExtV, DL: dl, VT: tyScalar(Ty: ResTy));
2650 ExtV = DAG.getBitcast(VT: ResTy, V: ExtV);
2651 return ExtV;
2652}
2653
2654SDValue
2655HexagonTargetLowering::extractVectorPred(SDValue VecV, SDValue IdxV,
2656 const SDLoc &dl, MVT ValTy, MVT ResTy,
2657 SelectionDAG &DAG) const {
2658 // Special case for v{8,4,2}i1 (the only boolean vectors legal in Hexagon
2659 // without any coprocessors).
2660 MVT VecTy = ty(Op: VecV);
2661 unsigned VecWidth = VecTy.getSizeInBits();
2662 unsigned ValWidth = ValTy.getSizeInBits();
2663 assert(VecWidth == VecTy.getVectorNumElements() &&
2664 "Vector elements should equal vector width size");
2665 assert(VecWidth == 8 || VecWidth == 4 || VecWidth == 2);
2666
2667 // Check if this is an extract of the lowest bit.
2668 if (isNullConstant(V: IdxV) && ValTy.getSizeInBits() == 1) {
2669 // Extracting the lowest bit is a no-op, but it changes the type,
2670 // so it must be kept as an operation to avoid errors related to
2671 // type mismatches.
2672 return DAG.getNode(Opcode: HexagonISD::TYPECAST, DL: dl, VT: MVT::i1, Operand: VecV);
2673 }
2674
2675 // If the value extracted is a single bit, use tstbit.
2676 if (ValWidth == 1) {
2677 SDValue A0 = getInstr(MachineOpc: Hexagon::C2_tfrpr, dl, Ty: MVT::i32, Ops: {VecV}, DAG);
2678 SDValue M0 = DAG.getConstant(Val: 8 / VecWidth, DL: dl, VT: MVT::i32);
2679 SDValue I0 = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MVT::i32, N1: IdxV, N2: M0);
2680 return DAG.getNode(Opcode: HexagonISD::TSTBIT, DL: dl, VT: MVT::i1, N1: A0, N2: I0);
2681 }
2682
2683 // Each bool vector (v2i1, v4i1, v8i1) always occupies 8 bits in
2684 // a predicate register. The elements of the vector are repeated
2685 // in the register (if necessary) so that the total number is 8.
2686 // The extracted subvector will need to be expanded in such a way.
2687 unsigned Scale = VecWidth / ValWidth;
2688
2689 // Generate (p2d VecV) >> 8*Idx to move the interesting bytes to
2690 // position 0.
2691 assert(ty(IdxV) == MVT::i32);
2692 unsigned VecRep = 8 / VecWidth;
2693 SDValue S0 = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MVT::i32, N1: IdxV,
2694 N2: DAG.getConstant(Val: 8*VecRep, DL: dl, VT: MVT::i32));
2695 SDValue T0 = DAG.getNode(Opcode: HexagonISD::P2D, DL: dl, VT: MVT::i64, Operand: VecV);
2696 SDValue T1 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i64, N1: T0, N2: S0);
2697 while (Scale > 1) {
2698 // The longest possible subvector is at most 32 bits, so it is always
2699 // contained in the low subregister.
2700 T1 = LoHalf(V: T1, DAG);
2701 T1 = expandPredicate(Vec32: T1, dl, DAG);
2702 Scale /= 2;
2703 }
2704
2705 return DAG.getNode(Opcode: HexagonISD::D2P, DL: dl, VT: ResTy, Operand: T1);
2706}
2707
2708SDValue
2709HexagonTargetLowering::insertVector(SDValue VecV, SDValue ValV, SDValue IdxV,
2710 const SDLoc &dl, MVT ValTy,
2711 SelectionDAG &DAG) const {
2712 MVT VecTy = ty(Op: VecV);
2713 if (VecTy.getVectorElementType() == MVT::i1)
2714 return insertVectorPred(VecV, ValV, IdxV, dl, ValTy, DAG);
2715
2716 unsigned VecWidth = VecTy.getSizeInBits();
2717 unsigned ValWidth = ValTy.getSizeInBits();
2718 assert(VecWidth == 32 || VecWidth == 64);
2719 assert((VecWidth % ValWidth) == 0);
2720
2721 // Cast everything to scalar integer types.
2722 MVT ScalarTy = MVT::getIntegerVT(BitWidth: VecWidth);
2723 // The actual type of ValV may be different than ValTy (which is related
2724 // to the vector type).
2725 unsigned VW = ty(Op: ValV).getSizeInBits();
2726 ValV = DAG.getBitcast(VT: MVT::getIntegerVT(BitWidth: VW), V: ValV);
2727 VecV = DAG.getBitcast(VT: ScalarTy, V: VecV);
2728 if (VW != VecWidth)
2729 ValV = DAG.getAnyExtOrTrunc(Op: ValV, DL: dl, VT: ScalarTy);
2730
2731 SDValue WidthV = DAG.getConstant(Val: ValWidth, DL: dl, VT: MVT::i32);
2732 SDValue InsV;
2733
2734 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: IdxV)) {
2735 unsigned W = C->getZExtValue() * ValWidth;
2736 SDValue OffV = DAG.getConstant(Val: W, DL: dl, VT: MVT::i32);
2737 InsV = DAG.getNode(Opcode: HexagonISD::INSERT, DL: dl, VT: ScalarTy,
2738 Ops: {VecV, ValV, WidthV, OffV});
2739 } else {
2740 if (ty(Op: IdxV) != MVT::i32)
2741 IdxV = DAG.getZExtOrTrunc(Op: IdxV, DL: dl, VT: MVT::i32);
2742 SDValue OffV = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MVT::i32, N1: IdxV, N2: WidthV);
2743 InsV = DAG.getNode(Opcode: HexagonISD::INSERT, DL: dl, VT: ScalarTy,
2744 Ops: {VecV, ValV, WidthV, OffV});
2745 }
2746
2747 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: VecTy, Operand: InsV);
2748}
2749
2750SDValue
2751HexagonTargetLowering::insertVectorPred(SDValue VecV, SDValue ValV,
2752 SDValue IdxV, const SDLoc &dl,
2753 MVT ValTy, SelectionDAG &DAG) const {
2754 MVT VecTy = ty(Op: VecV);
2755 unsigned VecLen = VecTy.getVectorNumElements();
2756
2757 if (ValTy == MVT::i1) {
2758 SDValue ToReg = getInstr(MachineOpc: Hexagon::C2_tfrpr, dl, Ty: MVT::i32, Ops: {VecV}, DAG);
2759 SDValue Ext = DAG.getSExtOrTrunc(Op: ValV, DL: dl, VT: MVT::i32);
2760 SDValue Width = DAG.getConstant(Val: 8 / VecLen, DL: dl, VT: MVT::i32);
2761 SDValue Idx = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MVT::i32, N1: IdxV, N2: Width);
2762 SDValue Ins =
2763 DAG.getNode(Opcode: HexagonISD::INSERT, DL: dl, VT: MVT::i32, Ops: {ToReg, Ext, Width, Idx});
2764 return getInstr(MachineOpc: Hexagon::C2_tfrrp, dl, Ty: VecTy, Ops: {Ins}, DAG);
2765 }
2766
2767 assert(ValTy.getVectorElementType() == MVT::i1);
2768 SDValue ValR = ValTy.isVector()
2769 ? DAG.getNode(Opcode: HexagonISD::P2D, DL: dl, VT: MVT::i64, Operand: ValV)
2770 : DAG.getSExtOrTrunc(Op: ValV, DL: dl, VT: MVT::i64);
2771
2772 unsigned Scale = VecLen / ValTy.getVectorNumElements();
2773 assert(Scale > 1);
2774
2775 for (unsigned R = Scale; R > 1; R /= 2) {
2776 ValR = contractPredicate(Vec64: ValR, dl, DAG);
2777 ValR = getCombine(Hi: DAG.getUNDEF(VT: MVT::i32), Lo: ValR, dl, ResTy: MVT::i64, DAG);
2778 }
2779
2780 SDValue Width = DAG.getConstant(Val: 64 / Scale, DL: dl, VT: MVT::i32);
2781 SDValue Idx = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MVT::i32, N1: IdxV, N2: Width);
2782 SDValue VecR = DAG.getNode(Opcode: HexagonISD::P2D, DL: dl, VT: MVT::i64, Operand: VecV);
2783 SDValue Ins =
2784 DAG.getNode(Opcode: HexagonISD::INSERT, DL: dl, VT: MVT::i64, Ops: {VecR, ValR, Width, Idx});
2785 return DAG.getNode(Opcode: HexagonISD::D2P, DL: dl, VT: VecTy, Operand: Ins);
2786}
2787
2788SDValue
2789HexagonTargetLowering::expandPredicate(SDValue Vec32, const SDLoc &dl,
2790 SelectionDAG &DAG) const {
2791 assert(ty(Vec32).getSizeInBits() == 32);
2792 if (isUndef(Op: Vec32))
2793 return DAG.getUNDEF(VT: MVT::i64);
2794 SDValue P = DAG.getBitcast(VT: MVT::v4i8, V: Vec32);
2795 SDValue X = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: MVT::v4i16, Operand: P);
2796 return DAG.getBitcast(VT: MVT::i64, V: X);
2797}
2798
2799SDValue
2800HexagonTargetLowering::contractPredicate(SDValue Vec64, const SDLoc &dl,
2801 SelectionDAG &DAG) const {
2802 assert(ty(Vec64).getSizeInBits() == 64);
2803 if (isUndef(Op: Vec64))
2804 return DAG.getUNDEF(VT: MVT::i32);
2805 // Collect even bytes:
2806 SDValue A = DAG.getBitcast(VT: MVT::v8i8, V: Vec64);
2807 SDValue S = DAG.getVectorShuffle(VT: MVT::v8i8, dl, N1: A, N2: DAG.getUNDEF(VT: MVT::v8i8),
2808 Mask: {0, 2, 4, 6, 1, 3, 5, 7});
2809 return extractVector(VecV: S, IdxV: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32), dl, ValTy: MVT::v4i8,
2810 ResTy: MVT::i32, DAG);
2811}
2812
2813SDValue
2814HexagonTargetLowering::getZero(const SDLoc &dl, MVT Ty, SelectionDAG &DAG)
2815 const {
2816 if (Ty.isVector()) {
2817 unsigned W = Ty.getSizeInBits();
2818 if (W <= 64)
2819 return DAG.getBitcast(VT: Ty, V: DAG.getConstant(Val: 0, DL: dl, VT: MVT::getIntegerVT(BitWidth: W)));
2820 return DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: Ty, Operand: getZero(dl, Ty: MVT::i32, DAG));
2821 }
2822
2823 if (Ty.isInteger())
2824 return DAG.getConstant(Val: 0, DL: dl, VT: Ty);
2825 if (Ty.isFloatingPoint())
2826 return DAG.getConstantFP(Val: 0.0, DL: dl, VT: Ty);
2827 llvm_unreachable("Invalid type for zero");
2828}
2829
2830SDValue
2831HexagonTargetLowering::appendUndef(SDValue Val, MVT ResTy, SelectionDAG &DAG)
2832 const {
2833 MVT ValTy = ty(Op: Val);
2834 assert(ValTy.getVectorElementType() == ResTy.getVectorElementType());
2835
2836 unsigned ValLen = ValTy.getVectorNumElements();
2837 unsigned ResLen = ResTy.getVectorNumElements();
2838 if (ValLen == ResLen)
2839 return Val;
2840
2841 const SDLoc &dl(Val);
2842 assert(ValLen < ResLen);
2843 assert(ResLen % ValLen == 0);
2844
2845 SmallVector<SDValue, 4> Concats = {Val};
2846 for (unsigned i = 1, e = ResLen / ValLen; i < e; ++i)
2847 Concats.push_back(Elt: DAG.getUNDEF(VT: ValTy));
2848
2849 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResTy, Ops: Concats);
2850}
2851
2852SDValue
2853HexagonTargetLowering::getCombine(SDValue Hi, SDValue Lo, const SDLoc &dl,
2854 MVT ResTy, SelectionDAG &DAG) const {
2855 MVT ElemTy = ty(Op: Hi);
2856 assert(ElemTy == ty(Lo));
2857
2858 if (!ElemTy.isVector()) {
2859 assert(ElemTy.isScalarInteger());
2860 MVT PairTy = ElemTy.widenIntegerElementType();
2861 SDValue Pair = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: PairTy, N1: Lo, N2: Hi);
2862 return DAG.getBitcast(VT: ResTy, V: Pair);
2863 }
2864
2865 unsigned Width = ElemTy.getSizeInBits();
2866 MVT IntTy = MVT::getIntegerVT(BitWidth: Width);
2867 SDValue Pair =
2868 DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: IntTy.widenIntegerElementType(),
2869 Ops: {DAG.getBitcast(VT: IntTy, V: Lo), DAG.getBitcast(VT: IntTy, V: Hi)});
2870 return DAG.getBitcast(VT: ResTy, V: Pair);
2871}
2872
2873SDValue
2874HexagonTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
2875 MVT VecTy = ty(Op);
2876 unsigned BW = VecTy.getSizeInBits();
2877 const SDLoc &dl(Op);
2878 SmallVector<SDValue,8> Ops;
2879 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i)
2880 Ops.push_back(Elt: Op.getOperand(i));
2881
2882 if (BW == 32)
2883 return buildVector32(Elem: Ops, dl, VecTy, DAG);
2884 if (BW == 64)
2885 return buildVector64(Elem: Ops, dl, VecTy, DAG);
2886
2887 if (VecTy == MVT::v8i1 || VecTy == MVT::v4i1 || VecTy == MVT::v2i1) {
2888 // Check if this is a special case or all-0 or all-1.
2889 bool All0 = true, All1 = true;
2890 for (SDValue P : Ops) {
2891 auto *CN = dyn_cast<ConstantSDNode>(Val: P.getNode());
2892 if (CN == nullptr) {
2893 All0 = All1 = false;
2894 break;
2895 }
2896 uint32_t C = CN->getZExtValue();
2897 All0 &= (C == 0);
2898 All1 &= (C == 1);
2899 }
2900 if (All0)
2901 return DAG.getNode(Opcode: HexagonISD::PFALSE, DL: dl, VT: VecTy);
2902 if (All1)
2903 return DAG.getNode(Opcode: HexagonISD::PTRUE, DL: dl, VT: VecTy);
2904
2905 // For each i1 element in the resulting predicate register, put 1
2906 // shifted by the index of the element into a general-purpose register,
2907 // then or them together and transfer it back into a predicate register.
2908 SDValue Rs[8];
2909 SDValue Z = getZero(dl, Ty: MVT::i32, DAG);
2910 // Always produce 8 bits, repeat inputs if necessary.
2911 unsigned Rep = 8 / VecTy.getVectorNumElements();
2912 for (unsigned i = 0; i != 8; ++i) {
2913 SDValue S = DAG.getConstant(Val: 1ull << i, DL: dl, VT: MVT::i32);
2914 Rs[i] = DAG.getSelect(DL: dl, VT: MVT::i32, Cond: Ops[i/Rep], LHS: S, RHS: Z);
2915 }
2916 for (ArrayRef<SDValue> A(Rs); A.size() != 1; A = A.drop_back(N: A.size()/2)) {
2917 for (unsigned i = 0, e = A.size()/2; i != e; ++i)
2918 Rs[i] = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: MVT::i32, N1: Rs[2*i], N2: Rs[2*i+1]);
2919 }
2920 // Move the value directly to a predicate register.
2921 return getInstr(MachineOpc: Hexagon::C2_tfrrp, dl, Ty: VecTy, Ops: {Rs[0]}, DAG);
2922 }
2923
2924 return SDValue();
2925}
2926
2927SDValue
2928HexagonTargetLowering::LowerCONCAT_VECTORS(SDValue Op,
2929 SelectionDAG &DAG) const {
2930 MVT VecTy = ty(Op);
2931 const SDLoc &dl(Op);
2932 if (VecTy.getSizeInBits() == 64) {
2933 assert(Op.getNumOperands() == 2);
2934 return getCombine(Hi: Op.getOperand(i: 1), Lo: Op.getOperand(i: 0), dl, ResTy: VecTy, DAG);
2935 }
2936
2937 MVT ElemTy = VecTy.getVectorElementType();
2938 if (ElemTy == MVT::i1) {
2939 assert(VecTy == MVT::v2i1 || VecTy == MVT::v4i1 || VecTy == MVT::v8i1);
2940 MVT OpTy = ty(Op: Op.getOperand(i: 0));
2941 // Scale is how many times the operands need to be contracted to match
2942 // the representation in the target register.
2943 unsigned Scale = VecTy.getVectorNumElements() / OpTy.getVectorNumElements();
2944 assert(Scale == Op.getNumOperands() && Scale > 1);
2945
2946 // First, convert all bool vectors to integers, then generate pairwise
2947 // inserts to form values of doubled length. Up until there are only
2948 // two values left to concatenate, all of these values will fit in a
2949 // 32-bit integer, so keep them as i32 to use 32-bit inserts.
2950 SmallVector<SDValue,4> Words[2];
2951 unsigned IdxW = 0;
2952
2953 for (SDValue P : Op.getNode()->op_values()) {
2954 SDValue W = DAG.getNode(Opcode: HexagonISD::P2D, DL: dl, VT: MVT::i64, Operand: P);
2955 for (unsigned R = Scale; R > 1; R /= 2) {
2956 W = contractPredicate(Vec64: W, dl, DAG);
2957 W = getCombine(Hi: DAG.getUNDEF(VT: MVT::i32), Lo: W, dl, ResTy: MVT::i64, DAG);
2958 }
2959 W = LoHalf(V: W, DAG);
2960 Words[IdxW].push_back(Elt: W);
2961 }
2962
2963 while (Scale > 2) {
2964 SDValue WidthV = DAG.getConstant(Val: 64 / Scale, DL: dl, VT: MVT::i32);
2965 Words[IdxW ^ 1].clear();
2966
2967 for (unsigned i = 0, e = Words[IdxW].size(); i != e; i += 2) {
2968 SDValue W0 = Words[IdxW][i], W1 = Words[IdxW][i+1];
2969 // Insert W1 into W0 right next to the significant bits of W0.
2970 SDValue T = DAG.getNode(Opcode: HexagonISD::INSERT, DL: dl, VT: MVT::i32,
2971 Ops: {W0, W1, WidthV, WidthV});
2972 Words[IdxW ^ 1].push_back(Elt: T);
2973 }
2974 IdxW ^= 1;
2975 Scale /= 2;
2976 }
2977
2978 // At this point there should only be two words left, and Scale should be 2.
2979 assert(Scale == 2 && Words[IdxW].size() == 2);
2980
2981 SDValue WW = getCombine(Hi: Words[IdxW][1], Lo: Words[IdxW][0], dl, ResTy: MVT::i64, DAG);
2982 return DAG.getNode(Opcode: HexagonISD::D2P, DL: dl, VT: VecTy, Operand: WW);
2983 }
2984
2985 return SDValue();
2986}
2987
2988SDValue
2989HexagonTargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
2990 SelectionDAG &DAG) const {
2991 SDValue Vec = Op.getOperand(i: 0);
2992 MVT ElemTy = ty(Op: Vec).getVectorElementType();
2993 return extractVector(VecV: Vec, IdxV: Op.getOperand(i: 1), dl: SDLoc(Op), ValTy: ElemTy, ResTy: ty(Op), DAG);
2994}
2995
2996SDValue
2997HexagonTargetLowering::LowerEXTRACT_SUBVECTOR(SDValue Op,
2998 SelectionDAG &DAG) const {
2999 return extractVector(VecV: Op.getOperand(i: 0), IdxV: Op.getOperand(i: 1), dl: SDLoc(Op),
3000 ValTy: ty(Op), ResTy: ty(Op), DAG);
3001}
3002
3003SDValue
3004HexagonTargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
3005 SelectionDAG &DAG) const {
3006 return insertVector(VecV: Op.getOperand(i: 0), ValV: Op.getOperand(i: 1), IdxV: Op.getOperand(i: 2),
3007 dl: SDLoc(Op), ValTy: ty(Op).getVectorElementType(), DAG);
3008}
3009
3010SDValue
3011HexagonTargetLowering::LowerINSERT_SUBVECTOR(SDValue Op,
3012 SelectionDAG &DAG) const {
3013 SDValue ValV = Op.getOperand(i: 1);
3014 return insertVector(VecV: Op.getOperand(i: 0), ValV, IdxV: Op.getOperand(i: 2),
3015 dl: SDLoc(Op), ValTy: ty(Op: ValV), DAG);
3016}
3017
3018bool
3019HexagonTargetLowering::allowTruncateForTailCall(Type *Ty1, Type *Ty2) const {
3020 // Assuming the caller does not have either a signext or zeroext modifier, and
3021 // only one value is accepted, any reasonable truncation is allowed.
3022 if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy())
3023 return false;
3024
3025 // FIXME: in principle up to 64-bit could be made safe, but it would be very
3026 // fragile at the moment: any support for multiple value returns would be
3027 // liable to disallow tail calls involving i64 -> iN truncation in many cases.
3028 return Ty1->getPrimitiveSizeInBits() <= 32;
3029}
3030
3031SDValue
3032HexagonTargetLowering::LowerLoad(SDValue Op, SelectionDAG &DAG) const {
3033 MVT Ty = ty(Op);
3034 const SDLoc &dl(Op);
3035 LoadSDNode *LN = cast<LoadSDNode>(Val: Op.getNode());
3036 MVT MemTy = LN->getMemoryVT().getSimpleVT();
3037 ISD::LoadExtType ET = LN->getExtensionType();
3038
3039 bool LoadPred = MemTy == MVT::v2i1 || MemTy == MVT::v4i1 || MemTy == MVT::v8i1;
3040 if (LoadPred) {
3041 SDValue NL = DAG.getLoad(
3042 AM: LN->getAddressingMode(), ExtType: ISD::ZEXTLOAD, VT: MVT::i32, dl, Chain: LN->getChain(),
3043 Ptr: LN->getBasePtr(), Offset: LN->getOffset(), PtrInfo: LN->getPointerInfo(),
3044 /*MemoryVT*/ MemVT: MVT::i8, Alignment: LN->getAlign(), MMOFlags: LN->getMemOperand()->getFlags(),
3045 AAInfo: LN->getAAInfo(), Ranges: LN->getRanges());
3046 LN = cast<LoadSDNode>(Val: NL.getNode());
3047 }
3048
3049 Align ClaimAlign = LN->getAlign();
3050 if (!validateConstPtrAlignment(Ptr: LN->getBasePtr(), NeedAlign: ClaimAlign, dl, DAG))
3051 return replaceMemWithUndef(Op, DAG);
3052
3053 // Call LowerUnalignedLoad for all loads, it recognizes loads that
3054 // don't need extra aligning.
3055 SDValue LU = LowerUnalignedLoad(Op: SDValue(LN, 0), DAG);
3056 if (LoadPred) {
3057 SDValue TP = getInstr(MachineOpc: Hexagon::C2_tfrrp, dl, Ty: MemTy, Ops: {LU}, DAG);
3058 if (ET == ISD::SEXTLOAD) {
3059 TP = DAG.getSExtOrTrunc(Op: TP, DL: dl, VT: Ty);
3060 } else if (ET != ISD::NON_EXTLOAD) {
3061 TP = DAG.getZExtOrTrunc(Op: TP, DL: dl, VT: Ty);
3062 }
3063 SDValue Ch = cast<LoadSDNode>(Val: LU.getNode())->getChain();
3064 return DAG.getMergeValues(Ops: {TP, Ch}, dl);
3065 }
3066 return LU;
3067}
3068
3069SDValue
3070HexagonTargetLowering::LowerStore(SDValue Op, SelectionDAG &DAG) const {
3071 const SDLoc &dl(Op);
3072 StoreSDNode *SN = cast<StoreSDNode>(Val: Op.getNode());
3073 SDValue Val = SN->getValue();
3074 MVT Ty = ty(Op: Val);
3075
3076 if (Ty == MVT::v2i1 || Ty == MVT::v4i1 || Ty == MVT::v8i1) {
3077 // Store the exact predicate (all bits).
3078 SDValue TR = getInstr(MachineOpc: Hexagon::C2_tfrpr, dl, Ty: MVT::i32, Ops: {Val}, DAG);
3079 SDValue NS = DAG.getTruncStore(Chain: SN->getChain(), dl, Val: TR, Ptr: SN->getBasePtr(),
3080 SVT: MVT::i8, MMO: SN->getMemOperand());
3081 if (SN->isIndexed()) {
3082 NS = DAG.getIndexedStore(OrigStore: NS, dl, Base: SN->getBasePtr(), Offset: SN->getOffset(),
3083 AM: SN->getAddressingMode());
3084 }
3085 SN = cast<StoreSDNode>(Val: NS.getNode());
3086 }
3087
3088 Align ClaimAlign = SN->getAlign();
3089 if (!validateConstPtrAlignment(Ptr: SN->getBasePtr(), NeedAlign: ClaimAlign, dl, DAG))
3090 return replaceMemWithUndef(Op, DAG);
3091
3092 MVT StoreTy = SN->getMemoryVT().getSimpleVT();
3093 Align NeedAlign = Subtarget.getTypeAlignment(Ty: StoreTy);
3094 if (ClaimAlign < NeedAlign)
3095 return expandUnalignedStore(ST: SN, DAG);
3096 return SDValue(SN, 0);
3097}
3098
3099SDValue
3100HexagonTargetLowering::LowerUnalignedLoad(SDValue Op, SelectionDAG &DAG)
3101 const {
3102 LoadSDNode *LN = cast<LoadSDNode>(Val: Op.getNode());
3103 MVT LoadTy = ty(Op);
3104 unsigned NeedAlign = Subtarget.getTypeAlignment(Ty: LoadTy).value();
3105 unsigned HaveAlign = LN->getAlign().value();
3106 if (HaveAlign >= NeedAlign)
3107 return Op;
3108
3109 const SDLoc &dl(Op);
3110 const DataLayout &DL = DAG.getDataLayout();
3111 LLVMContext &Ctx = *DAG.getContext();
3112
3113 // If the load aligning is disabled or the load can be broken up into two
3114 // smaller legal loads, do the default (target-independent) expansion.
3115 bool DoDefault = false;
3116 // Handle it in the default way if this is an indexed load.
3117 if (!LN->isUnindexed())
3118 DoDefault = true;
3119
3120 if (!AlignLoads) {
3121 if (allowsMemoryAccessForAlignment(Context&: Ctx, DL, VT: LN->getMemoryVT(),
3122 MMO: *LN->getMemOperand()))
3123 return Op;
3124 DoDefault = true;
3125 }
3126 if (!DoDefault && (2 * HaveAlign) == NeedAlign) {
3127 // The PartTy is the equivalent of "getLoadableTypeOfSize(HaveAlign)".
3128 MVT PartTy = HaveAlign <= 8 ? MVT::getIntegerVT(BitWidth: 8 * HaveAlign)
3129 : MVT::getVectorVT(VT: MVT::i8, NumElements: HaveAlign);
3130 DoDefault =
3131 allowsMemoryAccessForAlignment(Context&: Ctx, DL, VT: PartTy, MMO: *LN->getMemOperand());
3132 }
3133 if (DoDefault) {
3134 std::pair<SDValue, SDValue> P = expandUnalignedLoad(LD: LN, DAG);
3135 return DAG.getMergeValues(Ops: {P.first, P.second}, dl);
3136 }
3137
3138 // The code below generates two loads, both aligned as NeedAlign, and
3139 // with the distance of NeedAlign between them. For that to cover the
3140 // bits that need to be loaded (and without overlapping), the size of
3141 // the loads should be equal to NeedAlign. This is true for all loadable
3142 // types, but add an assertion in case something changes in the future.
3143 assert(LoadTy.getSizeInBits() == 8*NeedAlign);
3144
3145 unsigned LoadLen = NeedAlign;
3146 SDValue Base = LN->getBasePtr();
3147 SDValue Chain = LN->getChain();
3148 auto BO = getBaseAndOffset(Addr: Base);
3149 unsigned BaseOpc = BO.first.getOpcode();
3150 if (BaseOpc == HexagonISD::VALIGNADDR && BO.second % LoadLen == 0)
3151 return Op;
3152
3153 if (BO.second % LoadLen != 0) {
3154 BO.first = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::i32, N1: BO.first,
3155 N2: DAG.getConstant(Val: BO.second % LoadLen, DL: dl, VT: MVT::i32));
3156 BO.second -= BO.second % LoadLen;
3157 }
3158 SDValue BaseNoOff = (BaseOpc != HexagonISD::VALIGNADDR)
3159 ? DAG.getNode(Opcode: HexagonISD::VALIGNADDR, DL: dl, VT: MVT::i32, N1: BO.first,
3160 N2: DAG.getConstant(Val: NeedAlign, DL: dl, VT: MVT::i32))
3161 : BO.first;
3162 SDValue Base0 =
3163 DAG.getMemBasePlusOffset(Base: BaseNoOff, Offset: TypeSize::getFixed(ExactSize: BO.second), DL: dl);
3164 SDValue Base1 = DAG.getMemBasePlusOffset(
3165 Base: BaseNoOff, Offset: TypeSize::getFixed(ExactSize: BO.second + LoadLen), DL: dl);
3166
3167 MachineMemOperand *WideMMO = nullptr;
3168 if (MachineMemOperand *MMO = LN->getMemOperand()) {
3169 MachineFunction &MF = DAG.getMachineFunction();
3170 WideMMO = MF.getMachineMemOperand(
3171 PtrInfo: MMO->getPointerInfo(), F: MMO->getFlags(), Size: 2 * LoadLen, BaseAlignment: Align(LoadLen),
3172 AAInfo: MMO->getAAInfo(), Ranges: MMO->getRanges(), SSID: MMO->getSyncScopeID(),
3173 Ordering: MMO->getSuccessOrdering(), FailureOrdering: MMO->getFailureOrdering());
3174 }
3175
3176 SDValue Load0 = DAG.getLoad(VT: LoadTy, dl, Chain, Ptr: Base0, MMO: WideMMO);
3177 SDValue Load1 = DAG.getLoad(VT: LoadTy, dl, Chain, Ptr: Base1, MMO: WideMMO);
3178
3179 SDValue Aligned = DAG.getNode(Opcode: HexagonISD::VALIGN, DL: dl, VT: LoadTy,
3180 Ops: {Load1, Load0, BaseNoOff.getOperand(i: 0)});
3181 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other,
3182 N1: Load0.getValue(R: 1), N2: Load1.getValue(R: 1));
3183 SDValue M = DAG.getMergeValues(Ops: {Aligned, NewChain}, dl);
3184 return M;
3185}
3186
3187SDValue
3188HexagonTargetLowering::LowerUAddSubO(SDValue Op, SelectionDAG &DAG) const {
3189 SDValue X = Op.getOperand(i: 0), Y = Op.getOperand(i: 1);
3190 auto *CY = dyn_cast<ConstantSDNode>(Val&: Y);
3191 if (!CY)
3192 return SDValue();
3193
3194 const SDLoc &dl(Op);
3195 SDVTList VTs = Op.getNode()->getVTList();
3196 assert(VTs.NumVTs == 2);
3197 assert(VTs.VTs[1] == MVT::i1);
3198 unsigned Opc = Op.getOpcode();
3199
3200 if (CY) {
3201 uint64_t VY = CY->getZExtValue();
3202 assert(VY != 0 && "This should have been folded");
3203 // X +/- 1
3204 if (VY != 1)
3205 return SDValue();
3206
3207 if (Opc == ISD::UADDO) {
3208 SDValue Op = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: VTs.VTs[0], Ops: {X, Y});
3209 SDValue Ov = DAG.getSetCC(DL: dl, VT: MVT::i1, LHS: Op, RHS: getZero(dl, Ty: ty(Op), DAG),
3210 Cond: ISD::SETEQ);
3211 return DAG.getMergeValues(Ops: {Op, Ov}, dl);
3212 }
3213 if (Opc == ISD::USUBO) {
3214 SDValue Op = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: VTs.VTs[0], Ops: {X, Y});
3215 SDValue Ov = DAG.getSetCC(DL: dl, VT: MVT::i1, LHS: Op,
3216 RHS: DAG.getAllOnesConstant(DL: dl, VT: ty(Op)), Cond: ISD::SETEQ);
3217 return DAG.getMergeValues(Ops: {Op, Ov}, dl);
3218 }
3219 }
3220
3221 return SDValue();
3222}
3223
3224SDValue HexagonTargetLowering::LowerUAddSubOCarry(SDValue Op,
3225 SelectionDAG &DAG) const {
3226 const SDLoc &dl(Op);
3227 unsigned Opc = Op.getOpcode();
3228 SDValue X = Op.getOperand(i: 0), Y = Op.getOperand(i: 1), C = Op.getOperand(i: 2);
3229
3230 if (Opc == ISD::UADDO_CARRY)
3231 return DAG.getNode(Opcode: HexagonISD::ADDC, DL: dl, VTList: Op.getNode()->getVTList(),
3232 Ops: { X, Y, C });
3233
3234 EVT CarryTy = C.getValueType();
3235 SDValue SubC = DAG.getNode(Opcode: HexagonISD::SUBC, DL: dl, VTList: Op.getNode()->getVTList(),
3236 Ops: { X, Y, DAG.getLogicalNOT(DL: dl, Val: C, VT: CarryTy) });
3237 SDValue Out[] = { SubC.getValue(R: 0),
3238 DAG.getLogicalNOT(DL: dl, Val: SubC.getValue(R: 1), VT: CarryTy) };
3239 return DAG.getMergeValues(Ops: Out, dl);
3240}
3241
3242SDValue
3243HexagonTargetLowering::LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const {
3244 SDValue Chain = Op.getOperand(i: 0);
3245 SDValue Offset = Op.getOperand(i: 1);
3246 SDValue Handler = Op.getOperand(i: 2);
3247 SDLoc dl(Op);
3248 auto PtrVT = getPointerTy(DL: DAG.getDataLayout());
3249
3250 // Mark function as containing a call to EH_RETURN.
3251 HexagonMachineFunctionInfo *FuncInfo =
3252 DAG.getMachineFunction().getInfo<HexagonMachineFunctionInfo>();
3253 FuncInfo->setHasEHReturn();
3254
3255 unsigned OffsetReg = Hexagon::R28;
3256
3257 SDValue StoreAddr =
3258 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: DAG.getRegister(Reg: Hexagon::R30, VT: PtrVT),
3259 N2: DAG.getIntPtrConstant(Val: 4, DL: dl));
3260 Chain = DAG.getStore(Chain, dl, Val: Handler, Ptr: StoreAddr, PtrInfo: MachinePointerInfo());
3261 Chain = DAG.getCopyToReg(Chain, dl, Reg: OffsetReg, N: Offset);
3262
3263 // Not needed we already use it as explicit input to EH_RETURN.
3264 // MF.getRegInfo().addLiveOut(OffsetReg);
3265
3266 return DAG.getNode(Opcode: HexagonISD::EH_RETURN, DL: dl, VT: MVT::Other, Operand: Chain);
3267}
3268
3269SDValue
3270HexagonTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
3271 unsigned Opc = Op.getOpcode();
3272 // Handle INLINEASM first.
3273 if (Opc == ISD::INLINEASM || Opc == ISD::INLINEASM_BR)
3274 return LowerINLINEASM(Op, DAG);
3275
3276 if (isHvxOperation(N: Op.getNode(), DAG)) {
3277 // If HVX lowering returns nothing, try the default lowering.
3278 if (SDValue V = LowerHvxOperation(Op, DAG))
3279 return V;
3280 }
3281
3282 switch (Opc) {
3283 default:
3284#ifndef NDEBUG
3285 Op.getNode()->dumpr(&DAG);
3286#endif
3287 llvm_unreachable("Should not custom lower this!");
3288
3289 case ISD::FDIV:
3290 return LowerFDIV(Op, DAG);
3291 case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG);
3292 case ISD::INSERT_SUBVECTOR: return LowerINSERT_SUBVECTOR(Op, DAG);
3293 case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG);
3294 case ISD::EXTRACT_SUBVECTOR: return LowerEXTRACT_SUBVECTOR(Op, DAG);
3295 case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG);
3296 case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG);
3297 case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG);
3298 case ISD::BITCAST: return LowerBITCAST(Op, DAG);
3299 case ISD::LOAD: return LowerLoad(Op, DAG);
3300 case ISD::STORE: return LowerStore(Op, DAG);
3301 case ISD::UADDO:
3302 case ISD::USUBO: return LowerUAddSubO(Op, DAG);
3303 case ISD::UADDO_CARRY:
3304 case ISD::USUBO_CARRY: return LowerUAddSubOCarry(Op, DAG);
3305 case ISD::SRA:
3306 case ISD::SHL:
3307 case ISD::SRL: return LowerVECTOR_SHIFT(Op, DAG);
3308 case ISD::ROTL: return LowerROTL(Op, DAG);
3309 case ISD::ConstantPool: return LowerConstantPool(Op, DAG);
3310 case ISD::JumpTable: return LowerJumpTable(Op, DAG);
3311 case ISD::EH_RETURN: return LowerEH_RETURN(Op, DAG);
3312 case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG);
3313 case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG);
3314 case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG);
3315 case ISD::ATOMIC_FENCE: return LowerATOMIC_FENCE(Op, DAG);
3316 case ISD::GlobalAddress: return LowerGLOBALADDRESS(Op, DAG);
3317 case ISD::BlockAddress: return LowerBlockAddress(Op, DAG);
3318 case ISD::GLOBAL_OFFSET_TABLE: return LowerGLOBAL_OFFSET_TABLE(Op, DAG);
3319 case ISD::VACOPY: return LowerVACOPY(Op, DAG);
3320 case ISD::VASTART: return LowerVASTART(Op, DAG);
3321 case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG);
3322 case ISD::SETCC: return LowerSETCC(Op, DAG);
3323 case ISD::VSELECT: return LowerVSELECT(Op, DAG);
3324 case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG);
3325 case ISD::INTRINSIC_VOID: return LowerINTRINSIC_VOID(Op, DAG);
3326 case ISD::PREFETCH:
3327 return LowerPREFETCH(Op, DAG);
3328 break;
3329 }
3330
3331 return SDValue();
3332}
3333
3334void
3335HexagonTargetLowering::LowerOperationWrapper(SDNode *N,
3336 SmallVectorImpl<SDValue> &Results,
3337 SelectionDAG &DAG) const {
3338 if (isHvxOperation(N, DAG)) {
3339 LowerHvxOperationWrapper(N, Results, DAG);
3340 if (!Results.empty())
3341 return;
3342 }
3343
3344 SDValue Op(N, 0);
3345 unsigned Opc = N->getOpcode();
3346
3347 switch (Opc) {
3348 case HexagonISD::SSAT:
3349 case HexagonISD::USAT:
3350 Results.push_back(Elt: opJoin(Ops: SplitVectorOp(Op, DAG), dl: SDLoc(Op), DAG));
3351 break;
3352 case ISD::STORE:
3353 // We are only custom-lowering stores to verify the alignment of the
3354 // address if it is a compile-time constant. Since a store can be
3355 // modified during type-legalization (the value being stored may need
3356 // legalization), return empty Results here to indicate that we don't
3357 // really make any changes in the custom lowering.
3358 return;
3359 default:
3360 TargetLowering::LowerOperationWrapper(N, Results, DAG);
3361 break;
3362 }
3363}
3364
3365void
3366HexagonTargetLowering::ReplaceNodeResults(SDNode *N,
3367 SmallVectorImpl<SDValue> &Results,
3368 SelectionDAG &DAG) const {
3369 if (isHvxOperation(N, DAG)) {
3370 ReplaceHvxNodeResults(N, Results, DAG);
3371 if (!Results.empty())
3372 return;
3373 }
3374
3375 const SDLoc &dl(N);
3376 switch (N->getOpcode()) {
3377 case ISD::SRL:
3378 case ISD::SRA:
3379 case ISD::SHL:
3380 return;
3381 case ISD::BITCAST:
3382 // Handle a bitcast from v8i1 to i8.
3383 if (N->getValueType(ResNo: 0) == MVT::i8) {
3384 if (N->getOperand(Num: 0).getValueType() == MVT::v8i1) {
3385 SDValue P = getInstr(MachineOpc: Hexagon::C2_tfrpr, dl, Ty: MVT::i32,
3386 Ops: N->getOperand(Num: 0), DAG);
3387 SDValue T = DAG.getAnyExtOrTrunc(Op: P, DL: dl, VT: MVT::i8);
3388 Results.push_back(Elt: T);
3389 }
3390 }
3391 break;
3392 }
3393}
3394
3395SDValue
3396HexagonTargetLowering::PerformDAGCombine(SDNode *N,
3397 DAGCombinerInfo &DCI) const {
3398 SDValue Op(N, 0);
3399 const SDLoc &dl(Op);
3400 unsigned Opc = Op.getOpcode();
3401
3402 // Combining transformations applicable for arbitrary vector sizes.
3403 if (DCI.isBeforeLegalizeOps()) {
3404 switch (Opc) {
3405 case ISD::VECREDUCE_ADD:
3406 if (SDValue V = splitVecReduceAdd(N, DAG&: DCI.DAG))
3407 return V;
3408 if (SDValue V = expandVecReduceAdd(N, DAG&: DCI.DAG))
3409 return V;
3410 return SDValue();
3411 case ISD::PARTIAL_REDUCE_SMLA:
3412 case ISD::PARTIAL_REDUCE_UMLA:
3413 case ISD::PARTIAL_REDUCE_SUMLA:
3414 if (SDValue V = splitExtendingPartialReduceMLA(N, DAG&: DCI.DAG))
3415 return V;
3416 return SDValue();
3417 }
3418 } else {
3419 switch (Opc) {
3420 case ISD::VSELECT: {
3421 // (vselect (xor x, ptrue), v0, v1) -> (vselect x, v1, v0)
3422 SDValue Cond = Op.getOperand(i: 0);
3423 if (Cond->getOpcode() == ISD::XOR) {
3424 SDValue C0 = Cond.getOperand(i: 0), C1 = Cond.getOperand(i: 1);
3425 if (C1->getOpcode() == HexagonISD::PTRUE) {
3426 SDValue VSel = DCI.DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: ty(Op), N1: C0,
3427 N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 1));
3428 return VSel;
3429 }
3430 }
3431 return SDValue();
3432 }
3433 }
3434 }
3435
3436 if (isHvxOperation(N, DAG&: DCI.DAG)) {
3437 if (SDValue V = PerformHvxDAGCombine(N, DCI))
3438 return V;
3439 return SDValue();
3440 }
3441
3442 if (Opc == ISD::TRUNCATE) {
3443 SDValue Op0 = Op.getOperand(i: 0);
3444 // fold (truncate (build pair x, y)) -> (truncate x) or x
3445 if (Op0.getOpcode() == ISD::BUILD_PAIR) {
3446 EVT TruncTy = Op.getValueType();
3447 SDValue Elem0 = Op0.getOperand(i: 0);
3448 // if we match the low element of the pair, just return it.
3449 if (Elem0.getValueType() == TruncTy)
3450 return Elem0;
3451 // otherwise, if the low part is still too large, apply the truncate.
3452 if (Elem0.getValueType().bitsGT(VT: TruncTy))
3453 return DCI.DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: TruncTy, Operand: Elem0);
3454 }
3455 }
3456
3457 if (DCI.isBeforeLegalizeOps())
3458 return SDValue();
3459
3460 switch (Opc) {
3461 case HexagonISD::P2D: {
3462 SDValue P = Op.getOperand(i: 0);
3463 switch (P.getOpcode()) {
3464 case HexagonISD::PTRUE:
3465 return DCI.DAG.getAllOnesConstant(DL: dl, VT: ty(Op));
3466 case HexagonISD::PFALSE:
3467 return getZero(dl, Ty: ty(Op), DAG&: DCI.DAG);
3468 default:
3469 break;
3470 }
3471 break;
3472 }
3473 case ISD::TRUNCATE: {
3474 SDValue Op0 = Op.getOperand(i: 0);
3475 // fold (truncate (build pair x, y)) -> (truncate x) or x
3476 if (Op0.getOpcode() == ISD::BUILD_PAIR) {
3477 MVT TruncTy = ty(Op);
3478 SDValue Elem0 = Op0.getOperand(i: 0);
3479 // if we match the low element of the pair, just return it.
3480 if (ty(Op: Elem0) == TruncTy)
3481 return Elem0;
3482 // otherwise, if the low part is still too large, apply the truncate.
3483 if (ty(Op: Elem0).bitsGT(VT: TruncTy))
3484 return DCI.DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: TruncTy, Operand: Elem0);
3485 }
3486 break;
3487 }
3488 case ISD::OR: {
3489 // fold (or (shl xx, s), (zext y)) -> (COMBINE (shl xx, s-32), y)
3490 // if s >= 32
3491 auto fold0 = [&, this](SDValue Op) {
3492 if (ty(Op) != MVT::i64)
3493 return SDValue();
3494 SDValue Shl = Op.getOperand(i: 0);
3495 SDValue Zxt = Op.getOperand(i: 1);
3496 if (Shl.getOpcode() != ISD::SHL)
3497 std::swap(a&: Shl, b&: Zxt);
3498
3499 if (Shl.getOpcode() != ISD::SHL || Zxt.getOpcode() != ISD::ZERO_EXTEND)
3500 return SDValue();
3501
3502 SDValue Z = Zxt.getOperand(i: 0);
3503 auto *Amt = dyn_cast<ConstantSDNode>(Val: Shl.getOperand(i: 1));
3504 if (Amt && Amt->getZExtValue() >= 32 && ty(Op: Z).getSizeInBits() <= 32) {
3505 unsigned A = Amt->getZExtValue();
3506 SDValue S = Shl.getOperand(i: 0);
3507 SDValue T0 = DCI.DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: ty(Op: S), N1: S,
3508 N2: DCI.DAG.getConstant(Val: A - 32, DL: dl, VT: MVT::i32));
3509 SDValue T1 = DCI.DAG.getZExtOrTrunc(Op: T0, DL: dl, VT: MVT::i32);
3510 SDValue T2 = DCI.DAG.getZExtOrTrunc(Op: Z, DL: dl, VT: MVT::i32);
3511 return DCI.DAG.getNode(Opcode: HexagonISD::COMBINE, DL: dl, VT: MVT::i64, Ops: {T1, T2});
3512 }
3513 return SDValue();
3514 };
3515
3516 if (SDValue R = fold0(Op))
3517 return R;
3518 break;
3519 }
3520 }
3521
3522 return SDValue();
3523}
3524
3525/// Returns relocation base for the given PIC jumptable.
3526SDValue
3527HexagonTargetLowering::getPICJumpTableRelocBase(SDValue Table,
3528 SelectionDAG &DAG) const {
3529 int Idx = cast<JumpTableSDNode>(Val&: Table)->getIndex();
3530 EVT VT = Table.getValueType();
3531 SDValue T = DAG.getTargetJumpTable(JTI: Idx, VT, TargetFlags: HexagonII::MO_PCREL);
3532 return DAG.getNode(Opcode: HexagonISD::AT_PCREL, DL: SDLoc(Table), VT, Operand: T);
3533}
3534
3535//===----------------------------------------------------------------------===//
3536// Inline Assembly Support
3537//===----------------------------------------------------------------------===//
3538
3539TargetLowering::ConstraintType
3540HexagonTargetLowering::getConstraintType(StringRef Constraint) const {
3541 if (Constraint.size() == 1) {
3542 switch (Constraint[0]) {
3543 case 'q':
3544 case 'v':
3545 if (Subtarget.useHVXOps())
3546 return C_RegisterClass;
3547 break;
3548 case 'a':
3549 return C_RegisterClass;
3550 default:
3551 break;
3552 }
3553 }
3554 return TargetLowering::getConstraintType(Constraint);
3555}
3556
3557std::pair<unsigned, const TargetRegisterClass*>
3558HexagonTargetLowering::getRegForInlineAsmConstraint(
3559 const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const {
3560
3561 if (Constraint.size() == 1) {
3562 switch (Constraint[0]) {
3563 case 'r': // R0-R31
3564 switch (VT.SimpleTy) {
3565 default:
3566 return {0u, nullptr};
3567 case MVT::i1:
3568 case MVT::i8:
3569 case MVT::i16:
3570 case MVT::i32:
3571 case MVT::f32:
3572 return {0u, &Hexagon::IntRegsRegClass};
3573 case MVT::i64:
3574 case MVT::f64:
3575 return {0u, &Hexagon::DoubleRegsRegClass};
3576 }
3577 break;
3578 case 'a': // M0-M1
3579 if (VT != MVT::i32)
3580 return {0u, nullptr};
3581 return {0u, &Hexagon::ModRegsRegClass};
3582 case 'q': // q0-q3
3583 switch (VT.getSizeInBits()) {
3584 default:
3585 return {0u, nullptr};
3586 case 64:
3587 case 128:
3588 return {0u, &Hexagon::HvxQRRegClass};
3589 }
3590 break;
3591 case 'v': // V0-V31
3592 switch (VT.getSizeInBits()) {
3593 default:
3594 return {0u, nullptr};
3595 case 512:
3596 return {0u, &Hexagon::HvxVRRegClass};
3597 case 1024:
3598 if (Subtarget.hasV60Ops() && Subtarget.useHVX128BOps())
3599 return {0u, &Hexagon::HvxVRRegClass};
3600 return {0u, &Hexagon::HvxWRRegClass};
3601 case 2048:
3602 return {0u, &Hexagon::HvxWRRegClass};
3603 }
3604 break;
3605 default:
3606 return {0u, nullptr};
3607 }
3608 }
3609
3610 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
3611}
3612
3613/// isFPImmLegal - Returns true if the target can instruction select the
3614/// specified FP immediate natively. If false, the legalizer will
3615/// materialize the FP immediate as a load from a constant pool.
3616bool HexagonTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
3617 bool ForCodeSize) const {
3618 return true;
3619}
3620
3621/// Returns true if it is beneficial to convert a load of a constant
3622/// to just the constant itself.
3623bool HexagonTargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
3624 Type *Ty) const {
3625 if (!ConstantLoadsToImm)
3626 return false;
3627
3628 assert(Ty->isIntegerTy());
3629 unsigned BitSize = Ty->getPrimitiveSizeInBits();
3630 return (BitSize > 0 && BitSize <= 64);
3631}
3632
3633/// isLegalAddressingMode - Return true if the addressing mode represented by
3634/// AM is legal for this target, for a load/store of the specified type.
3635bool HexagonTargetLowering::isLegalAddressingMode(const DataLayout &DL,
3636 const AddrMode &AM, Type *Ty,
3637 unsigned AS, Instruction *I) const {
3638 if (Ty->isSized()) {
3639 // When LSR detects uses of the same base address to access different
3640 // types (e.g. unions), it will assume a conservative type for these
3641 // uses:
3642 // LSR Use: Kind=Address of void in addrspace(4294967295), ...
3643 // The type Ty passed here would then be "void". Skip the alignment
3644 // checks, but do not return false right away, since that confuses
3645 // LSR into crashing.
3646 Align A = DL.getABITypeAlign(Ty);
3647 // The base offset must be a multiple of the alignment.
3648 if (!isAligned(Lhs: A, SizeInBytes: AM.BaseOffs))
3649 return false;
3650 // The shifted offset must fit in 11 bits.
3651 if (!isInt<11>(x: AM.BaseOffs >> Log2(A)))
3652 return false;
3653 }
3654
3655 // No global is ever allowed as a base.
3656 if (AM.BaseGV)
3657 return false;
3658
3659 int Scale = AM.Scale;
3660 if (Scale < 0)
3661 Scale = -Scale;
3662 switch (Scale) {
3663 case 0: // No scale reg, "r+i", "r", or just "i".
3664 break;
3665 default: // No scaled addressing mode.
3666 return false;
3667 }
3668 return true;
3669}
3670
3671/// Return true if folding a constant offset with the given GlobalAddress is
3672/// legal. It is frequently not legal in PIC relocation models.
3673bool HexagonTargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA)
3674 const {
3675 return HTM.getRelocationModel() == Reloc::Static;
3676}
3677
3678/// isLegalICmpImmediate - Return true if the specified immediate is legal
3679/// icmp immediate, that is the target has icmp instructions which can compare
3680/// a register against the immediate without having to materialize the
3681/// immediate into a register.
3682bool HexagonTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
3683 return Imm >= -512 && Imm <= 511;
3684}
3685
3686/// IsEligibleForTailCallOptimization - Check whether the call is eligible
3687/// for tail call optimization. Targets which want to do tail call
3688/// optimization should implement this function.
3689bool HexagonTargetLowering::IsEligibleForTailCallOptimization(
3690 SDValue Callee,
3691 CallingConv::ID CalleeCC,
3692 bool IsVarArg,
3693 bool IsCalleeStructRet,
3694 bool IsCallerStructRet,
3695 const SmallVectorImpl<ISD::OutputArg> &Outs,
3696 const SmallVectorImpl<SDValue> &OutVals,
3697 const SmallVectorImpl<ISD::InputArg> &Ins,
3698 SelectionDAG& DAG) const {
3699 const Function &CallerF = DAG.getMachineFunction().getFunction();
3700 CallingConv::ID CallerCC = CallerF.getCallingConv();
3701 bool CCMatch = CallerCC == CalleeCC;
3702
3703 // ***************************************************************************
3704 // Look for obvious safe cases to perform tail call optimization that do not
3705 // require ABI changes.
3706 // ***************************************************************************
3707
3708 // If this is a tail call via a function pointer, then don't do it!
3709 if (!isa<GlobalAddressSDNode>(Val: Callee) &&
3710 !isa<ExternalSymbolSDNode>(Val: Callee)) {
3711 return false;
3712 }
3713
3714 // Do not optimize if the calling conventions do not match and the conventions
3715 // used are not C or Fast.
3716 if (!CCMatch) {
3717 bool R = (CallerCC == CallingConv::C || CallerCC == CallingConv::Fast);
3718 bool E = (CalleeCC == CallingConv::C || CalleeCC == CallingConv::Fast);
3719 // If R & E, then ok.
3720 if (!R || !E)
3721 return false;
3722 }
3723
3724 // Do not tail call optimize vararg calls.
3725 if (IsVarArg)
3726 return false;
3727
3728 // Also avoid tail call optimization if either caller or callee uses struct
3729 // return semantics.
3730 if (IsCalleeStructRet || IsCallerStructRet)
3731 return false;
3732
3733 // In addition to the cases above, we also disable Tail Call Optimization if
3734 // the calling convention code that at least one outgoing argument needs to
3735 // go on the stack. We cannot check that here because at this point that
3736 // information is not available.
3737 return true;
3738}
3739
3740/// Returns the target specific optimal type for load and store operations as
3741/// a result of memset, memcpy, and memmove lowering.
3742///
3743/// If DstAlign is zero that means it's safe to destination alignment can
3744/// satisfy any constraint. Similarly if SrcAlign is zero it means there isn't
3745/// a need to check it against alignment requirement, probably because the
3746/// source does not need to be loaded. If 'IsMemset' is true, that means it's
3747/// expanding a memset. If 'ZeroMemset' is true, that means it's a memset of
3748/// zero. 'MemcpyStrSrc' indicates whether the memcpy source is constant so it
3749/// does not need to be loaded. It returns EVT::Other if the type should be
3750/// determined using generic target-independent logic.
3751EVT HexagonTargetLowering::getOptimalMemOpType(
3752 LLVMContext &Context, const MemOp &Op,
3753 const AttributeList &FuncAttributes) const {
3754 if (Op.size() >= 8 && Op.isAligned(AlignCheck: Align(8)))
3755 return MVT::i64;
3756 if (Op.size() >= 4 && Op.isAligned(AlignCheck: Align(4)))
3757 return MVT::i32;
3758 if (Op.size() >= 2 && Op.isAligned(AlignCheck: Align(2)))
3759 return MVT::i16;
3760 return MVT::Other;
3761}
3762
3763// The helpers below are versions of llvm::getShuffleReduction and
3764// llvm::getOrderedReduction, adapted to use during DAG passes and simplified as
3765// follows:
3766// - ICmp and FCmp are not handled;
3767// - in every step in getShuffleReduction, the input is split into halves (not
3768// pairwise).
3769
3770static SDValue getOrderedReduction(SDValue Vec, unsigned Op,
3771 SelectionDAG &DAG) {
3772 assert(Op != Instruction::ICmp && Op != Instruction::FCmp);
3773
3774 EVT VT = Vec.getValueType();
3775 EVT EltT = VT.getVectorElementType();
3776 unsigned VF = VT.getVectorNumElements();
3777 assert(VF > 0 &&
3778 "Reduction emission only supported for non-zero length vectors!");
3779
3780 SDLoc DL(Vec);
3781 SDValue Result = DAG.getExtractVectorElt(DL, VT: EltT, Vec, Idx: 0);
3782 for (unsigned ExtractIdx = 1; ExtractIdx < VF; ++ExtractIdx) {
3783 SDValue Ext = DAG.getExtractVectorElt(DL, VT: EltT, Vec, Idx: ExtractIdx);
3784 Result = DAG.getNode(Opcode: Op, DL, VT: EltT, Ops: {Result, Ext});
3785 }
3786
3787 return Result;
3788}
3789
3790static SDValue getShuffleReduction(SDValue Vec, unsigned Op,
3791 SelectionDAG &DAG) {
3792 assert(Op != Instruction::ICmp && Op != Instruction::FCmp);
3793
3794 EVT VT = Vec.getValueType();
3795 unsigned VF = VT.getVectorNumElements();
3796 if (VF == 0)
3797 llvm_unreachable("Vector must be non-zero length");
3798 // VF is a power of 2 so we can emit the reduction using log2(VF) shuffles
3799 // and vector ops, reducing the set of values being computed by half each
3800 // round.
3801 assert(isPowerOf2_32(VF) &&
3802 "Reduction emission only supported for pow2 vectors!");
3803
3804 SDLoc DL(Vec);
3805 // TODO: Is it correct to create double-vector shuffle and fill 3/4 of it with
3806 // undefs?
3807 SmallVector<int, 32> ShuffleMask(VF);
3808 for (unsigned i = VF; i > 1; i >>= 1) {
3809 // Move the upper half of the vector to the lower half.
3810 for (unsigned j = 0; j != i / 2; ++j)
3811 ShuffleMask[j] = i / 2 + j;
3812 // Fill the rest of the mask with undef.
3813 std::fill(first: &ShuffleMask[i / 2], last: ShuffleMask.end(), value: -1);
3814
3815 SDValue Shuf =
3816 DAG.getVectorShuffle(VT, dl: DL, N1: Vec, N2: DAG.getUNDEF(VT), Mask: ShuffleMask);
3817
3818 Vec = DAG.getNode(Opcode: Op, DL, VT, Ops: {Vec, Shuf});
3819 }
3820 // The result is in the first element of the vector.
3821 return DAG.getExtractVectorElt(DL, VT: VT.getVectorElementType(), Vec, Idx: 0);
3822}
3823
3824SDValue HexagonTargetLowering::expandVecReduceAdd(SDNode *N,
3825 SelectionDAG &DAG) const {
3826 // Since we disabled automatic reduction expansion, generate log2 ladder code
3827 // if the vector is of a power-of-two length.
3828 SDValue Input = N->getOperand(Num: 0);
3829 if (isPowerOf2_32(Value: Input.getValueType().getVectorNumElements()))
3830 return getShuffleReduction(Vec: Input, Op: ISD::ADD, DAG);
3831 // Otherwise, reduction will be scalarized.
3832 return getOrderedReduction(Vec: Input, Op: ISD::ADD, DAG);
3833}
3834
3835bool HexagonTargetLowering::allowsMemoryAccess(
3836 LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace,
3837 Align Alignment, MachineMemOperand::Flags Flags, unsigned *Fast) const {
3838 if (!VT.isSimple())
3839 return false;
3840 MVT SVT = VT.getSimpleVT();
3841 if (Subtarget.isHVXVectorType(VecTy: SVT, IncludeBool: true))
3842 return allowsHvxMemoryAccess(VecTy: SVT, Flags, Fast);
3843 return TargetLoweringBase::allowsMemoryAccess(
3844 Context, DL, VT, AddrSpace, Alignment, Flags, Fast);
3845}
3846
3847bool HexagonTargetLowering::allowsMisalignedMemoryAccesses(
3848 EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags,
3849 unsigned *Fast) const {
3850 if (!VT.isSimple())
3851 return false;
3852 MVT SVT = VT.getSimpleVT();
3853 if (Subtarget.isHVXVectorType(VecTy: SVT, IncludeBool: true))
3854 return allowsHvxMisalignedMemoryAccesses(VecTy: SVT, Flags, Fast);
3855 if (Fast)
3856 *Fast = 0;
3857 return false;
3858}
3859
3860std::pair<const TargetRegisterClass*, uint8_t>
3861HexagonTargetLowering::findRepresentativeClass(const TargetRegisterInfo *TRI,
3862 MVT VT) const {
3863 if (Subtarget.isHVXVectorType(VecTy: VT, IncludeBool: true)) {
3864 unsigned BitWidth = VT.getSizeInBits();
3865 unsigned VecWidth = Subtarget.getVectorLength() * 8;
3866
3867 if (VT.getVectorElementType() == MVT::i1)
3868 return std::make_pair(x: &Hexagon::HvxQRRegClass, y: 1);
3869 if (BitWidth == VecWidth)
3870 return std::make_pair(x: &Hexagon::HvxVRRegClass, y: 1);
3871 assert(BitWidth == 2 * VecWidth);
3872 return std::make_pair(x: &Hexagon::HvxWRRegClass, y: 1);
3873 }
3874
3875 return TargetLowering::findRepresentativeClass(TRI, VT);
3876}
3877
3878bool HexagonTargetLowering::shouldReduceLoadWidth(
3879 SDNode *Load, ISD::LoadExtType ExtTy, EVT NewVT,
3880 std::optional<unsigned> ByteOffset) const {
3881 // TODO: This may be worth removing. Check regression tests for diffs.
3882 if (!TargetLoweringBase::shouldReduceLoadWidth(Load, ExtTy, NewVT,
3883 ByteOffset))
3884 return false;
3885
3886 auto *L = cast<LoadSDNode>(Val: Load);
3887 std::pair<SDValue, int> BO = getBaseAndOffset(Addr: L->getBasePtr());
3888 // Small-data object, do not shrink.
3889 if (BO.first.getOpcode() == HexagonISD::CONST32_GP)
3890 return false;
3891 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Val&: BO.first)) {
3892 auto &HTM = static_cast<const HexagonTargetMachine &>(getTargetMachine());
3893 const auto *GO = dyn_cast_or_null<const GlobalObject>(Val: GA->getGlobal());
3894 return !GO || !HTM.getObjFileLowering()->isGlobalInSmallSection(GO, TM: HTM);
3895 }
3896 return true;
3897}
3898
3899void HexagonTargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI,
3900 SDNode *Node) const {
3901 AdjustHvxInstrPostInstrSelection(MI, Node);
3902}
3903
3904Value *HexagonTargetLowering::emitLoadLinked(IRBuilderBase &Builder,
3905 Type *ValueTy, Value *Addr,
3906 AtomicOrdering Ord) const {
3907 unsigned SZ = ValueTy->getPrimitiveSizeInBits();
3908 assert((SZ == 32 || SZ == 64) && "Only 32/64-bit atomic loads supported");
3909 Intrinsic::ID IntID = (SZ == 32) ? Intrinsic::hexagon_L2_loadw_locked
3910 : Intrinsic::hexagon_L4_loadd_locked;
3911
3912 Value *Call =
3913 Builder.CreateIntrinsic(ID: IntID, Args: Addr, /*FMFSource=*/nullptr, Name: "larx");
3914
3915 return Builder.CreateBitCast(V: Call, DestTy: ValueTy);
3916}
3917
3918/// Perform a store-conditional operation to Addr. Return the status of the
3919/// store. This should be 0 if the store succeeded, non-zero otherwise.
3920Value *HexagonTargetLowering::emitStoreConditional(IRBuilderBase &Builder,
3921 Value *Val, Value *Addr,
3922 AtomicOrdering Ord) const {
3923 BasicBlock *BB = Builder.GetInsertBlock();
3924 Module *M = BB->getParent()->getParent();
3925 Type *Ty = Val->getType();
3926 unsigned SZ = Ty->getPrimitiveSizeInBits();
3927
3928 Type *CastTy = Builder.getIntNTy(N: SZ);
3929 assert((SZ == 32 || SZ == 64) && "Only 32/64-bit atomic stores supported");
3930 Intrinsic::ID IntID = (SZ == 32) ? Intrinsic::hexagon_S2_storew_locked
3931 : Intrinsic::hexagon_S4_stored_locked;
3932
3933 Val = Builder.CreateBitCast(V: Val, DestTy: CastTy);
3934
3935 Value *Call = Builder.CreateIntrinsic(ID: IntID, Args: {Addr, Val},
3936 /*FMFSource=*/nullptr, Name: "stcx");
3937 Value *Cmp = Builder.CreateICmpEQ(LHS: Call, RHS: Builder.getInt32(C: 0), Name: "");
3938 Value *Ext = Builder.CreateZExt(V: Cmp, DestTy: Type::getInt32Ty(C&: M->getContext()));
3939 return Ext;
3940}
3941
3942TargetLowering::AtomicExpansionKind
3943HexagonTargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const {
3944 // Do not expand loads and stores that don't exceed 64 bits.
3945 return LI->getType()->getPrimitiveSizeInBits() > 64
3946 ? AtomicExpansionKind::LLOnly
3947 : AtomicExpansionKind::None;
3948}
3949
3950TargetLowering::AtomicExpansionKind
3951HexagonTargetLowering::shouldExpandAtomicStoreInIR(StoreInst *SI) const {
3952 // Do not expand loads and stores that don't exceed 64 bits.
3953 return SI->getValueOperand()->getType()->getPrimitiveSizeInBits() > 64
3954 ? AtomicExpansionKind::Expand
3955 : AtomicExpansionKind::None;
3956}
3957
3958TargetLowering::AtomicExpansionKind
3959HexagonTargetLowering::shouldExpandAtomicCmpXchgInIR(
3960 const AtomicCmpXchgInst *AI) const {
3961 return AtomicExpansionKind::LLSC;
3962}
3963
3964MachineBasicBlock *HexagonTargetLowering::EmitInstrWithCustomInserter(
3965 MachineInstr &MI, MachineBasicBlock *BB) const {
3966 switch (MI.getOpcode()) {
3967 case TargetOpcode::PATCHABLE_EVENT_CALL:
3968 case TargetOpcode::PATCHABLE_TYPED_EVENT_CALL:
3969 // These are lowered in the AsmPrinter.
3970 return BB;
3971 default:
3972 llvm_unreachable("Unexpected instruction with custom inserter");
3973 }
3974}
3975
3976MachineInstr *
3977HexagonTargetLowering::EmitKCFICheck(MachineBasicBlock &MBB,
3978 MachineBasicBlock::instr_iterator &MBBI,
3979 const TargetInstrInfo *TII) const {
3980 assert(MBBI->isCall() && MBBI->getCFIType() &&
3981 "Invalid call instruction for a KCFI check");
3982
3983 switch (MBBI->getOpcode()) {
3984 case Hexagon::J2_callr:
3985 case Hexagon::PS_callr_nr:
3986 break;
3987 default:
3988 llvm_unreachable("Unexpected CFI call opcode");
3989 }
3990
3991 MachineOperand &Target = MBBI->getOperand(i: 0);
3992 assert(Target.isReg() && "Invalid target operand for an indirect call");
3993 Target.setIsRenamable(false);
3994
3995 return BuildMI(BB&: MBB, I: MBBI, MIMD: MBBI->getDebugLoc(), MCID: TII->get(Opcode: Hexagon::KCFI_CHECK))
3996 .addReg(RegNo: Target.getReg())
3997 .addImm(Val: MBBI->getCFIType())
3998 .getInstr();
3999}
4000
4001bool HexagonTargetLowering::isMaskAndCmp0FoldingBeneficial(
4002 const Instruction &AndI) const {
4003 // Only sink 'and' mask to cmp use block if it is masking a single bit since
4004 // this will fold the and/cmp/br into a single tstbit instruction.
4005 ConstantInt *Mask = dyn_cast<ConstantInt>(Val: AndI.getOperand(i: 1));
4006 if (!Mask)
4007 return false;
4008 return Mask->getValue().isPowerOf2();
4009}
4010
4011// Check if the result of the node is only used as a return value, as
4012// otherwise we can't perform a tail-call.
4013bool HexagonTargetLowering::isUsedByReturnOnly(SDNode *N,
4014 SDValue &Chain) const {
4015 if (N->getNumValues() != 1)
4016 return false;
4017 if (!N->hasNUsesOfValue(NUses: 1, Value: 0))
4018 return false;
4019
4020 SDNode *Copy = *N->user_begin();
4021
4022 if (Copy->getOpcode() == ISD::BITCAST) {
4023 return isUsedByReturnOnly(N: Copy, Chain);
4024 }
4025
4026 if (Copy->getOpcode() != ISD::CopyToReg) {
4027 return false;
4028 }
4029
4030 // If the ISD::CopyToReg has a glue operand, we conservatively assume it
4031 // isn't safe to perform a tail call.
4032 if (Copy->getOperand(Num: Copy->getNumOperands() - 1).getValueType() == MVT::Glue)
4033 return false;
4034
4035 // The copy must be used by a HexagonISD::RET_GLUE, and nothing else.
4036 bool HasRet = false;
4037 for (SDNode *Node : Copy->users()) {
4038 if (Node->getOpcode() != HexagonISD::RET_GLUE)
4039 return false;
4040 HasRet = true;
4041 }
4042 if (!HasRet)
4043 return false;
4044
4045 Chain = Copy->getOperand(Num: 0);
4046 return true;
4047}
4048
4049bool HexagonTargetLowering::hasInlineStackProbe(
4050 const MachineFunction &MF) const {
4051 if (MF.getFunction().hasFnAttribute(Kind: "probe-stack"))
4052 return MF.getFunction().getFnAttribute(Kind: "probe-stack").getValueAsString() ==
4053 "inline-asm";
4054 return false;
4055}
4056
4057unsigned HexagonTargetLowering::getStackProbeSize(const MachineFunction &MF,
4058 Align StackAlign) const {
4059 const Function &Fn = MF.getFunction();
4060 unsigned StackProbeSize =
4061 Fn.getFnAttributeAsParsedInteger(Kind: "stack-probe-size", Default: 4096);
4062 // Round down to the stack alignment.
4063 StackProbeSize = alignDown(Value: StackProbeSize, Align: StackAlign.value());
4064 return StackProbeSize ? StackProbeSize : StackAlign.value();
4065}
4066