1//===-- HexagonISelLowering.cpp - Hexagon DAG Lowering Implementation -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the interfaces that Hexagon uses to lower LLVM code
10// into a selection DAG.
11//
12//===----------------------------------------------------------------------===//
13
14#include "HexagonISelLowering.h"
15#include "Hexagon.h"
16#include "HexagonMachineFunctionInfo.h"
17#include "HexagonRegisterInfo.h"
18#include "HexagonSubtarget.h"
19#include "HexagonTargetMachine.h"
20#include "HexagonTargetObjectFile.h"
21#include "llvm/ADT/APInt.h"
22#include "llvm/ADT/ArrayRef.h"
23#include "llvm/ADT/SmallVector.h"
24#include "llvm/ADT/StringSwitch.h"
25#include "llvm/CodeGen/CallingConvLower.h"
26#include "llvm/CodeGen/MachineFrameInfo.h"
27#include "llvm/CodeGen/MachineFunction.h"
28#include "llvm/CodeGen/MachineMemOperand.h"
29#include "llvm/CodeGen/MachineRegisterInfo.h"
30#include "llvm/CodeGen/SelectionDAG.h"
31#include "llvm/CodeGen/TargetCallingConv.h"
32#include "llvm/CodeGen/ValueTypes.h"
33#include "llvm/IR/BasicBlock.h"
34#include "llvm/IR/CallingConv.h"
35#include "llvm/IR/DataLayout.h"
36#include "llvm/IR/DerivedTypes.h"
37#include "llvm/IR/DiagnosticInfo.h"
38#include "llvm/IR/DiagnosticPrinter.h"
39#include "llvm/IR/Function.h"
40#include "llvm/IR/GlobalValue.h"
41#include "llvm/IR/IRBuilder.h"
42#include "llvm/IR/InlineAsm.h"
43#include "llvm/IR/Instructions.h"
44#include "llvm/IR/IntrinsicInst.h"
45#include "llvm/IR/Intrinsics.h"
46#include "llvm/IR/IntrinsicsHexagon.h"
47#include "llvm/IR/Module.h"
48#include "llvm/IR/Type.h"
49#include "llvm/IR/Value.h"
50#include "llvm/Support/Casting.h"
51#include "llvm/Support/CodeGen.h"
52#include "llvm/Support/CommandLine.h"
53#include "llvm/Support/Debug.h"
54#include "llvm/Support/ErrorHandling.h"
55#include "llvm/Support/MathExtras.h"
56#include "llvm/Support/raw_ostream.h"
57#include "llvm/Target/TargetMachine.h"
58#include <algorithm>
59#include <cassert>
60#include <cstdint>
61#include <limits>
62#include <utility>
63
64using namespace llvm;
65
66#define DEBUG_TYPE "hexagon-lowering"
67
68static cl::opt<bool> EmitJumpTables("hexagon-emit-jump-tables",
69 cl::init(Val: true), cl::Hidden,
70 cl::desc("Control jump table emission on Hexagon target"));
71
72static cl::opt<bool>
73 EnableHexSDNodeSched("enable-hexagon-sdnode-sched", cl::Hidden,
74 cl::desc("Enable Hexagon SDNode scheduling"));
75
76static cl::opt<int> MinimumJumpTables("minimum-jump-tables", cl::Hidden,
77 cl::init(Val: 5),
78 cl::desc("Set minimum jump tables"));
79
80static cl::opt<bool>
81 ConstantLoadsToImm("constant-loads-to-imm", cl::Hidden, cl::init(Val: true),
82 cl::desc("Convert constant loads to immediate values."));
83
84static cl::opt<bool> AlignLoads("hexagon-align-loads",
85 cl::Hidden, cl::init(Val: false),
86 cl::desc("Rewrite unaligned loads as a pair of aligned loads"));
87
88static cl::opt<bool>
89 DisableArgsMinAlignment("hexagon-disable-args-min-alignment", cl::Hidden,
90 cl::init(Val: false),
91 cl::desc("Disable minimum alignment of 1 for "
92 "arguments passed by value on stack"));
93
94// Implement calling convention for Hexagon.
95
96static bool CC_SkipOdd(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
97 CCValAssign::LocInfo &LocInfo,
98 ISD::ArgFlagsTy &ArgFlags, CCState &State) {
99 static const MCPhysReg ArgRegs[] = {
100 Hexagon::R0, Hexagon::R1, Hexagon::R2,
101 Hexagon::R3, Hexagon::R4, Hexagon::R5
102 };
103 const unsigned NumArgRegs = std::size(ArgRegs);
104 unsigned RegNum = State.getFirstUnallocated(Regs: ArgRegs);
105
106 // RegNum is an index into ArgRegs: skip a register if RegNum is odd.
107 if (RegNum != NumArgRegs && RegNum % 2 == 1)
108 State.AllocateReg(Reg: ArgRegs[RegNum]);
109
110 // Always return false here, as this function only makes sure that the first
111 // unallocated register has an even register number and does not actually
112 // allocate a register for the current argument.
113 return false;
114}
115
116#define GET_CALLING_CONV_IMPL
117#include "HexagonGenCallingConv.inc"
118
119unsigned HexagonTargetLowering::getVectorTypeBreakdownForCallingConv(
120 LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT,
121 unsigned &NumIntermediates, MVT &RegisterVT) const {
122
123 bool isBoolVector = VT.getVectorElementType() == MVT::i1;
124 bool isPowerOf2 = VT.isPow2VectorType();
125 unsigned NumElts = VT.getVectorNumElements();
126
127 // Split vectors of type vXi1 into (X/8) vectors of type v8i1,
128 // where X is divisible by 8.
129 if (isBoolVector && !Subtarget.useHVXOps() && isPowerOf2 && NumElts >= 8) {
130 RegisterVT = MVT::v8i8;
131 IntermediateVT = MVT::v8i1;
132 NumIntermediates = NumElts / 8;
133 return NumIntermediates;
134 }
135
136 // In HVX 64-byte mode, vectors of type vXi1 are split into (X / 64) vectors
137 // of type v64i1, provided that X is divisible by 64.
138 if (isBoolVector && Subtarget.useHVX64BOps() && isPowerOf2 && NumElts >= 64) {
139 RegisterVT = MVT::v64i8;
140 IntermediateVT = MVT::v64i1;
141 NumIntermediates = NumElts / 64;
142 return NumIntermediates;
143 }
144
145 // In HVX 128-byte mode, vectors of type vXi1 are split into (X / 128) vectors
146 // of type v128i1, provided that X is divisible by 128.
147 if (isBoolVector && Subtarget.useHVX128BOps() && isPowerOf2 &&
148 NumElts >= 128) {
149 RegisterVT = MVT::v128i8;
150 IntermediateVT = MVT::v128i1;
151 NumIntermediates = NumElts / 128;
152 return NumIntermediates;
153 }
154
155 return TargetLowering::getVectorTypeBreakdownForCallingConv(
156 Context, CC, VT, IntermediateVT, NumIntermediates, RegisterVT);
157}
158
159std::pair<MVT, unsigned>
160HexagonTargetLowering::handleMaskRegisterForCallingConv(
161 const HexagonSubtarget &Subtarget, EVT VT) const {
162 assert(VT.getVectorElementType() == MVT::i1);
163
164 const unsigned NumElems = VT.getVectorNumElements();
165
166 if (!VT.isPow2VectorType())
167 return {MVT::INVALID_SIMPLE_VALUE_TYPE, 0};
168
169 if (!Subtarget.useHVXOps() && NumElems >= 8)
170 return {MVT::v8i8, NumElems / 8};
171
172 if (Subtarget.useHVX64BOps() && NumElems >= 64)
173 return {MVT::v64i8, NumElems / 64};
174
175 if (Subtarget.useHVX128BOps() && NumElems >= 128)
176 return {MVT::v128i8, NumElems / 128};
177
178 return {MVT::INVALID_SIMPLE_VALUE_TYPE, 0};
179}
180
181MVT HexagonTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
182 CallingConv::ID CC,
183 EVT VT) const {
184
185 if (VT.isVectorOf(EltVT: MVT::i1)) {
186 auto [RegisterVT, NumRegisters] =
187 handleMaskRegisterForCallingConv(Subtarget, VT);
188 if (RegisterVT != MVT::INVALID_SIMPLE_VALUE_TYPE)
189 return RegisterVT;
190 }
191
192 return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
193}
194
195SDValue
196HexagonTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG)
197 const {
198 unsigned IntNo = Op.getConstantOperandVal(i: 0);
199 SDLoc dl(Op);
200 switch (IntNo) {
201 default:
202 return SDValue(); // Don't custom lower most intrinsics.
203 case Intrinsic::thread_pointer: {
204 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
205 return DAG.getNode(Opcode: HexagonISD::THREAD_POINTER, DL: dl, VT: PtrVT);
206 }
207 }
208}
209
210/// CreateCopyOfByValArgument - Make a copy of an aggregate at address specified
211/// by "Src" to address "Dst" of size "Size". Alignment information is
212/// specified by the specific parameter attribute. The copy will be passed as
213/// a byval function parameter. Sometimes what we are copying is the end of a
214/// larger object, the part that does not fit in registers.
215static SDValue CreateCopyOfByValArgument(SDValue Src, SDValue Dst,
216 SDValue Chain, ISD::ArgFlagsTy Flags,
217 SelectionDAG &DAG, const SDLoc &dl) {
218 SDValue SizeNode = DAG.getConstant(Val: Flags.getByValSize(), DL: dl, VT: MVT::i32);
219 Align Alignment = Flags.getNonZeroByValAlign();
220 return DAG.getMemcpy(Chain, dl, Dst, Src, Size: SizeNode, DstAlign: Alignment, SrcAlign: Alignment,
221 /*isVolatile=*/isVol: false, /*AlwaysInline=*/false,
222 /*CI=*/nullptr, OverrideTailCall: std::nullopt, DstPtrInfo: MachinePointerInfo(),
223 SrcPtrInfo: MachinePointerInfo());
224}
225
226bool
227HexagonTargetLowering::CanLowerReturn(
228 CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg,
229 const SmallVectorImpl<ISD::OutputArg> &Outs,
230 LLVMContext &Context, const Type *RetTy) const {
231 SmallVector<CCValAssign, 16> RVLocs;
232 CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
233
234 if (MF.getSubtarget<HexagonSubtarget>().useHVXOps())
235 return CCInfo.CheckReturn(Outs, Fn: RetCC_Hexagon_HVX);
236 return CCInfo.CheckReturn(Outs, Fn: RetCC_Hexagon);
237}
238
239// LowerReturn - Lower ISD::RET. If a struct is larger than 8 bytes and is
240// passed by value, the function prototype is modified to return void and
241// the value is stored in memory pointed by a pointer passed by caller.
242SDValue
243HexagonTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
244 bool IsVarArg,
245 const SmallVectorImpl<ISD::OutputArg> &Outs,
246 const SmallVectorImpl<SDValue> &OutVals,
247 const SDLoc &dl, SelectionDAG &DAG) const {
248 // CCValAssign - represent the assignment of the return value to locations.
249 SmallVector<CCValAssign, 16> RVLocs;
250
251 // CCState - Info about the registers and stack slot.
252 CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
253 *DAG.getContext());
254
255 // Analyze return values of ISD::RET
256 if (Subtarget.useHVXOps())
257 CCInfo.AnalyzeReturn(Outs, Fn: RetCC_Hexagon_HVX);
258 else
259 CCInfo.AnalyzeReturn(Outs, Fn: RetCC_Hexagon);
260
261 SDValue Glue;
262 SmallVector<SDValue, 4> RetOps(1, Chain);
263
264 // Copy the result values into the output registers.
265 for (unsigned i = 0; i != RVLocs.size(); ++i) {
266 CCValAssign &VA = RVLocs[i];
267 SDValue Val = OutVals[i];
268
269 switch (VA.getLocInfo()) {
270 default:
271 // Loc info must be one of Full, BCvt, SExt, ZExt, or AExt.
272 llvm_unreachable("Unknown loc info!");
273 case CCValAssign::Full:
274 break;
275 case CCValAssign::BCvt:
276 Val = DAG.getBitcast(VT: VA.getLocVT(), V: Val);
277 break;
278 case CCValAssign::SExt:
279 Val = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: Val);
280 break;
281 case CCValAssign::ZExt:
282 Val = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: Val);
283 break;
284 case CCValAssign::AExt:
285 Val = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: Val);
286 break;
287 }
288
289 Chain = DAG.getCopyToReg(Chain, dl, Reg: VA.getLocReg(), N: Val, Glue);
290
291 // Guarantee that all emitted copies are stuck together with flags.
292 Glue = Chain.getValue(R: 1);
293 RetOps.push_back(Elt: DAG.getRegister(Reg: VA.getLocReg(), VT: VA.getLocVT()));
294 }
295
296 RetOps[0] = Chain; // Update chain.
297
298 // Add the glue if we have it.
299 if (Glue.getNode())
300 RetOps.push_back(Elt: Glue);
301
302 return DAG.getNode(Opcode: HexagonISD::RET_GLUE, DL: dl, VT: MVT::Other, Ops: RetOps);
303}
304
305bool HexagonTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
306 // If either no tail call or told not to tail call at all, don't.
307 return CI->isTailCall();
308}
309
310Register HexagonTargetLowering::getRegisterByName(
311 const char* RegName, LLT VT, const MachineFunction &) const {
312 // Just support r19, the linux kernel uses it.
313 Register Reg = StringSwitch<Register>(RegName)
314 .Case(S: "r0", Value: Hexagon::R0)
315 .Case(S: "r1", Value: Hexagon::R1)
316 .Case(S: "r2", Value: Hexagon::R2)
317 .Case(S: "r3", Value: Hexagon::R3)
318 .Case(S: "r4", Value: Hexagon::R4)
319 .Case(S: "r5", Value: Hexagon::R5)
320 .Case(S: "r6", Value: Hexagon::R6)
321 .Case(S: "r7", Value: Hexagon::R7)
322 .Case(S: "r8", Value: Hexagon::R8)
323 .Case(S: "r9", Value: Hexagon::R9)
324 .Case(S: "r10", Value: Hexagon::R10)
325 .Case(S: "r11", Value: Hexagon::R11)
326 .Case(S: "r12", Value: Hexagon::R12)
327 .Case(S: "r13", Value: Hexagon::R13)
328 .Case(S: "r14", Value: Hexagon::R14)
329 .Case(S: "r15", Value: Hexagon::R15)
330 .Case(S: "r16", Value: Hexagon::R16)
331 .Case(S: "r17", Value: Hexagon::R17)
332 .Case(S: "r18", Value: Hexagon::R18)
333 .Case(S: "r19", Value: Hexagon::R19)
334 .Case(S: "r20", Value: Hexagon::R20)
335 .Case(S: "r21", Value: Hexagon::R21)
336 .Case(S: "r22", Value: Hexagon::R22)
337 .Case(S: "r23", Value: Hexagon::R23)
338 .Case(S: "r24", Value: Hexagon::R24)
339 .Case(S: "r25", Value: Hexagon::R25)
340 .Case(S: "r26", Value: Hexagon::R26)
341 .Case(S: "r27", Value: Hexagon::R27)
342 .Case(S: "r28", Value: Hexagon::R28)
343 .Case(S: "r29", Value: Hexagon::R29)
344 .Case(S: "r30", Value: Hexagon::R30)
345 .Case(S: "r31", Value: Hexagon::R31)
346 .Case(S: "r1:0", Value: Hexagon::D0)
347 .Case(S: "r3:2", Value: Hexagon::D1)
348 .Case(S: "r5:4", Value: Hexagon::D2)
349 .Case(S: "r7:6", Value: Hexagon::D3)
350 .Case(S: "r9:8", Value: Hexagon::D4)
351 .Case(S: "r11:10", Value: Hexagon::D5)
352 .Case(S: "r13:12", Value: Hexagon::D6)
353 .Case(S: "r15:14", Value: Hexagon::D7)
354 .Case(S: "r17:16", Value: Hexagon::D8)
355 .Case(S: "r19:18", Value: Hexagon::D9)
356 .Case(S: "r21:20", Value: Hexagon::D10)
357 .Case(S: "r23:22", Value: Hexagon::D11)
358 .Case(S: "r25:24", Value: Hexagon::D12)
359 .Case(S: "r27:26", Value: Hexagon::D13)
360 .Case(S: "r29:28", Value: Hexagon::D14)
361 .Case(S: "r31:30", Value: Hexagon::D15)
362 .Case(S: "sp", Value: Hexagon::R29)
363 .Case(S: "fp", Value: Hexagon::R30)
364 .Case(S: "lr", Value: Hexagon::R31)
365 .Case(S: "p0", Value: Hexagon::P0)
366 .Case(S: "p1", Value: Hexagon::P1)
367 .Case(S: "p2", Value: Hexagon::P2)
368 .Case(S: "p3", Value: Hexagon::P3)
369 .Case(S: "sa0", Value: Hexagon::SA0)
370 .Case(S: "lc0", Value: Hexagon::LC0)
371 .Case(S: "sa1", Value: Hexagon::SA1)
372 .Case(S: "lc1", Value: Hexagon::LC1)
373 .Case(S: "m0", Value: Hexagon::M0)
374 .Case(S: "m1", Value: Hexagon::M1)
375 .Case(S: "usr", Value: Hexagon::USR)
376 .Case(S: "ugp", Value: Hexagon::UGP)
377 .Case(S: "cs0", Value: Hexagon::CS0)
378 .Case(S: "cs1", Value: Hexagon::CS1)
379 .Default(Value: Register());
380 return Reg;
381}
382
383/// LowerCallResult - Lower the result values of an ISD::CALL into the
384/// appropriate copies out of appropriate physical registers. This assumes that
385/// Chain/Glue are the input chain/glue to use, and that TheCall is the call
386/// being lowered. Returns a SDNode with the same number of values as the
387/// ISD::CALL.
388SDValue HexagonTargetLowering::LowerCallResult(
389 SDValue Chain, SDValue Glue, CallingConv::ID CallConv, bool IsVarArg,
390 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
391 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals,
392 const SmallVectorImpl<SDValue> &OutVals, SDValue Callee) const {
393 // Assign locations to each value returned by this call.
394 SmallVector<CCValAssign, 16> RVLocs;
395
396 CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
397 *DAG.getContext());
398
399 if (Subtarget.useHVXOps())
400 CCInfo.AnalyzeCallResult(Ins, Fn: RetCC_Hexagon_HVX);
401 else
402 CCInfo.AnalyzeCallResult(Ins, Fn: RetCC_Hexagon);
403
404 // Copy all of the result registers out of their specified physreg.
405 for (unsigned i = 0; i != RVLocs.size(); ++i) {
406 SDValue RetVal;
407 if (RVLocs[i].getValVT() == MVT::i1) {
408 // Return values of type MVT::i1 require special handling. The reason
409 // is that MVT::i1 is associated with the PredRegs register class, but
410 // values of that type are still returned in R0. Generate an explicit
411 // copy into a predicate register from R0, and treat the value of the
412 // predicate register as the call result.
413 auto &MRI = DAG.getMachineFunction().getRegInfo();
414 SDValue FR0 = DAG.getCopyFromReg(Chain, dl, Reg: RVLocs[i].getLocReg(),
415 VT: MVT::i32, Glue);
416 // FR0 = (Value, Chain, Glue)
417 Register PredR = MRI.createVirtualRegister(RegClass: &Hexagon::PredRegsRegClass);
418 SDValue TPR = DAG.getCopyToReg(Chain: FR0.getValue(R: 1), dl, Reg: PredR,
419 N: FR0.getValue(R: 0), Glue: FR0.getValue(R: 2));
420 // TPR = (Chain, Glue)
421 // Don't glue this CopyFromReg, because it copies from a virtual
422 // register. If it is glued to the call, InstrEmitter will add it
423 // as an implicit def to the call (EmitMachineNode).
424 RetVal = DAG.getCopyFromReg(Chain: TPR.getValue(R: 0), dl, Reg: PredR, VT: MVT::i1);
425 Glue = TPR.getValue(R: 1);
426 Chain = TPR.getValue(R: 0);
427 } else {
428 RetVal = DAG.getCopyFromReg(Chain, dl, Reg: RVLocs[i].getLocReg(),
429 VT: RVLocs[i].getValVT(), Glue);
430 Glue = RetVal.getValue(R: 2);
431 Chain = RetVal.getValue(R: 1);
432 }
433 InVals.push_back(Elt: RetVal.getValue(R: 0));
434 }
435
436 return Chain;
437}
438
439/// LowerCall - Functions arguments are copied from virtual regs to
440/// (physical regs)/(stack frame), CALLSEQ_START and CALLSEQ_END are emitted.
441SDValue
442HexagonTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
443 SmallVectorImpl<SDValue> &InVals) const {
444 SelectionDAG &DAG = CLI.DAG;
445 SDLoc &dl = CLI.DL;
446 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
447 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
448 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
449 SDValue Chain = CLI.Chain;
450 SDValue Callee = CLI.Callee;
451 CallingConv::ID CallConv = CLI.CallConv;
452 bool IsVarArg = CLI.IsVarArg;
453 bool DoesNotReturn = CLI.DoesNotReturn;
454
455 bool IsStructRet = Outs.empty() ? false : Outs[0].Flags.isSRet();
456 MachineFunction &MF = DAG.getMachineFunction();
457 MachineFrameInfo &MFI = MF.getFrameInfo();
458 auto PtrVT = getPointerTy(DL: MF.getDataLayout());
459
460 if (GlobalAddressSDNode *GAN = dyn_cast<GlobalAddressSDNode>(Val&: Callee))
461 Callee = DAG.getTargetGlobalAddress(GV: GAN->getGlobal(), DL: dl, VT: MVT::i32);
462
463 // Linux ABI treats var-arg calls the same way as regular ones.
464 bool TreatAsVarArg = !Subtarget.isEnvironmentMusl() && IsVarArg;
465
466 // Analyze operands of the call, assigning locations to each operand.
467 SmallVector<CCValAssign, 16> ArgLocs;
468 CCState CCInfo(CallConv, TreatAsVarArg, MF, ArgLocs, *DAG.getContext());
469
470 if (Subtarget.useHVXOps())
471 CCInfo.AnalyzeCallOperands(Outs, Fn: CC_Hexagon_HVX);
472 else if (DisableArgsMinAlignment)
473 CCInfo.AnalyzeCallOperands(Outs, Fn: CC_Hexagon_Legacy);
474 else
475 CCInfo.AnalyzeCallOperands(Outs, Fn: CC_Hexagon);
476
477 if (CLI.IsTailCall) {
478 bool StructAttrFlag = MF.getFunction().hasStructRetAttr();
479 CLI.IsTailCall = IsEligibleForTailCallOptimization(Callee, CalleeCC: CallConv,
480 isVarArg: IsVarArg, isCalleeStructRet: IsStructRet, isCallerStructRet: StructAttrFlag, Outs,
481 OutVals, Ins, DAG);
482 for (const CCValAssign &VA : ArgLocs) {
483 if (VA.isMemLoc()) {
484 CLI.IsTailCall = false;
485 break;
486 }
487 }
488 LLVM_DEBUG(dbgs() << (CLI.IsTailCall ? "Eligible for Tail Call\n"
489 : "Argument must be passed on stack. "
490 "Not eligible for Tail Call\n"));
491 }
492 // Get a count of how many bytes are to be pushed on the stack.
493 unsigned NumBytes = CCInfo.getStackSize();
494 SmallVector<std::pair<unsigned, SDValue>, 16> RegsToPass;
495 SmallVector<SDValue, 8> MemOpChains;
496
497 const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo();
498 SDValue StackPtr =
499 DAG.getCopyFromReg(Chain, dl, Reg: HRI.getStackRegister(), VT: PtrVT);
500
501 bool NeedsArgAlign = false;
502 Align LargestAlignSeen;
503 // Walk the register/memloc assignments, inserting copies/loads.
504 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
505 CCValAssign &VA = ArgLocs[i];
506 SDValue Arg = OutVals[i];
507 ISD::ArgFlagsTy Flags = Outs[i].Flags;
508 // Record if we need > 8 byte alignment on an argument.
509 bool ArgAlign = Subtarget.isHVXVectorType(VecTy: VA.getValVT());
510 NeedsArgAlign |= ArgAlign;
511
512 // Promote the value if needed.
513 switch (VA.getLocInfo()) {
514 default:
515 // Loc info must be one of Full, BCvt, SExt, ZExt, or AExt.
516 llvm_unreachable("Unknown loc info!");
517 case CCValAssign::Full:
518 break;
519 case CCValAssign::BCvt:
520 Arg = DAG.getBitcast(VT: VA.getLocVT(), V: Arg);
521 break;
522 case CCValAssign::SExt:
523 Arg = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: Arg);
524 break;
525 case CCValAssign::ZExt:
526 Arg = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: Arg);
527 break;
528 case CCValAssign::AExt:
529 Arg = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: Arg);
530 break;
531 }
532
533 if (VA.isMemLoc()) {
534 unsigned LocMemOffset = VA.getLocMemOffset();
535 SDValue MemAddr = DAG.getConstant(Val: LocMemOffset, DL: dl,
536 VT: StackPtr.getValueType());
537 MemAddr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::i32, N1: StackPtr, N2: MemAddr);
538 if (ArgAlign)
539 LargestAlignSeen = std::max(
540 a: LargestAlignSeen, b: Align(VA.getLocVT().getStoreSizeInBits() / 8));
541 if (Flags.isByVal()) {
542 // The argument is a struct passed by value. According to LLVM, "Arg"
543 // is a pointer.
544 MemOpChains.push_back(Elt: CreateCopyOfByValArgument(Src: Arg, Dst: MemAddr, Chain,
545 Flags, DAG, dl));
546 } else {
547 MachinePointerInfo LocPI = MachinePointerInfo::getStack(
548 MF&: DAG.getMachineFunction(), Offset: LocMemOffset);
549 SDValue S = DAG.getStore(Chain, dl, Val: Arg, Ptr: MemAddr, PtrInfo: LocPI);
550 MemOpChains.push_back(Elt: S);
551 }
552 continue;
553 }
554
555 // Arguments that can be passed on register must be kept at RegsToPass
556 // vector.
557 if (VA.isRegLoc())
558 RegsToPass.push_back(Elt: std::make_pair(x: VA.getLocReg(), y&: Arg));
559 }
560
561 if (NeedsArgAlign && Subtarget.hasV60Ops()) {
562 LLVM_DEBUG(dbgs() << "Function needs byte stack align due to call args\n");
563 Align VecAlign = HRI.getSpillAlign(RC: Hexagon::HvxVRRegClass);
564 LargestAlignSeen = std::max(a: LargestAlignSeen, b: VecAlign);
565 MFI.ensureMaxAlignment(Alignment: LargestAlignSeen);
566 }
567 // Transform all store nodes into one single node because all store
568 // nodes are independent of each other.
569 if (!MemOpChains.empty())
570 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: MemOpChains);
571
572 SDValue Glue;
573 if (!CLI.IsTailCall) {
574 Chain = DAG.getCALLSEQ_START(Chain, InSize: NumBytes, OutSize: 0, DL: dl);
575 Glue = Chain.getValue(R: 1);
576 }
577
578 // Build a sequence of copy-to-reg nodes chained together with token
579 // chain and flag operands which copy the outgoing args into registers.
580 // The Glue is necessary since all emitted instructions must be
581 // stuck together.
582 if (!CLI.IsTailCall) {
583 for (const auto &R : RegsToPass) {
584 Chain = DAG.getCopyToReg(Chain, dl, Reg: R.first, N: R.second, Glue);
585 Glue = Chain.getValue(R: 1);
586 }
587 } else {
588 // For tail calls lower the arguments to the 'real' stack slot.
589 //
590 // Force all the incoming stack arguments to be loaded from the stack
591 // before any new outgoing arguments are stored to the stack, because the
592 // outgoing stack slots may alias the incoming argument stack slots, and
593 // the alias isn't otherwise explicit. This is slightly more conservative
594 // than necessary, because it means that each store effectively depends
595 // on every argument instead of just those arguments it would clobber.
596 //
597 // Do not flag preceding copytoreg stuff together with the following stuff.
598 Glue = SDValue();
599 for (const auto &R : RegsToPass) {
600 Chain = DAG.getCopyToReg(Chain, dl, Reg: R.first, N: R.second, Glue);
601 Glue = Chain.getValue(R: 1);
602 }
603 Glue = SDValue();
604 }
605
606 bool LongCalls = MF.getSubtarget<HexagonSubtarget>().useLongCalls();
607 unsigned Flags = LongCalls ? HexagonII::HMOTF_ConstExtended : 0;
608
609 // If the callee is a GlobalAddress/ExternalSymbol node (quite common, every
610 // direct call is) turn it into a TargetGlobalAddress/TargetExternalSymbol
611 // node so that legalize doesn't hack it.
612 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Val&: Callee)) {
613 Callee = DAG.getTargetGlobalAddress(GV: G->getGlobal(), DL: dl, VT: PtrVT, offset: 0, TargetFlags: Flags);
614 } else if (ExternalSymbolSDNode *S =
615 dyn_cast<ExternalSymbolSDNode>(Val&: Callee)) {
616 Callee = DAG.getTargetExternalSymbol(Sym: S->getSymbol(), VT: PtrVT, TargetFlags: Flags);
617 }
618
619 // Returns a chain & a flag for retval copy to use.
620 SmallVector<SDValue, 8> Ops;
621 Ops.push_back(Elt: Chain);
622 Ops.push_back(Elt: Callee);
623
624 // Add argument registers to the end of the list so that they are
625 // known live into the call.
626 for (const auto &R : RegsToPass)
627 Ops.push_back(Elt: DAG.getRegister(Reg: R.first, VT: R.second.getValueType()));
628
629 const uint32_t *Mask = HRI.getCallPreservedMask(MF, CallConv);
630 assert(Mask && "Missing call preserved mask for calling convention");
631 Ops.push_back(Elt: DAG.getRegisterMask(RegMask: Mask));
632
633 if (Glue.getNode())
634 Ops.push_back(Elt: Glue);
635
636 if (CLI.IsTailCall) {
637 MFI.setHasTailCall();
638 return DAG.getNode(Opcode: HexagonISD::TC_RETURN, DL: dl, VT: MVT::Other, Ops);
639 }
640
641 // Set this here because we need to know this for "hasFP" in frame lowering.
642 // The target-independent code calls getFrameRegister before setting it, and
643 // getFrameRegister uses hasFP to determine whether the function has FP.
644 MFI.setHasCalls(true);
645
646 unsigned OpCode = DoesNotReturn ? HexagonISD::CALLnr : HexagonISD::CALL;
647 Chain = DAG.getNode(Opcode: OpCode, DL: dl, ResultTys: {MVT::Other, MVT::Glue}, Ops);
648 if (CLI.CFIType)
649 Chain.getNode()->setCFIType(CLI.CFIType->getZExtValue());
650 Glue = Chain.getValue(R: 1);
651
652 // Create the CALLSEQ_END node.
653 Chain = DAG.getCALLSEQ_END(Chain, Size1: NumBytes, Size2: 0, Glue, DL: dl);
654 Glue = Chain.getValue(R: 1);
655
656 // Handle result values, copying them out of physregs into vregs that we
657 // return.
658 return LowerCallResult(Chain, Glue, CallConv, IsVarArg, Ins, dl, DAG,
659 InVals, OutVals, Callee);
660}
661
662/// Returns true by value, base pointer and offset pointer and addressing
663/// mode by reference if this node can be combined with a load / store to
664/// form a post-indexed load / store.
665bool HexagonTargetLowering::getPostIndexedAddressParts(SDNode *N, SDNode *Op,
666 SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM,
667 SelectionDAG &DAG) const {
668 LSBaseSDNode *LSN = dyn_cast<LSBaseSDNode>(Val: N);
669 if (!LSN)
670 return false;
671 EVT VT = LSN->getMemoryVT();
672 if (!VT.isSimple())
673 return false;
674 bool IsLegalType = VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32 ||
675 VT == MVT::i64 || VT == MVT::f32 || VT == MVT::f64 ||
676 VT == MVT::v2i16 || VT == MVT::v2i32 || VT == MVT::v4i8 ||
677 VT == MVT::v4i16 || VT == MVT::v8i8 ||
678 Subtarget.isHVXVectorType(VecTy: VT.getSimpleVT());
679 if (!IsLegalType)
680 return false;
681
682 if (Op->getOpcode() != ISD::ADD)
683 return false;
684 Base = Op->getOperand(Num: 0);
685 Offset = Op->getOperand(Num: 1);
686 if (!isa<ConstantSDNode>(Val: Offset.getNode()))
687 return false;
688 AM = ISD::POST_INC;
689
690 int32_t V = cast<ConstantSDNode>(Val: Offset.getNode())->getSExtValue();
691 return Subtarget.getInstrInfo()->isValidAutoIncImm(VT, Offset: V);
692}
693
694SDValue HexagonTargetLowering::LowerFDIV(SDValue Op, SelectionDAG &DAG) const {
695 if (DAG.getMachineFunction().getFunction().hasOptSize())
696 return SDValue();
697 else
698 return Op;
699}
700
701SDValue
702HexagonTargetLowering::LowerINLINEASM(SDValue Op, SelectionDAG &DAG) const {
703 MachineFunction &MF = DAG.getMachineFunction();
704 auto &HMFI = *MF.getInfo<HexagonMachineFunctionInfo>();
705 const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo();
706 unsigned LR = HRI.getRARegister();
707
708 if ((Op.getOpcode() != ISD::INLINEASM &&
709 Op.getOpcode() != ISD::INLINEASM_BR) || HMFI.hasClobberLR())
710 return Op;
711
712 unsigned NumOps = Op.getNumOperands();
713 if (Op.getOperand(i: NumOps-1).getValueType() == MVT::Glue)
714 --NumOps; // Ignore the flag operand.
715
716 for (unsigned i = InlineAsm::Op_FirstOperand; i != NumOps;) {
717 const InlineAsm::Flag Flags(Op.getConstantOperandVal(i));
718 unsigned NumVals = Flags.getNumOperandRegisters();
719 ++i; // Skip the ID value.
720
721 switch (Flags.getKind()) {
722 default:
723 llvm_unreachable("Bad flags!");
724 case InlineAsm::Kind::RegUse:
725 case InlineAsm::Kind::Imm:
726 case InlineAsm::Kind::Mem:
727 i += NumVals;
728 break;
729 case InlineAsm::Kind::Clobber:
730 case InlineAsm::Kind::RegDef:
731 case InlineAsm::Kind::RegDefEarlyClobber: {
732 for (; NumVals; --NumVals, ++i) {
733 Register Reg = cast<RegisterSDNode>(Val: Op.getOperand(i))->getReg();
734 if (Reg != LR)
735 continue;
736 HMFI.setHasClobberLR(true);
737 return Op;
738 }
739 break;
740 }
741 }
742 }
743
744 return Op;
745}
746
747// Need to transform ISD::PREFETCH into something that doesn't inherit
748// all of the properties of ISD::PREFETCH, specifically SDNPMayLoad and
749// SDNPMayStore.
750SDValue HexagonTargetLowering::LowerPREFETCH(SDValue Op,
751 SelectionDAG &DAG) const {
752 SDValue Chain = Op.getOperand(i: 0);
753 SDValue Addr = Op.getOperand(i: 1);
754 // Lower it to DCFETCH($reg, #0). A "pat" will try to merge the offset in,
755 // if the "reg" is fed by an "add".
756 SDLoc DL(Op);
757 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
758 return DAG.getNode(Opcode: HexagonISD::DCFETCH, DL, VT: MVT::Other, N1: Chain, N2: Addr, N3: Zero);
759}
760
761SDValue HexagonTargetLowering::LowerINTRINSIC_VOID(SDValue Op,
762 SelectionDAG &DAG) const {
763 SDValue Chain = Op.getOperand(i: 0);
764 unsigned IntNo = Op.getConstantOperandVal(i: 1);
765 // Lower the hexagon_prefetch builtin to DCFETCH, as above.
766 if (IntNo == Intrinsic::hexagon_prefetch) {
767 SDValue Addr = Op.getOperand(i: 2);
768 SDLoc DL(Op);
769 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
770 return DAG.getNode(Opcode: HexagonISD::DCFETCH, DL, VT: MVT::Other, N1: Chain, N2: Addr, N3: Zero);
771 }
772 return SDValue();
773}
774
775SDValue
776HexagonTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
777 SelectionDAG &DAG) const {
778 SDValue Chain = Op.getOperand(i: 0);
779 SDValue Size = Op.getOperand(i: 1);
780 SDValue Align = Op.getOperand(i: 2);
781 SDLoc dl(Op);
782
783 ConstantSDNode *AlignConst = dyn_cast<ConstantSDNode>(Val&: Align);
784 assert(AlignConst && "Non-constant Align in LowerDYNAMIC_STACKALLOC");
785
786 unsigned A = AlignConst->getSExtValue();
787 auto &HFI = *Subtarget.getFrameLowering();
788 // "Zero" means natural stack alignment.
789 if (A == 0)
790 A = HFI.getStackAlign().value();
791
792 LLVM_DEBUG({
793 dbgs () << __func__ << " Align: " << A << " Size: ";
794 Size.getNode()->dump(&DAG);
795 dbgs() << "\n";
796 });
797
798 SDValue AC = DAG.getConstant(Val: A, DL: dl, VT: MVT::i32);
799 SDVTList VTs = DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other);
800 SDValue AA = DAG.getNode(Opcode: HexagonISD::ALLOCA, DL: dl, VTList: VTs, N1: Chain, N2: Size, N3: AC);
801
802 DAG.ReplaceAllUsesOfValueWith(From: Op, To: AA);
803 return AA;
804}
805
806SDValue HexagonTargetLowering::LowerFMINFMAX(SDValue Op,
807 SelectionDAG &DAG) const {
808 EVT OpVT = Op.getValueType();
809 MVT SimpleVT = OpVT.getSimpleVT();
810 SDLoc DL(Op);
811
812 // Check if any of the inputs are NaN. If so, propagate the NaN
813 // to the output, otherwise return the maximum/minimum of the inputs.
814 // We can safely use ISD::FMINNUM/ISD::FMAXNUM to run
815 // Hexagon's F2_sfmin/F2_sfmax, when no operand is NaN.
816 // Note: We cannot directly compare nodes against NaN node to find NaNs,
817 // because comparing NaN with anything always returns False (except for !=0
818 // which always return True). To work around that, we compare input operands
819 // with themselves under ISD::SETUO, which only returns true if the operand is
820 // NaN.
821
822 SDValue Op1 = Op.getOperand(i: 0);
823 SDValue Op2 = Op.getOperand(i: 1);
824 SDValue isOp1NaN = DAG.getSetCC(DL, VT: MVT::i1, LHS: Op1, RHS: Op1, Cond: ISD::SETUO);
825 SDValue isOp2NaN = DAG.getSetCC(DL, VT: MVT::i1, LHS: Op2, RHS: Op2, Cond: ISD::SETUO);
826
827 switch (Op.getOpcode()) {
828 case ISD::FMAXIMUM: {
829 SDValue FmaxNode = DAG.getNode(Opcode: ISD::FMAXNUM, DL, VT: SimpleVT, N1: Op1, N2: Op2);
830 SDValue result =
831 DAG.getNode(Opcode: ISD::SELECT, DL, VT: SimpleVT, N1: isOp2NaN, N2: Op2, N3: FmaxNode);
832 return DAG.getNode(Opcode: ISD::SELECT, DL, VT: SimpleVT, N1: isOp1NaN, N2: Op1, N3: result);
833 }
834 case ISD::FMINIMUM: {
835 SDValue FminNode = DAG.getNode(Opcode: ISD::FMINNUM, DL, VT: SimpleVT, N1: Op1, N2: Op2);
836 SDValue result =
837 DAG.getNode(Opcode: ISD::SELECT, DL, VT: SimpleVT, N1: isOp2NaN, N2: Op2, N3: FminNode);
838 return DAG.getNode(Opcode: ISD::SELECT, DL, VT: SimpleVT, N1: isOp1NaN, N2: Op1, N3: result);
839 }
840 default:
841 llvm_unreachable("Invalid opcode for LowerFMINFMAX");
842 }
843}
844
845SDValue HexagonTargetLowering::LowerFormalArguments(
846 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
847 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
848 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
849 MachineFunction &MF = DAG.getMachineFunction();
850 MachineFrameInfo &MFI = MF.getFrameInfo();
851 MachineRegisterInfo &MRI = MF.getRegInfo();
852
853 // Linux ABI treats var-arg calls the same way as regular ones.
854 bool TreatAsVarArg = !Subtarget.isEnvironmentMusl() && IsVarArg;
855
856 // Assign locations to all of the incoming arguments.
857 SmallVector<CCValAssign, 16> ArgLocs;
858 CCState CCInfo(CallConv, TreatAsVarArg, MF, ArgLocs, *DAG.getContext());
859
860 if (Subtarget.useHVXOps())
861 CCInfo.AnalyzeFormalArguments(Ins, Fn: CC_Hexagon_HVX);
862 else if (DisableArgsMinAlignment)
863 CCInfo.AnalyzeFormalArguments(Ins, Fn: CC_Hexagon_Legacy);
864 else
865 CCInfo.AnalyzeFormalArguments(Ins, Fn: CC_Hexagon);
866
867 // For LLVM, in the case when returning a struct by value (>8byte),
868 // the first argument is a pointer that points to the location on caller's
869 // stack where the return value will be stored. For Hexagon, the location on
870 // caller's stack is passed only when the struct size is smaller than (and
871 // equal to) 8 bytes. If not, no address will be passed into callee and
872 // callee return the result directly through R0/R1.
873 auto NextSingleReg = [] (const TargetRegisterClass &RC, unsigned Reg) {
874 switch (RC.getID()) {
875 case Hexagon::IntRegsRegClassID:
876 return Reg - Hexagon::R0 + 1;
877 case Hexagon::DoubleRegsRegClassID:
878 return (Reg - Hexagon::D0 + 1) * 2;
879 case Hexagon::HvxVRRegClassID:
880 return Reg - Hexagon::V0 + 1;
881 case Hexagon::HvxWRRegClassID:
882 return (Reg - Hexagon::W0 + 1) * 2;
883 }
884 llvm_unreachable("Unexpected register class");
885 };
886
887 auto &HFL = const_cast<HexagonFrameLowering&>(*Subtarget.getFrameLowering());
888 auto &HMFI = *MF.getInfo<HexagonMachineFunctionInfo>();
889 HFL.FirstVarArgSavedReg = 0;
890 HMFI.setFirstNamedArgFrameIndex(-int(MFI.getNumFixedObjects()));
891
892 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
893 CCValAssign &VA = ArgLocs[i];
894 ISD::ArgFlagsTy Flags = Ins[i].Flags;
895 bool ByVal = Flags.isByVal();
896
897 // Arguments passed in registers:
898 // 1. 32- and 64-bit values and HVX vectors are passed directly,
899 // 2. Large structs are passed via an address, and the address is
900 // passed in a register.
901 if (VA.isRegLoc() && ByVal && Flags.getByValSize() <= 8)
902 llvm_unreachable("ByValSize must be bigger than 8 bytes");
903
904 bool InReg = VA.isRegLoc() &&
905 (!ByVal || (ByVal && Flags.getByValSize() > 8));
906
907 if (InReg) {
908 MVT RegVT = VA.getLocVT();
909 if (VA.getLocInfo() == CCValAssign::BCvt)
910 RegVT = VA.getValVT();
911
912 const TargetRegisterClass *RC = getRegClassFor(VT: RegVT);
913 Register VReg = MRI.createVirtualRegister(RegClass: RC);
914 SDValue Copy = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: RegVT);
915
916 // Treat values of type MVT::i1 specially: they are passed in
917 // registers of type i32, but they need to remain as values of
918 // type i1 for consistency of the argument lowering.
919 if (VA.getValVT() == MVT::i1) {
920 assert(RegVT.getSizeInBits() <= 32);
921 SDValue T = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: RegVT,
922 N1: Copy, N2: DAG.getConstant(Val: 1, DL: dl, VT: RegVT));
923 Copy = DAG.getSetCC(DL: dl, VT: MVT::i1, LHS: T, RHS: DAG.getConstant(Val: 0, DL: dl, VT: RegVT),
924 Cond: ISD::SETNE);
925 } else {
926#ifndef NDEBUG
927 unsigned RegSize = RegVT.getSizeInBits();
928 assert(RegSize == 32 || RegSize == 64 ||
929 Subtarget.isHVXVectorType(RegVT));
930#endif
931 }
932 InVals.push_back(Elt: Copy);
933 MRI.addLiveIn(Reg: VA.getLocReg(), vreg: VReg);
934 HFL.FirstVarArgSavedReg = NextSingleReg(*RC, VA.getLocReg());
935 } else {
936 assert(VA.isMemLoc() && "Argument should be passed in memory");
937
938 // If it's a byval parameter, then we need to compute the
939 // "real" size, not the size of the pointer.
940 unsigned ObjSize = Flags.isByVal()
941 ? Flags.getByValSize()
942 : VA.getLocVT().getStoreSizeInBits() / 8;
943
944 // Create the frame index object for this incoming parameter.
945 int Offset = HEXAGON_LRFP_SIZE + VA.getLocMemOffset();
946 int FI = MFI.CreateFixedObject(Size: ObjSize, SPOffset: Offset, IsImmutable: true);
947 SDValue FIN = DAG.getFrameIndex(FI, VT: MVT::i32);
948
949 if (Flags.isByVal()) {
950 // If it's a pass-by-value aggregate, then do not dereference the stack
951 // location. Instead, we should generate a reference to the stack
952 // location.
953 InVals.push_back(Elt: FIN);
954 } else {
955 SDValue L = DAG.getLoad(VT: VA.getValVT(), dl, Chain, Ptr: FIN,
956 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI, Offset: 0));
957 InVals.push_back(Elt: L);
958 }
959 }
960 }
961
962 if (IsVarArg && Subtarget.isEnvironmentMusl()) {
963 for (int i = HFL.FirstVarArgSavedReg; i < 6; i++)
964 MRI.addLiveIn(Reg: Hexagon::R0+i);
965 }
966
967 if (IsVarArg && Subtarget.isEnvironmentMusl()) {
968 HMFI.setFirstNamedArgFrameIndex(HMFI.getFirstNamedArgFrameIndex() - 1);
969 HMFI.setLastNamedArgFrameIndex(-int(MFI.getNumFixedObjects()));
970
971 // Create Frame index for the start of register saved area.
972 int NumVarArgRegs = 6 - HFL.FirstVarArgSavedReg;
973 bool RequiresPadding = (NumVarArgRegs & 1);
974 int RegSaveAreaSizePlusPadding = RequiresPadding
975 ? (NumVarArgRegs + 1) * 4
976 : NumVarArgRegs * 4;
977
978 if (RegSaveAreaSizePlusPadding > 0) {
979 // The offset to saved register area should be 8 byte aligned.
980 int RegAreaStart = HEXAGON_LRFP_SIZE + CCInfo.getStackSize();
981 if (!(RegAreaStart % 8))
982 RegAreaStart = (RegAreaStart + 7) & -8;
983
984 int RegSaveAreaFrameIndex =
985 MFI.CreateFixedObject(Size: RegSaveAreaSizePlusPadding, SPOffset: RegAreaStart, IsImmutable: true);
986 HMFI.setRegSavedAreaStartFrameIndex(RegSaveAreaFrameIndex);
987
988 // This will point to the next argument passed via stack.
989 int Offset = RegAreaStart + RegSaveAreaSizePlusPadding;
990 int FI = MFI.CreateFixedObject(Hexagon_PointerSize, SPOffset: Offset, IsImmutable: true);
991 HMFI.setVarArgsFrameIndex(FI);
992 } else {
993 // This will point to the next argument passed via stack, when
994 // there is no saved register area.
995 int Offset = HEXAGON_LRFP_SIZE + CCInfo.getStackSize();
996 int FI = MFI.CreateFixedObject(Hexagon_PointerSize, SPOffset: Offset, IsImmutable: true);
997 HMFI.setRegSavedAreaStartFrameIndex(FI);
998 HMFI.setVarArgsFrameIndex(FI);
999 }
1000 }
1001
1002
1003 if (IsVarArg && !Subtarget.isEnvironmentMusl()) {
1004 // This will point to the next argument passed via stack.
1005 int Offset = HEXAGON_LRFP_SIZE + CCInfo.getStackSize();
1006 int FI = MFI.CreateFixedObject(Hexagon_PointerSize, SPOffset: Offset, IsImmutable: true);
1007 HMFI.setVarArgsFrameIndex(FI);
1008 }
1009
1010 return Chain;
1011}
1012
1013SDValue
1014HexagonTargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {
1015 // VASTART stores the address of the VarArgsFrameIndex slot into the
1016 // memory location argument.
1017 MachineFunction &MF = DAG.getMachineFunction();
1018 HexagonMachineFunctionInfo *QFI = MF.getInfo<HexagonMachineFunctionInfo>();
1019 SDValue Addr = DAG.getFrameIndex(FI: QFI->getVarArgsFrameIndex(), VT: MVT::i32);
1020 const Value *SV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 2))->getValue();
1021
1022 if (!Subtarget.isEnvironmentMusl()) {
1023 return DAG.getStore(Chain: Op.getOperand(i: 0), dl: SDLoc(Op), Val: Addr, Ptr: Op.getOperand(i: 1),
1024 PtrInfo: MachinePointerInfo(SV));
1025 }
1026 auto &FuncInfo = *MF.getInfo<HexagonMachineFunctionInfo>();
1027 auto &HFL = *Subtarget.getFrameLowering();
1028 SDLoc DL(Op);
1029 SmallVector<SDValue, 8> MemOps;
1030
1031 // Get frame index of va_list.
1032 SDValue FIN = Op.getOperand(i: 1);
1033
1034 // If first Vararg register is odd, add 4 bytes to start of
1035 // saved register area to point to the first register location.
1036 // This is because the saved register area has to be 8 byte aligned.
1037 // In case of an odd start register, there will be 4 bytes of padding in
1038 // the beginning of saved register area. If all registers area used up,
1039 // the following condition will handle it correctly.
1040 SDValue SavedRegAreaStartFrameIndex =
1041 DAG.getFrameIndex(FI: FuncInfo.getRegSavedAreaStartFrameIndex(), VT: MVT::i32);
1042
1043 auto PtrVT = getPointerTy(DL: DAG.getDataLayout());
1044
1045 if (HFL.FirstVarArgSavedReg & 1)
1046 SavedRegAreaStartFrameIndex =
1047 DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT,
1048 N1: DAG.getFrameIndex(FI: FuncInfo.getRegSavedAreaStartFrameIndex(),
1049 VT: MVT::i32),
1050 N2: DAG.getIntPtrConstant(Val: 4, DL));
1051
1052 // Store the saved register area start pointer.
1053 SDValue Store =
1054 DAG.getStore(Chain: Op.getOperand(i: 0), dl: DL,
1055 Val: SavedRegAreaStartFrameIndex,
1056 Ptr: FIN, PtrInfo: MachinePointerInfo(SV));
1057 MemOps.push_back(Elt: Store);
1058
1059 // Store saved register area end pointer.
1060 FIN = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT,
1061 N1: FIN, N2: DAG.getIntPtrConstant(Val: 4, DL));
1062 Store = DAG.getStore(Chain: Op.getOperand(i: 0), dl: DL,
1063 Val: DAG.getFrameIndex(FI: FuncInfo.getVarArgsFrameIndex(),
1064 VT: PtrVT),
1065 Ptr: FIN, PtrInfo: MachinePointerInfo(SV, 4));
1066 MemOps.push_back(Elt: Store);
1067
1068 // Store overflow area pointer.
1069 FIN = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT,
1070 N1: FIN, N2: DAG.getIntPtrConstant(Val: 4, DL));
1071 Store = DAG.getStore(Chain: Op.getOperand(i: 0), dl: DL,
1072 Val: DAG.getFrameIndex(FI: FuncInfo.getVarArgsFrameIndex(),
1073 VT: PtrVT),
1074 Ptr: FIN, PtrInfo: MachinePointerInfo(SV, 8));
1075 MemOps.push_back(Elt: Store);
1076
1077 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: MemOps);
1078}
1079
1080SDValue
1081HexagonTargetLowering::LowerVACOPY(SDValue Op, SelectionDAG &DAG) const {
1082 // Assert that the linux ABI is enabled for the current compilation.
1083 assert(Subtarget.isEnvironmentMusl() && "Linux ABI should be enabled");
1084 SDValue Chain = Op.getOperand(i: 0);
1085 SDValue DestPtr = Op.getOperand(i: 1);
1086 SDValue SrcPtr = Op.getOperand(i: 2);
1087 const Value *DestSV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 3))->getValue();
1088 const Value *SrcSV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 4))->getValue();
1089 SDLoc DL(Op);
1090 // Size of the va_list is 12 bytes as it has 3 pointers. Therefore,
1091 // we need to memcopy 12 bytes from va_list to another similar list.
1092 return DAG.getMemcpy(Chain, dl: DL, Dst: DestPtr, Src: SrcPtr,
1093 Size: DAG.getIntPtrConstant(Val: 12, DL), DstAlign: Align(4), SrcAlign: Align(4),
1094 /*isVolatile*/ isVol: false, AlwaysInline: false, /*CI=*/nullptr,
1095 OverrideTailCall: std::nullopt, DstPtrInfo: MachinePointerInfo(DestSV),
1096 SrcPtrInfo: MachinePointerInfo(SrcSV));
1097}
1098
1099SDValue HexagonTargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
1100 const SDLoc &dl(Op);
1101 SDValue LHS = Op.getOperand(i: 0);
1102 SDValue RHS = Op.getOperand(i: 1);
1103 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 2))->get();
1104 MVT ResTy = ty(Op);
1105 MVT OpTy = ty(Op: LHS);
1106
1107 if (OpTy == MVT::v2i16 || OpTy == MVT::v4i8) {
1108 assert(OpTy.getVectorElementType().isScalarInteger());
1109 MVT WideTy = OpTy.widenIntegerElementType();
1110 return DAG.getSetCC(DL: dl, VT: ResTy,
1111 LHS: DAG.getSExtOrTrunc(Op: LHS, DL: SDLoc(LHS), VT: WideTy),
1112 RHS: DAG.getSExtOrTrunc(Op: RHS, DL: SDLoc(RHS), VT: WideTy), Cond: CC);
1113 }
1114
1115 // Treat all other vector types as legal.
1116 if (ResTy.isVector())
1117 return Op;
1118
1119 // Equality comparisons of short integers should use sign-extend, not
1120 // zero-extend, since we can represent small negative values in the compare
1121 // instructions. The LLVM default is to use zero-extend arbitrarily in
1122 // these cases.
1123 auto isSExtFree = [this](SDValue N) {
1124 switch (N.getOpcode()) {
1125 case ISD::TRUNCATE: {
1126 // A sign-extend of a truncate of a sign-extend is free.
1127 SDValue Op = N.getOperand(i: 0);
1128 if (Op.getOpcode() != ISD::AssertSext)
1129 return false;
1130 EVT OrigTy = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
1131 unsigned ThisBW = ty(Op: N).getSizeInBits();
1132 unsigned OrigBW = OrigTy.getSizeInBits();
1133 // The type that was sign-extended to get the AssertSext must be
1134 // narrower than the type of N (so that N has still the same value
1135 // as the original).
1136 return ThisBW >= OrigBW;
1137 }
1138 case ISD::LOAD:
1139 // We have sign-extended loads.
1140 return true;
1141 }
1142 return false;
1143 };
1144
1145 // Only do this for equality comparisons. Signed comparisons are already
1146 // sign-extended by the generic operand promotion, and for unsigned
1147 // comparisons a sign-extension is never profitable: it does not change the
1148 // result (sign-extension preserves the unsigned ordering of the values of
1149 // the narrower type), but it turns constants with the sign bit of the
1150 // narrower type set into large 32-bit values, which then have to be
1151 // materialized in a register or use a constant extender.
1152 if ((OpTy == MVT::i8 || OpTy == MVT::i16) && ISD::isIntEqualitySetCC(Code: CC)) {
1153 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: RHS);
1154 bool IsNegative = C && C->getAPIntValue().isNegative();
1155 if (IsNegative || isSExtFree(LHS) || isSExtFree(RHS))
1156 return DAG.getSetCC(DL: dl, VT: ResTy,
1157 LHS: DAG.getSExtOrTrunc(Op: LHS, DL: SDLoc(LHS), VT: MVT::i32),
1158 RHS: DAG.getSExtOrTrunc(Op: RHS, DL: SDLoc(RHS), VT: MVT::i32), Cond: CC);
1159 }
1160
1161 return SDValue();
1162}
1163
1164SDValue
1165HexagonTargetLowering::LowerVSELECT(SDValue Op, SelectionDAG &DAG) const {
1166 SDValue PredOp = Op.getOperand(i: 0);
1167 SDValue Op1 = Op.getOperand(i: 1), Op2 = Op.getOperand(i: 2);
1168 MVT OpTy = ty(Op: Op1);
1169 const SDLoc &dl(Op);
1170
1171 if (OpTy == MVT::v2i16 || OpTy == MVT::v4i8) {
1172 assert(OpTy.getVectorElementType().isScalarInteger());
1173 MVT WideTy = OpTy.widenIntegerElementType();
1174 // Generate (trunc (select (_, sext, sext))).
1175 return DAG.getSExtOrTrunc(
1176 Op: DAG.getSelect(DL: dl, VT: WideTy, Cond: PredOp,
1177 LHS: DAG.getSExtOrTrunc(Op: Op1, DL: dl, VT: WideTy),
1178 RHS: DAG.getSExtOrTrunc(Op: Op2, DL: dl, VT: WideTy)),
1179 DL: dl, VT: OpTy);
1180 }
1181
1182 return SDValue();
1183}
1184
1185SDValue
1186HexagonTargetLowering::LowerConstantPool(SDValue Op, SelectionDAG &DAG) const {
1187 EVT ValTy = Op.getValueType();
1188 ConstantPoolSDNode *CPN = cast<ConstantPoolSDNode>(Val&: Op);
1189 Constant *CVal = nullptr;
1190 bool isVTi1Type = false;
1191 if (auto *CV = dyn_cast<ConstantVector>(Val: CPN->getConstVal())) {
1192 if (cast<VectorType>(Val: CV->getType())->getElementType()->isIntegerTy(BitWidth: 1)) {
1193 IRBuilder<> IRB(CV->getContext());
1194 SmallVector<Constant*, 128> NewConst;
1195 unsigned VecLen = CV->getNumOperands();
1196 assert(isPowerOf2_32(VecLen) &&
1197 "conversion only supported for pow2 VectorSize");
1198 for (unsigned i = 0; i < VecLen; ++i)
1199 NewConst.push_back(Elt: IRB.getInt8(C: CV->getOperand(i_nocapture: i)->isNullValue()));
1200
1201 CVal = ConstantVector::get(V: NewConst);
1202 isVTi1Type = true;
1203 }
1204 }
1205 Align Alignment = CPN->getAlign();
1206 bool IsPositionIndependent = isPositionIndependent();
1207 unsigned char TF = IsPositionIndependent ? HexagonII::MO_PCREL : 0;
1208
1209 unsigned Offset = 0;
1210 SDValue T;
1211 if (CPN->isMachineConstantPoolEntry())
1212 T = DAG.getTargetConstantPool(C: CPN->getMachineCPVal(), VT: ValTy, Align: Alignment,
1213 Offset, TargetFlags: TF);
1214 else if (isVTi1Type)
1215 T = DAG.getTargetConstantPool(C: CVal, VT: ValTy, Align: Alignment, Offset, TargetFlags: TF);
1216 else
1217 T = DAG.getTargetConstantPool(C: CPN->getConstVal(), VT: ValTy, Align: Alignment, Offset,
1218 TargetFlags: TF);
1219
1220 assert(cast<ConstantPoolSDNode>(T)->getTargetFlags() == TF &&
1221 "Inconsistent target flag encountered");
1222
1223 if (IsPositionIndependent)
1224 return DAG.getNode(Opcode: HexagonISD::AT_PCREL, DL: SDLoc(Op), VT: ValTy, Operand: T);
1225 return DAG.getNode(Opcode: HexagonISD::CP, DL: SDLoc(Op), VT: ValTy, Operand: T);
1226}
1227
1228SDValue
1229HexagonTargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const {
1230 EVT VT = Op.getValueType();
1231 int Idx = cast<JumpTableSDNode>(Val&: Op)->getIndex();
1232 if (isPositionIndependent()) {
1233 SDValue T = DAG.getTargetJumpTable(JTI: Idx, VT, TargetFlags: HexagonII::MO_PCREL);
1234 return DAG.getNode(Opcode: HexagonISD::AT_PCREL, DL: SDLoc(Op), VT, Operand: T);
1235 }
1236
1237 SDValue T = DAG.getTargetJumpTable(JTI: Idx, VT);
1238 return DAG.getNode(Opcode: HexagonISD::JT, DL: SDLoc(Op), VT, Operand: T);
1239}
1240
1241SDValue
1242HexagonTargetLowering::LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const {
1243 const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo();
1244 MachineFunction &MF = DAG.getMachineFunction();
1245 MachineFrameInfo &MFI = MF.getFrameInfo();
1246 MFI.setReturnAddressIsTaken(true);
1247
1248 EVT VT = Op.getValueType();
1249 SDLoc dl(Op);
1250 unsigned Depth = Op.getConstantOperandVal(i: 0);
1251 if (Depth) {
1252 SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
1253 SDValue Offset = DAG.getConstant(Val: 4, DL: dl, VT: MVT::i32);
1254 return DAG.getLoad(VT, dl, Chain: DAG.getEntryNode(),
1255 Ptr: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: FrameAddr, N2: Offset),
1256 PtrInfo: MachinePointerInfo());
1257 }
1258
1259 // Return LR, which contains the return address. Mark it an implicit live-in.
1260 Register Reg = MF.addLiveIn(PReg: HRI.getRARegister(), RC: getRegClassFor(VT: MVT::i32));
1261 return DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl, Reg, VT);
1262}
1263
1264SDValue
1265HexagonTargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const {
1266 const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo();
1267 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
1268 MFI.setFrameAddressIsTaken(true);
1269
1270 EVT VT = Op.getValueType();
1271 SDLoc dl(Op);
1272 unsigned Depth = Op.getConstantOperandVal(i: 0);
1273 SDValue FrameAddr = DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl,
1274 Reg: HRI.getFrameRegister(), VT);
1275 while (Depth--)
1276 FrameAddr = DAG.getLoad(VT, dl, Chain: DAG.getEntryNode(), Ptr: FrameAddr,
1277 PtrInfo: MachinePointerInfo());
1278 return FrameAddr;
1279}
1280
1281SDValue
1282HexagonTargetLowering::LowerATOMIC_FENCE(SDValue Op, SelectionDAG& DAG) const {
1283 SDLoc dl(Op);
1284 return DAG.getNode(Opcode: HexagonISD::BARRIER, DL: dl, VT: MVT::Other, Operand: Op.getOperand(i: 0));
1285}
1286
1287SDValue
1288HexagonTargetLowering::LowerGLOBALADDRESS(SDValue Op, SelectionDAG &DAG) const {
1289 SDLoc dl(Op);
1290 auto *GAN = cast<GlobalAddressSDNode>(Val&: Op);
1291 auto PtrVT = getPointerTy(DL: DAG.getDataLayout());
1292 auto *GV = GAN->getGlobal();
1293 int64_t Offset = GAN->getOffset();
1294
1295 auto &HLOF = *HTM.getObjFileLowering();
1296 Reloc::Model RM = HTM.getRelocationModel();
1297
1298 if (RM == Reloc::Static) {
1299 SDValue GA = DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: Offset);
1300 const GlobalObject *GO = GV->getAliaseeObject();
1301 if (GO && Subtarget.useSmallData() && HLOF.isGlobalInSmallSection(GO, TM: HTM))
1302 return DAG.getNode(Opcode: HexagonISD::CONST32_GP, DL: dl, VT: PtrVT, Operand: GA);
1303 return DAG.getNode(Opcode: HexagonISD::CONST32, DL: dl, VT: PtrVT, Operand: GA);
1304 }
1305
1306 bool UsePCRel = getTargetMachine().shouldAssumeDSOLocal(GV);
1307 if (UsePCRel) {
1308 SDValue GA = DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: Offset,
1309 TargetFlags: HexagonII::MO_PCREL);
1310 return DAG.getNode(Opcode: HexagonISD::AT_PCREL, DL: dl, VT: PtrVT, Operand: GA);
1311 }
1312
1313 // Use GOT index.
1314 SDValue GOT = DAG.getGLOBAL_OFFSET_TABLE(VT: PtrVT);
1315 SDValue GA = DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: 0, TargetFlags: HexagonII::MO_GOT);
1316 SDValue Off = DAG.getConstant(Val: Offset, DL: dl, VT: MVT::i32);
1317 return DAG.getNode(Opcode: HexagonISD::AT_GOT, DL: dl, VT: PtrVT, N1: GOT, N2: GA, N3: Off);
1318}
1319
1320// Specifies that for loads and stores VT can be promoted to PromotedLdStVT.
1321SDValue
1322HexagonTargetLowering::LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const {
1323 const BlockAddress *BA = cast<BlockAddressSDNode>(Val&: Op)->getBlockAddress();
1324 SDLoc dl(Op);
1325 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
1326
1327 Reloc::Model RM = HTM.getRelocationModel();
1328 if (RM == Reloc::Static) {
1329 SDValue A = DAG.getTargetBlockAddress(BA, VT: PtrVT);
1330 return DAG.getNode(Opcode: HexagonISD::CONST32_GP, DL: dl, VT: PtrVT, Operand: A);
1331 }
1332
1333 SDValue A = DAG.getTargetBlockAddress(BA, VT: PtrVT, Offset: 0, TargetFlags: HexagonII::MO_PCREL);
1334 return DAG.getNode(Opcode: HexagonISD::AT_PCREL, DL: dl, VT: PtrVT, Operand: A);
1335}
1336
1337SDValue
1338HexagonTargetLowering::LowerGLOBAL_OFFSET_TABLE(SDValue Op, SelectionDAG &DAG)
1339 const {
1340 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
1341 SDValue GOTSym = DAG.getTargetExternalSymbol(HEXAGON_GOT_SYM_NAME, VT: PtrVT,
1342 TargetFlags: HexagonII::MO_PCREL);
1343 return DAG.getNode(Opcode: HexagonISD::AT_PCREL, DL: SDLoc(Op), VT: PtrVT, Operand: GOTSym);
1344}
1345
1346SDValue
1347HexagonTargetLowering::GetDynamicTLSAddr(SelectionDAG &DAG, SDValue Chain,
1348 GlobalAddressSDNode *GA, SDValue Glue, EVT PtrVT, unsigned ReturnReg,
1349 unsigned char OperandFlags) const {
1350 MachineFunction &MF = DAG.getMachineFunction();
1351 MachineFrameInfo &MFI = MF.getFrameInfo();
1352 SDVTList NodeTys = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
1353 SDLoc dl(GA);
1354 SDValue TGA = DAG.getTargetGlobalAddress(GV: GA->getGlobal(), DL: dl,
1355 VT: GA->getValueType(ResNo: 0),
1356 offset: GA->getOffset(),
1357 TargetFlags: OperandFlags);
1358 // Create Operands for the call.The Operands should have the following:
1359 // 1. Chain SDValue
1360 // 2. Callee which in this case is the Global address value.
1361 // 3. Registers live into the call.In this case its R0, as we
1362 // have just one argument to be passed.
1363 // 4. Glue.
1364 // Note: The order is important.
1365
1366 const auto &HRI = *Subtarget.getRegisterInfo();
1367 const uint32_t *Mask = HRI.getCallPreservedMask(MF, CallingConv::C);
1368 assert(Mask && "Missing call preserved mask for calling convention");
1369 SDValue Ops[] = { Chain, TGA, DAG.getRegister(Reg: Hexagon::R0, VT: PtrVT),
1370 DAG.getRegisterMask(RegMask: Mask), Glue };
1371 Chain = DAG.getNode(Opcode: HexagonISD::CALL, DL: dl, VTList: NodeTys, Ops);
1372
1373 // Inform MFI that function has calls.
1374 MFI.setAdjustsStack(true);
1375
1376 Glue = Chain.getValue(R: 1);
1377 return DAG.getCopyFromReg(Chain, dl, Reg: ReturnReg, VT: PtrVT, Glue);
1378}
1379
1380//
1381// Lower using the initial executable model for TLS addresses
1382//
1383SDValue
1384HexagonTargetLowering::LowerToTLSInitialExecModel(GlobalAddressSDNode *GA,
1385 SelectionDAG &DAG) const {
1386 SDLoc dl(GA);
1387 int64_t Offset = GA->getOffset();
1388 auto PtrVT = getPointerTy(DL: DAG.getDataLayout());
1389
1390 // Get the thread pointer.
1391 SDValue TP = DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl, Reg: Hexagon::UGP, VT: PtrVT);
1392
1393 bool IsPositionIndependent = isPositionIndependent();
1394 unsigned char TF =
1395 IsPositionIndependent ? HexagonII::MO_IEGOT : HexagonII::MO_IE;
1396
1397 // First generate the TLS symbol address
1398 SDValue TGA = DAG.getTargetGlobalAddress(GV: GA->getGlobal(), DL: dl, VT: PtrVT,
1399 offset: Offset, TargetFlags: TF);
1400
1401 SDValue Sym = DAG.getNode(Opcode: HexagonISD::CONST32, DL: dl, VT: PtrVT, Operand: TGA);
1402
1403 if (IsPositionIndependent) {
1404 // Generate the GOT pointer in case of position independent code
1405 SDValue GOT = LowerGLOBAL_OFFSET_TABLE(Op: Sym, DAG);
1406
1407 // Add the TLS Symbol address to GOT pointer.This gives
1408 // GOT relative relocation for the symbol.
1409 Sym = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: GOT, N2: Sym);
1410 }
1411
1412 // Load the offset value for TLS symbol.This offset is relative to
1413 // thread pointer.
1414 SDValue LoadOffset =
1415 DAG.getLoad(VT: PtrVT, dl, Chain: DAG.getEntryNode(), Ptr: Sym, PtrInfo: MachinePointerInfo());
1416
1417 // Address of the thread local variable is the add of thread
1418 // pointer and the offset of the variable.
1419 return DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: TP, N2: LoadOffset);
1420}
1421
1422//
1423// Lower using the local executable model for TLS addresses
1424//
1425SDValue
1426HexagonTargetLowering::LowerToTLSLocalExecModel(GlobalAddressSDNode *GA,
1427 SelectionDAG &DAG) const {
1428 SDLoc dl(GA);
1429 int64_t Offset = GA->getOffset();
1430 auto PtrVT = getPointerTy(DL: DAG.getDataLayout());
1431
1432 // Get the thread pointer.
1433 SDValue TP = DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl, Reg: Hexagon::UGP, VT: PtrVT);
1434 // Generate the TLS symbol address
1435 SDValue TGA = DAG.getTargetGlobalAddress(GV: GA->getGlobal(), DL: dl, VT: PtrVT, offset: Offset,
1436 TargetFlags: HexagonII::MO_TPREL);
1437 SDValue Sym = DAG.getNode(Opcode: HexagonISD::CONST32, DL: dl, VT: PtrVT, Operand: TGA);
1438
1439 // Address of the thread local variable is the add of thread
1440 // pointer and the offset of the variable.
1441 return DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: TP, N2: Sym);
1442}
1443
1444//
1445// Lower using the general dynamic model for TLS addresses
1446//
1447SDValue
1448HexagonTargetLowering::LowerToTLSGeneralDynamicModel(GlobalAddressSDNode *GA,
1449 SelectionDAG &DAG) const {
1450 SDLoc dl(GA);
1451 int64_t Offset = GA->getOffset();
1452 auto PtrVT = getPointerTy(DL: DAG.getDataLayout());
1453
1454 // First generate the TLS symbol address
1455 SDValue TGA = DAG.getTargetGlobalAddress(GV: GA->getGlobal(), DL: dl, VT: PtrVT, offset: Offset,
1456 TargetFlags: HexagonII::MO_GDGOT);
1457
1458 // Then, generate the GOT pointer
1459 SDValue GOT = LowerGLOBAL_OFFSET_TABLE(Op: TGA, DAG);
1460
1461 // Add the TLS symbol and the GOT pointer
1462 SDValue Sym = DAG.getNode(Opcode: HexagonISD::CONST32, DL: dl, VT: PtrVT, Operand: TGA);
1463 SDValue Chain = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: GOT, N2: Sym);
1464
1465 // Copy over the argument to R0
1466 SDValue InGlue;
1467 Chain = DAG.getCopyToReg(Chain: DAG.getEntryNode(), dl, Reg: Hexagon::R0, N: Chain, Glue: InGlue);
1468 InGlue = Chain.getValue(R: 1);
1469
1470 unsigned Flags = DAG.getSubtarget<HexagonSubtarget>().useLongCalls()
1471 ? HexagonII::MO_GDPLT | HexagonII::HMOTF_ConstExtended
1472 : HexagonII::MO_GDPLT;
1473
1474 return GetDynamicTLSAddr(DAG, Chain, GA, Glue: InGlue, PtrVT,
1475 ReturnReg: Hexagon::R0, OperandFlags: Flags);
1476}
1477
1478//
1479// Lower TLS addresses.
1480//
1481// For now for dynamic models, we only support the general dynamic model.
1482//
1483SDValue
1484HexagonTargetLowering::LowerGlobalTLSAddress(SDValue Op,
1485 SelectionDAG &DAG) const {
1486 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Val&: Op);
1487
1488 switch (HTM.getTLSModel(GV: GA->getGlobal())) {
1489 case TLSModel::GeneralDynamic:
1490 case TLSModel::LocalDynamic:
1491 return LowerToTLSGeneralDynamicModel(GA, DAG);
1492 case TLSModel::InitialExec:
1493 return LowerToTLSInitialExecModel(GA, DAG);
1494 case TLSModel::LocalExec:
1495 return LowerToTLSLocalExecModel(GA, DAG);
1496 }
1497 llvm_unreachable("Bogus TLS model");
1498}
1499
1500//===----------------------------------------------------------------------===//
1501// TargetLowering Implementation
1502//===----------------------------------------------------------------------===//
1503
1504HexagonTargetLowering::HexagonTargetLowering(const TargetMachine &TM,
1505 const HexagonSubtarget &ST)
1506 : TargetLowering(TM, ST),
1507 HTM(static_cast<const HexagonTargetMachine &>(TM)), Subtarget(ST) {
1508 auto &HRI = *Subtarget.getRegisterInfo();
1509
1510 setPrefLoopAlignment(Align(16));
1511 setMinFunctionAlignment(Align(4));
1512 setPrefFunctionAlignment(Align(16));
1513 setStackPointerRegisterToSaveRestore(HRI.getStackRegister());
1514 setBooleanContents(TargetLoweringBase::UndefinedBooleanContent);
1515 setBooleanVectorContents(TargetLoweringBase::UndefinedBooleanContent);
1516
1517 setMaxAtomicSizeInBitsSupported(64);
1518 setMinCmpXchgSizeInBits(32);
1519
1520 if (EnableHexSDNodeSched)
1521 setSchedulingPreference(Sched::VLIW);
1522 else
1523 setSchedulingPreference(Sched::Source);
1524
1525 // Limits for inline expansion of memcpy/memmove
1526 MaxStoresPerMemcpy = 6;
1527 MaxStoresPerMemcpyOptSize = 4;
1528 MaxStoresPerMemmove = 6;
1529 MaxStoresPerMemmoveOptSize = 4;
1530 MaxStoresPerMemset = 8;
1531 MaxStoresPerMemsetOptSize = 4;
1532
1533 setTargetDAGCombine(ISD::VECREDUCE_ADD);
1534
1535 //
1536 // Set up register classes.
1537 //
1538
1539 addRegisterClass(VT: MVT::i1, RC: &Hexagon::PredRegsRegClass);
1540 addRegisterClass(VT: MVT::v2i1, RC: &Hexagon::PredRegsRegClass); // bbbbaaaa
1541 addRegisterClass(VT: MVT::v4i1, RC: &Hexagon::PredRegsRegClass); // ddccbbaa
1542 addRegisterClass(VT: MVT::v8i1, RC: &Hexagon::PredRegsRegClass); // hgfedcba
1543 addRegisterClass(VT: MVT::i32, RC: &Hexagon::IntRegsRegClass);
1544 addRegisterClass(VT: MVT::v2i16, RC: &Hexagon::IntRegsRegClass);
1545 addRegisterClass(VT: MVT::v4i8, RC: &Hexagon::IntRegsRegClass);
1546 addRegisterClass(VT: MVT::i64, RC: &Hexagon::DoubleRegsRegClass);
1547 addRegisterClass(VT: MVT::v8i8, RC: &Hexagon::DoubleRegsRegClass);
1548 addRegisterClass(VT: MVT::v4i16, RC: &Hexagon::DoubleRegsRegClass);
1549 addRegisterClass(VT: MVT::v2i32, RC: &Hexagon::DoubleRegsRegClass);
1550
1551 addRegisterClass(VT: MVT::f32, RC: &Hexagon::IntRegsRegClass);
1552 addRegisterClass(VT: MVT::f64, RC: &Hexagon::DoubleRegsRegClass);
1553
1554 //
1555 // Handling of scalar operations.
1556 //
1557 // All operations default to "legal", except:
1558 // - indexed loads and stores (pre-/post-incremented),
1559 // - ANY_EXTEND_VECTOR_INREG, ATOMIC_CMP_SWAP_WITH_SUCCESS, CONCAT_VECTORS,
1560 // ConstantFP, FCEIL, FCOPYSIGN, FEXP, FEXP2, FFLOOR, FGETSIGN,
1561 // FLOG, FLOG2, FLOG10, FMAXIMUMNUM, FMINIMUMNUM, FNEARBYINT, FRINT, FROUND,
1562 // TRAP, FTRUNC, PREFETCH, SIGN_EXTEND_VECTOR_INREG,
1563 // ZERO_EXTEND_VECTOR_INREG,
1564 // which default to "expand" for at least one type.
1565
1566 // Misc operations.
1567 setOperationAction(Op: ISD::ConstantFP, VT: MVT::f32, Action: Legal);
1568 setOperationAction(Op: ISD::ConstantFP, VT: MVT::f64, Action: Legal);
1569 setOperationAction(Op: ISD::TRAP, VT: MVT::Other, Action: Legal);
1570 setOperationAction(Op: ISD::DEBUGTRAP, VT: MVT::Other, Action: Legal);
1571 setOperationAction(Op: ISD::ConstantPool, VT: MVT::i32, Action: Custom);
1572 setOperationAction(Op: ISD::JumpTable, VT: MVT::i32, Action: Custom);
1573 setOperationAction(Op: ISD::BUILD_PAIR, VT: MVT::i64, Action: Expand);
1574 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i1, Action: Expand);
1575 setOperationAction(Op: ISD::INLINEASM, VT: MVT::Other, Action: Custom);
1576 setOperationAction(Op: ISD::INLINEASM_BR, VT: MVT::Other, Action: Custom);
1577 setOperationAction(Op: ISD::PREFETCH, VT: MVT::Other, Action: Custom);
1578 setOperationAction(Op: ISD::READCYCLECOUNTER, VT: MVT::i64, Action: Legal);
1579 setOperationAction(Op: ISD::READSTEADYCOUNTER, VT: MVT::i64, Action: Legal);
1580 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::Other, Action: Custom);
1581 setOperationAction(Op: ISD::INTRINSIC_VOID, VT: MVT::Other, Action: Custom);
1582 setOperationAction(Op: ISD::EH_RETURN, VT: MVT::Other, Action: Custom);
1583 setOperationAction(Op: ISD::GLOBAL_OFFSET_TABLE, VT: MVT::i32, Action: Custom);
1584 setOperationAction(Op: ISD::GlobalTLSAddress, VT: MVT::i32, Action: Custom);
1585 setOperationAction(Op: ISD::ATOMIC_FENCE, VT: MVT::Other, Action: Custom);
1586
1587 // Custom legalize GlobalAddress nodes into CONST32.
1588 setOperationAction(Op: ISD::GlobalAddress, VT: MVT::i32, Action: Custom);
1589 setOperationAction(Op: ISD::GlobalAddress, VT: MVT::i8, Action: Custom);
1590 setOperationAction(Op: ISD::BlockAddress, VT: MVT::i32, Action: Custom);
1591
1592 // Hexagon needs to optimize cases with negative constants.
1593 setOperationAction(Op: ISD::SETCC, VT: MVT::i8, Action: Custom);
1594 setOperationAction(Op: ISD::SETCC, VT: MVT::i16, Action: Custom);
1595 setOperationAction(Op: ISD::SETCC, VT: MVT::v4i8, Action: Custom);
1596 setOperationAction(Op: ISD::SETCC, VT: MVT::v2i16, Action: Custom);
1597
1598 // VASTART needs to be custom lowered to use the VarArgsFrameIndex.
1599 setOperationAction(Op: ISD::VASTART, VT: MVT::Other, Action: Custom);
1600 setOperationAction(Op: ISD::VAEND, VT: MVT::Other, Action: Expand);
1601 setOperationAction(Op: ISD::VAARG, VT: MVT::Other, Action: Expand);
1602 if (Subtarget.isEnvironmentMusl())
1603 setOperationAction(Op: ISD::VACOPY, VT: MVT::Other, Action: Custom);
1604 else
1605 setOperationAction(Op: ISD::VACOPY, VT: MVT::Other, Action: Expand);
1606
1607 setOperationAction(Op: ISD::STACKSAVE, VT: MVT::Other, Action: Expand);
1608 setOperationAction(Op: ISD::STACKRESTORE, VT: MVT::Other, Action: Expand);
1609 setOperationAction(Op: ISD::DYNAMIC_STACKALLOC, VT: MVT::i32, Action: Custom);
1610
1611 if (EmitJumpTables)
1612 setMinimumJumpTableEntries(MinimumJumpTables);
1613 else
1614 setMinimumJumpTableEntries(std::numeric_limits<unsigned>::max());
1615 setOperationAction(Op: ISD::BR_JT, VT: MVT::Other, Action: Expand);
1616
1617 for (unsigned LegalIntOp :
1618 {ISD::ABS, ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX}) {
1619 setOperationAction(Op: LegalIntOp, VT: MVT::i32, Action: Legal);
1620 setOperationAction(Op: LegalIntOp, VT: MVT::i64, Action: Legal);
1621 }
1622
1623 // Hexagon has A4_addp_c and A4_subp_c that take and generate a carry bit,
1624 // but they only operate on i64.
1625 for (MVT VT : MVT::integer_valuetypes()) {
1626 setOperationAction(Op: ISD::UADDO, VT, Action: Custom);
1627 setOperationAction(Op: ISD::USUBO, VT, Action: Custom);
1628 setOperationAction(Op: ISD::SADDO, VT, Action: Expand);
1629 setOperationAction(Op: ISD::SSUBO, VT, Action: Expand);
1630 setOperationAction(Op: ISD::UADDO_CARRY, VT, Action: Expand);
1631 setOperationAction(Op: ISD::USUBO_CARRY, VT, Action: Expand);
1632 }
1633 setOperationAction(Op: ISD::UADDO_CARRY, VT: MVT::i64, Action: Custom);
1634 setOperationAction(Op: ISD::USUBO_CARRY, VT: MVT::i64, Action: Custom);
1635
1636 setOperationAction(Op: ISD::CTLZ, VT: MVT::i8, Action: Promote);
1637 setOperationAction(Op: ISD::CTLZ, VT: MVT::i16, Action: Promote);
1638 setOperationAction(Op: ISD::CTTZ, VT: MVT::i8, Action: Promote);
1639 setOperationAction(Op: ISD::CTTZ, VT: MVT::i16, Action: Promote);
1640
1641 // Popcount can count # of 1s in i64 but returns i32.
1642 setOperationAction(Op: ISD::CTPOP, VT: MVT::i8, Action: Promote);
1643 setOperationAction(Op: ISD::CTPOP, VT: MVT::i16, Action: Promote);
1644 setOperationAction(Op: ISD::CTPOP, VT: MVT::i32, Action: Promote);
1645 setOperationAction(Op: ISD::CTPOP, VT: MVT::i64, Action: Legal);
1646
1647 setOperationAction(Op: ISD::BITREVERSE, VT: MVT::i32, Action: Legal);
1648 setOperationAction(Op: ISD::BITREVERSE, VT: MVT::i64, Action: Legal);
1649 setOperationAction(Op: ISD::BSWAP, VT: MVT::i32, Action: Legal);
1650 setOperationAction(Op: ISD::BSWAP, VT: MVT::i64, Action: Legal);
1651
1652 setOperationAction(Op: ISD::FSHL, VT: MVT::i32, Action: Legal);
1653 setOperationAction(Op: ISD::FSHL, VT: MVT::i64, Action: Legal);
1654 setOperationAction(Op: ISD::FSHR, VT: MVT::i32, Action: Legal);
1655 setOperationAction(Op: ISD::FSHR, VT: MVT::i64, Action: Legal);
1656
1657 for (unsigned IntExpOp :
1658 {ISD::SDIV, ISD::UDIV, ISD::SREM, ISD::UREM,
1659 ISD::SDIVREM, ISD::UDIVREM, ISD::ROTL, ISD::ROTR,
1660 ISD::SHL_PARTS, ISD::SRA_PARTS, ISD::SRL_PARTS,
1661 ISD::SMUL_LOHI, ISD::UMUL_LOHI}) {
1662 for (MVT VT : MVT::integer_valuetypes())
1663 setOperationAction(Op: IntExpOp, VT, Action: Expand);
1664 }
1665 for (MVT VT : MVT::fp_valuetypes()) {
1666 for (unsigned FPExpOp : {ISD::FDIV, ISD::FSQRT, ISD::FSIN, ISD::FCOS,
1667 ISD::FSINCOS, ISD::FPOW, ISD::FCOPYSIGN})
1668 setOperationAction(Op: FPExpOp, VT, Action: Expand);
1669
1670 setOperationAction(Op: ISD::FREM, VT, Action: LibCall);
1671 }
1672
1673 // No extending loads from i32.
1674 for (MVT VT : MVT::integer_valuetypes()) {
1675 setLoadExtAction(ExtType: ISD::ZEXTLOAD, ValVT: VT, MemVT: MVT::i32, Action: Expand);
1676 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: VT, MemVT: MVT::i32, Action: Expand);
1677 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: MVT::i32, Action: Expand);
1678 }
1679 // Turn FP truncstore into trunc + store.
1680 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f32, Action: Expand);
1681 setTruncStoreAction(ValVT: MVT::f32, MemVT: MVT::bf16, Action: Expand);
1682 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::bf16, Action: Expand);
1683 // Turn FP extload into load/fpextend.
1684 for (MVT VT : MVT::fp_valuetypes())
1685 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: MVT::f32, Action: Expand);
1686
1687 // Expand BR_CC and SELECT_CC for all integer and fp types.
1688 for (MVT VT : MVT::integer_valuetypes()) {
1689 setOperationAction(Op: ISD::BR_CC, VT, Action: Expand);
1690 setOperationAction(Op: ISD::SELECT_CC, VT, Action: Expand);
1691 }
1692 for (MVT VT : MVT::fp_valuetypes()) {
1693 setOperationAction(Op: ISD::BR_CC, VT, Action: Expand);
1694 setOperationAction(Op: ISD::SELECT_CC, VT, Action: Expand);
1695 }
1696 setOperationAction(Op: ISD::BR_CC, VT: MVT::Other, Action: Expand);
1697
1698 //
1699 // Handling of vector operations.
1700 //
1701
1702 // Set the action for vector operations to "expand", then override it with
1703 // either "custom" or "legal" for specific cases.
1704 // clang-format off
1705 static const unsigned VectExpOps[] = {
1706 // Integer arithmetic:
1707 ISD::ADD, ISD::SUB, ISD::MUL, ISD::SDIV, ISD::UDIV,
1708 ISD::SREM, ISD::UREM, ISD::SDIVREM, ISD::UDIVREM, ISD::SADDO,
1709 ISD::UADDO, ISD::SSUBO, ISD::USUBO, ISD::SMUL_LOHI, ISD::UMUL_LOHI,
1710 // Logical/bit:
1711 ISD::AND, ISD::OR, ISD::XOR, ISD::ROTL, ISD::ROTR,
1712 ISD::CTPOP, ISD::CTLZ, ISD::CTTZ, ISD::BSWAP, ISD::BITREVERSE,
1713 // Floating point arithmetic/math functions:
1714 ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FMA, ISD::FDIV,
1715 ISD::FREM, ISD::FNEG, ISD::FABS, ISD::FSQRT, ISD::FSIN,
1716 ISD::FCOS, ISD::FPOW, ISD::FLOG, ISD::FLOG2,
1717 ISD::FLOG10, ISD::FEXP, ISD::FEXP2, ISD::FCEIL, ISD::FTRUNC,
1718 ISD::FRINT, ISD::FNEARBYINT, ISD::FROUND, ISD::FFLOOR,
1719 ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM,
1720 ISD::FSINCOS, ISD::FLDEXP,
1721 // Misc:
1722 ISD::BR_CC, ISD::SELECT_CC, ISD::ConstantPool,
1723 // Vector:
1724 ISD::BUILD_VECTOR, ISD::SCALAR_TO_VECTOR,
1725 ISD::EXTRACT_VECTOR_ELT, ISD::INSERT_VECTOR_ELT,
1726 ISD::EXTRACT_SUBVECTOR, ISD::INSERT_SUBVECTOR,
1727 ISD::CONCAT_VECTORS, ISD::VECTOR_SHUFFLE,
1728 ISD::SPLAT_VECTOR,
1729 };
1730 // clang-format on
1731
1732 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
1733 for (unsigned VectExpOp : VectExpOps)
1734 setOperationAction(Op: VectExpOp, VT, Action: Expand);
1735
1736 // Expand all extending loads and truncating stores:
1737 for (MVT TargetVT : MVT::fixedlen_vector_valuetypes()) {
1738 if (TargetVT == VT)
1739 continue;
1740 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: TargetVT, MemVT: VT, Action: Expand);
1741 setLoadExtAction(ExtType: ISD::ZEXTLOAD, ValVT: TargetVT, MemVT: VT, Action: Expand);
1742 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: TargetVT, MemVT: VT, Action: Expand);
1743 setTruncStoreAction(ValVT: VT, MemVT: TargetVT, Action: Expand);
1744 }
1745
1746 // Normalize all inputs to SELECT to be vectors of i32.
1747 if (VT.getVectorElementType() != MVT::i32) {
1748 MVT VT32 = MVT::getVectorVT(VT: MVT::i32, NumElements: VT.getSizeInBits()/32);
1749 setOperationAction(Op: ISD::SELECT, VT, Action: Promote);
1750 AddPromotedToType(Opc: ISD::SELECT, OrigVT: VT, DestVT: VT32);
1751 }
1752 setOperationAction(Op: ISD::SRA, VT, Action: Custom);
1753 setOperationAction(Op: ISD::SHL, VT, Action: Custom);
1754 setOperationAction(Op: ISD::SRL, VT, Action: Custom);
1755 }
1756
1757 setOperationAction(Op: ISD::SADDSAT, VT: MVT::i32, Action: Legal);
1758 setOperationAction(Op: ISD::SADDSAT, VT: MVT::i64, Action: Legal);
1759
1760 // Extending loads from (native) vectors of i8 into (native) vectors of i16
1761 // are legal.
1762 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v2i16, MemVT: MVT::v2i8, Action: Legal);
1763 setLoadExtAction(ExtType: ISD::ZEXTLOAD, ValVT: MVT::v2i16, MemVT: MVT::v2i8, Action: Legal);
1764 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: MVT::v2i16, MemVT: MVT::v2i8, Action: Legal);
1765 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v4i16, MemVT: MVT::v4i8, Action: Legal);
1766 setLoadExtAction(ExtType: ISD::ZEXTLOAD, ValVT: MVT::v4i16, MemVT: MVT::v4i8, Action: Legal);
1767 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: MVT::v4i16, MemVT: MVT::v4i8, Action: Legal);
1768
1769 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::v2i8, Action: Legal);
1770 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::v2i16, Action: Legal);
1771 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::v2i32, Action: Legal);
1772
1773 // Types natively supported:
1774 for (MVT NativeVT : {MVT::v8i1, MVT::v4i1, MVT::v2i1, MVT::v4i8,
1775 MVT::v8i8, MVT::v2i16, MVT::v4i16, MVT::v2i32}) {
1776 setOperationAction(Op: ISD::BUILD_VECTOR, VT: NativeVT, Action: Custom);
1777 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: NativeVT, Action: Custom);
1778 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: NativeVT, Action: Custom);
1779 setOperationAction(Op: ISD::EXTRACT_SUBVECTOR, VT: NativeVT, Action: Custom);
1780 setOperationAction(Op: ISD::INSERT_SUBVECTOR, VT: NativeVT, Action: Custom);
1781 setOperationAction(Op: ISD::CONCAT_VECTORS, VT: NativeVT, Action: Custom);
1782
1783 setOperationAction(Op: ISD::ADD, VT: NativeVT, Action: Legal);
1784 setOperationAction(Op: ISD::SUB, VT: NativeVT, Action: Legal);
1785 setOperationAction(Op: ISD::MUL, VT: NativeVT, Action: Legal);
1786 setOperationAction(Op: ISD::AND, VT: NativeVT, Action: Legal);
1787 setOperationAction(Op: ISD::OR, VT: NativeVT, Action: Legal);
1788 setOperationAction(Op: ISD::XOR, VT: NativeVT, Action: Legal);
1789
1790 if (NativeVT.getVectorElementType() != MVT::i1) {
1791 setOperationAction(Op: ISD::SPLAT_VECTOR, VT: NativeVT, Action: Legal);
1792 setOperationAction(Op: ISD::BSWAP, VT: NativeVT, Action: Legal);
1793 setOperationAction(Op: ISD::BITREVERSE, VT: NativeVT, Action: Legal);
1794 }
1795 }
1796
1797 for (MVT VT : {MVT::v8i8, MVT::v4i16, MVT::v2i32}) {
1798 setOperationAction(Op: ISD::SMIN, VT, Action: Legal);
1799 setOperationAction(Op: ISD::SMAX, VT, Action: Legal);
1800 setOperationAction(Op: ISD::UMIN, VT, Action: Legal);
1801 setOperationAction(Op: ISD::UMAX, VT, Action: Legal);
1802 }
1803
1804 // Custom lower unaligned loads.
1805 // Also, for both loads and stores, verify the alignment of the address
1806 // in case it is a compile-time constant. This is a usability feature to
1807 // provide a meaningful error message to users.
1808 for (MVT VT : {MVT::i16, MVT::i32, MVT::v4i8, MVT::i64, MVT::v8i8,
1809 MVT::v2i16, MVT::v4i16, MVT::v2i32}) {
1810 setOperationAction(Op: ISD::LOAD, VT, Action: Custom);
1811 setOperationAction(Op: ISD::STORE, VT, Action: Custom);
1812 }
1813
1814 // Custom-lower load/stores of boolean vectors.
1815 for (MVT VT : {MVT::v2i1, MVT::v4i1, MVT::v8i1}) {
1816 setOperationAction(Op: ISD::LOAD, VT, Action: Custom);
1817 setOperationAction(Op: ISD::STORE, VT, Action: Custom);
1818 }
1819
1820 // Normalize integer compares to EQ/GT/UGT
1821 for (MVT VT : {MVT::v2i16, MVT::v4i8, MVT::v8i8, MVT::v2i32, MVT::v4i16,
1822 MVT::v2i32}) {
1823 setCondCodeAction(CCs: ISD::SETNE, VT, Action: Expand);
1824 setCondCodeAction(CCs: ISD::SETLE, VT, Action: Expand);
1825 setCondCodeAction(CCs: ISD::SETGE, VT, Action: Expand);
1826 setCondCodeAction(CCs: ISD::SETLT, VT, Action: Expand);
1827 setCondCodeAction(CCs: ISD::SETULE, VT, Action: Expand);
1828 setCondCodeAction(CCs: ISD::SETUGE, VT, Action: Expand);
1829 setCondCodeAction(CCs: ISD::SETULT, VT, Action: Expand);
1830 }
1831
1832 // Normalize boolean compares to [U]LE/[U]LT
1833 for (MVT VT : {MVT::i1, MVT::v2i1, MVT::v4i1, MVT::v8i1}) {
1834 setCondCodeAction(CCs: ISD::SETGE, VT, Action: Expand);
1835 setCondCodeAction(CCs: ISD::SETGT, VT, Action: Expand);
1836 setCondCodeAction(CCs: ISD::SETUGE, VT, Action: Expand);
1837 setCondCodeAction(CCs: ISD::SETUGT, VT, Action: Expand);
1838 }
1839
1840 // Custom-lower bitcasts from i8 to v8i1.
1841 setOperationAction(Op: ISD::BITCAST, VT: MVT::i8, Action: Custom);
1842 setOperationAction(Op: ISD::SETCC, VT: MVT::v2i16, Action: Custom);
1843 setOperationAction(Op: ISD::VSELECT, VT: MVT::v4i8, Action: Custom);
1844 setOperationAction(Op: ISD::VSELECT, VT: MVT::v2i16, Action: Custom);
1845 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT: MVT::v4i8, Action: Custom);
1846 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT: MVT::v4i16, Action: Custom);
1847 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT: MVT::v8i8, Action: Custom);
1848
1849 // V5+.
1850 setOperationAction(Op: ISD::FMA, VT: MVT::f64, Action: Expand);
1851 setOperationAction(Op: ISD::FADD, VT: MVT::f64, Action: Expand);
1852 setOperationAction(Op: ISD::FSUB, VT: MVT::f64, Action: Expand);
1853 setOperationAction(Op: ISD::FMUL, VT: MVT::f64, Action: Expand);
1854 setOperationAction(Op: ISD::FDIV, VT: MVT::f32, Action: Custom);
1855
1856 setOperationAction(Op: ISD::FMINIMUMNUM, VT: MVT::f32, Action: Legal);
1857 setOperationAction(Op: ISD::FMAXIMUMNUM, VT: MVT::f32, Action: Legal);
1858 setOperationAction(Op: ISD::FMINNUM, VT: MVT::f32, Action: Legal);
1859 setOperationAction(Op: ISD::FMAXNUM, VT: MVT::f32, Action: Legal);
1860 setOperationAction(Op: ISD::FMAXIMUM, VT: MVT::f32, Action: Custom);
1861 setOperationAction(Op: ISD::FMAXIMUM, VT: MVT::f16, Action: Custom);
1862 setOperationAction(Op: ISD::FMINIMUM, VT: MVT::f32, Action: Custom);
1863 setOperationAction(Op: ISD::FMINIMUM, VT: MVT::f16, Action: Custom);
1864
1865 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i1, Action: Promote);
1866 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i8, Action: Promote);
1867 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i16, Action: Promote);
1868 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::i1, Action: Promote);
1869 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::i8, Action: Promote);
1870 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::i16, Action: Promote);
1871 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::i1, Action: Promote);
1872 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::i8, Action: Promote);
1873 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::i16, Action: Promote);
1874 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i1, Action: Promote);
1875 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i8, Action: Promote);
1876 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i16, Action: Promote);
1877
1878 // Special handling for half-precision floating point conversions.
1879 // Lower half float conversions into library calls.
1880 setOperationAction(Op: ISD::FP16_TO_FP, VT: MVT::f32, Action: Expand);
1881 setOperationAction(Op: ISD::FP16_TO_FP, VT: MVT::f64, Action: Expand);
1882 setOperationAction(Op: ISD::FP_TO_FP16, VT: MVT::f32, Action: Expand);
1883 setOperationAction(Op: ISD::FP_TO_FP16, VT: MVT::f64, Action: Expand);
1884 setOperationAction(Op: ISD::BF16_TO_FP, VT: MVT::f32, Action: Expand);
1885 setOperationAction(Op: ISD::BF16_TO_FP, VT: MVT::f64, Action: Expand);
1886 setOperationAction(Op: ISD::FP_TO_BF16, VT: MVT::f64, Action: Expand);
1887
1888 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f32, MemVT: MVT::f16, Action: Expand);
1889 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::f16, Action: Expand);
1890 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f32, MemVT: MVT::bf16, Action: Expand);
1891 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::bf16, Action: Expand);
1892
1893 setTruncStoreAction(ValVT: MVT::f32, MemVT: MVT::f16, Action: Expand);
1894 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f16, Action: Expand);
1895
1896 // Handling of indexed loads/stores: default is "expand".
1897 //
1898 for (MVT VT : {MVT::i8, MVT::i16, MVT::i32, MVT::i64, MVT::f32, MVT::f64,
1899 MVT::v2i16, MVT::v2i32, MVT::v4i8, MVT::v4i16, MVT::v8i8}) {
1900 setIndexedLoadAction(IdxModes: ISD::POST_INC, VT, Action: Legal);
1901 setIndexedStoreAction(IdxModes: ISD::POST_INC, VT, Action: Legal);
1902 }
1903
1904 // Subtarget-specific operation actions.
1905 //
1906 if (Subtarget.hasV60Ops()) {
1907 setOperationAction(Op: ISD::ROTL, VT: MVT::i32, Action: Legal);
1908 setOperationAction(Op: ISD::ROTL, VT: MVT::i64, Action: Legal);
1909 setOperationAction(Op: ISD::ROTR, VT: MVT::i32, Action: Legal);
1910 setOperationAction(Op: ISD::ROTR, VT: MVT::i64, Action: Legal);
1911 }
1912 if (Subtarget.hasV66Ops()) {
1913 setOperationAction(Op: ISD::FADD, VT: MVT::f64, Action: Legal);
1914 setOperationAction(Op: ISD::FSUB, VT: MVT::f64, Action: Legal);
1915 }
1916 if (Subtarget.hasV67Ops()) {
1917 setOperationAction(Op: ISD::FMINIMUMNUM, VT: MVT::f64, Action: Legal);
1918 setOperationAction(Op: ISD::FMAXIMUMNUM, VT: MVT::f64, Action: Legal);
1919 setOperationAction(Op: ISD::FMINNUM, VT: MVT::f64, Action: Legal);
1920 setOperationAction(Op: ISD::FMAXNUM, VT: MVT::f64, Action: Legal);
1921 setOperationAction(Op: ISD::FMUL, VT: MVT::f64, Action: Legal);
1922 }
1923
1924 setTargetDAGCombine(ISD::OR);
1925 setTargetDAGCombine(ISD::TRUNCATE);
1926 setTargetDAGCombine(ISD::VSELECT);
1927
1928 if (Subtarget.useHVXOps())
1929 initializeHVXLowering();
1930
1931 computeRegisterProperties(TRI: &HRI);
1932}
1933
1934bool
1935HexagonTargetLowering::validateConstPtrAlignment(SDValue Ptr, Align NeedAlign,
1936 const SDLoc &dl, SelectionDAG &DAG) const {
1937 auto *CA = dyn_cast<ConstantSDNode>(Val&: Ptr);
1938 if (!CA)
1939 return true;
1940 unsigned Addr = CA->getZExtValue();
1941 Align HaveAlign =
1942 Addr != 0 ? Align(1ull << llvm::countr_zero(Val: Addr)) : NeedAlign;
1943 if (HaveAlign >= NeedAlign)
1944 return true;
1945
1946 static int DK_MisalignedTrap = llvm::getNextAvailablePluginDiagnosticKind();
1947
1948 struct DiagnosticInfoMisalignedTrap : public DiagnosticInfo {
1949 DiagnosticInfoMisalignedTrap(StringRef M)
1950 : DiagnosticInfo(DK_MisalignedTrap, DS_Remark), Msg(M) {}
1951 void print(DiagnosticPrinter &DP) const override {
1952 DP << Msg;
1953 }
1954 static bool classof(const DiagnosticInfo *DI) {
1955 return DI->getKind() == DK_MisalignedTrap;
1956 }
1957 StringRef Msg;
1958 };
1959
1960 std::string ErrMsg;
1961 raw_string_ostream O(ErrMsg);
1962 O << "Misaligned constant address: " << format_hex(N: Addr, Width: 10)
1963 << " has alignment " << HaveAlign.value()
1964 << ", but the memory access requires " << NeedAlign.value();
1965 if (DebugLoc DL = dl.getDebugLoc())
1966 DL.print(OS&: O << ", at ");
1967 O << ". The instruction has been replaced with a trap.";
1968
1969 DAG.getContext()->diagnose(DI: DiagnosticInfoMisalignedTrap(O.str()));
1970 return false;
1971}
1972
1973SDValue
1974HexagonTargetLowering::replaceMemWithUndef(SDValue Op, SelectionDAG &DAG)
1975 const {
1976 const SDLoc &dl(Op);
1977 auto *LS = cast<LSBaseSDNode>(Val: Op.getNode());
1978 assert(!LS->isIndexed() && "Not expecting indexed ops on constant address");
1979
1980 SDValue Chain = LS->getChain();
1981 SDValue Trap = DAG.getNode(Opcode: ISD::TRAP, DL: dl, VT: MVT::Other, Operand: Chain);
1982 if (LS->getOpcode() == ISD::LOAD)
1983 return DAG.getMergeValues(Ops: {DAG.getUNDEF(VT: ty(Op)), Trap}, dl);
1984 return Trap;
1985}
1986
1987// Bit-reverse Load Intrinsic: Check if the instruction is a bit reverse load
1988// intrinsic.
1989static bool isBrevLdIntrinsic(const Value *Inst) {
1990 unsigned ID = cast<IntrinsicInst>(Val: Inst)->getIntrinsicID();
1991 return (ID == Intrinsic::hexagon_L2_loadrd_pbr ||
1992 ID == Intrinsic::hexagon_L2_loadri_pbr ||
1993 ID == Intrinsic::hexagon_L2_loadrh_pbr ||
1994 ID == Intrinsic::hexagon_L2_loadruh_pbr ||
1995 ID == Intrinsic::hexagon_L2_loadrb_pbr ||
1996 ID == Intrinsic::hexagon_L2_loadrub_pbr);
1997}
1998
1999// Bit-reverse Load Intrinsic :Crawl up and figure out the object from previous
2000// instruction. So far we only handle bitcast, extract value and bit reverse
2001// load intrinsic instructions. Should we handle CGEP ?
2002static Value *getBrevLdObject(Value *V) {
2003 if (Operator::getOpcode(V) == Instruction::ExtractValue ||
2004 Operator::getOpcode(V) == Instruction::BitCast)
2005 V = cast<Operator>(Val: V)->getOperand(i: 0);
2006 else if (isa<IntrinsicInst>(Val: V) && isBrevLdIntrinsic(Inst: V))
2007 V = cast<Instruction>(Val: V)->getOperand(i: 0);
2008 return V;
2009}
2010
2011// Bit-reverse Load Intrinsic: For a PHI Node return either an incoming edge or
2012// a back edge. If the back edge comes from the intrinsic itself, the incoming
2013// edge is returned.
2014static Value *returnEdge(const PHINode *PN, Value *IntrBaseVal) {
2015 const BasicBlock *Parent = PN->getParent();
2016 int Idx = -1;
2017 for (unsigned i = 0, e = PN->getNumIncomingValues(); i < e; ++i) {
2018 BasicBlock *Blk = PN->getIncomingBlock(i);
2019 // Determine if the back edge is originated from intrinsic.
2020 if (Blk == Parent) {
2021 Value *BackEdgeVal = PN->getIncomingValue(i);
2022 Value *BaseVal;
2023 // Loop over till we return the same Value or we hit the IntrBaseVal.
2024 do {
2025 BaseVal = BackEdgeVal;
2026 BackEdgeVal = getBrevLdObject(V: BackEdgeVal);
2027 } while ((BaseVal != BackEdgeVal) && (IntrBaseVal != BackEdgeVal));
2028 // If the getBrevLdObject returns IntrBaseVal, we should return the
2029 // incoming edge.
2030 if (IntrBaseVal == BackEdgeVal)
2031 continue;
2032 Idx = i;
2033 break;
2034 } else // Set the node to incoming edge.
2035 Idx = i;
2036 }
2037 assert(Idx >= 0 && "Unexpected index to incoming argument in PHI");
2038 return PN->getIncomingValue(i: Idx);
2039}
2040
2041// Bit-reverse Load Intrinsic: Figure out the underlying object the base
2042// pointer points to, for the bit-reverse load intrinsic. Setting this to
2043// memoperand might help alias analysis to figure out the dependencies.
2044// A bit-reverse load accesses the base pointer with its low 16 bits reversed,
2045// and post-increments the base pointer by the modifier value. For a chain of
2046// bit-reverse loads, the offset that a load accesses relative to the
2047// underlying object is the bit-reverse of the sum of the modifiers of the
2048// preceding loads in the chain. Offset is set to that value, and HasOffset is
2049// set to true, when the sum is known, that is when all of those modifiers are
2050// constants and the sum fits in 16 unsigned bits. Otherwise HasOffset is set
2051// to false and Offset is left unchanged.
2052static Value *getUnderLyingObjectForBrevLdIntr(Value *V, int &Offset,
2053 bool &HasOffset) {
2054 Value *IntrBaseVal = V;
2055 Value *BaseVal;
2056 int64_t Sum = 0;
2057 HasOffset = true;
2058 // Loop over till we return the same Value, implies we either figure out
2059 // the object or we hit a PHI
2060 do {
2061 BaseVal = V;
2062 V = getBrevLdObject(V);
2063 // Identify if this is part of a chain of bit-reverse loads, and accumulate
2064 // the modifier of the preceding load in the chain.
2065 if (HasOffset && BaseVal != V && isa<IntrinsicInst>(Val: V) &&
2066 isBrevLdIntrinsic(Inst: V)) {
2067 Value *Modifier = cast<IntrinsicInst>(Val: V)->getOperand(i_nocapture: 1);
2068 if (auto *CN = dyn_cast<ConstantInt>(Val: Modifier))
2069 Sum += CN->getSExtValue();
2070 else
2071 HasOffset = false;
2072 }
2073 } while (BaseVal != V);
2074
2075 // Only the low 16 bits of the base pointer take part in the bit-reverse. A
2076 // sum that does not fit in them would also change the remaining bits.
2077 if (HasOffset && Sum >= 0 && isUInt<16>(x: Sum))
2078 Offset = APInt(16, Sum).reverseBits().getZExtValue();
2079 else
2080 HasOffset = false;
2081
2082 // Identify the object from PHINode.
2083 if (const PHINode *PN = dyn_cast<PHINode>(Val: V))
2084 return returnEdge(PN, IntrBaseVal);
2085 // For non PHI nodes, the object is the last value returned by getBrevLdObject
2086 else
2087 return V;
2088}
2089
2090/// Given an intrinsic, checks if on the target the intrinsic will need to map
2091/// to a MemIntrinsicNode (touches memory). If this is the case, it stores
2092/// the intrinsic information into the Infos vector.
2093void HexagonTargetLowering::getTgtMemIntrinsic(
2094 SmallVectorImpl<IntrinsicInfo> &Infos, const CallBase &I,
2095 MachineFunction &MF, unsigned Intrinsic) const {
2096 IntrinsicInfo Info;
2097 switch (Intrinsic) {
2098 case Intrinsic::hexagon_L2_loadrd_pbr:
2099 case Intrinsic::hexagon_L2_loadri_pbr:
2100 case Intrinsic::hexagon_L2_loadrh_pbr:
2101 case Intrinsic::hexagon_L2_loadruh_pbr:
2102 case Intrinsic::hexagon_L2_loadrb_pbr:
2103 case Intrinsic::hexagon_L2_loadrub_pbr: {
2104 Info.opc = ISD::INTRINSIC_W_CHAIN;
2105 auto &DL = I.getDataLayout();
2106 auto &Cont = I.getCalledFunction()->getParent()->getContext();
2107 // The intrinsic function call is of the form { ElTy, i8* }
2108 // @llvm.hexagon.L2.loadXX.pbr(i8*, i32). The pointer and memory access type
2109 // should be derived from ElTy.
2110 Type *ElTy = I.getCalledFunction()->getReturnType()->getStructElementType(N: 0);
2111 Info.memVT = MVT::getVT(Ty: ElTy);
2112 llvm::Value *BasePtrVal = I.getOperand(i_nocapture: 0);
2113 // The offset value comes through the Modifier register. Determine the
2114 // offset that is going to be accessed relative to the underlying object.
2115 // If it cannot be determined, leave the pointer information out of the
2116 // memory operand, so that alias analysis stays conservative.
2117 bool HasOffset = false;
2118 Info.offset = 0;
2119 Value *UnderlyingObj =
2120 getUnderLyingObjectForBrevLdIntr(V: BasePtrVal, Offset&: Info.offset, HasOffset);
2121 // The underlying object is unknown if the base pointer could not be traced
2122 // back to a pointer value. Also, unless the object is aligned to 64K, the
2123 // low 16 bits of the base pointer are not known, and reversing them can
2124 // produce an address anywhere in the surrounding 64K region, possibly
2125 // outside of the object.
2126 if (!UnderlyingObj->getType()->isPointerTy() ||
2127 UnderlyingObj->getPointerAlignment(DL) < Align(65536))
2128 HasOffset = false;
2129 if (HasOffset)
2130 Info.ptrVal = UnderlyingObj;
2131 Info.align = DL.getABITypeAlign(Ty: Info.memVT.getTypeForEVT(Context&: Cont));
2132 Info.flags = MachineMemOperand::MOLoad;
2133 Infos.push_back(Elt: Info);
2134 return;
2135 }
2136 case Intrinsic::hexagon_V6_vgathermw:
2137 case Intrinsic::hexagon_V6_vgathermw_128B:
2138 case Intrinsic::hexagon_V6_vgathermh:
2139 case Intrinsic::hexagon_V6_vgathermh_128B:
2140 case Intrinsic::hexagon_V6_vgathermhw:
2141 case Intrinsic::hexagon_V6_vgathermhw_128B:
2142 case Intrinsic::hexagon_V6_vgathermwq:
2143 case Intrinsic::hexagon_V6_vgathermwq_128B:
2144 case Intrinsic::hexagon_V6_vgathermhq:
2145 case Intrinsic::hexagon_V6_vgathermhq_128B:
2146 case Intrinsic::hexagon_V6_vgathermhwq:
2147 case Intrinsic::hexagon_V6_vgathermhwq_128B:
2148 case Intrinsic::hexagon_V6_vgather_vscattermh:
2149 case Intrinsic::hexagon_V6_vgather_vscattermh_128B: {
2150 const Module &M = *I.getParent()->getParent()->getParent();
2151 Info.opc = ISD::INTRINSIC_W_CHAIN;
2152 Type *VecTy = I.getArgOperand(i: I.arg_size() - 1)->getType();
2153 assert(VecTy->isVectorTy() && "Expected vector operand for vgather");
2154 Info.memVT = MVT::getVT(Ty: VecTy);
2155 Info.ptrVal = I.getArgOperand(i: 0);
2156 Info.offset = 0;
2157 Info.align =
2158 MaybeAlign(M.getDataLayout().getTypeAllocSizeInBits(Ty: VecTy) / 8);
2159 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
2160 MachineMemOperand::MOVolatile;
2161 Infos.push_back(Elt: Info);
2162 return;
2163 }
2164 default:
2165 break;
2166 }
2167}
2168
2169bool HexagonTargetLowering::hasBitTest(SDValue X, SDValue Y) const {
2170 return X.getValueType().isScalarInteger(); // 'tstbit'
2171}
2172
2173bool HexagonTargetLowering::isTruncateFree(Type *Ty1, Type *Ty2) const {
2174 return isTruncateFree(VT1: EVT::getEVT(Ty: Ty1), VT2: EVT::getEVT(Ty: Ty2));
2175}
2176
2177bool HexagonTargetLowering::isTruncateFree(EVT VT1, EVT VT2) const {
2178 if (!VT1.isSimple() || !VT2.isSimple())
2179 return false;
2180 return VT1.getSimpleVT() == MVT::i64 && VT2.getSimpleVT() == MVT::i32;
2181}
2182
2183bool HexagonTargetLowering::isFMAFasterThanFMulAndFAdd(
2184 const MachineFunction &MF, EVT VT) const {
2185 return isOperationLegalOrCustom(Op: ISD::FMA, VT);
2186}
2187
2188// Should we expand the build vector with shuffles?
2189bool HexagonTargetLowering::shouldExpandBuildVectorWithShuffles(EVT VT,
2190 unsigned DefinedValues) const {
2191 return false;
2192}
2193
2194TargetLowering::ExtractSubvectorCost
2195HexagonTargetLowering::getExtractSubvectorCost(EVT ResVT, EVT SrcVT,
2196 unsigned Index) const {
2197 assert(ResVT.getVectorElementType() == SrcVT.getVectorElementType());
2198 if (!ResVT.isSimple() || !SrcVT.isSimple())
2199 return ExtractSubvectorCost::Expensive;
2200
2201 MVT ResTy = ResVT.getSimpleVT(), SrcTy = SrcVT.getSimpleVT();
2202 if (ResTy.getVectorElementType() != MVT::i1)
2203 return ExtractSubvectorCost::Free;
2204
2205 // Non-HVX bool vectors are relatively cheap.
2206 if (SrcTy.getVectorNumElements() <= 8)
2207 return ExtractSubvectorCost::Free;
2208 return ExtractSubvectorCost::Expensive;
2209}
2210
2211bool HexagonTargetLowering::isTargetCanonicalConstantNode(SDValue Op) const {
2212 return Op.getOpcode() == ISD::CONCAT_VECTORS ||
2213 TargetLowering::isTargetCanonicalConstantNode(Op);
2214}
2215
2216bool HexagonTargetLowering::isShuffleMaskLegal(ArrayRef<int> Mask,
2217 EVT VT) const {
2218 return true;
2219}
2220
2221TargetLoweringBase::LegalizeTypeAction
2222HexagonTargetLowering::getPreferredVectorAction(MVT VT) const {
2223 unsigned VecLen = VT.getVectorMinNumElements();
2224 MVT ElemTy = VT.getVectorElementType();
2225
2226 if (VecLen == 1 || VT.isScalableVector())
2227 return TargetLoweringBase::TypeScalarizeVector;
2228
2229 if (Subtarget.useHVXOps()) {
2230 unsigned Action = getPreferredHvxVectorAction(VecTy: VT);
2231 if (Action != ~0u)
2232 return static_cast<TargetLoweringBase::LegalizeTypeAction>(Action);
2233 }
2234
2235 // Always widen (remaining) vectors of i1.
2236 if (ElemTy == MVT::i1)
2237 return TargetLoweringBase::TypeWidenVector;
2238 // Widen non-power-of-2 vectors. Such types cannot be split right now,
2239 // and computeRegisterProperties will override "split" with "widen",
2240 // which can cause other issues.
2241 if (!isPowerOf2_32(Value: VecLen))
2242 return TargetLoweringBase::TypeWidenVector;
2243
2244 return TargetLoweringBase::TypeSplitVector;
2245}
2246
2247TargetLoweringBase::LegalizeAction
2248HexagonTargetLowering::getCustomOperationAction(SDNode &Op) const {
2249 if (Subtarget.useHVXOps()) {
2250 unsigned Action = getCustomHvxOperationAction(Op);
2251 if (Action != ~0u)
2252 return static_cast<TargetLoweringBase::LegalizeAction>(Action);
2253 }
2254 return TargetLoweringBase::Legal;
2255}
2256
2257std::pair<SDValue, int>
2258HexagonTargetLowering::getBaseAndOffset(SDValue Addr) const {
2259 if (Addr.getOpcode() == ISD::ADD) {
2260 SDValue Op1 = Addr.getOperand(i: 1);
2261 if (auto *CN = dyn_cast<const ConstantSDNode>(Val: Op1.getNode()))
2262 return { Addr.getOperand(i: 0), CN->getSExtValue() };
2263 }
2264 return { Addr, 0 };
2265}
2266
2267// Lower a vector shuffle (V1, V2, V3). V1 and V2 are the two vectors
2268// to select data from, V3 is the permutation.
2269SDValue
2270HexagonTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG)
2271 const {
2272 const auto *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
2273 ArrayRef<int> AM = SVN->getMask();
2274 assert(AM.size() <= 8 && "Unexpected shuffle mask");
2275 unsigned VecLen = AM.size();
2276
2277 MVT VecTy = ty(Op);
2278 assert(!Subtarget.isHVXVectorType(VecTy, true) &&
2279 "HVX shuffles should be legal");
2280 assert(VecTy.getSizeInBits() <= 64 && "Unexpected vector length");
2281
2282 SDValue Op0 = Op.getOperand(i: 0);
2283 SDValue Op1 = Op.getOperand(i: 1);
2284 const SDLoc &dl(Op);
2285
2286 // If the inputs are not the same as the output, bail. This is not an
2287 // error situation, but complicates the handling and the default expansion
2288 // (into BUILD_VECTOR) should be adequate.
2289 if (ty(Op: Op0) != VecTy || ty(Op: Op1) != VecTy)
2290 return SDValue();
2291
2292 // Normalize the mask so that the first non-negative index comes from
2293 // the first operand.
2294 SmallVector<int, 8> Mask(AM);
2295 unsigned F = llvm::find_if(Range&: AM, P: [](int M) { return M >= 0; }) - AM.data();
2296 if (F == AM.size())
2297 return DAG.getUNDEF(VT: VecTy);
2298 if (AM[F] >= int(VecLen)) {
2299 ShuffleVectorSDNode::commuteMask(Mask);
2300 std::swap(a&: Op0, b&: Op1);
2301 }
2302
2303 // Express the shuffle mask in terms of bytes.
2304 SmallVector<int,8> ByteMask;
2305 unsigned ElemBytes = VecTy.getVectorElementType().getSizeInBits() / 8;
2306 for (int M : Mask) {
2307 if (M < 0) {
2308 for (unsigned j = 0; j != ElemBytes; ++j)
2309 ByteMask.push_back(Elt: -1);
2310 } else {
2311 for (unsigned j = 0; j != ElemBytes; ++j)
2312 ByteMask.push_back(Elt: M*ElemBytes + j);
2313 }
2314 }
2315 assert(ByteMask.size() <= 8);
2316
2317 // All non-undef (non-negative) indexes are well within [0..127], so they
2318 // fit in a single byte. Build two 64-bit words:
2319 // - MaskIdx where each byte is the corresponding index (for non-negative
2320 // indexes), and 0xFF for negative indexes, and
2321 // - MaskUnd that has 0xFF for each negative index.
2322 uint64_t MaskIdx = 0;
2323 uint64_t MaskUnd = 0;
2324 for (unsigned i = 0, e = ByteMask.size(); i != e; ++i) {
2325 unsigned S = 8*i;
2326 uint64_t M = ByteMask[i] & 0xFF;
2327 if (M == 0xFF)
2328 MaskUnd |= M << S;
2329 MaskIdx |= M << S;
2330 }
2331
2332 if (ByteMask.size() == 4) {
2333 // Identity.
2334 if (MaskIdx == (0x03020100 | MaskUnd))
2335 return Op0;
2336 // Byte swap.
2337 if (MaskIdx == (0x00010203 | MaskUnd)) {
2338 SDValue T0 = DAG.getBitcast(VT: MVT::i32, V: Op0);
2339 SDValue T1 = DAG.getNode(Opcode: ISD::BSWAP, DL: dl, VT: MVT::i32, Operand: T0);
2340 return DAG.getBitcast(VT: VecTy, V: T1);
2341 }
2342
2343 // Byte packs.
2344 SDValue Concat10 =
2345 getCombine(Hi: Op1, Lo: Op0, dl, ResTy: typeJoin(Tys: {ty(Op: Op1), ty(Op: Op0)}), DAG);
2346 if (MaskIdx == (0x06040200 | MaskUnd))
2347 return getInstr(MachineOpc: Hexagon::S2_vtrunehb, dl, Ty: VecTy, Ops: {Concat10}, DAG);
2348 if (MaskIdx == (0x07050301 | MaskUnd))
2349 return getInstr(MachineOpc: Hexagon::S2_vtrunohb, dl, Ty: VecTy, Ops: {Concat10}, DAG);
2350
2351 SDValue Concat01 =
2352 getCombine(Hi: Op0, Lo: Op1, dl, ResTy: typeJoin(Tys: {ty(Op: Op0), ty(Op: Op1)}), DAG);
2353 if (MaskIdx == (0x02000604 | MaskUnd))
2354 return getInstr(MachineOpc: Hexagon::S2_vtrunehb, dl, Ty: VecTy, Ops: {Concat01}, DAG);
2355 if (MaskIdx == (0x03010705 | MaskUnd))
2356 return getInstr(MachineOpc: Hexagon::S2_vtrunohb, dl, Ty: VecTy, Ops: {Concat01}, DAG);
2357 }
2358
2359 if (ByteMask.size() == 8) {
2360 // Identity.
2361 if (MaskIdx == (0x0706050403020100ull | MaskUnd))
2362 return Op0;
2363 // Byte swap.
2364 if (MaskIdx == (0x0001020304050607ull | MaskUnd)) {
2365 SDValue T0 = DAG.getBitcast(VT: MVT::i64, V: Op0);
2366 SDValue T1 = DAG.getNode(Opcode: ISD::BSWAP, DL: dl, VT: MVT::i64, Operand: T0);
2367 return DAG.getBitcast(VT: VecTy, V: T1);
2368 }
2369
2370 // Halfword picks.
2371 if (MaskIdx == (0x0d0c050409080100ull | MaskUnd))
2372 return getInstr(MachineOpc: Hexagon::S2_shuffeh, dl, Ty: VecTy, Ops: {Op1, Op0}, DAG);
2373 if (MaskIdx == (0x0f0e07060b0a0302ull | MaskUnd))
2374 return getInstr(MachineOpc: Hexagon::S2_shuffoh, dl, Ty: VecTy, Ops: {Op1, Op0}, DAG);
2375 if (MaskIdx == (0x0d0c090805040100ull | MaskUnd))
2376 return getInstr(MachineOpc: Hexagon::S2_vtrunewh, dl, Ty: VecTy, Ops: {Op1, Op0}, DAG);
2377 if (MaskIdx == (0x0f0e0b0a07060302ull | MaskUnd))
2378 return getInstr(MachineOpc: Hexagon::S2_vtrunowh, dl, Ty: VecTy, Ops: {Op1, Op0}, DAG);
2379 if (MaskIdx == (0x0706030205040100ull | MaskUnd)) {
2380 VectorPair P = opSplit(Vec: Op0, dl, DAG);
2381 return getInstr(MachineOpc: Hexagon::S2_packhl, dl, Ty: VecTy, Ops: {P.second, P.first}, DAG);
2382 }
2383
2384 // Byte packs.
2385 if (MaskIdx == (0x0e060c040a020800ull | MaskUnd))
2386 return getInstr(MachineOpc: Hexagon::S2_shuffeb, dl, Ty: VecTy, Ops: {Op1, Op0}, DAG);
2387 if (MaskIdx == (0x0f070d050b030901ull | MaskUnd))
2388 return getInstr(MachineOpc: Hexagon::S2_shuffob, dl, Ty: VecTy, Ops: {Op1, Op0}, DAG);
2389 }
2390
2391 return SDValue();
2392}
2393
2394SDValue
2395HexagonTargetLowering::getSplatValue(SDValue Op, SelectionDAG &DAG) const {
2396 switch (Op.getOpcode()) {
2397 case ISD::BUILD_VECTOR:
2398 if (SDValue S = cast<BuildVectorSDNode>(Val&: Op)->getSplatValue())
2399 return S;
2400 break;
2401 case ISD::SPLAT_VECTOR:
2402 return Op.getOperand(i: 0);
2403 }
2404 return SDValue();
2405}
2406
2407// Create a Hexagon-specific node for shifting a vector by an integer.
2408SDValue
2409HexagonTargetLowering::getVectorShiftByInt(SDValue Op, SelectionDAG &DAG)
2410 const {
2411 unsigned NewOpc;
2412 switch (Op.getOpcode()) {
2413 case ISD::SHL:
2414 NewOpc = HexagonISD::VASL;
2415 break;
2416 case ISD::SRA:
2417 NewOpc = HexagonISD::VASR;
2418 break;
2419 case ISD::SRL:
2420 NewOpc = HexagonISD::VLSR;
2421 break;
2422 default:
2423 llvm_unreachable("Unexpected shift opcode");
2424 }
2425 if (SDValue Sp = getSplatValue(Op: Op.getOperand(i: 1), DAG)) {
2426 const SDLoc dl(Op);
2427 // Canonicalize shift amount to i32 as required.
2428 SDValue Sh = Sp;
2429 if (Sh.getValueType() != MVT::i32)
2430 Sh = DAG.getZExtOrTrunc(Op: Sh, DL: dl, VT: MVT::i32);
2431
2432 assert(Sh.getValueType() == MVT::i32 &&
2433 "Hexagon vector shift-by-int must use i32 shift operand");
2434 return DAG.getNode(Opcode: NewOpc, DL: dl, VT: ty(Op), N1: Op.getOperand(i: 0), N2: Sh);
2435 }
2436
2437 return SDValue();
2438}
2439
2440SDValue
2441HexagonTargetLowering::LowerVECTOR_SHIFT(SDValue Op, SelectionDAG &DAG) const {
2442 const SDLoc &dl(Op);
2443
2444 // First try to convert the shift (by vector) to a shift by a scalar.
2445 // If we first split the shift, the shift amount will become 'extract
2446 // subvector', and will no longer be recognized as scalar.
2447 SDValue Res = Op;
2448 if (SDValue S = getVectorShiftByInt(Op, DAG))
2449 Res = S;
2450
2451 unsigned Opc = Res.getOpcode();
2452 switch (Opc) {
2453 case HexagonISD::VASR:
2454 case HexagonISD::VLSR:
2455 case HexagonISD::VASL:
2456 break;
2457 default:
2458 // No instructions for shifts by non-scalars.
2459 return SDValue();
2460 }
2461
2462 MVT ResTy = ty(Op: Res);
2463 if (ResTy.getVectorElementType() != MVT::i8)
2464 return Res;
2465
2466 // For shifts of i8, extend the inputs to i16, then truncate back to i8.
2467 assert(ResTy.getVectorElementType() == MVT::i8);
2468 SDValue Val = Res.getOperand(i: 0), Amt = Res.getOperand(i: 1);
2469
2470 auto ShiftPartI8 = [&dl, &DAG, this](unsigned Opc, SDValue V, SDValue A) {
2471 MVT Ty = ty(Op: V);
2472 MVT ExtTy = MVT::getVectorVT(VT: MVT::i16, NumElements: Ty.getVectorNumElements());
2473 SDValue ExtV = Opc == HexagonISD::VASR ? DAG.getSExtOrTrunc(Op: V, DL: dl, VT: ExtTy)
2474 : DAG.getZExtOrTrunc(Op: V, DL: dl, VT: ExtTy);
2475 SDValue ExtS = DAG.getNode(Opcode: Opc, DL: dl, VT: ExtTy, Ops: {ExtV, A});
2476 return DAG.getZExtOrTrunc(Op: ExtS, DL: dl, VT: Ty);
2477 };
2478
2479 if (ResTy.getSizeInBits() == 32)
2480 return ShiftPartI8(Opc, Val, Amt);
2481
2482 auto [LoV, HiV] = opSplit(Vec: Val, dl, DAG);
2483 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResTy,
2484 Ops: {ShiftPartI8(Opc, LoV, Amt), ShiftPartI8(Opc, HiV, Amt)});
2485}
2486
2487SDValue
2488HexagonTargetLowering::LowerROTL(SDValue Op, SelectionDAG &DAG) const {
2489 if (isa<ConstantSDNode>(Val: Op.getOperand(i: 1).getNode()))
2490 return Op;
2491 return SDValue();
2492}
2493
2494SDValue
2495HexagonTargetLowering::LowerBITCAST(SDValue Op, SelectionDAG &DAG) const {
2496 MVT ResTy = ty(Op);
2497 SDValue InpV = Op.getOperand(i: 0);
2498 MVT InpTy = ty(Op: InpV);
2499 assert(ResTy.getSizeInBits() == InpTy.getSizeInBits());
2500 const SDLoc &dl(Op);
2501
2502 // Handle conversion from i8 to v8i1.
2503 if (InpTy == MVT::i8) {
2504 if (ResTy == MVT::v8i1) {
2505 SDValue Sc = DAG.getBitcast(VT: tyScalar(Ty: InpTy), V: InpV);
2506 SDValue Ext = DAG.getZExtOrTrunc(Op: Sc, DL: dl, VT: MVT::i32);
2507 return getInstr(MachineOpc: Hexagon::C2_tfrrp, dl, Ty: ResTy, Ops: Ext, DAG);
2508 }
2509 return SDValue();
2510 }
2511
2512 return Op;
2513}
2514
2515bool
2516HexagonTargetLowering::getBuildVectorConstInts(ArrayRef<SDValue> Values,
2517 MVT VecTy, SelectionDAG &DAG,
2518 MutableArrayRef<ConstantInt*> Consts) const {
2519 MVT ElemTy = VecTy.getVectorElementType();
2520 unsigned ElemWidth = ElemTy.getSizeInBits();
2521 IntegerType *IntTy = IntegerType::get(C&: *DAG.getContext(), NumBits: ElemWidth);
2522 bool AllConst = true;
2523
2524 for (unsigned i = 0, e = Values.size(); i != e; ++i) {
2525 SDValue V = Values[i];
2526 if (V.isUndef()) {
2527 Consts[i] = ConstantInt::get(Ty: IntTy, V: 0);
2528 continue;
2529 }
2530 // Make sure to always cast to IntTy.
2531 if (auto *CN = dyn_cast<ConstantSDNode>(Val: V.getNode())) {
2532 const ConstantInt *CI = CN->getConstantIntValue();
2533 Consts[i] = cast<ConstantInt>(
2534 Val: ConstantInt::get(Ty: IntTy, V: CI->getValue().trunc(width: ElemWidth)));
2535 } else if (auto *CN = dyn_cast<ConstantFPSDNode>(Val: V.getNode())) {
2536 const ConstantFP *CF = CN->getConstantFPValue();
2537 APInt A = CF->getValueAPF().bitcastToAPInt();
2538 Consts[i] = ConstantInt::get(Ty: IntTy, V: A.getZExtValue());
2539 } else {
2540 AllConst = false;
2541 }
2542 }
2543 return AllConst;
2544}
2545
2546SDValue
2547HexagonTargetLowering::buildVector32(ArrayRef<SDValue> Elem, const SDLoc &dl,
2548 MVT VecTy, SelectionDAG &DAG) const {
2549 MVT ElemTy = VecTy.getVectorElementType();
2550 assert(VecTy.getVectorNumElements() == Elem.size());
2551
2552 SmallVector<ConstantInt*,4> Consts(Elem.size());
2553 bool AllConst = getBuildVectorConstInts(Values: Elem, VecTy, DAG, Consts);
2554
2555 unsigned First, Num = Elem.size();
2556 for (First = 0; First != Num; ++First) {
2557 if (!isUndef(Op: Elem[First]))
2558 break;
2559 }
2560 if (First == Num)
2561 return DAG.getUNDEF(VT: VecTy);
2562
2563 if (AllConst &&
2564 llvm::all_of(Range&: Consts, P: [](ConstantInt *CI) { return CI->isZero(); }))
2565 return getZero(dl, Ty: VecTy, DAG);
2566
2567 if (ElemTy == MVT::i16 || ElemTy == MVT::f16) {
2568 assert(Elem.size() == 2);
2569 if (AllConst) {
2570 // The 'Consts' array will have all values as integers regardless
2571 // of the vector element type.
2572 uint32_t V = (Consts[0]->getZExtValue() & 0xFFFF) |
2573 Consts[1]->getZExtValue() << 16;
2574 return DAG.getBitcast(VT: VecTy, V: DAG.getConstant(Val: V, DL: dl, VT: MVT::i32));
2575 }
2576 SDValue E0, E1;
2577 if (ElemTy == MVT::f16) {
2578 E0 = DAG.getZExtOrTrunc(Op: DAG.getBitcast(VT: MVT::i16, V: Elem[0]), DL: dl, VT: MVT::i32);
2579 E1 = DAG.getZExtOrTrunc(Op: DAG.getBitcast(VT: MVT::i16, V: Elem[1]), DL: dl, VT: MVT::i32);
2580 } else {
2581 E0 = Elem[0];
2582 E1 = Elem[1];
2583 }
2584 SDValue N = getInstr(MachineOpc: Hexagon::A2_combine_ll, dl, Ty: MVT::i32, Ops: {E1, E0}, DAG);
2585 return DAG.getBitcast(VT: VecTy, V: N);
2586 }
2587
2588 if (ElemTy == MVT::i8) {
2589 // First try generating a constant.
2590 if (AllConst) {
2591 uint32_t V = (Consts[0]->getZExtValue() & 0xFF) |
2592 (Consts[1]->getZExtValue() & 0xFF) << 8 |
2593 (Consts[2]->getZExtValue() & 0xFF) << 16 |
2594 Consts[3]->getZExtValue() << 24;
2595 return DAG.getBitcast(VT: MVT::v4i8, V: DAG.getConstant(Val: V, DL: dl, VT: MVT::i32));
2596 }
2597
2598 // Then try splat.
2599 bool IsSplat = true;
2600 for (unsigned i = First+1; i != Num; ++i) {
2601 if (Elem[i] == Elem[First] || isUndef(Op: Elem[i]))
2602 continue;
2603 IsSplat = false;
2604 break;
2605 }
2606 if (IsSplat) {
2607 // Legalize the operand of SPLAT_VECTOR.
2608 SDValue Ext = DAG.getZExtOrTrunc(Op: Elem[First], DL: dl, VT: MVT::i32);
2609 return DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: VecTy, Operand: Ext);
2610 }
2611
2612 // Generate
2613 // (zxtb(Elem[0]) | (zxtb(Elem[1]) << 8)) |
2614 // (zxtb(Elem[2]) | (zxtb(Elem[3]) << 8)) << 16
2615 assert(Elem.size() == 4);
2616 SDValue Vs[4];
2617 for (unsigned i = 0; i != 4; ++i) {
2618 Vs[i] = DAG.getZExtOrTrunc(Op: Elem[i], DL: dl, VT: MVT::i32);
2619 Vs[i] = DAG.getZeroExtendInReg(Op: Vs[i], DL: dl, VT: MVT::i8);
2620 }
2621 SDValue S8 = DAG.getConstant(Val: 8, DL: dl, VT: MVT::i32);
2622 SDValue T0 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: MVT::i32, Ops: {Vs[1], S8});
2623 SDValue T1 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: MVT::i32, Ops: {Vs[3], S8});
2624 SDValue B0 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: MVT::i32, Ops: {Vs[0], T0});
2625 SDValue B1 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: MVT::i32, Ops: {Vs[2], T1});
2626
2627 SDValue R = getInstr(MachineOpc: Hexagon::A2_combine_ll, dl, Ty: MVT::i32, Ops: {B1, B0}, DAG);
2628 return DAG.getBitcast(VT: MVT::v4i8, V: R);
2629 }
2630
2631#ifndef NDEBUG
2632 dbgs() << "VecTy: " << VecTy << '\n';
2633#endif
2634 llvm_unreachable("Unexpected vector element type");
2635}
2636
2637SDValue
2638HexagonTargetLowering::buildVector64(ArrayRef<SDValue> Elem, const SDLoc &dl,
2639 MVT VecTy, SelectionDAG &DAG) const {
2640 MVT ElemTy = VecTy.getVectorElementType();
2641 assert(VecTy.getVectorNumElements() == Elem.size());
2642
2643 SmallVector<ConstantInt*,8> Consts(Elem.size());
2644 bool AllConst = getBuildVectorConstInts(Values: Elem, VecTy, DAG, Consts);
2645
2646 unsigned First, Num = Elem.size();
2647 for (First = 0; First != Num; ++First) {
2648 if (!isUndef(Op: Elem[First]))
2649 break;
2650 }
2651 if (First == Num)
2652 return DAG.getUNDEF(VT: VecTy);
2653
2654 if (AllConst &&
2655 llvm::all_of(Range&: Consts, P: [](ConstantInt *CI) { return CI->isZero(); }))
2656 return getZero(dl, Ty: VecTy, DAG);
2657
2658 // First try splat if possible.
2659 if (ElemTy == MVT::i16 || ElemTy == MVT::f16) {
2660 bool IsSplat = true;
2661 for (unsigned i = First+1; i != Num; ++i) {
2662 if (Elem[i] == Elem[First] || isUndef(Op: Elem[i]))
2663 continue;
2664 IsSplat = false;
2665 break;
2666 }
2667 if (IsSplat) {
2668 // Legalize the operand of SPLAT_VECTOR
2669 SDValue S = ElemTy == MVT::f16 ? DAG.getBitcast(VT: MVT::i16, V: Elem[First])
2670 : Elem[First];
2671 SDValue Ext = DAG.getZExtOrTrunc(Op: S, DL: dl, VT: MVT::i32);
2672 return DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: VecTy, Operand: Ext);
2673 }
2674 }
2675
2676 // Then try constant.
2677 if (AllConst) {
2678 uint64_t Val = 0;
2679 unsigned W = ElemTy.getSizeInBits();
2680 uint64_t Mask = (1ull << W) - 1;
2681 for (unsigned i = 0; i != Num; ++i)
2682 Val = (Val << W) | (Consts[Num-1-i]->getZExtValue() & Mask);
2683 SDValue V0 = DAG.getConstant(Val, DL: dl, VT: MVT::i64);
2684 return DAG.getBitcast(VT: VecTy, V: V0);
2685 }
2686
2687 // Build two 32-bit vectors and concatenate.
2688 MVT HalfTy = MVT::getVectorVT(VT: ElemTy, NumElements: Num/2);
2689 SDValue L = (ElemTy == MVT::i32)
2690 ? Elem[0]
2691 : buildVector32(Elem: Elem.take_front(N: Num/2), dl, VecTy: HalfTy, DAG);
2692 SDValue H = (ElemTy == MVT::i32)
2693 ? Elem[1]
2694 : buildVector32(Elem: Elem.drop_front(N: Num/2), dl, VecTy: HalfTy, DAG);
2695 return getCombine(Hi: H, Lo: L, dl, ResTy: VecTy, DAG);
2696}
2697
2698SDValue
2699HexagonTargetLowering::extractVector(SDValue VecV, SDValue IdxV,
2700 const SDLoc &dl, MVT ValTy, MVT ResTy,
2701 SelectionDAG &DAG) const {
2702 MVT VecTy = ty(Op: VecV);
2703 assert(!ValTy.isVector() ||
2704 VecTy.getVectorElementType() == ValTy.getVectorElementType());
2705 if (VecTy.getVectorElementType() == MVT::i1)
2706 return extractVectorPred(VecV, IdxV, dl, ValTy, ResTy, DAG);
2707
2708 unsigned VecWidth = VecTy.getSizeInBits();
2709 unsigned ValWidth = ValTy.getSizeInBits();
2710 unsigned ElemWidth = VecTy.getVectorElementType().getSizeInBits();
2711 assert((VecWidth % ElemWidth) == 0);
2712 assert(VecWidth == 32 || VecWidth == 64);
2713
2714 // Cast everything to scalar integer types.
2715 MVT ScalarTy = tyScalar(Ty: VecTy);
2716 VecV = DAG.getBitcast(VT: ScalarTy, V: VecV);
2717
2718 SDValue WidthV = DAG.getConstant(Val: ValWidth, DL: dl, VT: MVT::i32);
2719 SDValue ExtV;
2720
2721 if (auto *IdxN = dyn_cast<ConstantSDNode>(Val&: IdxV)) {
2722 unsigned Off = IdxN->getZExtValue() * ElemWidth;
2723 if (VecWidth == 64 && ValWidth == 32) {
2724 assert(Off == 0 || Off == 32);
2725 ExtV = Off == 0 ? LoHalf(V: VecV, DAG) : HiHalf(V: VecV, DAG);
2726 } else if (Off == 0 && (ValWidth % 8) == 0) {
2727 ExtV = DAG.getZeroExtendInReg(Op: VecV, DL: dl, VT: tyScalar(Ty: ValTy));
2728 } else {
2729 SDValue OffV = DAG.getConstant(Val: Off, DL: dl, VT: MVT::i32);
2730 // The return type of EXTRACTU must be the same as the type of the
2731 // input vector.
2732 ExtV = DAG.getNode(Opcode: HexagonISD::EXTRACTU, DL: dl, VT: ScalarTy,
2733 Ops: {VecV, WidthV, OffV});
2734 }
2735 } else {
2736 if (ty(Op: IdxV) != MVT::i32)
2737 IdxV = DAG.getZExtOrTrunc(Op: IdxV, DL: dl, VT: MVT::i32);
2738 SDValue OffV = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MVT::i32, N1: IdxV,
2739 N2: DAG.getConstant(Val: ElemWidth, DL: dl, VT: MVT::i32));
2740 ExtV = DAG.getNode(Opcode: HexagonISD::EXTRACTU, DL: dl, VT: ScalarTy,
2741 Ops: {VecV, WidthV, OffV});
2742 }
2743
2744 // Cast ExtV to the requested result type.
2745 ExtV = DAG.getZExtOrTrunc(Op: ExtV, DL: dl, VT: tyScalar(Ty: ResTy));
2746 ExtV = DAG.getBitcast(VT: ResTy, V: ExtV);
2747 return ExtV;
2748}
2749
2750SDValue
2751HexagonTargetLowering::extractVectorPred(SDValue VecV, SDValue IdxV,
2752 const SDLoc &dl, MVT ValTy, MVT ResTy,
2753 SelectionDAG &DAG) const {
2754 // Special case for v{8,4,2}i1 (the only boolean vectors legal in Hexagon
2755 // without any coprocessors).
2756 MVT VecTy = ty(Op: VecV);
2757 unsigned VecWidth = VecTy.getSizeInBits();
2758 unsigned ValWidth = ValTy.getSizeInBits();
2759 assert(VecWidth == VecTy.getVectorNumElements() &&
2760 "Vector elements should equal vector width size");
2761 assert(VecWidth == 8 || VecWidth == 4 || VecWidth == 2);
2762
2763 // Check if this is an extract of the lowest bit.
2764 if (isNullConstant(V: IdxV) && ValTy.getSizeInBits() == 1) {
2765 // Extracting the lowest bit is a no-op, but it changes the type,
2766 // so it must be kept as an operation to avoid errors related to
2767 // type mismatches.
2768 return DAG.getNode(Opcode: HexagonISD::TYPECAST, DL: dl, VT: MVT::i1, Operand: VecV);
2769 }
2770
2771 // If the value extracted is a single bit, use tstbit.
2772 if (ValWidth == 1) {
2773 SDValue A0 = getInstr(MachineOpc: Hexagon::C2_tfrpr, dl, Ty: MVT::i32, Ops: {VecV}, DAG);
2774 SDValue M0 = DAG.getConstant(Val: 8 / VecWidth, DL: dl, VT: MVT::i32);
2775 SDValue I0 = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MVT::i32, N1: IdxV, N2: M0);
2776 return DAG.getNode(Opcode: HexagonISD::TSTBIT, DL: dl, VT: MVT::i1, N1: A0, N2: I0);
2777 }
2778
2779 // Each bool vector (v2i1, v4i1, v8i1) always occupies 8 bits in
2780 // a predicate register. The elements of the vector are repeated
2781 // in the register (if necessary) so that the total number is 8.
2782 // The extracted subvector will need to be expanded in such a way.
2783 unsigned Scale = VecWidth / ValWidth;
2784
2785 // Generate (p2d VecV) >> 8*Idx to move the interesting bytes to
2786 // position 0.
2787 assert(ty(IdxV) == MVT::i32);
2788 unsigned VecRep = 8 / VecWidth;
2789 SDValue S0 = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MVT::i32, N1: IdxV,
2790 N2: DAG.getConstant(Val: 8*VecRep, DL: dl, VT: MVT::i32));
2791 SDValue T0 = DAG.getNode(Opcode: HexagonISD::P2D, DL: dl, VT: MVT::i64, Operand: VecV);
2792 SDValue T1 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i64, N1: T0, N2: S0);
2793 while (Scale > 1) {
2794 // The longest possible subvector is at most 32 bits, so it is always
2795 // contained in the low subregister.
2796 T1 = LoHalf(V: T1, DAG);
2797 T1 = expandPredicate(Vec32: T1, dl, DAG);
2798 Scale /= 2;
2799 }
2800
2801 return DAG.getNode(Opcode: HexagonISD::D2P, DL: dl, VT: ResTy, Operand: T1);
2802}
2803
2804SDValue
2805HexagonTargetLowering::insertVector(SDValue VecV, SDValue ValV, SDValue IdxV,
2806 const SDLoc &dl, MVT ValTy,
2807 SelectionDAG &DAG) const {
2808 MVT VecTy = ty(Op: VecV);
2809 if (VecTy.getVectorElementType() == MVT::i1)
2810 return insertVectorPred(VecV, ValV, IdxV, dl, ValTy, DAG);
2811
2812 unsigned VecWidth = VecTy.getSizeInBits();
2813 unsigned ValWidth = ValTy.getSizeInBits();
2814 assert(VecWidth == 32 || VecWidth == 64);
2815 assert((VecWidth % ValWidth) == 0);
2816
2817 // Cast everything to scalar integer types.
2818 MVT ScalarTy = MVT::getIntegerVT(BitWidth: VecWidth);
2819 // The actual type of ValV may be different than ValTy (which is related
2820 // to the vector type).
2821 unsigned VW = ty(Op: ValV).getSizeInBits();
2822 ValV = DAG.getBitcast(VT: MVT::getIntegerVT(BitWidth: VW), V: ValV);
2823 VecV = DAG.getBitcast(VT: ScalarTy, V: VecV);
2824 if (VW != VecWidth)
2825 ValV = DAG.getAnyExtOrTrunc(Op: ValV, DL: dl, VT: ScalarTy);
2826
2827 SDValue WidthV = DAG.getConstant(Val: ValWidth, DL: dl, VT: MVT::i32);
2828 SDValue InsV;
2829
2830 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: IdxV)) {
2831 unsigned W = C->getZExtValue() * ValWidth;
2832 SDValue OffV = DAG.getConstant(Val: W, DL: dl, VT: MVT::i32);
2833 InsV = DAG.getNode(Opcode: HexagonISD::INSERT, DL: dl, VT: ScalarTy,
2834 Ops: {VecV, ValV, WidthV, OffV});
2835 } else {
2836 if (ty(Op: IdxV) != MVT::i32)
2837 IdxV = DAG.getZExtOrTrunc(Op: IdxV, DL: dl, VT: MVT::i32);
2838 SDValue OffV = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MVT::i32, N1: IdxV, N2: WidthV);
2839 InsV = DAG.getNode(Opcode: HexagonISD::INSERT, DL: dl, VT: ScalarTy,
2840 Ops: {VecV, ValV, WidthV, OffV});
2841 }
2842
2843 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: VecTy, Operand: InsV);
2844}
2845
2846SDValue
2847HexagonTargetLowering::insertVectorPred(SDValue VecV, SDValue ValV,
2848 SDValue IdxV, const SDLoc &dl,
2849 MVT ValTy, SelectionDAG &DAG) const {
2850 MVT VecTy = ty(Op: VecV);
2851 unsigned VecLen = VecTy.getVectorNumElements();
2852
2853 if (ValTy == MVT::i1) {
2854 SDValue ToReg = getInstr(MachineOpc: Hexagon::C2_tfrpr, dl, Ty: MVT::i32, Ops: {VecV}, DAG);
2855 SDValue Ext = DAG.getSExtOrTrunc(Op: ValV, DL: dl, VT: MVT::i32);
2856 SDValue Width = DAG.getConstant(Val: 8 / VecLen, DL: dl, VT: MVT::i32);
2857 SDValue Idx = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MVT::i32, N1: IdxV, N2: Width);
2858 SDValue Ins =
2859 DAG.getNode(Opcode: HexagonISD::INSERT, DL: dl, VT: MVT::i32, Ops: {ToReg, Ext, Width, Idx});
2860 return getInstr(MachineOpc: Hexagon::C2_tfrrp, dl, Ty: VecTy, Ops: {Ins}, DAG);
2861 }
2862
2863 assert(ValTy.getVectorElementType() == MVT::i1);
2864 SDValue ValR = ValTy.isVector()
2865 ? DAG.getNode(Opcode: HexagonISD::P2D, DL: dl, VT: MVT::i64, Operand: ValV)
2866 : DAG.getSExtOrTrunc(Op: ValV, DL: dl, VT: MVT::i64);
2867
2868 unsigned Scale = VecLen / ValTy.getVectorNumElements();
2869 assert(Scale > 1);
2870
2871 for (unsigned R = Scale; R > 1; R /= 2) {
2872 ValR = contractPredicate(Vec64: ValR, dl, DAG);
2873 ValR = getCombine(Hi: DAG.getUNDEF(VT: MVT::i32), Lo: ValR, dl, ResTy: MVT::i64, DAG);
2874 }
2875
2876 SDValue Width = DAG.getConstant(Val: 64 / Scale, DL: dl, VT: MVT::i32);
2877 SDValue Idx = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MVT::i32, N1: IdxV, N2: Width);
2878 SDValue VecR = DAG.getNode(Opcode: HexagonISD::P2D, DL: dl, VT: MVT::i64, Operand: VecV);
2879 SDValue Ins =
2880 DAG.getNode(Opcode: HexagonISD::INSERT, DL: dl, VT: MVT::i64, Ops: {VecR, ValR, Width, Idx});
2881 return DAG.getNode(Opcode: HexagonISD::D2P, DL: dl, VT: VecTy, Operand: Ins);
2882}
2883
2884SDValue
2885HexagonTargetLowering::expandPredicate(SDValue Vec32, const SDLoc &dl,
2886 SelectionDAG &DAG) const {
2887 assert(ty(Vec32).getSizeInBits() == 32);
2888 if (isUndef(Op: Vec32))
2889 return DAG.getUNDEF(VT: MVT::i64);
2890 SDValue P = DAG.getBitcast(VT: MVT::v4i8, V: Vec32);
2891 SDValue X = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: MVT::v4i16, Operand: P);
2892 return DAG.getBitcast(VT: MVT::i64, V: X);
2893}
2894
2895SDValue
2896HexagonTargetLowering::contractPredicate(SDValue Vec64, const SDLoc &dl,
2897 SelectionDAG &DAG) const {
2898 assert(ty(Vec64).getSizeInBits() == 64);
2899 if (isUndef(Op: Vec64))
2900 return DAG.getUNDEF(VT: MVT::i32);
2901 // Collect even bytes:
2902 SDValue A = DAG.getBitcast(VT: MVT::v8i8, V: Vec64);
2903 SDValue S = DAG.getVectorShuffle(VT: MVT::v8i8, dl, N1: A, N2: DAG.getUNDEF(VT: MVT::v8i8),
2904 Mask: {0, 2, 4, 6, 1, 3, 5, 7});
2905 return extractVector(VecV: S, IdxV: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32), dl, ValTy: MVT::v4i8,
2906 ResTy: MVT::i32, DAG);
2907}
2908
2909SDValue
2910HexagonTargetLowering::getZero(const SDLoc &dl, MVT Ty, SelectionDAG &DAG)
2911 const {
2912 if (Ty.isVector()) {
2913 unsigned W = Ty.getSizeInBits();
2914 if (W <= 64)
2915 return DAG.getBitcast(VT: Ty, V: DAG.getConstant(Val: 0, DL: dl, VT: MVT::getIntegerVT(BitWidth: W)));
2916 return DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: Ty, Operand: getZero(dl, Ty: MVT::i32, DAG));
2917 }
2918
2919 if (Ty.isInteger())
2920 return DAG.getConstant(Val: 0, DL: dl, VT: Ty);
2921 if (Ty.isFloatingPoint())
2922 return DAG.getConstantFP(Val: 0.0, DL: dl, VT: Ty);
2923 llvm_unreachable("Invalid type for zero");
2924}
2925
2926SDValue
2927HexagonTargetLowering::appendUndef(SDValue Val, MVT ResTy, SelectionDAG &DAG)
2928 const {
2929 MVT ValTy = ty(Op: Val);
2930 assert(ValTy.getVectorElementType() == ResTy.getVectorElementType());
2931
2932 unsigned ValLen = ValTy.getVectorNumElements();
2933 unsigned ResLen = ResTy.getVectorNumElements();
2934 if (ValLen == ResLen)
2935 return Val;
2936
2937 const SDLoc &dl(Val);
2938 assert(ValLen < ResLen);
2939 assert(ResLen % ValLen == 0);
2940
2941 SmallVector<SDValue, 4> Concats = {Val};
2942 for (unsigned i = 1, e = ResLen / ValLen; i < e; ++i)
2943 Concats.push_back(Elt: DAG.getUNDEF(VT: ValTy));
2944
2945 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResTy, Ops: Concats);
2946}
2947
2948SDValue
2949HexagonTargetLowering::getCombine(SDValue Hi, SDValue Lo, const SDLoc &dl,
2950 MVT ResTy, SelectionDAG &DAG) const {
2951 MVT ElemTy = ty(Op: Hi);
2952 assert(ElemTy == ty(Lo));
2953
2954 if (!ElemTy.isVector()) {
2955 assert(ElemTy.isScalarInteger());
2956 MVT PairTy = ElemTy.widenIntegerElementType();
2957 SDValue Pair = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: PairTy, N1: Lo, N2: Hi);
2958 return DAG.getBitcast(VT: ResTy, V: Pair);
2959 }
2960
2961 unsigned Width = ElemTy.getSizeInBits();
2962 MVT IntTy = MVT::getIntegerVT(BitWidth: Width);
2963 SDValue Pair =
2964 DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: IntTy.widenIntegerElementType(),
2965 Ops: {DAG.getBitcast(VT: IntTy, V: Lo), DAG.getBitcast(VT: IntTy, V: Hi)});
2966 return DAG.getBitcast(VT: ResTy, V: Pair);
2967}
2968
2969SDValue
2970HexagonTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
2971 MVT VecTy = ty(Op);
2972 unsigned BW = VecTy.getSizeInBits();
2973 const SDLoc &dl(Op);
2974 SmallVector<SDValue,8> Ops;
2975 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i)
2976 Ops.push_back(Elt: Op.getOperand(i));
2977
2978 if (BW == 32)
2979 return buildVector32(Elem: Ops, dl, VecTy, DAG);
2980 if (BW == 64)
2981 return buildVector64(Elem: Ops, dl, VecTy, DAG);
2982
2983 if (VecTy == MVT::v8i1 || VecTy == MVT::v4i1 || VecTy == MVT::v2i1) {
2984 // Check if this is a special case or all-0 or all-1.
2985 bool All0 = true, All1 = true;
2986 for (SDValue P : Ops) {
2987 auto *CN = dyn_cast<ConstantSDNode>(Val: P.getNode());
2988 if (CN == nullptr) {
2989 All0 = All1 = false;
2990 break;
2991 }
2992 uint32_t C = CN->getZExtValue();
2993 All0 &= (C == 0);
2994 All1 &= (C == 1);
2995 }
2996 if (All0)
2997 return DAG.getNode(Opcode: HexagonISD::PFALSE, DL: dl, VT: VecTy);
2998 if (All1)
2999 return DAG.getNode(Opcode: HexagonISD::PTRUE, DL: dl, VT: VecTy);
3000
3001 // For each i1 element in the resulting predicate register, put 1
3002 // shifted by the index of the element into a general-purpose register,
3003 // then or them together and transfer it back into a predicate register.
3004 SDValue Rs[8];
3005 SDValue Z = getZero(dl, Ty: MVT::i32, DAG);
3006 // Always produce 8 bits, repeat inputs if necessary.
3007 unsigned Rep = 8 / VecTy.getVectorNumElements();
3008 for (unsigned i = 0; i != 8; ++i) {
3009 SDValue S = DAG.getConstant(Val: 1ull << i, DL: dl, VT: MVT::i32);
3010 Rs[i] = DAG.getSelect(DL: dl, VT: MVT::i32, Cond: Ops[i/Rep], LHS: S, RHS: Z);
3011 }
3012 for (ArrayRef<SDValue> A(Rs); A.size() != 1; A = A.drop_back(N: A.size()/2)) {
3013 for (unsigned i = 0, e = A.size()/2; i != e; ++i)
3014 Rs[i] = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: MVT::i32, N1: Rs[2*i], N2: Rs[2*i+1]);
3015 }
3016 // Move the value directly to a predicate register.
3017 return getInstr(MachineOpc: Hexagon::C2_tfrrp, dl, Ty: VecTy, Ops: {Rs[0]}, DAG);
3018 }
3019
3020 return SDValue();
3021}
3022
3023SDValue
3024HexagonTargetLowering::LowerCONCAT_VECTORS(SDValue Op,
3025 SelectionDAG &DAG) const {
3026 MVT VecTy = ty(Op);
3027 const SDLoc &dl(Op);
3028 if (VecTy.getSizeInBits() == 64) {
3029 assert(Op.getNumOperands() == 2);
3030 return getCombine(Hi: Op.getOperand(i: 1), Lo: Op.getOperand(i: 0), dl, ResTy: VecTy, DAG);
3031 }
3032
3033 MVT ElemTy = VecTy.getVectorElementType();
3034 if (ElemTy == MVT::i1) {
3035 assert(VecTy == MVT::v2i1 || VecTy == MVT::v4i1 || VecTy == MVT::v8i1);
3036 MVT OpTy = ty(Op: Op.getOperand(i: 0));
3037 // Scale is how many times the operands need to be contracted to match
3038 // the representation in the target register.
3039 unsigned Scale = VecTy.getVectorNumElements() / OpTy.getVectorNumElements();
3040 assert(Scale == Op.getNumOperands() && Scale > 1);
3041
3042 // First, convert all bool vectors to integers, then generate pairwise
3043 // inserts to form values of doubled length. Up until there are only
3044 // two values left to concatenate, all of these values will fit in a
3045 // 32-bit integer, so keep them as i32 to use 32-bit inserts.
3046 SmallVector<SDValue,4> Words[2];
3047 unsigned IdxW = 0;
3048
3049 for (SDValue P : Op.getNode()->op_values()) {
3050 SDValue W = DAG.getNode(Opcode: HexagonISD::P2D, DL: dl, VT: MVT::i64, Operand: P);
3051 for (unsigned R = Scale; R > 1; R /= 2) {
3052 W = contractPredicate(Vec64: W, dl, DAG);
3053 W = getCombine(Hi: DAG.getUNDEF(VT: MVT::i32), Lo: W, dl, ResTy: MVT::i64, DAG);
3054 }
3055 W = LoHalf(V: W, DAG);
3056 Words[IdxW].push_back(Elt: W);
3057 }
3058
3059 while (Scale > 2) {
3060 SDValue WidthV = DAG.getConstant(Val: 64 / Scale, DL: dl, VT: MVT::i32);
3061 Words[IdxW ^ 1].clear();
3062
3063 for (unsigned i = 0, e = Words[IdxW].size(); i != e; i += 2) {
3064 SDValue W0 = Words[IdxW][i], W1 = Words[IdxW][i+1];
3065 // Insert W1 into W0 right next to the significant bits of W0.
3066 SDValue T = DAG.getNode(Opcode: HexagonISD::INSERT, DL: dl, VT: MVT::i32,
3067 Ops: {W0, W1, WidthV, WidthV});
3068 Words[IdxW ^ 1].push_back(Elt: T);
3069 }
3070 IdxW ^= 1;
3071 Scale /= 2;
3072 }
3073
3074 // At this point there should only be two words left, and Scale should be 2.
3075 assert(Scale == 2 && Words[IdxW].size() == 2);
3076
3077 SDValue WW = getCombine(Hi: Words[IdxW][1], Lo: Words[IdxW][0], dl, ResTy: MVT::i64, DAG);
3078 return DAG.getNode(Opcode: HexagonISD::D2P, DL: dl, VT: VecTy, Operand: WW);
3079 }
3080
3081 return SDValue();
3082}
3083
3084SDValue
3085HexagonTargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
3086 SelectionDAG &DAG) const {
3087 SDValue Vec = Op.getOperand(i: 0);
3088 MVT ElemTy = ty(Op: Vec).getVectorElementType();
3089 return extractVector(VecV: Vec, IdxV: Op.getOperand(i: 1), dl: SDLoc(Op), ValTy: ElemTy, ResTy: ty(Op), DAG);
3090}
3091
3092SDValue
3093HexagonTargetLowering::LowerEXTRACT_SUBVECTOR(SDValue Op,
3094 SelectionDAG &DAG) const {
3095 return extractVector(VecV: Op.getOperand(i: 0), IdxV: Op.getOperand(i: 1), dl: SDLoc(Op),
3096 ValTy: ty(Op), ResTy: ty(Op), DAG);
3097}
3098
3099SDValue
3100HexagonTargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
3101 SelectionDAG &DAG) const {
3102 return insertVector(VecV: Op.getOperand(i: 0), ValV: Op.getOperand(i: 1), IdxV: Op.getOperand(i: 2),
3103 dl: SDLoc(Op), ValTy: ty(Op).getVectorElementType(), DAG);
3104}
3105
3106SDValue
3107HexagonTargetLowering::LowerINSERT_SUBVECTOR(SDValue Op,
3108 SelectionDAG &DAG) const {
3109 SDValue ValV = Op.getOperand(i: 1);
3110 return insertVector(VecV: Op.getOperand(i: 0), ValV, IdxV: Op.getOperand(i: 2),
3111 dl: SDLoc(Op), ValTy: ty(Op: ValV), DAG);
3112}
3113
3114bool
3115HexagonTargetLowering::allowTruncateForTailCall(Type *Ty1, Type *Ty2) const {
3116 // Assuming the caller does not have either a signext or zeroext modifier, and
3117 // only one value is accepted, any reasonable truncation is allowed.
3118 if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy())
3119 return false;
3120
3121 // FIXME: in principle up to 64-bit could be made safe, but it would be very
3122 // fragile at the moment: any support for multiple value returns would be
3123 // liable to disallow tail calls involving i64 -> iN truncation in many cases.
3124 return Ty1->getPrimitiveSizeInBits() <= 32;
3125}
3126
3127SDValue
3128HexagonTargetLowering::LowerLoad(SDValue Op, SelectionDAG &DAG) const {
3129 MVT Ty = ty(Op);
3130 const SDLoc &dl(Op);
3131 LoadSDNode *LN = cast<LoadSDNode>(Val: Op.getNode());
3132 MVT MemTy = LN->getMemoryVT().getSimpleVT();
3133 ISD::LoadExtType ET = LN->getExtensionType();
3134
3135 bool LoadPred = MemTy == MVT::v2i1 || MemTy == MVT::v4i1 || MemTy == MVT::v8i1;
3136 if (LoadPred) {
3137 SDValue NL = DAG.getLoad(
3138 AM: LN->getAddressingMode(), ExtType: ISD::ZEXTLOAD, VT: MVT::i32, dl, Chain: LN->getChain(),
3139 Ptr: LN->getBasePtr(), Offset: LN->getOffset(), PtrInfo: LN->getPointerInfo(),
3140 /*MemoryVT*/ MemVT: MVT::i8, Alignment: LN->getAlign(), MMOFlags: LN->getMemOperand()->getFlags(),
3141 Metadata: MMOMetadata(LN->getAAInfo(), LN->getRanges()));
3142 LN = cast<LoadSDNode>(Val: NL.getNode());
3143 }
3144
3145 Align ClaimAlign = LN->getAlign();
3146 if (!validateConstPtrAlignment(Ptr: LN->getBasePtr(), NeedAlign: ClaimAlign, dl, DAG))
3147 return replaceMemWithUndef(Op, DAG);
3148
3149 // Call LowerUnalignedLoad for all loads, it recognizes loads that
3150 // don't need extra aligning.
3151 SDValue LU = LowerUnalignedLoad(Op: SDValue(LN, 0), DAG);
3152 if (LoadPred) {
3153 SDValue TP = getInstr(MachineOpc: Hexagon::C2_tfrrp, dl, Ty: MemTy, Ops: {LU}, DAG);
3154 if (ET == ISD::SEXTLOAD) {
3155 TP = DAG.getSExtOrTrunc(Op: TP, DL: dl, VT: Ty);
3156 } else if (ET != ISD::NON_EXTLOAD) {
3157 TP = DAG.getZExtOrTrunc(Op: TP, DL: dl, VT: Ty);
3158 }
3159 SDValue Ch = cast<LoadSDNode>(Val: LU.getNode())->getChain();
3160 return DAG.getMergeValues(Ops: {TP, Ch}, dl);
3161 }
3162 return LU;
3163}
3164
3165SDValue
3166HexagonTargetLowering::LowerStore(SDValue Op, SelectionDAG &DAG) const {
3167 const SDLoc &dl(Op);
3168 StoreSDNode *SN = cast<StoreSDNode>(Val: Op.getNode());
3169 SDValue Val = SN->getValue();
3170 MVT Ty = ty(Op: Val);
3171
3172 if (Ty == MVT::v2i1 || Ty == MVT::v4i1 || Ty == MVT::v8i1) {
3173 // Store the exact predicate (all bits).
3174 SDValue TR = getInstr(MachineOpc: Hexagon::C2_tfrpr, dl, Ty: MVT::i32, Ops: {Val}, DAG);
3175 SDValue NS = DAG.getTruncStore(Chain: SN->getChain(), dl, Val: TR, Ptr: SN->getBasePtr(),
3176 SVT: MVT::i8, MMO: SN->getMemOperand());
3177 if (SN->isIndexed()) {
3178 NS = DAG.getIndexedStore(OrigStore: NS, dl, Base: SN->getBasePtr(), Offset: SN->getOffset(),
3179 AM: SN->getAddressingMode());
3180 }
3181 SN = cast<StoreSDNode>(Val: NS.getNode());
3182 }
3183
3184 Align ClaimAlign = SN->getAlign();
3185 if (!validateConstPtrAlignment(Ptr: SN->getBasePtr(), NeedAlign: ClaimAlign, dl, DAG))
3186 return replaceMemWithUndef(Op, DAG);
3187
3188 MVT StoreTy = SN->getMemoryVT().getSimpleVT();
3189 Align NeedAlign = Subtarget.getTypeAlignment(Ty: StoreTy);
3190 if (ClaimAlign < NeedAlign)
3191 return expandUnalignedStore(ST: SN, DAG);
3192 return SDValue(SN, 0);
3193}
3194
3195SDValue
3196HexagonTargetLowering::LowerUnalignedLoad(SDValue Op, SelectionDAG &DAG)
3197 const {
3198 LoadSDNode *LN = cast<LoadSDNode>(Val: Op.getNode());
3199 MVT LoadTy = ty(Op);
3200 unsigned NeedAlign = Subtarget.getTypeAlignment(Ty: LoadTy).value();
3201 unsigned HaveAlign = LN->getAlign().value();
3202 if (HaveAlign >= NeedAlign)
3203 return Op;
3204
3205 const SDLoc &dl(Op);
3206 const DataLayout &DL = DAG.getDataLayout();
3207 LLVMContext &Ctx = *DAG.getContext();
3208
3209 // If the load aligning is disabled or the load can be broken up into two
3210 // smaller legal loads, do the default (target-independent) expansion.
3211 bool DoDefault = false;
3212 // Handle it in the default way if this is an indexed load.
3213 if (!LN->isUnindexed())
3214 DoDefault = true;
3215
3216 if (!AlignLoads) {
3217 if (allowsMemoryAccessForAlignment(Context&: Ctx, DL, VT: LN->getMemoryVT(),
3218 MMO: *LN->getMemOperand()))
3219 return Op;
3220 DoDefault = true;
3221 }
3222 if (!DoDefault && (2 * HaveAlign) == NeedAlign) {
3223 // The PartTy is the equivalent of "getLoadableTypeOfSize(HaveAlign)".
3224 MVT PartTy = HaveAlign <= 8 ? MVT::getIntegerVT(BitWidth: 8 * HaveAlign)
3225 : MVT::getVectorVT(VT: MVT::i8, NumElements: HaveAlign);
3226 DoDefault =
3227 allowsMemoryAccessForAlignment(Context&: Ctx, DL, VT: PartTy, MMO: *LN->getMemOperand());
3228 }
3229 if (DoDefault) {
3230 std::pair<SDValue, SDValue> P = expandUnalignedLoad(LD: LN, DAG);
3231 return DAG.getMergeValues(Ops: {P.first, P.second}, dl);
3232 }
3233
3234 // The code below generates two loads, both aligned as NeedAlign, and
3235 // with the distance of NeedAlign between them. For that to cover the
3236 // bits that need to be loaded (and without overlapping), the size of
3237 // the loads should be equal to NeedAlign. This is true for all loadable
3238 // types, but add an assertion in case something changes in the future.
3239 assert(LoadTy.getSizeInBits() == 8*NeedAlign);
3240
3241 unsigned LoadLen = NeedAlign;
3242 SDValue Base = LN->getBasePtr();
3243 SDValue Chain = LN->getChain();
3244 auto BO = getBaseAndOffset(Addr: Base);
3245 unsigned BaseOpc = BO.first.getOpcode();
3246 if (BaseOpc == HexagonISD::VALIGNADDR && BO.second % LoadLen == 0)
3247 return Op;
3248
3249 if (BO.second % LoadLen != 0) {
3250 BO.first = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::i32, N1: BO.first,
3251 N2: DAG.getConstant(Val: BO.second % LoadLen, DL: dl, VT: MVT::i32));
3252 BO.second -= BO.second % LoadLen;
3253 }
3254 SDValue BaseNoOff = (BaseOpc != HexagonISD::VALIGNADDR)
3255 ? DAG.getNode(Opcode: HexagonISD::VALIGNADDR, DL: dl, VT: MVT::i32, N1: BO.first,
3256 N2: DAG.getConstant(Val: NeedAlign, DL: dl, VT: MVT::i32))
3257 : BO.first;
3258 SDValue Base0 =
3259 DAG.getMemBasePlusOffset(Base: BaseNoOff, Offset: TypeSize::getFixed(ExactSize: BO.second), DL: dl);
3260 SDValue Base1 = DAG.getMemBasePlusOffset(
3261 Base: BaseNoOff, Offset: TypeSize::getFixed(ExactSize: BO.second + LoadLen), DL: dl);
3262
3263 MachineMemOperand *WideMMO = nullptr;
3264 if (MachineMemOperand *MMO = LN->getMemOperand()) {
3265 MachineFunction &MF = DAG.getMachineFunction();
3266 WideMMO = MF.getMachineMemOperand(
3267 PtrInfo: MMO->getPointerInfo(), F: MMO->getFlags(), Size: 2 * LoadLen, BaseAlignment: Align(LoadLen),
3268 Metadata: MMOMetadata(MMO->getAAInfo(), MMO->getRanges()), SSID: MMO->getSyncScopeID(),
3269 Ordering: MMO->getSuccessOrdering(), FailureOrdering: MMO->getFailureOrdering());
3270 }
3271
3272 SDValue Load0 = DAG.getLoad(VT: LoadTy, dl, Chain, Ptr: Base0, MMO: WideMMO);
3273 SDValue Load1 = DAG.getLoad(VT: LoadTy, dl, Chain, Ptr: Base1, MMO: WideMMO);
3274
3275 SDValue Aligned = DAG.getNode(Opcode: HexagonISD::VALIGN, DL: dl, VT: LoadTy,
3276 Ops: {Load1, Load0, BaseNoOff.getOperand(i: 0)});
3277 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other,
3278 N1: Load0.getValue(R: 1), N2: Load1.getValue(R: 1));
3279 SDValue M = DAG.getMergeValues(Ops: {Aligned, NewChain}, dl);
3280 return M;
3281}
3282
3283SDValue
3284HexagonTargetLowering::LowerUAddSubO(SDValue Op, SelectionDAG &DAG) const {
3285 SDValue X = Op.getOperand(i: 0), Y = Op.getOperand(i: 1);
3286 auto *CY = dyn_cast<ConstantSDNode>(Val&: Y);
3287 if (!CY)
3288 return SDValue();
3289
3290 const SDLoc &dl(Op);
3291 SDVTList VTs = Op.getNode()->getVTList();
3292 assert(VTs.NumVTs == 2);
3293 assert(VTs.VTs[1] == MVT::i1);
3294 unsigned Opc = Op.getOpcode();
3295
3296 if (CY) {
3297 uint64_t VY = CY->getZExtValue();
3298 assert(VY != 0 && "This should have been folded");
3299 // X +/- 1
3300 if (VY != 1)
3301 return SDValue();
3302
3303 if (Opc == ISD::UADDO) {
3304 SDValue Op = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: VTs.VTs[0], Ops: {X, Y});
3305 SDValue Ov = DAG.getSetCC(DL: dl, VT: MVT::i1, LHS: Op, RHS: getZero(dl, Ty: ty(Op), DAG),
3306 Cond: ISD::SETEQ);
3307 return DAG.getMergeValues(Ops: {Op, Ov}, dl);
3308 }
3309 if (Opc == ISD::USUBO) {
3310 SDValue Op = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: VTs.VTs[0], Ops: {X, Y});
3311 SDValue Ov = DAG.getSetCC(DL: dl, VT: MVT::i1, LHS: Op,
3312 RHS: DAG.getAllOnesConstant(DL: dl, VT: ty(Op)), Cond: ISD::SETEQ);
3313 return DAG.getMergeValues(Ops: {Op, Ov}, dl);
3314 }
3315 }
3316
3317 return SDValue();
3318}
3319
3320SDValue HexagonTargetLowering::LowerUAddSubOCarry(SDValue Op,
3321 SelectionDAG &DAG) const {
3322 const SDLoc &dl(Op);
3323 unsigned Opc = Op.getOpcode();
3324 SDValue X = Op.getOperand(i: 0), Y = Op.getOperand(i: 1), C = Op.getOperand(i: 2);
3325
3326 if (Opc == ISD::UADDO_CARRY)
3327 return DAG.getNode(Opcode: HexagonISD::ADDC, DL: dl, VTList: Op.getNode()->getVTList(),
3328 Ops: { X, Y, C });
3329
3330 EVT CarryTy = C.getValueType();
3331 SDValue SubC = DAG.getNode(Opcode: HexagonISD::SUBC, DL: dl, VTList: Op.getNode()->getVTList(),
3332 Ops: { X, Y, DAG.getLogicalNOT(DL: dl, Val: C, VT: CarryTy) });
3333 SDValue Out[] = { SubC.getValue(R: 0),
3334 DAG.getLogicalNOT(DL: dl, Val: SubC.getValue(R: 1), VT: CarryTy) };
3335 return DAG.getMergeValues(Ops: Out, dl);
3336}
3337
3338SDValue
3339HexagonTargetLowering::LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const {
3340 SDValue Chain = Op.getOperand(i: 0);
3341 SDValue Offset = Op.getOperand(i: 1);
3342 SDValue Handler = Op.getOperand(i: 2);
3343 SDLoc dl(Op);
3344 auto PtrVT = getPointerTy(DL: DAG.getDataLayout());
3345
3346 // Mark function as containing a call to EH_RETURN.
3347 HexagonMachineFunctionInfo *FuncInfo =
3348 DAG.getMachineFunction().getInfo<HexagonMachineFunctionInfo>();
3349 FuncInfo->setHasEHReturn();
3350
3351 unsigned OffsetReg = Hexagon::R28;
3352
3353 SDValue StoreAddr =
3354 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: DAG.getRegister(Reg: Hexagon::R30, VT: PtrVT),
3355 N2: DAG.getIntPtrConstant(Val: 4, DL: dl));
3356 Chain = DAG.getStore(Chain, dl, Val: Handler, Ptr: StoreAddr, PtrInfo: MachinePointerInfo());
3357 Chain = DAG.getCopyToReg(Chain, dl, Reg: OffsetReg, N: Offset);
3358
3359 // Not needed we already use it as explicit input to EH_RETURN.
3360 // MF.getRegInfo().addLiveOut(OffsetReg);
3361
3362 return DAG.getNode(Opcode: HexagonISD::EH_RETURN, DL: dl, VT: MVT::Other, Operand: Chain);
3363}
3364
3365SDValue
3366HexagonTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
3367 unsigned Opc = Op.getOpcode();
3368 // Handle INLINEASM first.
3369 if (Opc == ISD::INLINEASM || Opc == ISD::INLINEASM_BR)
3370 return LowerINLINEASM(Op, DAG);
3371
3372 if (isHvxOperation(N: Op.getNode(), DAG)) {
3373 // If HVX lowering returns nothing, try the default lowering.
3374 if (SDValue V = LowerHvxOperation(Op, DAG))
3375 return V;
3376 }
3377
3378 switch (Opc) {
3379 default:
3380#ifndef NDEBUG
3381 Op.getNode()->dumpr(&DAG);
3382#endif
3383 llvm_unreachable("Should not custom lower this!");
3384
3385 case ISD::FDIV:
3386 return LowerFDIV(Op, DAG);
3387 case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG);
3388 case ISD::INSERT_SUBVECTOR: return LowerINSERT_SUBVECTOR(Op, DAG);
3389 case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG);
3390 case ISD::EXTRACT_SUBVECTOR: return LowerEXTRACT_SUBVECTOR(Op, DAG);
3391 case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG);
3392 case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG);
3393 case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG);
3394 case ISD::BITCAST: return LowerBITCAST(Op, DAG);
3395 case ISD::LOAD: return LowerLoad(Op, DAG);
3396 case ISD::STORE: return LowerStore(Op, DAG);
3397 case ISD::UADDO:
3398 case ISD::USUBO: return LowerUAddSubO(Op, DAG);
3399 case ISD::UADDO_CARRY:
3400 case ISD::USUBO_CARRY: return LowerUAddSubOCarry(Op, DAG);
3401 case ISD::SRA:
3402 case ISD::SHL:
3403 case ISD::SRL: return LowerVECTOR_SHIFT(Op, DAG);
3404 case ISD::ROTL: return LowerROTL(Op, DAG);
3405 case ISD::ConstantPool: return LowerConstantPool(Op, DAG);
3406 case ISD::JumpTable: return LowerJumpTable(Op, DAG);
3407 case ISD::EH_RETURN: return LowerEH_RETURN(Op, DAG);
3408 case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG);
3409 case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG);
3410 case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG);
3411 case ISD::ATOMIC_FENCE: return LowerATOMIC_FENCE(Op, DAG);
3412 case ISD::GlobalAddress: return LowerGLOBALADDRESS(Op, DAG);
3413 case ISD::BlockAddress: return LowerBlockAddress(Op, DAG);
3414 case ISD::GLOBAL_OFFSET_TABLE: return LowerGLOBAL_OFFSET_TABLE(Op, DAG);
3415 case ISD::VACOPY: return LowerVACOPY(Op, DAG);
3416 case ISD::VASTART: return LowerVASTART(Op, DAG);
3417 case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG);
3418 case ISD::SETCC: return LowerSETCC(Op, DAG);
3419 case ISD::VSELECT: return LowerVSELECT(Op, DAG);
3420 case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG);
3421 case ISD::INTRINSIC_VOID: return LowerINTRINSIC_VOID(Op, DAG);
3422 case ISD::PREFETCH:
3423 return LowerPREFETCH(Op, DAG);
3424 case ISD::FMAXIMUM:
3425 case ISD::FMINIMUM:
3426 return LowerFMINFMAX(Op, DAG);
3427 break;
3428 }
3429
3430 return SDValue();
3431}
3432
3433void
3434HexagonTargetLowering::LowerOperationWrapper(SDNode *N,
3435 SmallVectorImpl<SDValue> &Results,
3436 SelectionDAG &DAG) const {
3437 if (isHvxOperation(N, DAG)) {
3438 LowerHvxOperationWrapper(N, Results, DAG);
3439 if (!Results.empty())
3440 return;
3441 }
3442
3443 SDValue Op(N, 0);
3444 unsigned Opc = N->getOpcode();
3445
3446 switch (Opc) {
3447 case HexagonISD::SSAT:
3448 case HexagonISD::USAT:
3449 Results.push_back(Elt: opJoin(Ops: SplitVectorOp(Op, DAG), dl: SDLoc(Op), DAG));
3450 break;
3451 case ISD::STORE:
3452 // We are only custom-lowering stores to verify the alignment of the
3453 // address if it is a compile-time constant. Since a store can be
3454 // modified during type-legalization (the value being stored may need
3455 // legalization), return empty Results here to indicate that we don't
3456 // really make any changes in the custom lowering.
3457 return;
3458 default:
3459 TargetLowering::LowerOperationWrapper(N, Results, DAG);
3460 break;
3461 }
3462}
3463
3464void
3465HexagonTargetLowering::ReplaceNodeResults(SDNode *N,
3466 SmallVectorImpl<SDValue> &Results,
3467 SelectionDAG &DAG) const {
3468 if (isHvxOperation(N, DAG)) {
3469 ReplaceHvxNodeResults(N, Results, DAG);
3470 if (!Results.empty())
3471 return;
3472 }
3473
3474 const SDLoc &dl(N);
3475 switch (N->getOpcode()) {
3476 case ISD::SRL:
3477 case ISD::SRA:
3478 case ISD::SHL:
3479 return;
3480 case ISD::BITCAST:
3481 // Handle a bitcast from v8i1 to i8.
3482 if (N->getValueType(ResNo: 0) == MVT::i8) {
3483 if (N->getOperand(Num: 0).getValueType() == MVT::v8i1) {
3484 SDValue P = getInstr(MachineOpc: Hexagon::C2_tfrpr, dl, Ty: MVT::i32,
3485 Ops: N->getOperand(Num: 0), DAG);
3486 SDValue T = DAG.getAnyExtOrTrunc(Op: P, DL: dl, VT: MVT::i8);
3487 Results.push_back(Elt: T);
3488 }
3489 }
3490 break;
3491 }
3492}
3493
3494SDValue
3495HexagonTargetLowering::PerformDAGCombine(SDNode *N,
3496 DAGCombinerInfo &DCI) const {
3497 SDValue Op(N, 0);
3498 const SDLoc &dl(Op);
3499 unsigned Opc = Op.getOpcode();
3500
3501 // Combining transformations applicable for arbitrary vector sizes.
3502 if (DCI.isBeforeLegalizeOps()) {
3503 switch (Opc) {
3504 case ISD::VECREDUCE_ADD:
3505 if (SDValue V = splitVecReduceAdd(N, DAG&: DCI.DAG))
3506 return V;
3507 if (SDValue V = expandVecReduceAdd(N, DAG&: DCI.DAG))
3508 return V;
3509 return SDValue();
3510 case ISD::PARTIAL_REDUCE_SMLA:
3511 case ISD::PARTIAL_REDUCE_UMLA:
3512 case ISD::PARTIAL_REDUCE_SUMLA:
3513 if (SDValue V = splitExtendingPartialReduceMLA(N, DAG&: DCI.DAG))
3514 return V;
3515 return SDValue();
3516 }
3517 } else {
3518 switch (Opc) {
3519 case ISD::VSELECT: {
3520 // (vselect (xor x, ptrue), v0, v1) -> (vselect x, v1, v0)
3521 SDValue Cond = Op.getOperand(i: 0);
3522 if (Cond->getOpcode() == ISD::XOR) {
3523 SDValue C0 = Cond.getOperand(i: 0), C1 = Cond.getOperand(i: 1);
3524 if (C1->getOpcode() == HexagonISD::PTRUE) {
3525 SDValue VSel = DCI.DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: ty(Op), N1: C0,
3526 N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 1));
3527 return VSel;
3528 }
3529 }
3530 return SDValue();
3531 }
3532 }
3533 }
3534
3535 if (isHvxOperation(N, DAG&: DCI.DAG)) {
3536 if (SDValue V = PerformHvxDAGCombine(N, DCI))
3537 return V;
3538 return SDValue();
3539 }
3540
3541 if (Opc == ISD::TRUNCATE) {
3542 SDValue Op0 = Op.getOperand(i: 0);
3543 // fold (truncate (build pair x, y)) -> (truncate x) or x
3544 if (Op0.getOpcode() == ISD::BUILD_PAIR) {
3545 EVT TruncTy = Op.getValueType();
3546 SDValue Elem0 = Op0.getOperand(i: 0);
3547 // if we match the low element of the pair, just return it.
3548 if (Elem0.getValueType() == TruncTy)
3549 return Elem0;
3550 // otherwise, if the low part is still too large, apply the truncate.
3551 if (Elem0.getValueType().bitsGT(VT: TruncTy))
3552 return DCI.DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: TruncTy, Operand: Elem0);
3553 }
3554 }
3555
3556 if (DCI.isBeforeLegalizeOps())
3557 return SDValue();
3558
3559 switch (Opc) {
3560 case HexagonISD::P2D: {
3561 SDValue P = Op.getOperand(i: 0);
3562 switch (P.getOpcode()) {
3563 case HexagonISD::PTRUE:
3564 return DCI.DAG.getAllOnesConstant(DL: dl, VT: ty(Op));
3565 case HexagonISD::PFALSE:
3566 return getZero(dl, Ty: ty(Op), DAG&: DCI.DAG);
3567 default:
3568 break;
3569 }
3570 break;
3571 }
3572 case ISD::TRUNCATE: {
3573 SDValue Op0 = Op.getOperand(i: 0);
3574 // fold (truncate (build pair x, y)) -> (truncate x) or x
3575 if (Op0.getOpcode() == ISD::BUILD_PAIR) {
3576 MVT TruncTy = ty(Op);
3577 SDValue Elem0 = Op0.getOperand(i: 0);
3578 // if we match the low element of the pair, just return it.
3579 if (ty(Op: Elem0) == TruncTy)
3580 return Elem0;
3581 // otherwise, if the low part is still too large, apply the truncate.
3582 if (ty(Op: Elem0).bitsGT(VT: TruncTy))
3583 return DCI.DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: TruncTy, Operand: Elem0);
3584 }
3585 break;
3586 }
3587 case ISD::OR: {
3588 // fold (or (shl xx, s), (zext y)) -> (COMBINE (shl xx, s-32), y)
3589 // if s >= 32
3590 auto fold0 = [&, this](SDValue Op) {
3591 if (ty(Op) != MVT::i64)
3592 return SDValue();
3593 SDValue Shl = Op.getOperand(i: 0);
3594 SDValue Zxt = Op.getOperand(i: 1);
3595 if (Shl.getOpcode() != ISD::SHL)
3596 std::swap(a&: Shl, b&: Zxt);
3597
3598 if (Shl.getOpcode() != ISD::SHL || Zxt.getOpcode() != ISD::ZERO_EXTEND)
3599 return SDValue();
3600
3601 SDValue Z = Zxt.getOperand(i: 0);
3602 auto *Amt = dyn_cast<ConstantSDNode>(Val: Shl.getOperand(i: 1));
3603 if (Amt && Amt->getZExtValue() >= 32 && ty(Op: Z).getSizeInBits() <= 32) {
3604 unsigned A = Amt->getZExtValue();
3605 SDValue S = Shl.getOperand(i: 0);
3606 SDValue T0 = DCI.DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: ty(Op: S), N1: S,
3607 N2: DCI.DAG.getConstant(Val: A - 32, DL: dl, VT: MVT::i32));
3608 SDValue T1 = DCI.DAG.getZExtOrTrunc(Op: T0, DL: dl, VT: MVT::i32);
3609 SDValue T2 = DCI.DAG.getZExtOrTrunc(Op: Z, DL: dl, VT: MVT::i32);
3610 return DCI.DAG.getNode(Opcode: HexagonISD::COMBINE, DL: dl, VT: MVT::i64, Ops: {T1, T2});
3611 }
3612 return SDValue();
3613 };
3614
3615 if (SDValue R = fold0(Op))
3616 return R;
3617 break;
3618 }
3619 }
3620
3621 return SDValue();
3622}
3623
3624/// Returns relocation base for the given PIC jumptable.
3625SDValue
3626HexagonTargetLowering::getPICJumpTableRelocBase(SDValue Table,
3627 SelectionDAG &DAG) const {
3628 int Idx = cast<JumpTableSDNode>(Val&: Table)->getIndex();
3629 EVT VT = Table.getValueType();
3630 SDValue T = DAG.getTargetJumpTable(JTI: Idx, VT, TargetFlags: HexagonII::MO_PCREL);
3631 return DAG.getNode(Opcode: HexagonISD::AT_PCREL, DL: SDLoc(Table), VT, Operand: T);
3632}
3633
3634//===----------------------------------------------------------------------===//
3635// Inline Assembly Support
3636//===----------------------------------------------------------------------===//
3637
3638TargetLowering::ConstraintType
3639HexagonTargetLowering::getConstraintType(StringRef Constraint) const {
3640 if (Constraint.size() == 1) {
3641 switch (Constraint[0]) {
3642 case 'q':
3643 case 'v':
3644 if (Subtarget.useHVXOps())
3645 return C_RegisterClass;
3646 break;
3647 case 'a':
3648 return C_RegisterClass;
3649 default:
3650 break;
3651 }
3652 }
3653 return TargetLowering::getConstraintType(Constraint);
3654}
3655
3656std::pair<unsigned, const TargetRegisterClass*>
3657HexagonTargetLowering::getRegForInlineAsmConstraint(
3658 const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const {
3659
3660 if (Constraint.size() == 1) {
3661 switch (Constraint[0]) {
3662 case 'r': // R0-R31
3663 switch (VT.SimpleTy) {
3664 default:
3665 return {0u, nullptr};
3666 case MVT::i1:
3667 case MVT::i8:
3668 case MVT::i16:
3669 case MVT::i32:
3670 case MVT::f32:
3671 return {0u, &Hexagon::IntRegsRegClass};
3672 case MVT::i64:
3673 case MVT::f64:
3674 return {0u, &Hexagon::DoubleRegsRegClass};
3675 }
3676 break;
3677 case 'a': // M0-M1
3678 if (VT != MVT::i32)
3679 return {0u, nullptr};
3680 return {0u, &Hexagon::ModRegsRegClass};
3681 case 'q': // q0-q3
3682 switch (VT.getSizeInBits()) {
3683 default:
3684 return {0u, nullptr};
3685 case 64:
3686 case 128:
3687 return {0u, &Hexagon::HvxQRRegClass};
3688 }
3689 break;
3690 case 'v': // V0-V31
3691 switch (VT.getSizeInBits()) {
3692 default:
3693 return {0u, nullptr};
3694 case 512:
3695 return {0u, &Hexagon::HvxVRRegClass};
3696 case 1024:
3697 if (Subtarget.hasV60Ops() && Subtarget.useHVX128BOps())
3698 return {0u, &Hexagon::HvxVRRegClass};
3699 return {0u, &Hexagon::HvxWRRegClass};
3700 case 2048:
3701 return {0u, &Hexagon::HvxWRRegClass};
3702 }
3703 break;
3704 default:
3705 return {0u, nullptr};
3706 }
3707 }
3708
3709 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
3710}
3711
3712/// isFPImmLegal - Returns true if the target can instruction select the
3713/// specified FP immediate natively. If false, the legalizer will
3714/// materialize the FP immediate as a load from a constant pool.
3715bool HexagonTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
3716 bool ForCodeSize) const {
3717 return true;
3718}
3719
3720/// Returns true if it is beneficial to convert a load of a constant
3721/// to just the constant itself.
3722bool HexagonTargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
3723 Type *Ty) const {
3724 if (!ConstantLoadsToImm)
3725 return false;
3726
3727 assert(Ty->isIntegerTy());
3728 unsigned BitSize = Ty->getPrimitiveSizeInBits();
3729 return (BitSize > 0 && BitSize <= 64);
3730}
3731
3732/// isLegalAddressingMode - Return true if the addressing mode represented by
3733/// AM is legal for this target, for a load/store of the specified type.
3734bool HexagonTargetLowering::isLegalAddressingMode(const DataLayout &DL,
3735 const AddrMode &AM, Type *Ty,
3736 unsigned AS, Instruction *I) const {
3737 if (Ty->isSized()) {
3738 // When LSR detects uses of the same base address to access different
3739 // types (e.g. unions), it will assume a conservative type for these
3740 // uses:
3741 // LSR Use: Kind=Address of void in addrspace(4294967295), ...
3742 // The type Ty passed here would then be "void". Skip the alignment
3743 // checks, but do not return false right away, since that confuses
3744 // LSR into crashing.
3745 Align A = DL.getABITypeAlign(Ty);
3746 // The base offset must be a multiple of the alignment.
3747 if (!isAligned(Lhs: A, SizeInBytes: AM.BaseOffs))
3748 return false;
3749 // The shifted offset must fit in 11 bits.
3750 if (!isInt<11>(x: AM.BaseOffs >> Log2(A)))
3751 return false;
3752 }
3753
3754 // No global is ever allowed as a base.
3755 if (AM.BaseGV)
3756 return false;
3757
3758 int Scale = AM.Scale;
3759 if (Scale < 0)
3760 Scale = -Scale;
3761 switch (Scale) {
3762 case 0: // No scale reg, "r+i", "r", or just "i".
3763 break;
3764 default: // No scaled addressing mode.
3765 return false;
3766 }
3767 return true;
3768}
3769
3770/// Return true if folding a constant offset with the given GlobalAddress is
3771/// legal. It is frequently not legal in PIC relocation models.
3772bool HexagonTargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA)
3773 const {
3774 return HTM.getRelocationModel() == Reloc::Static;
3775}
3776
3777/// isLegalICmpImmediate - Return true if the specified immediate is legal
3778/// icmp immediate, that is the target has icmp instructions which can compare
3779/// a register against the immediate without having to materialize the
3780/// immediate into a register.
3781bool HexagonTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
3782 return Imm >= -512 && Imm <= 511;
3783}
3784
3785/// IsEligibleForTailCallOptimization - Check whether the call is eligible
3786/// for tail call optimization. Targets which want to do tail call
3787/// optimization should implement this function.
3788bool HexagonTargetLowering::IsEligibleForTailCallOptimization(
3789 SDValue Callee,
3790 CallingConv::ID CalleeCC,
3791 bool IsVarArg,
3792 bool IsCalleeStructRet,
3793 bool IsCallerStructRet,
3794 const SmallVectorImpl<ISD::OutputArg> &Outs,
3795 const SmallVectorImpl<SDValue> &OutVals,
3796 const SmallVectorImpl<ISD::InputArg> &Ins,
3797 SelectionDAG& DAG) const {
3798 const Function &CallerF = DAG.getMachineFunction().getFunction();
3799 CallingConv::ID CallerCC = CallerF.getCallingConv();
3800 bool CCMatch = CallerCC == CalleeCC;
3801
3802 // ***************************************************************************
3803 // Look for obvious safe cases to perform tail call optimization that do not
3804 // require ABI changes.
3805 // ***************************************************************************
3806
3807 // If this is a tail call via a function pointer, then don't do it!
3808 if (!isa<GlobalAddressSDNode>(Val: Callee) &&
3809 !isa<ExternalSymbolSDNode>(Val: Callee)) {
3810 return false;
3811 }
3812
3813 // Do not optimize if the calling conventions do not match and the conventions
3814 // used are not C or Fast.
3815 if (!CCMatch) {
3816 bool R = (CallerCC == CallingConv::C || CallerCC == CallingConv::Fast);
3817 bool E = (CalleeCC == CallingConv::C || CalleeCC == CallingConv::Fast);
3818 // If R & E, then ok.
3819 if (!R || !E)
3820 return false;
3821 }
3822
3823 // Do not tail call optimize vararg calls.
3824 if (IsVarArg)
3825 return false;
3826
3827 // Also avoid tail call optimization if either caller or callee uses struct
3828 // return semantics.
3829 if (IsCalleeStructRet || IsCallerStructRet)
3830 return false;
3831
3832 // In addition to the cases above, we also disable Tail Call Optimization if
3833 // the calling convention code that at least one outgoing argument needs to
3834 // go on the stack. We cannot check that here because at this point that
3835 // information is not available.
3836 return true;
3837}
3838
3839/// Returns the target specific optimal type for load and store operations as
3840/// a result of memset, memcpy, and memmove lowering.
3841///
3842/// If DstAlign is zero that means it's safe to destination alignment can
3843/// satisfy any constraint. Similarly if SrcAlign is zero it means there isn't
3844/// a need to check it against alignment requirement, probably because the
3845/// source does not need to be loaded. If 'IsMemset' is true, that means it's
3846/// expanding a memset. If 'ZeroMemset' is true, that means it's a memset of
3847/// zero. 'MemcpyStrSrc' indicates whether the memcpy source is constant so it
3848/// does not need to be loaded. It returns EVT::Other if the type should be
3849/// determined using generic target-independent logic.
3850EVT HexagonTargetLowering::getOptimalMemOpType(
3851 LLVMContext &Context, const MemOp &Op,
3852 const AttributeList &FuncAttributes) const {
3853 if (Op.size() >= 8 && Op.isAligned(AlignCheck: Align(8)))
3854 return MVT::i64;
3855 if (Op.size() >= 4 && Op.isAligned(AlignCheck: Align(4)))
3856 return MVT::i32;
3857 if (Op.size() >= 2 && Op.isAligned(AlignCheck: Align(2)))
3858 return MVT::i16;
3859 return MVT::Other;
3860}
3861
3862// The helpers below are versions of llvm::getShuffleReduction and
3863// llvm::getOrderedReduction, adapted to use during DAG passes and simplified as
3864// follows:
3865// - ICmp and FCmp are not handled;
3866// - in every step in getShuffleReduction, the input is split into halves (not
3867// pairwise).
3868
3869static SDValue getOrderedReduction(SDValue Vec, unsigned Op,
3870 SelectionDAG &DAG) {
3871 assert(Op != Instruction::ICmp && Op != Instruction::FCmp);
3872
3873 EVT VT = Vec.getValueType();
3874 EVT EltT = VT.getVectorElementType();
3875 unsigned VF = VT.getVectorNumElements();
3876 assert(VF > 0 &&
3877 "Reduction emission only supported for non-zero length vectors!");
3878
3879 SDLoc DL(Vec);
3880 SDValue Result = DAG.getExtractVectorElt(DL, VT: EltT, Vec, Idx: 0);
3881 for (unsigned ExtractIdx = 1; ExtractIdx < VF; ++ExtractIdx) {
3882 SDValue Ext = DAG.getExtractVectorElt(DL, VT: EltT, Vec, Idx: ExtractIdx);
3883 Result = DAG.getNode(Opcode: Op, DL, VT: EltT, Ops: {Result, Ext});
3884 }
3885
3886 return Result;
3887}
3888
3889static SDValue getShuffleReduction(SDValue Vec, unsigned Op,
3890 SelectionDAG &DAG) {
3891 assert(Op != Instruction::ICmp && Op != Instruction::FCmp);
3892
3893 EVT VT = Vec.getValueType();
3894 unsigned VF = VT.getVectorNumElements();
3895 if (VF == 0)
3896 llvm_unreachable("Vector must be non-zero length");
3897 // VF is a power of 2 so we can emit the reduction using log2(VF) shuffles
3898 // and vector ops, reducing the set of values being computed by half each
3899 // round.
3900 assert(isPowerOf2_32(VF) &&
3901 "Reduction emission only supported for pow2 vectors!");
3902
3903 SDLoc DL(Vec);
3904 // TODO: Is it correct to create double-vector shuffle and fill 3/4 of it with
3905 // undefs?
3906 SmallVector<int, 32> ShuffleMask(VF);
3907 for (unsigned i = VF; i > 1; i >>= 1) {
3908 // Move the upper half of the vector to the lower half.
3909 for (unsigned j = 0; j != i / 2; ++j)
3910 ShuffleMask[j] = i / 2 + j;
3911 // Fill the rest of the mask with undef.
3912 std::fill(first: &ShuffleMask[i / 2], last: ShuffleMask.end(), value: -1);
3913
3914 SDValue Shuf =
3915 DAG.getVectorShuffle(VT, dl: DL, N1: Vec, N2: DAG.getUNDEF(VT), Mask: ShuffleMask);
3916
3917 Vec = DAG.getNode(Opcode: Op, DL, VT, Ops: {Vec, Shuf});
3918 }
3919 // The result is in the first element of the vector.
3920 return DAG.getExtractVectorElt(DL, VT: VT.getVectorElementType(), Vec, Idx: 0);
3921}
3922
3923SDValue HexagonTargetLowering::expandVecReduceAdd(SDNode *N,
3924 SelectionDAG &DAG) const {
3925 // Since we disabled automatic reduction expansion, generate log2 ladder code
3926 // if the vector is of a power-of-two length.
3927 SDValue Input = N->getOperand(Num: 0);
3928 if (isPowerOf2_32(Value: Input.getValueType().getVectorNumElements()))
3929 return getShuffleReduction(Vec: Input, Op: ISD::ADD, DAG);
3930 // Otherwise, reduction will be scalarized.
3931 return getOrderedReduction(Vec: Input, Op: ISD::ADD, DAG);
3932}
3933
3934bool HexagonTargetLowering::allowsMemoryAccess(
3935 LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace,
3936 Align Alignment, MachineMemOperand::Flags Flags, unsigned *Fast) const {
3937 if (!VT.isSimple())
3938 return false;
3939 MVT SVT = VT.getSimpleVT();
3940 if (Subtarget.isHVXVectorType(VecTy: SVT, IncludeBool: true))
3941 return allowsHvxMemoryAccess(VecTy: SVT, Flags, Fast);
3942 return TargetLoweringBase::allowsMemoryAccess(
3943 Context, DL, VT, AddrSpace, Alignment, Flags, Fast);
3944}
3945
3946bool HexagonTargetLowering::allowsMisalignedMemoryAccesses(
3947 EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags,
3948 unsigned *Fast) const {
3949 if (!VT.isSimple())
3950 return false;
3951 MVT SVT = VT.getSimpleVT();
3952 if (Subtarget.isHVXVectorType(VecTy: SVT, IncludeBool: true))
3953 return allowsHvxMisalignedMemoryAccesses(VecTy: SVT, Flags, Fast);
3954 if (Fast)
3955 *Fast = 0;
3956 return false;
3957}
3958
3959std::pair<const TargetRegisterClass*, uint8_t>
3960HexagonTargetLowering::findRepresentativeClass(const TargetRegisterInfo *TRI,
3961 MVT VT) const {
3962 if (Subtarget.isHVXVectorType(VecTy: VT, IncludeBool: true)) {
3963 unsigned BitWidth = VT.getSizeInBits();
3964 unsigned VecWidth = Subtarget.getVectorLength() * 8;
3965
3966 if (VT.getVectorElementType() == MVT::i1)
3967 return std::make_pair(x: &Hexagon::HvxQRRegClass, y: 1);
3968 if (BitWidth == VecWidth)
3969 return std::make_pair(x: &Hexagon::HvxVRRegClass, y: 1);
3970 assert(BitWidth == 2 * VecWidth);
3971 return std::make_pair(x: &Hexagon::HvxWRRegClass, y: 1);
3972 }
3973
3974 return TargetLowering::findRepresentativeClass(TRI, VT);
3975}
3976
3977bool HexagonTargetLowering::shouldReduceLoadWidth(
3978 SDNode *Load, ISD::LoadExtType ExtTy, EVT NewVT,
3979 std::optional<unsigned> ByteOffset) const {
3980 // TODO: This may be worth removing. Check regression tests for diffs.
3981 if (!TargetLoweringBase::shouldReduceLoadWidth(Load, ExtTy, NewVT,
3982 ByteOffset))
3983 return false;
3984
3985 auto *L = cast<LoadSDNode>(Val: Load);
3986 std::pair<SDValue, int> BO = getBaseAndOffset(Addr: L->getBasePtr());
3987 // Small-data object, do not shrink.
3988 if (BO.first.getOpcode() == HexagonISD::CONST32_GP)
3989 return false;
3990 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Val&: BO.first)) {
3991 auto &HTM = static_cast<const HexagonTargetMachine &>(getTargetMachine());
3992 const auto *GO = dyn_cast_or_null<const GlobalObject>(Val: GA->getGlobal());
3993 return !GO || !HTM.getObjFileLowering()->isGlobalInSmallSection(GO, TM: HTM);
3994 }
3995 return true;
3996}
3997
3998void HexagonTargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI,
3999 SDNode *Node) const {
4000 AdjustHvxInstrPostInstrSelection(MI, Node);
4001}
4002
4003Value *HexagonTargetLowering::emitLoadLinked(IRBuilderBase &Builder,
4004 Type *ValueTy, Value *Addr,
4005 AtomicOrdering Ord) const {
4006 unsigned SZ = ValueTy->getPrimitiveSizeInBits();
4007 assert((SZ == 32 || SZ == 64) && "Only 32/64-bit atomic loads supported");
4008 Intrinsic::ID IntID = (SZ == 32) ? Intrinsic::hexagon_L2_loadw_locked
4009 : Intrinsic::hexagon_L4_loadd_locked;
4010
4011 Value *Call =
4012 Builder.CreateIntrinsic(ID: IntID, Args: Addr, /*FMFSource=*/nullptr, Name: "larx");
4013
4014 return Builder.CreateBitCast(V: Call, DestTy: ValueTy);
4015}
4016
4017/// Perform a store-conditional operation to Addr. Return the status of the
4018/// store. This should be 0 if the store succeeded, non-zero otherwise.
4019Value *HexagonTargetLowering::emitStoreConditional(IRBuilderBase &Builder,
4020 Value *Val, Value *Addr,
4021 AtomicOrdering Ord) const {
4022 BasicBlock *BB = Builder.GetInsertBlock();
4023 Module *M = BB->getParent()->getParent();
4024 Type *Ty = Val->getType();
4025 unsigned SZ = Ty->getPrimitiveSizeInBits();
4026
4027 Type *CastTy = Builder.getIntNTy(N: SZ);
4028 assert((SZ == 32 || SZ == 64) && "Only 32/64-bit atomic stores supported");
4029 Intrinsic::ID IntID = (SZ == 32) ? Intrinsic::hexagon_S2_storew_locked
4030 : Intrinsic::hexagon_S4_stored_locked;
4031
4032 Val = Builder.CreateBitCast(V: Val, DestTy: CastTy);
4033
4034 Value *Call = Builder.CreateIntrinsic(ID: IntID, Args: {Addr, Val},
4035 /*FMFSource=*/nullptr, Name: "stcx");
4036 Value *Cmp = Builder.CreateICmpEQ(LHS: Call, RHS: Builder.getInt32(C: 0), Name: "");
4037 Value *Ext = Builder.CreateZExt(V: Cmp, DestTy: Type::getInt32Ty(C&: M->getContext()));
4038 return Ext;
4039}
4040
4041TargetLowering::AtomicExpansionKind
4042HexagonTargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const {
4043 // Do not expand loads and stores that don't exceed 64 bits.
4044 return LI->getType()->getPrimitiveSizeInBits() > 64
4045 ? AtomicExpansionKind::LLOnly
4046 : AtomicExpansionKind::None;
4047}
4048
4049TargetLowering::AtomicExpansionKind
4050HexagonTargetLowering::shouldExpandAtomicStoreInIR(StoreInst *SI) const {
4051 // Do not expand loads and stores that don't exceed 64 bits.
4052 return SI->getValueOperand()->getType()->getPrimitiveSizeInBits() > 64
4053 ? AtomicExpansionKind::Expand
4054 : AtomicExpansionKind::None;
4055}
4056
4057TargetLowering::AtomicExpansionKind
4058HexagonTargetLowering::shouldExpandAtomicCmpXchgInIR(
4059 const AtomicCmpXchgInst *AI) const {
4060 return AtomicExpansionKind::LLSC;
4061}
4062
4063MachineBasicBlock *HexagonTargetLowering::EmitInstrWithCustomInserter(
4064 MachineInstr &MI, MachineBasicBlock *BB) const {
4065 switch (MI.getOpcode()) {
4066 case TargetOpcode::PATCHABLE_EVENT_CALL:
4067 case TargetOpcode::PATCHABLE_TYPED_EVENT_CALL:
4068 // These are lowered in the AsmPrinter.
4069 return BB;
4070 default:
4071 llvm_unreachable("Unexpected instruction with custom inserter");
4072 }
4073}
4074
4075MachineInstr *
4076HexagonTargetLowering::EmitKCFICheck(MachineBasicBlock &MBB,
4077 MachineBasicBlock::instr_iterator &MBBI,
4078 const TargetInstrInfo *TII) const {
4079 assert(MBBI->isCall() && MBBI->getCFIType() &&
4080 "Invalid call instruction for a KCFI check");
4081
4082 switch (MBBI->getOpcode()) {
4083 case Hexagon::J2_callr:
4084 case Hexagon::PS_callr_nr:
4085 break;
4086 default:
4087 llvm_unreachable("Unexpected CFI call opcode");
4088 }
4089
4090 MachineOperand &Target = MBBI->getOperand(i: 0);
4091 assert(Target.isReg() && "Invalid target operand for an indirect call");
4092 Target.setIsRenamable(false);
4093
4094 return BuildMI(BB&: MBB, I: MBBI, MIMD: MBBI->getDebugLoc(), MCID: TII->get(Opcode: Hexagon::KCFI_CHECK))
4095 .addReg(RegNo: Target.getReg())
4096 .addImm(Val: MBBI->getCFIType())
4097 .getInstr();
4098}
4099
4100bool HexagonTargetLowering::isMaskAndCmp0FoldingBeneficial(
4101 const Instruction &AndI) const {
4102 // Only sink 'and' mask to cmp use block if it is masking a single bit since
4103 // this will fold the and/cmp/br into a single tstbit instruction.
4104 ConstantInt *Mask = dyn_cast<ConstantInt>(Val: AndI.getOperand(i: 1));
4105 if (!Mask)
4106 return false;
4107 return Mask->getValue().isPowerOf2();
4108}
4109
4110// Check if the result of the node is only used as a return value, as
4111// otherwise we can't perform a tail-call.
4112bool HexagonTargetLowering::isUsedByReturnOnly(SDNode *N,
4113 SDValue &Chain) const {
4114 if (N->getNumValues() != 1)
4115 return false;
4116 if (!N->hasNUsesOfValue(NUses: 1, Value: 0))
4117 return false;
4118
4119 SDNode *Copy = *N->user_begin();
4120
4121 if (Copy->getOpcode() == ISD::BITCAST) {
4122 return isUsedByReturnOnly(N: Copy, Chain);
4123 }
4124
4125 if (Copy->getOpcode() != ISD::CopyToReg) {
4126 return false;
4127 }
4128
4129 // If the ISD::CopyToReg has a glue operand, we conservatively assume it
4130 // isn't safe to perform a tail call.
4131 if (Copy->getOperand(Num: Copy->getNumOperands() - 1).getValueType() == MVT::Glue)
4132 return false;
4133
4134 // The copy must be used by a HexagonISD::RET_GLUE, and nothing else.
4135 bool HasRet = false;
4136 for (SDNode *Node : Copy->users()) {
4137 if (Node->getOpcode() != HexagonISD::RET_GLUE)
4138 return false;
4139 HasRet = true;
4140 }
4141 if (!HasRet)
4142 return false;
4143
4144 Chain = Copy->getOperand(Num: 0);
4145 return true;
4146}
4147
4148bool HexagonTargetLowering::hasInlineStackProbe(
4149 const MachineFunction &MF) const {
4150 if (MF.getFunction().hasFnAttribute(Kind: "probe-stack"))
4151 return MF.getFunction().getFnAttribute(Kind: "probe-stack").getValueAsString() ==
4152 "inline-asm";
4153 return false;
4154}
4155
4156unsigned HexagonTargetLowering::getStackProbeSize(const MachineFunction &MF,
4157 Align StackAlign) const {
4158 const Function &Fn = MF.getFunction();
4159 unsigned StackProbeSize =
4160 Fn.getFnAttributeAsParsedInteger(Kind: "stack-probe-size", Default: 4096);
4161 // Round down to the stack alignment.
4162 StackProbeSize = alignDown(Value: StackProbeSize, Align: StackAlign.value());
4163 return StackProbeSize ? StackProbeSize : StackAlign.value();
4164}
4165