1//===-- HexagonISelLowering.cpp - Hexagon DAG Lowering Implementation -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the interfaces that Hexagon uses to lower LLVM code
10// into a selection DAG.
11//
12//===----------------------------------------------------------------------===//
13
14#include "HexagonISelLowering.h"
15#include "Hexagon.h"
16#include "HexagonMachineFunctionInfo.h"
17#include "HexagonRegisterInfo.h"
18#include "HexagonSubtarget.h"
19#include "HexagonTargetMachine.h"
20#include "HexagonTargetObjectFile.h"
21#include "llvm/ADT/APInt.h"
22#include "llvm/ADT/ArrayRef.h"
23#include "llvm/ADT/SmallVector.h"
24#include "llvm/ADT/StringSwitch.h"
25#include "llvm/CodeGen/CallingConvLower.h"
26#include "llvm/CodeGen/MachineFrameInfo.h"
27#include "llvm/CodeGen/MachineFunction.h"
28#include "llvm/CodeGen/MachineMemOperand.h"
29#include "llvm/CodeGen/MachineRegisterInfo.h"
30#include "llvm/CodeGen/SelectionDAG.h"
31#include "llvm/CodeGen/TargetCallingConv.h"
32#include "llvm/CodeGen/ValueTypes.h"
33#include "llvm/IR/BasicBlock.h"
34#include "llvm/IR/CallingConv.h"
35#include "llvm/IR/DataLayout.h"
36#include "llvm/IR/DerivedTypes.h"
37#include "llvm/IR/DiagnosticInfo.h"
38#include "llvm/IR/DiagnosticPrinter.h"
39#include "llvm/IR/Function.h"
40#include "llvm/IR/GlobalValue.h"
41#include "llvm/IR/IRBuilder.h"
42#include "llvm/IR/InlineAsm.h"
43#include "llvm/IR/Instructions.h"
44#include "llvm/IR/IntrinsicInst.h"
45#include "llvm/IR/Intrinsics.h"
46#include "llvm/IR/IntrinsicsHexagon.h"
47#include "llvm/IR/Module.h"
48#include "llvm/IR/Type.h"
49#include "llvm/IR/Value.h"
50#include "llvm/Support/Casting.h"
51#include "llvm/Support/CodeGen.h"
52#include "llvm/Support/CommandLine.h"
53#include "llvm/Support/Debug.h"
54#include "llvm/Support/ErrorHandling.h"
55#include "llvm/Support/MathExtras.h"
56#include "llvm/Support/raw_ostream.h"
57#include "llvm/Target/TargetMachine.h"
58#include <algorithm>
59#include <cassert>
60#include <cstdint>
61#include <limits>
62#include <utility>
63
64using namespace llvm;
65
66#define DEBUG_TYPE "hexagon-lowering"
67
68static cl::opt<bool> EmitJumpTables("hexagon-emit-jump-tables",
69 cl::init(Val: true), cl::Hidden,
70 cl::desc("Control jump table emission on Hexagon target"));
71
72static cl::opt<bool>
73 EnableHexSDNodeSched("enable-hexagon-sdnode-sched", cl::Hidden,
74 cl::desc("Enable Hexagon SDNode scheduling"));
75
76static cl::opt<int> MinimumJumpTables("minimum-jump-tables", cl::Hidden,
77 cl::init(Val: 5),
78 cl::desc("Set minimum jump tables"));
79
80static cl::opt<bool>
81 ConstantLoadsToImm("constant-loads-to-imm", cl::Hidden, cl::init(Val: true),
82 cl::desc("Convert constant loads to immediate values."));
83
84static cl::opt<bool> AlignLoads("hexagon-align-loads",
85 cl::Hidden, cl::init(Val: false),
86 cl::desc("Rewrite unaligned loads as a pair of aligned loads"));
87
88static cl::opt<bool>
89 DisableArgsMinAlignment("hexagon-disable-args-min-alignment", cl::Hidden,
90 cl::init(Val: false),
91 cl::desc("Disable minimum alignment of 1 for "
92 "arguments passed by value on stack"));
93
94// Implement calling convention for Hexagon.
95
96static bool CC_SkipOdd(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
97 CCValAssign::LocInfo &LocInfo,
98 ISD::ArgFlagsTy &ArgFlags, CCState &State) {
99 static const MCPhysReg ArgRegs[] = {
100 Hexagon::R0, Hexagon::R1, Hexagon::R2,
101 Hexagon::R3, Hexagon::R4, Hexagon::R5
102 };
103 const unsigned NumArgRegs = std::size(ArgRegs);
104 unsigned RegNum = State.getFirstUnallocated(Regs: ArgRegs);
105
106 // RegNum is an index into ArgRegs: skip a register if RegNum is odd.
107 if (RegNum != NumArgRegs && RegNum % 2 == 1)
108 State.AllocateReg(Reg: ArgRegs[RegNum]);
109
110 // Always return false here, as this function only makes sure that the first
111 // unallocated register has an even register number and does not actually
112 // allocate a register for the current argument.
113 return false;
114}
115
116#include "HexagonGenCallingConv.inc"
117
118unsigned HexagonTargetLowering::getVectorTypeBreakdownForCallingConv(
119 LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT,
120 unsigned &NumIntermediates, MVT &RegisterVT) const {
121
122 bool isBoolVector = VT.getVectorElementType() == MVT::i1;
123 bool isPowerOf2 = VT.isPow2VectorType();
124 unsigned NumElts = VT.getVectorNumElements();
125
126 // Split vectors of type vXi1 into (X/8) vectors of type v8i1,
127 // where X is divisible by 8.
128 if (isBoolVector && !Subtarget.useHVXOps() && isPowerOf2 && NumElts >= 8) {
129 RegisterVT = MVT::v8i8;
130 IntermediateVT = MVT::v8i1;
131 NumIntermediates = NumElts / 8;
132 return NumIntermediates;
133 }
134
135 // In HVX 64-byte mode, vectors of type vXi1 are split into (X / 64) vectors
136 // of type v64i1, provided that X is divisible by 64.
137 if (isBoolVector && Subtarget.useHVX64BOps() && isPowerOf2 && NumElts >= 64) {
138 RegisterVT = MVT::v64i8;
139 IntermediateVT = MVT::v64i1;
140 NumIntermediates = NumElts / 64;
141 return NumIntermediates;
142 }
143
144 // In HVX 128-byte mode, vectors of type vXi1 are split into (X / 128) vectors
145 // of type v128i1, provided that X is divisible by 128.
146 if (isBoolVector && Subtarget.useHVX128BOps() && isPowerOf2 &&
147 NumElts >= 128) {
148 RegisterVT = MVT::v128i8;
149 IntermediateVT = MVT::v128i1;
150 NumIntermediates = NumElts / 128;
151 return NumIntermediates;
152 }
153
154 return TargetLowering::getVectorTypeBreakdownForCallingConv(
155 Context, CC, VT, IntermediateVT, NumIntermediates, RegisterVT);
156}
157
158std::pair<MVT, unsigned>
159HexagonTargetLowering::handleMaskRegisterForCallingConv(
160 const HexagonSubtarget &Subtarget, EVT VT) const {
161 assert(VT.getVectorElementType() == MVT::i1);
162
163 const unsigned NumElems = VT.getVectorNumElements();
164
165 if (!VT.isPow2VectorType())
166 return {MVT::INVALID_SIMPLE_VALUE_TYPE, 0};
167
168 if (!Subtarget.useHVXOps() && NumElems >= 8)
169 return {MVT::v8i8, NumElems / 8};
170
171 if (Subtarget.useHVX64BOps() && NumElems >= 64)
172 return {MVT::v64i8, NumElems / 64};
173
174 if (Subtarget.useHVX128BOps() && NumElems >= 128)
175 return {MVT::v128i8, NumElems / 128};
176
177 return {MVT::INVALID_SIMPLE_VALUE_TYPE, 0};
178}
179
180MVT HexagonTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
181 CallingConv::ID CC,
182 EVT VT) const {
183
184 if (VT.isVector() && VT.getVectorElementType() == MVT::i1) {
185 auto [RegisterVT, NumRegisters] =
186 handleMaskRegisterForCallingConv(Subtarget, VT);
187 if (RegisterVT != MVT::INVALID_SIMPLE_VALUE_TYPE)
188 return RegisterVT;
189 }
190
191 return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
192}
193
194SDValue
195HexagonTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG)
196 const {
197 unsigned IntNo = Op.getConstantOperandVal(i: 0);
198 SDLoc dl(Op);
199 switch (IntNo) {
200 default:
201 return SDValue(); // Don't custom lower most intrinsics.
202 case Intrinsic::thread_pointer: {
203 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
204 return DAG.getNode(Opcode: HexagonISD::THREAD_POINTER, DL: dl, VT: PtrVT);
205 }
206 }
207}
208
209/// CreateCopyOfByValArgument - Make a copy of an aggregate at address specified
210/// by "Src" to address "Dst" of size "Size". Alignment information is
211/// specified by the specific parameter attribute. The copy will be passed as
212/// a byval function parameter. Sometimes what we are copying is the end of a
213/// larger object, the part that does not fit in registers.
214static SDValue CreateCopyOfByValArgument(SDValue Src, SDValue Dst,
215 SDValue Chain, ISD::ArgFlagsTy Flags,
216 SelectionDAG &DAG, const SDLoc &dl) {
217 SDValue SizeNode = DAG.getConstant(Val: Flags.getByValSize(), DL: dl, VT: MVT::i32);
218 return DAG.getMemcpy(
219 Chain, dl, Dst, Src, Size: SizeNode, Alignment: Flags.getNonZeroByValAlign(),
220 /*isVolatile=*/isVol: false, /*AlwaysInline=*/false,
221 /*CI=*/nullptr, OverrideTailCall: std::nullopt, DstPtrInfo: MachinePointerInfo(), SrcPtrInfo: MachinePointerInfo());
222}
223
224bool
225HexagonTargetLowering::CanLowerReturn(
226 CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg,
227 const SmallVectorImpl<ISD::OutputArg> &Outs,
228 LLVMContext &Context, const Type *RetTy) const {
229 SmallVector<CCValAssign, 16> RVLocs;
230 CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
231
232 if (MF.getSubtarget<HexagonSubtarget>().useHVXOps())
233 return CCInfo.CheckReturn(Outs, Fn: RetCC_Hexagon_HVX);
234 return CCInfo.CheckReturn(Outs, Fn: RetCC_Hexagon);
235}
236
237// LowerReturn - Lower ISD::RET. If a struct is larger than 8 bytes and is
238// passed by value, the function prototype is modified to return void and
239// the value is stored in memory pointed by a pointer passed by caller.
240SDValue
241HexagonTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
242 bool IsVarArg,
243 const SmallVectorImpl<ISD::OutputArg> &Outs,
244 const SmallVectorImpl<SDValue> &OutVals,
245 const SDLoc &dl, SelectionDAG &DAG) const {
246 // CCValAssign - represent the assignment of the return value to locations.
247 SmallVector<CCValAssign, 16> RVLocs;
248
249 // CCState - Info about the registers and stack slot.
250 CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
251 *DAG.getContext());
252
253 // Analyze return values of ISD::RET
254 if (Subtarget.useHVXOps())
255 CCInfo.AnalyzeReturn(Outs, Fn: RetCC_Hexagon_HVX);
256 else
257 CCInfo.AnalyzeReturn(Outs, Fn: RetCC_Hexagon);
258
259 SDValue Glue;
260 SmallVector<SDValue, 4> RetOps(1, Chain);
261
262 // Copy the result values into the output registers.
263 for (unsigned i = 0; i != RVLocs.size(); ++i) {
264 CCValAssign &VA = RVLocs[i];
265 SDValue Val = OutVals[i];
266
267 switch (VA.getLocInfo()) {
268 default:
269 // Loc info must be one of Full, BCvt, SExt, ZExt, or AExt.
270 llvm_unreachable("Unknown loc info!");
271 case CCValAssign::Full:
272 break;
273 case CCValAssign::BCvt:
274 Val = DAG.getBitcast(VT: VA.getLocVT(), V: Val);
275 break;
276 case CCValAssign::SExt:
277 Val = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: Val);
278 break;
279 case CCValAssign::ZExt:
280 Val = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: Val);
281 break;
282 case CCValAssign::AExt:
283 Val = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: Val);
284 break;
285 }
286
287 Chain = DAG.getCopyToReg(Chain, dl, Reg: VA.getLocReg(), N: Val, Glue);
288
289 // Guarantee that all emitted copies are stuck together with flags.
290 Glue = Chain.getValue(R: 1);
291 RetOps.push_back(Elt: DAG.getRegister(Reg: VA.getLocReg(), VT: VA.getLocVT()));
292 }
293
294 RetOps[0] = Chain; // Update chain.
295
296 // Add the glue if we have it.
297 if (Glue.getNode())
298 RetOps.push_back(Elt: Glue);
299
300 return DAG.getNode(Opcode: HexagonISD::RET_GLUE, DL: dl, VT: MVT::Other, Ops: RetOps);
301}
302
303bool HexagonTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
304 // If either no tail call or told not to tail call at all, don't.
305 return CI->isTailCall();
306}
307
308Register HexagonTargetLowering::getRegisterByName(
309 const char* RegName, LLT VT, const MachineFunction &) const {
310 // Just support r19, the linux kernel uses it.
311 Register Reg = StringSwitch<Register>(RegName)
312 .Case(S: "r0", Value: Hexagon::R0)
313 .Case(S: "r1", Value: Hexagon::R1)
314 .Case(S: "r2", Value: Hexagon::R2)
315 .Case(S: "r3", Value: Hexagon::R3)
316 .Case(S: "r4", Value: Hexagon::R4)
317 .Case(S: "r5", Value: Hexagon::R5)
318 .Case(S: "r6", Value: Hexagon::R6)
319 .Case(S: "r7", Value: Hexagon::R7)
320 .Case(S: "r8", Value: Hexagon::R8)
321 .Case(S: "r9", Value: Hexagon::R9)
322 .Case(S: "r10", Value: Hexagon::R10)
323 .Case(S: "r11", Value: Hexagon::R11)
324 .Case(S: "r12", Value: Hexagon::R12)
325 .Case(S: "r13", Value: Hexagon::R13)
326 .Case(S: "r14", Value: Hexagon::R14)
327 .Case(S: "r15", Value: Hexagon::R15)
328 .Case(S: "r16", Value: Hexagon::R16)
329 .Case(S: "r17", Value: Hexagon::R17)
330 .Case(S: "r18", Value: Hexagon::R18)
331 .Case(S: "r19", Value: Hexagon::R19)
332 .Case(S: "r20", Value: Hexagon::R20)
333 .Case(S: "r21", Value: Hexagon::R21)
334 .Case(S: "r22", Value: Hexagon::R22)
335 .Case(S: "r23", Value: Hexagon::R23)
336 .Case(S: "r24", Value: Hexagon::R24)
337 .Case(S: "r25", Value: Hexagon::R25)
338 .Case(S: "r26", Value: Hexagon::R26)
339 .Case(S: "r27", Value: Hexagon::R27)
340 .Case(S: "r28", Value: Hexagon::R28)
341 .Case(S: "r29", Value: Hexagon::R29)
342 .Case(S: "r30", Value: Hexagon::R30)
343 .Case(S: "r31", Value: Hexagon::R31)
344 .Case(S: "r1:0", Value: Hexagon::D0)
345 .Case(S: "r3:2", Value: Hexagon::D1)
346 .Case(S: "r5:4", Value: Hexagon::D2)
347 .Case(S: "r7:6", Value: Hexagon::D3)
348 .Case(S: "r9:8", Value: Hexagon::D4)
349 .Case(S: "r11:10", Value: Hexagon::D5)
350 .Case(S: "r13:12", Value: Hexagon::D6)
351 .Case(S: "r15:14", Value: Hexagon::D7)
352 .Case(S: "r17:16", Value: Hexagon::D8)
353 .Case(S: "r19:18", Value: Hexagon::D9)
354 .Case(S: "r21:20", Value: Hexagon::D10)
355 .Case(S: "r23:22", Value: Hexagon::D11)
356 .Case(S: "r25:24", Value: Hexagon::D12)
357 .Case(S: "r27:26", Value: Hexagon::D13)
358 .Case(S: "r29:28", Value: Hexagon::D14)
359 .Case(S: "r31:30", Value: Hexagon::D15)
360 .Case(S: "sp", Value: Hexagon::R29)
361 .Case(S: "fp", Value: Hexagon::R30)
362 .Case(S: "lr", Value: Hexagon::R31)
363 .Case(S: "p0", Value: Hexagon::P0)
364 .Case(S: "p1", Value: Hexagon::P1)
365 .Case(S: "p2", Value: Hexagon::P2)
366 .Case(S: "p3", Value: Hexagon::P3)
367 .Case(S: "sa0", Value: Hexagon::SA0)
368 .Case(S: "lc0", Value: Hexagon::LC0)
369 .Case(S: "sa1", Value: Hexagon::SA1)
370 .Case(S: "lc1", Value: Hexagon::LC1)
371 .Case(S: "m0", Value: Hexagon::M0)
372 .Case(S: "m1", Value: Hexagon::M1)
373 .Case(S: "usr", Value: Hexagon::USR)
374 .Case(S: "ugp", Value: Hexagon::UGP)
375 .Case(S: "cs0", Value: Hexagon::CS0)
376 .Case(S: "cs1", Value: Hexagon::CS1)
377 .Default(Value: Register());
378 return Reg;
379}
380
381/// LowerCallResult - Lower the result values of an ISD::CALL into the
382/// appropriate copies out of appropriate physical registers. This assumes that
383/// Chain/Glue are the input chain/glue to use, and that TheCall is the call
384/// being lowered. Returns a SDNode with the same number of values as the
385/// ISD::CALL.
386SDValue HexagonTargetLowering::LowerCallResult(
387 SDValue Chain, SDValue Glue, CallingConv::ID CallConv, bool IsVarArg,
388 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
389 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals,
390 const SmallVectorImpl<SDValue> &OutVals, SDValue Callee) const {
391 // Assign locations to each value returned by this call.
392 SmallVector<CCValAssign, 16> RVLocs;
393
394 CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
395 *DAG.getContext());
396
397 if (Subtarget.useHVXOps())
398 CCInfo.AnalyzeCallResult(Ins, Fn: RetCC_Hexagon_HVX);
399 else
400 CCInfo.AnalyzeCallResult(Ins, Fn: RetCC_Hexagon);
401
402 // Copy all of the result registers out of their specified physreg.
403 for (unsigned i = 0; i != RVLocs.size(); ++i) {
404 SDValue RetVal;
405 if (RVLocs[i].getValVT() == MVT::i1) {
406 // Return values of type MVT::i1 require special handling. The reason
407 // is that MVT::i1 is associated with the PredRegs register class, but
408 // values of that type are still returned in R0. Generate an explicit
409 // copy into a predicate register from R0, and treat the value of the
410 // predicate register as the call result.
411 auto &MRI = DAG.getMachineFunction().getRegInfo();
412 SDValue FR0 = DAG.getCopyFromReg(Chain, dl, Reg: RVLocs[i].getLocReg(),
413 VT: MVT::i32, Glue);
414 // FR0 = (Value, Chain, Glue)
415 Register PredR = MRI.createVirtualRegister(RegClass: &Hexagon::PredRegsRegClass);
416 SDValue TPR = DAG.getCopyToReg(Chain: FR0.getValue(R: 1), dl, Reg: PredR,
417 N: FR0.getValue(R: 0), Glue: FR0.getValue(R: 2));
418 // TPR = (Chain, Glue)
419 // Don't glue this CopyFromReg, because it copies from a virtual
420 // register. If it is glued to the call, InstrEmitter will add it
421 // as an implicit def to the call (EmitMachineNode).
422 RetVal = DAG.getCopyFromReg(Chain: TPR.getValue(R: 0), dl, Reg: PredR, VT: MVT::i1);
423 Glue = TPR.getValue(R: 1);
424 Chain = TPR.getValue(R: 0);
425 } else {
426 RetVal = DAG.getCopyFromReg(Chain, dl, Reg: RVLocs[i].getLocReg(),
427 VT: RVLocs[i].getValVT(), Glue);
428 Glue = RetVal.getValue(R: 2);
429 Chain = RetVal.getValue(R: 1);
430 }
431 InVals.push_back(Elt: RetVal.getValue(R: 0));
432 }
433
434 return Chain;
435}
436
437/// LowerCall - Functions arguments are copied from virtual regs to
438/// (physical regs)/(stack frame), CALLSEQ_START and CALLSEQ_END are emitted.
439SDValue
440HexagonTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
441 SmallVectorImpl<SDValue> &InVals) const {
442 SelectionDAG &DAG = CLI.DAG;
443 SDLoc &dl = CLI.DL;
444 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
445 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
446 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
447 SDValue Chain = CLI.Chain;
448 SDValue Callee = CLI.Callee;
449 CallingConv::ID CallConv = CLI.CallConv;
450 bool IsVarArg = CLI.IsVarArg;
451 bool DoesNotReturn = CLI.DoesNotReturn;
452
453 bool IsStructRet = Outs.empty() ? false : Outs[0].Flags.isSRet();
454 MachineFunction &MF = DAG.getMachineFunction();
455 MachineFrameInfo &MFI = MF.getFrameInfo();
456 auto PtrVT = getPointerTy(DL: MF.getDataLayout());
457
458 if (GlobalAddressSDNode *GAN = dyn_cast<GlobalAddressSDNode>(Val&: Callee))
459 Callee = DAG.getTargetGlobalAddress(GV: GAN->getGlobal(), DL: dl, VT: MVT::i32);
460
461 // Linux ABI treats var-arg calls the same way as regular ones.
462 bool TreatAsVarArg = !Subtarget.isEnvironmentMusl() && IsVarArg;
463
464 // Analyze operands of the call, assigning locations to each operand.
465 SmallVector<CCValAssign, 16> ArgLocs;
466 CCState CCInfo(CallConv, TreatAsVarArg, MF, ArgLocs, *DAG.getContext());
467
468 if (Subtarget.useHVXOps())
469 CCInfo.AnalyzeCallOperands(Outs, Fn: CC_Hexagon_HVX);
470 else if (DisableArgsMinAlignment)
471 CCInfo.AnalyzeCallOperands(Outs, Fn: CC_Hexagon_Legacy);
472 else
473 CCInfo.AnalyzeCallOperands(Outs, Fn: CC_Hexagon);
474
475 if (CLI.IsTailCall) {
476 bool StructAttrFlag = MF.getFunction().hasStructRetAttr();
477 CLI.IsTailCall = IsEligibleForTailCallOptimization(Callee, CalleeCC: CallConv,
478 isVarArg: IsVarArg, isCalleeStructRet: IsStructRet, isCallerStructRet: StructAttrFlag, Outs,
479 OutVals, Ins, DAG);
480 for (const CCValAssign &VA : ArgLocs) {
481 if (VA.isMemLoc()) {
482 CLI.IsTailCall = false;
483 break;
484 }
485 }
486 LLVM_DEBUG(dbgs() << (CLI.IsTailCall ? "Eligible for Tail Call\n"
487 : "Argument must be passed on stack. "
488 "Not eligible for Tail Call\n"));
489 }
490 // Get a count of how many bytes are to be pushed on the stack.
491 unsigned NumBytes = CCInfo.getStackSize();
492 SmallVector<std::pair<unsigned, SDValue>, 16> RegsToPass;
493 SmallVector<SDValue, 8> MemOpChains;
494
495 const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo();
496 SDValue StackPtr =
497 DAG.getCopyFromReg(Chain, dl, Reg: HRI.getStackRegister(), VT: PtrVT);
498
499 bool NeedsArgAlign = false;
500 Align LargestAlignSeen;
501 // Walk the register/memloc assignments, inserting copies/loads.
502 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
503 CCValAssign &VA = ArgLocs[i];
504 SDValue Arg = OutVals[i];
505 ISD::ArgFlagsTy Flags = Outs[i].Flags;
506 // Record if we need > 8 byte alignment on an argument.
507 bool ArgAlign = Subtarget.isHVXVectorType(VecTy: VA.getValVT());
508 NeedsArgAlign |= ArgAlign;
509
510 // Promote the value if needed.
511 switch (VA.getLocInfo()) {
512 default:
513 // Loc info must be one of Full, BCvt, SExt, ZExt, or AExt.
514 llvm_unreachable("Unknown loc info!");
515 case CCValAssign::Full:
516 break;
517 case CCValAssign::BCvt:
518 Arg = DAG.getBitcast(VT: VA.getLocVT(), V: Arg);
519 break;
520 case CCValAssign::SExt:
521 Arg = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: Arg);
522 break;
523 case CCValAssign::ZExt:
524 Arg = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: Arg);
525 break;
526 case CCValAssign::AExt:
527 Arg = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: Arg);
528 break;
529 }
530
531 if (VA.isMemLoc()) {
532 unsigned LocMemOffset = VA.getLocMemOffset();
533 SDValue MemAddr = DAG.getConstant(Val: LocMemOffset, DL: dl,
534 VT: StackPtr.getValueType());
535 MemAddr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::i32, N1: StackPtr, N2: MemAddr);
536 if (ArgAlign)
537 LargestAlignSeen = std::max(
538 a: LargestAlignSeen, b: Align(VA.getLocVT().getStoreSizeInBits() / 8));
539 if (Flags.isByVal()) {
540 // The argument is a struct passed by value. According to LLVM, "Arg"
541 // is a pointer.
542 MemOpChains.push_back(Elt: CreateCopyOfByValArgument(Src: Arg, Dst: MemAddr, Chain,
543 Flags, DAG, dl));
544 } else {
545 MachinePointerInfo LocPI = MachinePointerInfo::getStack(
546 MF&: DAG.getMachineFunction(), Offset: LocMemOffset);
547 SDValue S = DAG.getStore(Chain, dl, Val: Arg, Ptr: MemAddr, PtrInfo: LocPI);
548 MemOpChains.push_back(Elt: S);
549 }
550 continue;
551 }
552
553 // Arguments that can be passed on register must be kept at RegsToPass
554 // vector.
555 if (VA.isRegLoc())
556 RegsToPass.push_back(Elt: std::make_pair(x: VA.getLocReg(), y&: Arg));
557 }
558
559 if (NeedsArgAlign && Subtarget.hasV60Ops()) {
560 LLVM_DEBUG(dbgs() << "Function needs byte stack align due to call args\n");
561 Align VecAlign = HRI.getSpillAlign(RC: Hexagon::HvxVRRegClass);
562 LargestAlignSeen = std::max(a: LargestAlignSeen, b: VecAlign);
563 MFI.ensureMaxAlignment(Alignment: LargestAlignSeen);
564 }
565 // Transform all store nodes into one single node because all store
566 // nodes are independent of each other.
567 if (!MemOpChains.empty())
568 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: MemOpChains);
569
570 SDValue Glue;
571 if (!CLI.IsTailCall) {
572 Chain = DAG.getCALLSEQ_START(Chain, InSize: NumBytes, OutSize: 0, DL: dl);
573 Glue = Chain.getValue(R: 1);
574 }
575
576 // Build a sequence of copy-to-reg nodes chained together with token
577 // chain and flag operands which copy the outgoing args into registers.
578 // The Glue is necessary since all emitted instructions must be
579 // stuck together.
580 if (!CLI.IsTailCall) {
581 for (const auto &R : RegsToPass) {
582 Chain = DAG.getCopyToReg(Chain, dl, Reg: R.first, N: R.second, Glue);
583 Glue = Chain.getValue(R: 1);
584 }
585 } else {
586 // For tail calls lower the arguments to the 'real' stack slot.
587 //
588 // Force all the incoming stack arguments to be loaded from the stack
589 // before any new outgoing arguments are stored to the stack, because the
590 // outgoing stack slots may alias the incoming argument stack slots, and
591 // the alias isn't otherwise explicit. This is slightly more conservative
592 // than necessary, because it means that each store effectively depends
593 // on every argument instead of just those arguments it would clobber.
594 //
595 // Do not flag preceding copytoreg stuff together with the following stuff.
596 Glue = SDValue();
597 for (const auto &R : RegsToPass) {
598 Chain = DAG.getCopyToReg(Chain, dl, Reg: R.first, N: R.second, Glue);
599 Glue = Chain.getValue(R: 1);
600 }
601 Glue = SDValue();
602 }
603
604 bool LongCalls = MF.getSubtarget<HexagonSubtarget>().useLongCalls();
605 unsigned Flags = LongCalls ? HexagonII::HMOTF_ConstExtended : 0;
606
607 // If the callee is a GlobalAddress/ExternalSymbol node (quite common, every
608 // direct call is) turn it into a TargetGlobalAddress/TargetExternalSymbol
609 // node so that legalize doesn't hack it.
610 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Val&: Callee)) {
611 Callee = DAG.getTargetGlobalAddress(GV: G->getGlobal(), DL: dl, VT: PtrVT, offset: 0, TargetFlags: Flags);
612 } else if (ExternalSymbolSDNode *S =
613 dyn_cast<ExternalSymbolSDNode>(Val&: Callee)) {
614 Callee = DAG.getTargetExternalSymbol(Sym: S->getSymbol(), VT: PtrVT, TargetFlags: Flags);
615 }
616
617 // Returns a chain & a flag for retval copy to use.
618 SmallVector<SDValue, 8> Ops;
619 Ops.push_back(Elt: Chain);
620 Ops.push_back(Elt: Callee);
621
622 // Add argument registers to the end of the list so that they are
623 // known live into the call.
624 for (const auto &R : RegsToPass)
625 Ops.push_back(Elt: DAG.getRegister(Reg: R.first, VT: R.second.getValueType()));
626
627 const uint32_t *Mask = HRI.getCallPreservedMask(MF, CallConv);
628 assert(Mask && "Missing call preserved mask for calling convention");
629 Ops.push_back(Elt: DAG.getRegisterMask(RegMask: Mask));
630
631 if (Glue.getNode())
632 Ops.push_back(Elt: Glue);
633
634 if (CLI.IsTailCall) {
635 MFI.setHasTailCall();
636 return DAG.getNode(Opcode: HexagonISD::TC_RETURN, DL: dl, VT: MVT::Other, Ops);
637 }
638
639 // Set this here because we need to know this for "hasFP" in frame lowering.
640 // The target-independent code calls getFrameRegister before setting it, and
641 // getFrameRegister uses hasFP to determine whether the function has FP.
642 MFI.setHasCalls(true);
643
644 unsigned OpCode = DoesNotReturn ? HexagonISD::CALLnr : HexagonISD::CALL;
645 Chain = DAG.getNode(Opcode: OpCode, DL: dl, ResultTys: {MVT::Other, MVT::Glue}, Ops);
646 Glue = Chain.getValue(R: 1);
647
648 // Create the CALLSEQ_END node.
649 Chain = DAG.getCALLSEQ_END(Chain, Size1: NumBytes, Size2: 0, Glue, DL: dl);
650 Glue = Chain.getValue(R: 1);
651
652 // Handle result values, copying them out of physregs into vregs that we
653 // return.
654 return LowerCallResult(Chain, Glue, CallConv, IsVarArg, Ins, dl, DAG,
655 InVals, OutVals, Callee);
656}
657
658/// Returns true by value, base pointer and offset pointer and addressing
659/// mode by reference if this node can be combined with a load / store to
660/// form a post-indexed load / store.
661bool HexagonTargetLowering::getPostIndexedAddressParts(SDNode *N, SDNode *Op,
662 SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM,
663 SelectionDAG &DAG) const {
664 LSBaseSDNode *LSN = dyn_cast<LSBaseSDNode>(Val: N);
665 if (!LSN)
666 return false;
667 EVT VT = LSN->getMemoryVT();
668 if (!VT.isSimple())
669 return false;
670 bool IsLegalType = VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32 ||
671 VT == MVT::i64 || VT == MVT::f32 || VT == MVT::f64 ||
672 VT == MVT::v2i16 || VT == MVT::v2i32 || VT == MVT::v4i8 ||
673 VT == MVT::v4i16 || VT == MVT::v8i8 ||
674 Subtarget.isHVXVectorType(VecTy: VT.getSimpleVT());
675 if (!IsLegalType)
676 return false;
677
678 if (Op->getOpcode() != ISD::ADD)
679 return false;
680 Base = Op->getOperand(Num: 0);
681 Offset = Op->getOperand(Num: 1);
682 if (!isa<ConstantSDNode>(Val: Offset.getNode()))
683 return false;
684 AM = ISD::POST_INC;
685
686 int32_t V = cast<ConstantSDNode>(Val: Offset.getNode())->getSExtValue();
687 return Subtarget.getInstrInfo()->isValidAutoIncImm(VT, Offset: V);
688}
689
690SDValue HexagonTargetLowering::LowerFDIV(SDValue Op, SelectionDAG &DAG) const {
691 if (DAG.getMachineFunction().getFunction().hasOptSize())
692 return SDValue();
693 else
694 return Op;
695}
696
697SDValue
698HexagonTargetLowering::LowerINLINEASM(SDValue Op, SelectionDAG &DAG) const {
699 MachineFunction &MF = DAG.getMachineFunction();
700 auto &HMFI = *MF.getInfo<HexagonMachineFunctionInfo>();
701 const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo();
702 unsigned LR = HRI.getRARegister();
703
704 if ((Op.getOpcode() != ISD::INLINEASM &&
705 Op.getOpcode() != ISD::INLINEASM_BR) || HMFI.hasClobberLR())
706 return Op;
707
708 unsigned NumOps = Op.getNumOperands();
709 if (Op.getOperand(i: NumOps-1).getValueType() == MVT::Glue)
710 --NumOps; // Ignore the flag operand.
711
712 for (unsigned i = InlineAsm::Op_FirstOperand; i != NumOps;) {
713 const InlineAsm::Flag Flags(Op.getConstantOperandVal(i));
714 unsigned NumVals = Flags.getNumOperandRegisters();
715 ++i; // Skip the ID value.
716
717 switch (Flags.getKind()) {
718 default:
719 llvm_unreachable("Bad flags!");
720 case InlineAsm::Kind::RegUse:
721 case InlineAsm::Kind::Imm:
722 case InlineAsm::Kind::Mem:
723 i += NumVals;
724 break;
725 case InlineAsm::Kind::Clobber:
726 case InlineAsm::Kind::RegDef:
727 case InlineAsm::Kind::RegDefEarlyClobber: {
728 for (; NumVals; --NumVals, ++i) {
729 Register Reg = cast<RegisterSDNode>(Val: Op.getOperand(i))->getReg();
730 if (Reg != LR)
731 continue;
732 HMFI.setHasClobberLR(true);
733 return Op;
734 }
735 break;
736 }
737 }
738 }
739
740 return Op;
741}
742
743// Need to transform ISD::PREFETCH into something that doesn't inherit
744// all of the properties of ISD::PREFETCH, specifically SDNPMayLoad and
745// SDNPMayStore.
746SDValue HexagonTargetLowering::LowerPREFETCH(SDValue Op,
747 SelectionDAG &DAG) const {
748 SDValue Chain = Op.getOperand(i: 0);
749 SDValue Addr = Op.getOperand(i: 1);
750 // Lower it to DCFETCH($reg, #0). A "pat" will try to merge the offset in,
751 // if the "reg" is fed by an "add".
752 SDLoc DL(Op);
753 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
754 return DAG.getNode(Opcode: HexagonISD::DCFETCH, DL, VT: MVT::Other, N1: Chain, N2: Addr, N3: Zero);
755}
756
757SDValue HexagonTargetLowering::LowerINTRINSIC_VOID(SDValue Op,
758 SelectionDAG &DAG) const {
759 SDValue Chain = Op.getOperand(i: 0);
760 unsigned IntNo = Op.getConstantOperandVal(i: 1);
761 // Lower the hexagon_prefetch builtin to DCFETCH, as above.
762 if (IntNo == Intrinsic::hexagon_prefetch) {
763 SDValue Addr = Op.getOperand(i: 2);
764 SDLoc DL(Op);
765 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
766 return DAG.getNode(Opcode: HexagonISD::DCFETCH, DL, VT: MVT::Other, N1: Chain, N2: Addr, N3: Zero);
767 }
768 return SDValue();
769}
770
771SDValue
772HexagonTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
773 SelectionDAG &DAG) const {
774 SDValue Chain = Op.getOperand(i: 0);
775 SDValue Size = Op.getOperand(i: 1);
776 SDValue Align = Op.getOperand(i: 2);
777 SDLoc dl(Op);
778
779 ConstantSDNode *AlignConst = dyn_cast<ConstantSDNode>(Val&: Align);
780 assert(AlignConst && "Non-constant Align in LowerDYNAMIC_STACKALLOC");
781
782 unsigned A = AlignConst->getSExtValue();
783 auto &HFI = *Subtarget.getFrameLowering();
784 // "Zero" means natural stack alignment.
785 if (A == 0)
786 A = HFI.getStackAlign().value();
787
788 LLVM_DEBUG({
789 dbgs () << __func__ << " Align: " << A << " Size: ";
790 Size.getNode()->dump(&DAG);
791 dbgs() << "\n";
792 });
793
794 SDValue AC = DAG.getConstant(Val: A, DL: dl, VT: MVT::i32);
795 SDVTList VTs = DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other);
796 SDValue AA = DAG.getNode(Opcode: HexagonISD::ALLOCA, DL: dl, VTList: VTs, N1: Chain, N2: Size, N3: AC);
797
798 DAG.ReplaceAllUsesOfValueWith(From: Op, To: AA);
799 return AA;
800}
801
802SDValue HexagonTargetLowering::LowerFormalArguments(
803 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
804 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
805 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
806 MachineFunction &MF = DAG.getMachineFunction();
807 MachineFrameInfo &MFI = MF.getFrameInfo();
808 MachineRegisterInfo &MRI = MF.getRegInfo();
809
810 // Linux ABI treats var-arg calls the same way as regular ones.
811 bool TreatAsVarArg = !Subtarget.isEnvironmentMusl() && IsVarArg;
812
813 // Assign locations to all of the incoming arguments.
814 SmallVector<CCValAssign, 16> ArgLocs;
815 CCState CCInfo(CallConv, TreatAsVarArg, MF, ArgLocs, *DAG.getContext());
816
817 if (Subtarget.useHVXOps())
818 CCInfo.AnalyzeFormalArguments(Ins, Fn: CC_Hexagon_HVX);
819 else if (DisableArgsMinAlignment)
820 CCInfo.AnalyzeFormalArguments(Ins, Fn: CC_Hexagon_Legacy);
821 else
822 CCInfo.AnalyzeFormalArguments(Ins, Fn: CC_Hexagon);
823
824 // For LLVM, in the case when returning a struct by value (>8byte),
825 // the first argument is a pointer that points to the location on caller's
826 // stack where the return value will be stored. For Hexagon, the location on
827 // caller's stack is passed only when the struct size is smaller than (and
828 // equal to) 8 bytes. If not, no address will be passed into callee and
829 // callee return the result directly through R0/R1.
830 auto NextSingleReg = [] (const TargetRegisterClass &RC, unsigned Reg) {
831 switch (RC.getID()) {
832 case Hexagon::IntRegsRegClassID:
833 return Reg - Hexagon::R0 + 1;
834 case Hexagon::DoubleRegsRegClassID:
835 return (Reg - Hexagon::D0 + 1) * 2;
836 case Hexagon::HvxVRRegClassID:
837 return Reg - Hexagon::V0 + 1;
838 case Hexagon::HvxWRRegClassID:
839 return (Reg - Hexagon::W0 + 1) * 2;
840 }
841 llvm_unreachable("Unexpected register class");
842 };
843
844 auto &HFL = const_cast<HexagonFrameLowering&>(*Subtarget.getFrameLowering());
845 auto &HMFI = *MF.getInfo<HexagonMachineFunctionInfo>();
846 HFL.FirstVarArgSavedReg = 0;
847 HMFI.setFirstNamedArgFrameIndex(-int(MFI.getNumFixedObjects()));
848
849 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
850 CCValAssign &VA = ArgLocs[i];
851 ISD::ArgFlagsTy Flags = Ins[i].Flags;
852 bool ByVal = Flags.isByVal();
853
854 // Arguments passed in registers:
855 // 1. 32- and 64-bit values and HVX vectors are passed directly,
856 // 2. Large structs are passed via an address, and the address is
857 // passed in a register.
858 if (VA.isRegLoc() && ByVal && Flags.getByValSize() <= 8)
859 llvm_unreachable("ByValSize must be bigger than 8 bytes");
860
861 bool InReg = VA.isRegLoc() &&
862 (!ByVal || (ByVal && Flags.getByValSize() > 8));
863
864 if (InReg) {
865 MVT RegVT = VA.getLocVT();
866 if (VA.getLocInfo() == CCValAssign::BCvt)
867 RegVT = VA.getValVT();
868
869 const TargetRegisterClass *RC = getRegClassFor(VT: RegVT);
870 Register VReg = MRI.createVirtualRegister(RegClass: RC);
871 SDValue Copy = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: RegVT);
872
873 // Treat values of type MVT::i1 specially: they are passed in
874 // registers of type i32, but they need to remain as values of
875 // type i1 for consistency of the argument lowering.
876 if (VA.getValVT() == MVT::i1) {
877 assert(RegVT.getSizeInBits() <= 32);
878 SDValue T = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: RegVT,
879 N1: Copy, N2: DAG.getConstant(Val: 1, DL: dl, VT: RegVT));
880 Copy = DAG.getSetCC(DL: dl, VT: MVT::i1, LHS: T, RHS: DAG.getConstant(Val: 0, DL: dl, VT: RegVT),
881 Cond: ISD::SETNE);
882 } else {
883#ifndef NDEBUG
884 unsigned RegSize = RegVT.getSizeInBits();
885 assert(RegSize == 32 || RegSize == 64 ||
886 Subtarget.isHVXVectorType(RegVT));
887#endif
888 }
889 InVals.push_back(Elt: Copy);
890 MRI.addLiveIn(Reg: VA.getLocReg(), vreg: VReg);
891 HFL.FirstVarArgSavedReg = NextSingleReg(*RC, VA.getLocReg());
892 } else {
893 assert(VA.isMemLoc() && "Argument should be passed in memory");
894
895 // If it's a byval parameter, then we need to compute the
896 // "real" size, not the size of the pointer.
897 unsigned ObjSize = Flags.isByVal()
898 ? Flags.getByValSize()
899 : VA.getLocVT().getStoreSizeInBits() / 8;
900
901 // Create the frame index object for this incoming parameter.
902 int Offset = HEXAGON_LRFP_SIZE + VA.getLocMemOffset();
903 int FI = MFI.CreateFixedObject(Size: ObjSize, SPOffset: Offset, IsImmutable: true);
904 SDValue FIN = DAG.getFrameIndex(FI, VT: MVT::i32);
905
906 if (Flags.isByVal()) {
907 // If it's a pass-by-value aggregate, then do not dereference the stack
908 // location. Instead, we should generate a reference to the stack
909 // location.
910 InVals.push_back(Elt: FIN);
911 } else {
912 SDValue L = DAG.getLoad(VT: VA.getValVT(), dl, Chain, Ptr: FIN,
913 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI, Offset: 0));
914 InVals.push_back(Elt: L);
915 }
916 }
917 }
918
919 if (IsVarArg && Subtarget.isEnvironmentMusl()) {
920 for (int i = HFL.FirstVarArgSavedReg; i < 6; i++)
921 MRI.addLiveIn(Reg: Hexagon::R0+i);
922 }
923
924 if (IsVarArg && Subtarget.isEnvironmentMusl()) {
925 HMFI.setFirstNamedArgFrameIndex(HMFI.getFirstNamedArgFrameIndex() - 1);
926 HMFI.setLastNamedArgFrameIndex(-int(MFI.getNumFixedObjects()));
927
928 // Create Frame index for the start of register saved area.
929 int NumVarArgRegs = 6 - HFL.FirstVarArgSavedReg;
930 bool RequiresPadding = (NumVarArgRegs & 1);
931 int RegSaveAreaSizePlusPadding = RequiresPadding
932 ? (NumVarArgRegs + 1) * 4
933 : NumVarArgRegs * 4;
934
935 if (RegSaveAreaSizePlusPadding > 0) {
936 // The offset to saved register area should be 8 byte aligned.
937 int RegAreaStart = HEXAGON_LRFP_SIZE + CCInfo.getStackSize();
938 if (!(RegAreaStart % 8))
939 RegAreaStart = (RegAreaStart + 7) & -8;
940
941 int RegSaveAreaFrameIndex =
942 MFI.CreateFixedObject(Size: RegSaveAreaSizePlusPadding, SPOffset: RegAreaStart, IsImmutable: true);
943 HMFI.setRegSavedAreaStartFrameIndex(RegSaveAreaFrameIndex);
944
945 // This will point to the next argument passed via stack.
946 int Offset = RegAreaStart + RegSaveAreaSizePlusPadding;
947 int FI = MFI.CreateFixedObject(Hexagon_PointerSize, SPOffset: Offset, IsImmutable: true);
948 HMFI.setVarArgsFrameIndex(FI);
949 } else {
950 // This will point to the next argument passed via stack, when
951 // there is no saved register area.
952 int Offset = HEXAGON_LRFP_SIZE + CCInfo.getStackSize();
953 int FI = MFI.CreateFixedObject(Hexagon_PointerSize, SPOffset: Offset, IsImmutable: true);
954 HMFI.setRegSavedAreaStartFrameIndex(FI);
955 HMFI.setVarArgsFrameIndex(FI);
956 }
957 }
958
959
960 if (IsVarArg && !Subtarget.isEnvironmentMusl()) {
961 // This will point to the next argument passed via stack.
962 int Offset = HEXAGON_LRFP_SIZE + CCInfo.getStackSize();
963 int FI = MFI.CreateFixedObject(Hexagon_PointerSize, SPOffset: Offset, IsImmutable: true);
964 HMFI.setVarArgsFrameIndex(FI);
965 }
966
967 return Chain;
968}
969
970SDValue
971HexagonTargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {
972 // VASTART stores the address of the VarArgsFrameIndex slot into the
973 // memory location argument.
974 MachineFunction &MF = DAG.getMachineFunction();
975 HexagonMachineFunctionInfo *QFI = MF.getInfo<HexagonMachineFunctionInfo>();
976 SDValue Addr = DAG.getFrameIndex(FI: QFI->getVarArgsFrameIndex(), VT: MVT::i32);
977 const Value *SV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 2))->getValue();
978
979 if (!Subtarget.isEnvironmentMusl()) {
980 return DAG.getStore(Chain: Op.getOperand(i: 0), dl: SDLoc(Op), Val: Addr, Ptr: Op.getOperand(i: 1),
981 PtrInfo: MachinePointerInfo(SV));
982 }
983 auto &FuncInfo = *MF.getInfo<HexagonMachineFunctionInfo>();
984 auto &HFL = *Subtarget.getFrameLowering();
985 SDLoc DL(Op);
986 SmallVector<SDValue, 8> MemOps;
987
988 // Get frame index of va_list.
989 SDValue FIN = Op.getOperand(i: 1);
990
991 // If first Vararg register is odd, add 4 bytes to start of
992 // saved register area to point to the first register location.
993 // This is because the saved register area has to be 8 byte aligned.
994 // In case of an odd start register, there will be 4 bytes of padding in
995 // the beginning of saved register area. If all registers area used up,
996 // the following condition will handle it correctly.
997 SDValue SavedRegAreaStartFrameIndex =
998 DAG.getFrameIndex(FI: FuncInfo.getRegSavedAreaStartFrameIndex(), VT: MVT::i32);
999
1000 auto PtrVT = getPointerTy(DL: DAG.getDataLayout());
1001
1002 if (HFL.FirstVarArgSavedReg & 1)
1003 SavedRegAreaStartFrameIndex =
1004 DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT,
1005 N1: DAG.getFrameIndex(FI: FuncInfo.getRegSavedAreaStartFrameIndex(),
1006 VT: MVT::i32),
1007 N2: DAG.getIntPtrConstant(Val: 4, DL));
1008
1009 // Store the saved register area start pointer.
1010 SDValue Store =
1011 DAG.getStore(Chain: Op.getOperand(i: 0), dl: DL,
1012 Val: SavedRegAreaStartFrameIndex,
1013 Ptr: FIN, PtrInfo: MachinePointerInfo(SV));
1014 MemOps.push_back(Elt: Store);
1015
1016 // Store saved register area end pointer.
1017 FIN = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT,
1018 N1: FIN, N2: DAG.getIntPtrConstant(Val: 4, DL));
1019 Store = DAG.getStore(Chain: Op.getOperand(i: 0), dl: DL,
1020 Val: DAG.getFrameIndex(FI: FuncInfo.getVarArgsFrameIndex(),
1021 VT: PtrVT),
1022 Ptr: FIN, PtrInfo: MachinePointerInfo(SV, 4));
1023 MemOps.push_back(Elt: Store);
1024
1025 // Store overflow area pointer.
1026 FIN = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT,
1027 N1: FIN, N2: DAG.getIntPtrConstant(Val: 4, DL));
1028 Store = DAG.getStore(Chain: Op.getOperand(i: 0), dl: DL,
1029 Val: DAG.getFrameIndex(FI: FuncInfo.getVarArgsFrameIndex(),
1030 VT: PtrVT),
1031 Ptr: FIN, PtrInfo: MachinePointerInfo(SV, 8));
1032 MemOps.push_back(Elt: Store);
1033
1034 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: MemOps);
1035}
1036
1037SDValue
1038HexagonTargetLowering::LowerVACOPY(SDValue Op, SelectionDAG &DAG) const {
1039 // Assert that the linux ABI is enabled for the current compilation.
1040 assert(Subtarget.isEnvironmentMusl() && "Linux ABI should be enabled");
1041 SDValue Chain = Op.getOperand(i: 0);
1042 SDValue DestPtr = Op.getOperand(i: 1);
1043 SDValue SrcPtr = Op.getOperand(i: 2);
1044 const Value *DestSV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 3))->getValue();
1045 const Value *SrcSV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 4))->getValue();
1046 SDLoc DL(Op);
1047 // Size of the va_list is 12 bytes as it has 3 pointers. Therefore,
1048 // we need to memcopy 12 bytes from va_list to another similar list.
1049 return DAG.getMemcpy(
1050 Chain, dl: DL, Dst: DestPtr, Src: SrcPtr, Size: DAG.getIntPtrConstant(Val: 12, DL), Alignment: Align(4),
1051 /*isVolatile*/ isVol: false, AlwaysInline: false, /*CI=*/nullptr, OverrideTailCall: std::nullopt,
1052 DstPtrInfo: MachinePointerInfo(DestSV), SrcPtrInfo: MachinePointerInfo(SrcSV));
1053}
1054
1055SDValue HexagonTargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
1056 const SDLoc &dl(Op);
1057 SDValue LHS = Op.getOperand(i: 0);
1058 SDValue RHS = Op.getOperand(i: 1);
1059 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 2))->get();
1060 MVT ResTy = ty(Op);
1061 MVT OpTy = ty(Op: LHS);
1062
1063 if (OpTy == MVT::v2i16 || OpTy == MVT::v4i8) {
1064 MVT ElemTy = OpTy.getVectorElementType();
1065 assert(ElemTy.isScalarInteger());
1066 MVT WideTy = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: 2*ElemTy.getSizeInBits()),
1067 NumElements: OpTy.getVectorNumElements());
1068 return DAG.getSetCC(DL: dl, VT: ResTy,
1069 LHS: DAG.getSExtOrTrunc(Op: LHS, DL: SDLoc(LHS), VT: WideTy),
1070 RHS: DAG.getSExtOrTrunc(Op: RHS, DL: SDLoc(RHS), VT: WideTy), Cond: CC);
1071 }
1072
1073 // Treat all other vector types as legal.
1074 if (ResTy.isVector())
1075 return Op;
1076
1077 // Comparisons of short integers should use sign-extend, not zero-extend,
1078 // since we can represent small negative values in the compare instructions.
1079 // The LLVM default is to use zero-extend arbitrarily in these cases.
1080 auto isSExtFree = [this](SDValue N) {
1081 switch (N.getOpcode()) {
1082 case ISD::TRUNCATE: {
1083 // A sign-extend of a truncate of a sign-extend is free.
1084 SDValue Op = N.getOperand(i: 0);
1085 if (Op.getOpcode() != ISD::AssertSext)
1086 return false;
1087 EVT OrigTy = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
1088 unsigned ThisBW = ty(Op: N).getSizeInBits();
1089 unsigned OrigBW = OrigTy.getSizeInBits();
1090 // The type that was sign-extended to get the AssertSext must be
1091 // narrower than the type of N (so that N has still the same value
1092 // as the original).
1093 return ThisBW >= OrigBW;
1094 }
1095 case ISD::LOAD:
1096 // We have sign-extended loads.
1097 return true;
1098 }
1099 return false;
1100 };
1101
1102 if (OpTy == MVT::i8 || OpTy == MVT::i16) {
1103 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: RHS);
1104 bool IsNegative = C && C->getAPIntValue().isNegative();
1105 if (IsNegative || isSExtFree(LHS) || isSExtFree(RHS))
1106 return DAG.getSetCC(DL: dl, VT: ResTy,
1107 LHS: DAG.getSExtOrTrunc(Op: LHS, DL: SDLoc(LHS), VT: MVT::i32),
1108 RHS: DAG.getSExtOrTrunc(Op: RHS, DL: SDLoc(RHS), VT: MVT::i32), Cond: CC);
1109 }
1110
1111 return SDValue();
1112}
1113
1114SDValue
1115HexagonTargetLowering::LowerVSELECT(SDValue Op, SelectionDAG &DAG) const {
1116 SDValue PredOp = Op.getOperand(i: 0);
1117 SDValue Op1 = Op.getOperand(i: 1), Op2 = Op.getOperand(i: 2);
1118 MVT OpTy = ty(Op: Op1);
1119 const SDLoc &dl(Op);
1120
1121 if (OpTy == MVT::v2i16 || OpTy == MVT::v4i8) {
1122 MVT ElemTy = OpTy.getVectorElementType();
1123 assert(ElemTy.isScalarInteger());
1124 MVT WideTy = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: 2*ElemTy.getSizeInBits()),
1125 NumElements: OpTy.getVectorNumElements());
1126 // Generate (trunc (select (_, sext, sext))).
1127 return DAG.getSExtOrTrunc(
1128 Op: DAG.getSelect(DL: dl, VT: WideTy, Cond: PredOp,
1129 LHS: DAG.getSExtOrTrunc(Op: Op1, DL: dl, VT: WideTy),
1130 RHS: DAG.getSExtOrTrunc(Op: Op2, DL: dl, VT: WideTy)),
1131 DL: dl, VT: OpTy);
1132 }
1133
1134 return SDValue();
1135}
1136
1137SDValue
1138HexagonTargetLowering::LowerConstantPool(SDValue Op, SelectionDAG &DAG) const {
1139 EVT ValTy = Op.getValueType();
1140 ConstantPoolSDNode *CPN = cast<ConstantPoolSDNode>(Val&: Op);
1141 Constant *CVal = nullptr;
1142 bool isVTi1Type = false;
1143 if (auto *CV = dyn_cast<ConstantVector>(Val: CPN->getConstVal())) {
1144 if (cast<VectorType>(Val: CV->getType())->getElementType()->isIntegerTy(Bitwidth: 1)) {
1145 IRBuilder<> IRB(CV->getContext());
1146 SmallVector<Constant*, 128> NewConst;
1147 unsigned VecLen = CV->getNumOperands();
1148 assert(isPowerOf2_32(VecLen) &&
1149 "conversion only supported for pow2 VectorSize");
1150 for (unsigned i = 0; i < VecLen; ++i)
1151 NewConst.push_back(Elt: IRB.getInt8(C: CV->getOperand(i_nocapture: i)->isNullValue()));
1152
1153 CVal = ConstantVector::get(V: NewConst);
1154 isVTi1Type = true;
1155 }
1156 }
1157 Align Alignment = CPN->getAlign();
1158 bool IsPositionIndependent = isPositionIndependent();
1159 unsigned char TF = IsPositionIndependent ? HexagonII::MO_PCREL : 0;
1160
1161 unsigned Offset = 0;
1162 SDValue T;
1163 if (CPN->isMachineConstantPoolEntry())
1164 T = DAG.getTargetConstantPool(C: CPN->getMachineCPVal(), VT: ValTy, Align: Alignment,
1165 Offset, TargetFlags: TF);
1166 else if (isVTi1Type)
1167 T = DAG.getTargetConstantPool(C: CVal, VT: ValTy, Align: Alignment, Offset, TargetFlags: TF);
1168 else
1169 T = DAG.getTargetConstantPool(C: CPN->getConstVal(), VT: ValTy, Align: Alignment, Offset,
1170 TargetFlags: TF);
1171
1172 assert(cast<ConstantPoolSDNode>(T)->getTargetFlags() == TF &&
1173 "Inconsistent target flag encountered");
1174
1175 if (IsPositionIndependent)
1176 return DAG.getNode(Opcode: HexagonISD::AT_PCREL, DL: SDLoc(Op), VT: ValTy, Operand: T);
1177 return DAG.getNode(Opcode: HexagonISD::CP, DL: SDLoc(Op), VT: ValTy, Operand: T);
1178}
1179
1180SDValue
1181HexagonTargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const {
1182 EVT VT = Op.getValueType();
1183 int Idx = cast<JumpTableSDNode>(Val&: Op)->getIndex();
1184 if (isPositionIndependent()) {
1185 SDValue T = DAG.getTargetJumpTable(JTI: Idx, VT, TargetFlags: HexagonII::MO_PCREL);
1186 return DAG.getNode(Opcode: HexagonISD::AT_PCREL, DL: SDLoc(Op), VT, Operand: T);
1187 }
1188
1189 SDValue T = DAG.getTargetJumpTable(JTI: Idx, VT);
1190 return DAG.getNode(Opcode: HexagonISD::JT, DL: SDLoc(Op), VT, Operand: T);
1191}
1192
1193SDValue
1194HexagonTargetLowering::LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const {
1195 const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo();
1196 MachineFunction &MF = DAG.getMachineFunction();
1197 MachineFrameInfo &MFI = MF.getFrameInfo();
1198 MFI.setReturnAddressIsTaken(true);
1199
1200 EVT VT = Op.getValueType();
1201 SDLoc dl(Op);
1202 unsigned Depth = Op.getConstantOperandVal(i: 0);
1203 if (Depth) {
1204 SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
1205 SDValue Offset = DAG.getConstant(Val: 4, DL: dl, VT: MVT::i32);
1206 return DAG.getLoad(VT, dl, Chain: DAG.getEntryNode(),
1207 Ptr: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: FrameAddr, N2: Offset),
1208 PtrInfo: MachinePointerInfo());
1209 }
1210
1211 // Return LR, which contains the return address. Mark it an implicit live-in.
1212 Register Reg = MF.addLiveIn(PReg: HRI.getRARegister(), RC: getRegClassFor(VT: MVT::i32));
1213 return DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl, Reg, VT);
1214}
1215
1216SDValue
1217HexagonTargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const {
1218 const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo();
1219 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
1220 MFI.setFrameAddressIsTaken(true);
1221
1222 EVT VT = Op.getValueType();
1223 SDLoc dl(Op);
1224 unsigned Depth = Op.getConstantOperandVal(i: 0);
1225 SDValue FrameAddr = DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl,
1226 Reg: HRI.getFrameRegister(), VT);
1227 while (Depth--)
1228 FrameAddr = DAG.getLoad(VT, dl, Chain: DAG.getEntryNode(), Ptr: FrameAddr,
1229 PtrInfo: MachinePointerInfo());
1230 return FrameAddr;
1231}
1232
1233SDValue
1234HexagonTargetLowering::LowerATOMIC_FENCE(SDValue Op, SelectionDAG& DAG) const {
1235 SDLoc dl(Op);
1236 return DAG.getNode(Opcode: HexagonISD::BARRIER, DL: dl, VT: MVT::Other, Operand: Op.getOperand(i: 0));
1237}
1238
1239SDValue
1240HexagonTargetLowering::LowerGLOBALADDRESS(SDValue Op, SelectionDAG &DAG) const {
1241 SDLoc dl(Op);
1242 auto *GAN = cast<GlobalAddressSDNode>(Val&: Op);
1243 auto PtrVT = getPointerTy(DL: DAG.getDataLayout());
1244 auto *GV = GAN->getGlobal();
1245 int64_t Offset = GAN->getOffset();
1246
1247 auto &HLOF = *HTM.getObjFileLowering();
1248 Reloc::Model RM = HTM.getRelocationModel();
1249
1250 if (RM == Reloc::Static) {
1251 SDValue GA = DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: Offset);
1252 const GlobalObject *GO = GV->getAliaseeObject();
1253 if (GO && Subtarget.useSmallData() && HLOF.isGlobalInSmallSection(GO, TM: HTM))
1254 return DAG.getNode(Opcode: HexagonISD::CONST32_GP, DL: dl, VT: PtrVT, Operand: GA);
1255 return DAG.getNode(Opcode: HexagonISD::CONST32, DL: dl, VT: PtrVT, Operand: GA);
1256 }
1257
1258 bool UsePCRel = getTargetMachine().shouldAssumeDSOLocal(GV);
1259 if (UsePCRel) {
1260 SDValue GA = DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: Offset,
1261 TargetFlags: HexagonII::MO_PCREL);
1262 return DAG.getNode(Opcode: HexagonISD::AT_PCREL, DL: dl, VT: PtrVT, Operand: GA);
1263 }
1264
1265 // Use GOT index.
1266 SDValue GOT = DAG.getGLOBAL_OFFSET_TABLE(VT: PtrVT);
1267 SDValue GA = DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: 0, TargetFlags: HexagonII::MO_GOT);
1268 SDValue Off = DAG.getConstant(Val: Offset, DL: dl, VT: MVT::i32);
1269 return DAG.getNode(Opcode: HexagonISD::AT_GOT, DL: dl, VT: PtrVT, N1: GOT, N2: GA, N3: Off);
1270}
1271
1272// Specifies that for loads and stores VT can be promoted to PromotedLdStVT.
1273SDValue
1274HexagonTargetLowering::LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const {
1275 const BlockAddress *BA = cast<BlockAddressSDNode>(Val&: Op)->getBlockAddress();
1276 SDLoc dl(Op);
1277 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
1278
1279 Reloc::Model RM = HTM.getRelocationModel();
1280 if (RM == Reloc::Static) {
1281 SDValue A = DAG.getTargetBlockAddress(BA, VT: PtrVT);
1282 return DAG.getNode(Opcode: HexagonISD::CONST32_GP, DL: dl, VT: PtrVT, Operand: A);
1283 }
1284
1285 SDValue A = DAG.getTargetBlockAddress(BA, VT: PtrVT, Offset: 0, TargetFlags: HexagonII::MO_PCREL);
1286 return DAG.getNode(Opcode: HexagonISD::AT_PCREL, DL: dl, VT: PtrVT, Operand: A);
1287}
1288
1289SDValue
1290HexagonTargetLowering::LowerGLOBAL_OFFSET_TABLE(SDValue Op, SelectionDAG &DAG)
1291 const {
1292 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
1293 SDValue GOTSym = DAG.getTargetExternalSymbol(HEXAGON_GOT_SYM_NAME, VT: PtrVT,
1294 TargetFlags: HexagonII::MO_PCREL);
1295 return DAG.getNode(Opcode: HexagonISD::AT_PCREL, DL: SDLoc(Op), VT: PtrVT, Operand: GOTSym);
1296}
1297
1298SDValue
1299HexagonTargetLowering::GetDynamicTLSAddr(SelectionDAG &DAG, SDValue Chain,
1300 GlobalAddressSDNode *GA, SDValue Glue, EVT PtrVT, unsigned ReturnReg,
1301 unsigned char OperandFlags) const {
1302 MachineFunction &MF = DAG.getMachineFunction();
1303 MachineFrameInfo &MFI = MF.getFrameInfo();
1304 SDVTList NodeTys = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
1305 SDLoc dl(GA);
1306 SDValue TGA = DAG.getTargetGlobalAddress(GV: GA->getGlobal(), DL: dl,
1307 VT: GA->getValueType(ResNo: 0),
1308 offset: GA->getOffset(),
1309 TargetFlags: OperandFlags);
1310 // Create Operands for the call.The Operands should have the following:
1311 // 1. Chain SDValue
1312 // 2. Callee which in this case is the Global address value.
1313 // 3. Registers live into the call.In this case its R0, as we
1314 // have just one argument to be passed.
1315 // 4. Glue.
1316 // Note: The order is important.
1317
1318 const auto &HRI = *Subtarget.getRegisterInfo();
1319 const uint32_t *Mask = HRI.getCallPreservedMask(MF, CallingConv::C);
1320 assert(Mask && "Missing call preserved mask for calling convention");
1321 SDValue Ops[] = { Chain, TGA, DAG.getRegister(Reg: Hexagon::R0, VT: PtrVT),
1322 DAG.getRegisterMask(RegMask: Mask), Glue };
1323 Chain = DAG.getNode(Opcode: HexagonISD::CALL, DL: dl, VTList: NodeTys, Ops);
1324
1325 // Inform MFI that function has calls.
1326 MFI.setAdjustsStack(true);
1327
1328 Glue = Chain.getValue(R: 1);
1329 return DAG.getCopyFromReg(Chain, dl, Reg: ReturnReg, VT: PtrVT, Glue);
1330}
1331
1332//
1333// Lower using the initial executable model for TLS addresses
1334//
1335SDValue
1336HexagonTargetLowering::LowerToTLSInitialExecModel(GlobalAddressSDNode *GA,
1337 SelectionDAG &DAG) const {
1338 SDLoc dl(GA);
1339 int64_t Offset = GA->getOffset();
1340 auto PtrVT = getPointerTy(DL: DAG.getDataLayout());
1341
1342 // Get the thread pointer.
1343 SDValue TP = DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl, Reg: Hexagon::UGP, VT: PtrVT);
1344
1345 bool IsPositionIndependent = isPositionIndependent();
1346 unsigned char TF =
1347 IsPositionIndependent ? HexagonII::MO_IEGOT : HexagonII::MO_IE;
1348
1349 // First generate the TLS symbol address
1350 SDValue TGA = DAG.getTargetGlobalAddress(GV: GA->getGlobal(), DL: dl, VT: PtrVT,
1351 offset: Offset, TargetFlags: TF);
1352
1353 SDValue Sym = DAG.getNode(Opcode: HexagonISD::CONST32, DL: dl, VT: PtrVT, Operand: TGA);
1354
1355 if (IsPositionIndependent) {
1356 // Generate the GOT pointer in case of position independent code
1357 SDValue GOT = LowerGLOBAL_OFFSET_TABLE(Op: Sym, DAG);
1358
1359 // Add the TLS Symbol address to GOT pointer.This gives
1360 // GOT relative relocation for the symbol.
1361 Sym = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: GOT, N2: Sym);
1362 }
1363
1364 // Load the offset value for TLS symbol.This offset is relative to
1365 // thread pointer.
1366 SDValue LoadOffset =
1367 DAG.getLoad(VT: PtrVT, dl, Chain: DAG.getEntryNode(), Ptr: Sym, PtrInfo: MachinePointerInfo());
1368
1369 // Address of the thread local variable is the add of thread
1370 // pointer and the offset of the variable.
1371 return DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: TP, N2: LoadOffset);
1372}
1373
1374//
1375// Lower using the local executable model for TLS addresses
1376//
1377SDValue
1378HexagonTargetLowering::LowerToTLSLocalExecModel(GlobalAddressSDNode *GA,
1379 SelectionDAG &DAG) const {
1380 SDLoc dl(GA);
1381 int64_t Offset = GA->getOffset();
1382 auto PtrVT = getPointerTy(DL: DAG.getDataLayout());
1383
1384 // Get the thread pointer.
1385 SDValue TP = DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl, Reg: Hexagon::UGP, VT: PtrVT);
1386 // Generate the TLS symbol address
1387 SDValue TGA = DAG.getTargetGlobalAddress(GV: GA->getGlobal(), DL: dl, VT: PtrVT, offset: Offset,
1388 TargetFlags: HexagonII::MO_TPREL);
1389 SDValue Sym = DAG.getNode(Opcode: HexagonISD::CONST32, DL: dl, VT: PtrVT, Operand: TGA);
1390
1391 // Address of the thread local variable is the add of thread
1392 // pointer and the offset of the variable.
1393 return DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: TP, N2: Sym);
1394}
1395
1396//
1397// Lower using the general dynamic model for TLS addresses
1398//
1399SDValue
1400HexagonTargetLowering::LowerToTLSGeneralDynamicModel(GlobalAddressSDNode *GA,
1401 SelectionDAG &DAG) const {
1402 SDLoc dl(GA);
1403 int64_t Offset = GA->getOffset();
1404 auto PtrVT = getPointerTy(DL: DAG.getDataLayout());
1405
1406 // First generate the TLS symbol address
1407 SDValue TGA = DAG.getTargetGlobalAddress(GV: GA->getGlobal(), DL: dl, VT: PtrVT, offset: Offset,
1408 TargetFlags: HexagonII::MO_GDGOT);
1409
1410 // Then, generate the GOT pointer
1411 SDValue GOT = LowerGLOBAL_OFFSET_TABLE(Op: TGA, DAG);
1412
1413 // Add the TLS symbol and the GOT pointer
1414 SDValue Sym = DAG.getNode(Opcode: HexagonISD::CONST32, DL: dl, VT: PtrVT, Operand: TGA);
1415 SDValue Chain = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: GOT, N2: Sym);
1416
1417 // Copy over the argument to R0
1418 SDValue InGlue;
1419 Chain = DAG.getCopyToReg(Chain: DAG.getEntryNode(), dl, Reg: Hexagon::R0, N: Chain, Glue: InGlue);
1420 InGlue = Chain.getValue(R: 1);
1421
1422 unsigned Flags = DAG.getSubtarget<HexagonSubtarget>().useLongCalls()
1423 ? HexagonII::MO_GDPLT | HexagonII::HMOTF_ConstExtended
1424 : HexagonII::MO_GDPLT;
1425
1426 return GetDynamicTLSAddr(DAG, Chain, GA, Glue: InGlue, PtrVT,
1427 ReturnReg: Hexagon::R0, OperandFlags: Flags);
1428}
1429
1430//
1431// Lower TLS addresses.
1432//
1433// For now for dynamic models, we only support the general dynamic model.
1434//
1435SDValue
1436HexagonTargetLowering::LowerGlobalTLSAddress(SDValue Op,
1437 SelectionDAG &DAG) const {
1438 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Val&: Op);
1439
1440 switch (HTM.getTLSModel(GV: GA->getGlobal())) {
1441 case TLSModel::GeneralDynamic:
1442 case TLSModel::LocalDynamic:
1443 return LowerToTLSGeneralDynamicModel(GA, DAG);
1444 case TLSModel::InitialExec:
1445 return LowerToTLSInitialExecModel(GA, DAG);
1446 case TLSModel::LocalExec:
1447 return LowerToTLSLocalExecModel(GA, DAG);
1448 }
1449 llvm_unreachable("Bogus TLS model");
1450}
1451
1452//===----------------------------------------------------------------------===//
1453// TargetLowering Implementation
1454//===----------------------------------------------------------------------===//
1455
1456HexagonTargetLowering::HexagonTargetLowering(const TargetMachine &TM,
1457 const HexagonSubtarget &ST)
1458 : TargetLowering(TM, ST),
1459 HTM(static_cast<const HexagonTargetMachine &>(TM)), Subtarget(ST) {
1460 auto &HRI = *Subtarget.getRegisterInfo();
1461
1462 setPrefLoopAlignment(Align(16));
1463 setMinFunctionAlignment(Align(4));
1464 setPrefFunctionAlignment(Align(16));
1465 setStackPointerRegisterToSaveRestore(HRI.getStackRegister());
1466 setBooleanContents(TargetLoweringBase::UndefinedBooleanContent);
1467 setBooleanVectorContents(TargetLoweringBase::UndefinedBooleanContent);
1468
1469 setMaxAtomicSizeInBitsSupported(64);
1470 setMinCmpXchgSizeInBits(32);
1471
1472 if (EnableHexSDNodeSched)
1473 setSchedulingPreference(Sched::VLIW);
1474 else
1475 setSchedulingPreference(Sched::Source);
1476
1477 // Limits for inline expansion of memcpy/memmove
1478 MaxStoresPerMemcpy = 6;
1479 MaxStoresPerMemcpyOptSize = 4;
1480 MaxStoresPerMemmove = 6;
1481 MaxStoresPerMemmoveOptSize = 4;
1482 MaxStoresPerMemset = 8;
1483 MaxStoresPerMemsetOptSize = 4;
1484
1485 setTargetDAGCombine(ISD::VECREDUCE_ADD);
1486
1487 //
1488 // Set up register classes.
1489 //
1490
1491 addRegisterClass(VT: MVT::i1, RC: &Hexagon::PredRegsRegClass);
1492 addRegisterClass(VT: MVT::v2i1, RC: &Hexagon::PredRegsRegClass); // bbbbaaaa
1493 addRegisterClass(VT: MVT::v4i1, RC: &Hexagon::PredRegsRegClass); // ddccbbaa
1494 addRegisterClass(VT: MVT::v8i1, RC: &Hexagon::PredRegsRegClass); // hgfedcba
1495 addRegisterClass(VT: MVT::i32, RC: &Hexagon::IntRegsRegClass);
1496 addRegisterClass(VT: MVT::v2i16, RC: &Hexagon::IntRegsRegClass);
1497 addRegisterClass(VT: MVT::v4i8, RC: &Hexagon::IntRegsRegClass);
1498 addRegisterClass(VT: MVT::i64, RC: &Hexagon::DoubleRegsRegClass);
1499 addRegisterClass(VT: MVT::v8i8, RC: &Hexagon::DoubleRegsRegClass);
1500 addRegisterClass(VT: MVT::v4i16, RC: &Hexagon::DoubleRegsRegClass);
1501 addRegisterClass(VT: MVT::v2i32, RC: &Hexagon::DoubleRegsRegClass);
1502
1503 addRegisterClass(VT: MVT::f32, RC: &Hexagon::IntRegsRegClass);
1504 addRegisterClass(VT: MVT::f64, RC: &Hexagon::DoubleRegsRegClass);
1505
1506 //
1507 // Handling of scalar operations.
1508 //
1509 // All operations default to "legal", except:
1510 // - indexed loads and stores (pre-/post-incremented),
1511 // - ANY_EXTEND_VECTOR_INREG, ATOMIC_CMP_SWAP_WITH_SUCCESS, CONCAT_VECTORS,
1512 // ConstantFP, FCEIL, FCOPYSIGN, FEXP, FEXP2, FFLOOR, FGETSIGN,
1513 // FLOG, FLOG2, FLOG10, FMAXIMUMNUM, FMINIMUMNUM, FNEARBYINT, FRINT, FROUND,
1514 // TRAP, FTRUNC, PREFETCH, SIGN_EXTEND_VECTOR_INREG,
1515 // ZERO_EXTEND_VECTOR_INREG,
1516 // which default to "expand" for at least one type.
1517
1518 // Misc operations.
1519 setOperationAction(Op: ISD::ConstantFP, VT: MVT::f32, Action: Legal);
1520 setOperationAction(Op: ISD::ConstantFP, VT: MVT::f64, Action: Legal);
1521 setOperationAction(Op: ISD::TRAP, VT: MVT::Other, Action: Legal);
1522 setOperationAction(Op: ISD::DEBUGTRAP, VT: MVT::Other, Action: Legal);
1523 setOperationAction(Op: ISD::ConstantPool, VT: MVT::i32, Action: Custom);
1524 setOperationAction(Op: ISD::JumpTable, VT: MVT::i32, Action: Custom);
1525 setOperationAction(Op: ISD::BUILD_PAIR, VT: MVT::i64, Action: Expand);
1526 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i1, Action: Expand);
1527 setOperationAction(Op: ISD::INLINEASM, VT: MVT::Other, Action: Custom);
1528 setOperationAction(Op: ISD::INLINEASM_BR, VT: MVT::Other, Action: Custom);
1529 setOperationAction(Op: ISD::PREFETCH, VT: MVT::Other, Action: Custom);
1530 setOperationAction(Op: ISD::READCYCLECOUNTER, VT: MVT::i64, Action: Legal);
1531 setOperationAction(Op: ISD::READSTEADYCOUNTER, VT: MVT::i64, Action: Legal);
1532 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::Other, Action: Custom);
1533 setOperationAction(Op: ISD::INTRINSIC_VOID, VT: MVT::Other, Action: Custom);
1534 setOperationAction(Op: ISD::EH_RETURN, VT: MVT::Other, Action: Custom);
1535 setOperationAction(Op: ISD::GLOBAL_OFFSET_TABLE, VT: MVT::i32, Action: Custom);
1536 setOperationAction(Op: ISD::GlobalTLSAddress, VT: MVT::i32, Action: Custom);
1537 setOperationAction(Op: ISD::ATOMIC_FENCE, VT: MVT::Other, Action: Custom);
1538
1539 // Custom legalize GlobalAddress nodes into CONST32.
1540 setOperationAction(Op: ISD::GlobalAddress, VT: MVT::i32, Action: Custom);
1541 setOperationAction(Op: ISD::GlobalAddress, VT: MVT::i8, Action: Custom);
1542 setOperationAction(Op: ISD::BlockAddress, VT: MVT::i32, Action: Custom);
1543
1544 // Hexagon needs to optimize cases with negative constants.
1545 setOperationAction(Op: ISD::SETCC, VT: MVT::i8, Action: Custom);
1546 setOperationAction(Op: ISD::SETCC, VT: MVT::i16, Action: Custom);
1547 setOperationAction(Op: ISD::SETCC, VT: MVT::v4i8, Action: Custom);
1548 setOperationAction(Op: ISD::SETCC, VT: MVT::v2i16, Action: Custom);
1549
1550 // VASTART needs to be custom lowered to use the VarArgsFrameIndex.
1551 setOperationAction(Op: ISD::VASTART, VT: MVT::Other, Action: Custom);
1552 setOperationAction(Op: ISD::VAEND, VT: MVT::Other, Action: Expand);
1553 setOperationAction(Op: ISD::VAARG, VT: MVT::Other, Action: Expand);
1554 if (Subtarget.isEnvironmentMusl())
1555 setOperationAction(Op: ISD::VACOPY, VT: MVT::Other, Action: Custom);
1556 else
1557 setOperationAction(Op: ISD::VACOPY, VT: MVT::Other, Action: Expand);
1558
1559 setOperationAction(Op: ISD::STACKSAVE, VT: MVT::Other, Action: Expand);
1560 setOperationAction(Op: ISD::STACKRESTORE, VT: MVT::Other, Action: Expand);
1561 setOperationAction(Op: ISD::DYNAMIC_STACKALLOC, VT: MVT::i32, Action: Custom);
1562
1563 if (EmitJumpTables)
1564 setMinimumJumpTableEntries(MinimumJumpTables);
1565 else
1566 setMinimumJumpTableEntries(std::numeric_limits<unsigned>::max());
1567 setOperationAction(Op: ISD::BR_JT, VT: MVT::Other, Action: Expand);
1568
1569 for (unsigned LegalIntOp :
1570 {ISD::ABS, ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX}) {
1571 setOperationAction(Op: LegalIntOp, VT: MVT::i32, Action: Legal);
1572 setOperationAction(Op: LegalIntOp, VT: MVT::i64, Action: Legal);
1573 }
1574
1575 // Hexagon has A4_addp_c and A4_subp_c that take and generate a carry bit,
1576 // but they only operate on i64.
1577 for (MVT VT : MVT::integer_valuetypes()) {
1578 setOperationAction(Op: ISD::UADDO, VT, Action: Custom);
1579 setOperationAction(Op: ISD::USUBO, VT, Action: Custom);
1580 setOperationAction(Op: ISD::SADDO, VT, Action: Expand);
1581 setOperationAction(Op: ISD::SSUBO, VT, Action: Expand);
1582 setOperationAction(Op: ISD::UADDO_CARRY, VT, Action: Expand);
1583 setOperationAction(Op: ISD::USUBO_CARRY, VT, Action: Expand);
1584 }
1585 setOperationAction(Op: ISD::UADDO_CARRY, VT: MVT::i64, Action: Custom);
1586 setOperationAction(Op: ISD::USUBO_CARRY, VT: MVT::i64, Action: Custom);
1587
1588 setOperationAction(Op: ISD::CTLZ, VT: MVT::i8, Action: Promote);
1589 setOperationAction(Op: ISD::CTLZ, VT: MVT::i16, Action: Promote);
1590 setOperationAction(Op: ISD::CTTZ, VT: MVT::i8, Action: Promote);
1591 setOperationAction(Op: ISD::CTTZ, VT: MVT::i16, Action: Promote);
1592
1593 // Popcount can count # of 1s in i64 but returns i32.
1594 setOperationAction(Op: ISD::CTPOP, VT: MVT::i8, Action: Promote);
1595 setOperationAction(Op: ISD::CTPOP, VT: MVT::i16, Action: Promote);
1596 setOperationAction(Op: ISD::CTPOP, VT: MVT::i32, Action: Promote);
1597 setOperationAction(Op: ISD::CTPOP, VT: MVT::i64, Action: Legal);
1598
1599 setOperationAction(Op: ISD::BITREVERSE, VT: MVT::i32, Action: Legal);
1600 setOperationAction(Op: ISD::BITREVERSE, VT: MVT::i64, Action: Legal);
1601 setOperationAction(Op: ISD::BSWAP, VT: MVT::i32, Action: Legal);
1602 setOperationAction(Op: ISD::BSWAP, VT: MVT::i64, Action: Legal);
1603
1604 setOperationAction(Op: ISD::FSHL, VT: MVT::i32, Action: Legal);
1605 setOperationAction(Op: ISD::FSHL, VT: MVT::i64, Action: Legal);
1606 setOperationAction(Op: ISD::FSHR, VT: MVT::i32, Action: Legal);
1607 setOperationAction(Op: ISD::FSHR, VT: MVT::i64, Action: Legal);
1608
1609 for (unsigned IntExpOp :
1610 {ISD::SDIV, ISD::UDIV, ISD::SREM, ISD::UREM,
1611 ISD::SDIVREM, ISD::UDIVREM, ISD::ROTL, ISD::ROTR,
1612 ISD::SHL_PARTS, ISD::SRA_PARTS, ISD::SRL_PARTS,
1613 ISD::SMUL_LOHI, ISD::UMUL_LOHI}) {
1614 for (MVT VT : MVT::integer_valuetypes())
1615 setOperationAction(Op: IntExpOp, VT, Action: Expand);
1616 }
1617 for (MVT VT : MVT::fp_valuetypes()) {
1618 for (unsigned FPExpOp : {ISD::FDIV, ISD::FSQRT, ISD::FSIN, ISD::FCOS,
1619 ISD::FSINCOS, ISD::FPOW, ISD::FCOPYSIGN})
1620 setOperationAction(Op: FPExpOp, VT, Action: Expand);
1621
1622 setOperationAction(Op: ISD::FREM, VT, Action: LibCall);
1623 }
1624
1625 // No extending loads from i32.
1626 for (MVT VT : MVT::integer_valuetypes()) {
1627 setLoadExtAction(ExtType: ISD::ZEXTLOAD, ValVT: VT, MemVT: MVT::i32, Action: Expand);
1628 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: VT, MemVT: MVT::i32, Action: Expand);
1629 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: MVT::i32, Action: Expand);
1630 }
1631 // Turn FP truncstore into trunc + store.
1632 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f32, Action: Expand);
1633 setTruncStoreAction(ValVT: MVT::f32, MemVT: MVT::bf16, Action: Expand);
1634 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::bf16, Action: Expand);
1635 // Turn FP extload into load/fpextend.
1636 for (MVT VT : MVT::fp_valuetypes())
1637 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: MVT::f32, Action: Expand);
1638
1639 // Expand BR_CC and SELECT_CC for all integer and fp types.
1640 for (MVT VT : MVT::integer_valuetypes()) {
1641 setOperationAction(Op: ISD::BR_CC, VT, Action: Expand);
1642 setOperationAction(Op: ISD::SELECT_CC, VT, Action: Expand);
1643 }
1644 for (MVT VT : MVT::fp_valuetypes()) {
1645 setOperationAction(Op: ISD::BR_CC, VT, Action: Expand);
1646 setOperationAction(Op: ISD::SELECT_CC, VT, Action: Expand);
1647 }
1648 setOperationAction(Op: ISD::BR_CC, VT: MVT::Other, Action: Expand);
1649
1650 //
1651 // Handling of vector operations.
1652 //
1653
1654 // Set the action for vector operations to "expand", then override it with
1655 // either "custom" or "legal" for specific cases.
1656 // clang-format off
1657 static const unsigned VectExpOps[] = {
1658 // Integer arithmetic:
1659 ISD::ADD, ISD::SUB, ISD::MUL, ISD::SDIV, ISD::UDIV,
1660 ISD::SREM, ISD::UREM, ISD::SDIVREM, ISD::UDIVREM, ISD::SADDO,
1661 ISD::UADDO, ISD::SSUBO, ISD::USUBO, ISD::SMUL_LOHI, ISD::UMUL_LOHI,
1662 // Logical/bit:
1663 ISD::AND, ISD::OR, ISD::XOR, ISD::ROTL, ISD::ROTR,
1664 ISD::CTPOP, ISD::CTLZ, ISD::CTTZ, ISD::BSWAP, ISD::BITREVERSE,
1665 // Floating point arithmetic/math functions:
1666 ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FMA, ISD::FDIV,
1667 ISD::FREM, ISD::FNEG, ISD::FABS, ISD::FSQRT, ISD::FSIN,
1668 ISD::FCOS, ISD::FPOW, ISD::FLOG, ISD::FLOG2,
1669 ISD::FLOG10, ISD::FEXP, ISD::FEXP2, ISD::FCEIL, ISD::FTRUNC,
1670 ISD::FRINT, ISD::FNEARBYINT, ISD::FROUND, ISD::FFLOOR,
1671 ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM,
1672 ISD::FSINCOS, ISD::FLDEXP,
1673 // Misc:
1674 ISD::BR_CC, ISD::SELECT_CC, ISD::ConstantPool,
1675 // Vector:
1676 ISD::BUILD_VECTOR, ISD::SCALAR_TO_VECTOR,
1677 ISD::EXTRACT_VECTOR_ELT, ISD::INSERT_VECTOR_ELT,
1678 ISD::EXTRACT_SUBVECTOR, ISD::INSERT_SUBVECTOR,
1679 ISD::CONCAT_VECTORS, ISD::VECTOR_SHUFFLE,
1680 ISD::SPLAT_VECTOR,
1681 };
1682 // clang-format on
1683
1684 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
1685 for (unsigned VectExpOp : VectExpOps)
1686 setOperationAction(Op: VectExpOp, VT, Action: Expand);
1687
1688 // Expand all extending loads and truncating stores:
1689 for (MVT TargetVT : MVT::fixedlen_vector_valuetypes()) {
1690 if (TargetVT == VT)
1691 continue;
1692 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: TargetVT, MemVT: VT, Action: Expand);
1693 setLoadExtAction(ExtType: ISD::ZEXTLOAD, ValVT: TargetVT, MemVT: VT, Action: Expand);
1694 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: TargetVT, MemVT: VT, Action: Expand);
1695 setTruncStoreAction(ValVT: VT, MemVT: TargetVT, Action: Expand);
1696 }
1697
1698 // Normalize all inputs to SELECT to be vectors of i32.
1699 if (VT.getVectorElementType() != MVT::i32) {
1700 MVT VT32 = MVT::getVectorVT(VT: MVT::i32, NumElements: VT.getSizeInBits()/32);
1701 setOperationAction(Op: ISD::SELECT, VT, Action: Promote);
1702 AddPromotedToType(Opc: ISD::SELECT, OrigVT: VT, DestVT: VT32);
1703 }
1704 setOperationAction(Op: ISD::SRA, VT, Action: Custom);
1705 setOperationAction(Op: ISD::SHL, VT, Action: Custom);
1706 setOperationAction(Op: ISD::SRL, VT, Action: Custom);
1707 }
1708
1709 setOperationAction(Op: ISD::SADDSAT, VT: MVT::i32, Action: Legal);
1710 setOperationAction(Op: ISD::SADDSAT, VT: MVT::i64, Action: Legal);
1711
1712 // Extending loads from (native) vectors of i8 into (native) vectors of i16
1713 // are legal.
1714 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v2i16, MemVT: MVT::v2i8, Action: Legal);
1715 setLoadExtAction(ExtType: ISD::ZEXTLOAD, ValVT: MVT::v2i16, MemVT: MVT::v2i8, Action: Legal);
1716 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: MVT::v2i16, MemVT: MVT::v2i8, Action: Legal);
1717 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v4i16, MemVT: MVT::v4i8, Action: Legal);
1718 setLoadExtAction(ExtType: ISD::ZEXTLOAD, ValVT: MVT::v4i16, MemVT: MVT::v4i8, Action: Legal);
1719 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: MVT::v4i16, MemVT: MVT::v4i8, Action: Legal);
1720
1721 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::v2i8, Action: Legal);
1722 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::v2i16, Action: Legal);
1723 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::v2i32, Action: Legal);
1724
1725 // Types natively supported:
1726 for (MVT NativeVT : {MVT::v8i1, MVT::v4i1, MVT::v2i1, MVT::v4i8,
1727 MVT::v8i8, MVT::v2i16, MVT::v4i16, MVT::v2i32}) {
1728 setOperationAction(Op: ISD::BUILD_VECTOR, VT: NativeVT, Action: Custom);
1729 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: NativeVT, Action: Custom);
1730 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: NativeVT, Action: Custom);
1731 setOperationAction(Op: ISD::EXTRACT_SUBVECTOR, VT: NativeVT, Action: Custom);
1732 setOperationAction(Op: ISD::INSERT_SUBVECTOR, VT: NativeVT, Action: Custom);
1733 setOperationAction(Op: ISD::CONCAT_VECTORS, VT: NativeVT, Action: Custom);
1734
1735 setOperationAction(Op: ISD::ADD, VT: NativeVT, Action: Legal);
1736 setOperationAction(Op: ISD::SUB, VT: NativeVT, Action: Legal);
1737 setOperationAction(Op: ISD::MUL, VT: NativeVT, Action: Legal);
1738 setOperationAction(Op: ISD::AND, VT: NativeVT, Action: Legal);
1739 setOperationAction(Op: ISD::OR, VT: NativeVT, Action: Legal);
1740 setOperationAction(Op: ISD::XOR, VT: NativeVT, Action: Legal);
1741
1742 if (NativeVT.getVectorElementType() != MVT::i1) {
1743 setOperationAction(Op: ISD::SPLAT_VECTOR, VT: NativeVT, Action: Legal);
1744 setOperationAction(Op: ISD::BSWAP, VT: NativeVT, Action: Legal);
1745 setOperationAction(Op: ISD::BITREVERSE, VT: NativeVT, Action: Legal);
1746 }
1747 }
1748
1749 for (MVT VT : {MVT::v8i8, MVT::v4i16, MVT::v2i32}) {
1750 setOperationAction(Op: ISD::SMIN, VT, Action: Legal);
1751 setOperationAction(Op: ISD::SMAX, VT, Action: Legal);
1752 setOperationAction(Op: ISD::UMIN, VT, Action: Legal);
1753 setOperationAction(Op: ISD::UMAX, VT, Action: Legal);
1754 }
1755
1756 // Custom lower unaligned loads.
1757 // Also, for both loads and stores, verify the alignment of the address
1758 // in case it is a compile-time constant. This is a usability feature to
1759 // provide a meaningful error message to users.
1760 for (MVT VT : {MVT::i16, MVT::i32, MVT::v4i8, MVT::i64, MVT::v8i8,
1761 MVT::v2i16, MVT::v4i16, MVT::v2i32}) {
1762 setOperationAction(Op: ISD::LOAD, VT, Action: Custom);
1763 setOperationAction(Op: ISD::STORE, VT, Action: Custom);
1764 }
1765
1766 // Custom-lower load/stores of boolean vectors.
1767 for (MVT VT : {MVT::v2i1, MVT::v4i1, MVT::v8i1}) {
1768 setOperationAction(Op: ISD::LOAD, VT, Action: Custom);
1769 setOperationAction(Op: ISD::STORE, VT, Action: Custom);
1770 }
1771
1772 // Normalize integer compares to EQ/GT/UGT
1773 for (MVT VT : {MVT::v2i16, MVT::v4i8, MVT::v8i8, MVT::v2i32, MVT::v4i16,
1774 MVT::v2i32}) {
1775 setCondCodeAction(CCs: ISD::SETNE, VT, Action: Expand);
1776 setCondCodeAction(CCs: ISD::SETLE, VT, Action: Expand);
1777 setCondCodeAction(CCs: ISD::SETGE, VT, Action: Expand);
1778 setCondCodeAction(CCs: ISD::SETLT, VT, Action: Expand);
1779 setCondCodeAction(CCs: ISD::SETULE, VT, Action: Expand);
1780 setCondCodeAction(CCs: ISD::SETUGE, VT, Action: Expand);
1781 setCondCodeAction(CCs: ISD::SETULT, VT, Action: Expand);
1782 }
1783
1784 // Normalize boolean compares to [U]LE/[U]LT
1785 for (MVT VT : {MVT::i1, MVT::v2i1, MVT::v4i1, MVT::v8i1}) {
1786 setCondCodeAction(CCs: ISD::SETGE, VT, Action: Expand);
1787 setCondCodeAction(CCs: ISD::SETGT, VT, Action: Expand);
1788 setCondCodeAction(CCs: ISD::SETUGE, VT, Action: Expand);
1789 setCondCodeAction(CCs: ISD::SETUGT, VT, Action: Expand);
1790 }
1791
1792 // Custom-lower bitcasts from i8 to v8i1.
1793 setOperationAction(Op: ISD::BITCAST, VT: MVT::i8, Action: Custom);
1794 setOperationAction(Op: ISD::SETCC, VT: MVT::v2i16, Action: Custom);
1795 setOperationAction(Op: ISD::VSELECT, VT: MVT::v4i8, Action: Custom);
1796 setOperationAction(Op: ISD::VSELECT, VT: MVT::v2i16, Action: Custom);
1797 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT: MVT::v4i8, Action: Custom);
1798 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT: MVT::v4i16, Action: Custom);
1799 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT: MVT::v8i8, Action: Custom);
1800
1801 // V5+.
1802 setOperationAction(Op: ISD::FMA, VT: MVT::f64, Action: Expand);
1803 setOperationAction(Op: ISD::FADD, VT: MVT::f64, Action: Expand);
1804 setOperationAction(Op: ISD::FSUB, VT: MVT::f64, Action: Expand);
1805 setOperationAction(Op: ISD::FMUL, VT: MVT::f64, Action: Expand);
1806 setOperationAction(Op: ISD::FDIV, VT: MVT::f32, Action: Custom);
1807
1808 setOperationAction(Op: ISD::FMINIMUMNUM, VT: MVT::f32, Action: Legal);
1809 setOperationAction(Op: ISD::FMAXIMUMNUM, VT: MVT::f32, Action: Legal);
1810
1811 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i1, Action: Promote);
1812 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i8, Action: Promote);
1813 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i16, Action: Promote);
1814 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::i1, Action: Promote);
1815 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::i8, Action: Promote);
1816 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::i16, Action: Promote);
1817 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::i1, Action: Promote);
1818 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::i8, Action: Promote);
1819 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::i16, Action: Promote);
1820 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i1, Action: Promote);
1821 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i8, Action: Promote);
1822 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i16, Action: Promote);
1823
1824 // Special handling for half-precision floating point conversions.
1825 // Lower half float conversions into library calls.
1826 setOperationAction(Op: ISD::FP16_TO_FP, VT: MVT::f32, Action: Expand);
1827 setOperationAction(Op: ISD::FP16_TO_FP, VT: MVT::f64, Action: Expand);
1828 setOperationAction(Op: ISD::FP_TO_FP16, VT: MVT::f32, Action: Expand);
1829 setOperationAction(Op: ISD::FP_TO_FP16, VT: MVT::f64, Action: Expand);
1830 setOperationAction(Op: ISD::BF16_TO_FP, VT: MVT::f32, Action: Expand);
1831 setOperationAction(Op: ISD::BF16_TO_FP, VT: MVT::f64, Action: Expand);
1832 setOperationAction(Op: ISD::FP_TO_BF16, VT: MVT::f64, Action: Expand);
1833
1834 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f32, MemVT: MVT::f16, Action: Expand);
1835 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::f16, Action: Expand);
1836 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f32, MemVT: MVT::bf16, Action: Expand);
1837 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::bf16, Action: Expand);
1838
1839 setTruncStoreAction(ValVT: MVT::f32, MemVT: MVT::f16, Action: Expand);
1840 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f16, Action: Expand);
1841
1842 // Handling of indexed loads/stores: default is "expand".
1843 //
1844 for (MVT VT : {MVT::i8, MVT::i16, MVT::i32, MVT::i64, MVT::f32, MVT::f64,
1845 MVT::v2i16, MVT::v2i32, MVT::v4i8, MVT::v4i16, MVT::v8i8}) {
1846 setIndexedLoadAction(IdxModes: ISD::POST_INC, VT, Action: Legal);
1847 setIndexedStoreAction(IdxModes: ISD::POST_INC, VT, Action: Legal);
1848 }
1849
1850 // Subtarget-specific operation actions.
1851 //
1852 if (Subtarget.hasV60Ops()) {
1853 setOperationAction(Op: ISD::ROTL, VT: MVT::i32, Action: Legal);
1854 setOperationAction(Op: ISD::ROTL, VT: MVT::i64, Action: Legal);
1855 setOperationAction(Op: ISD::ROTR, VT: MVT::i32, Action: Legal);
1856 setOperationAction(Op: ISD::ROTR, VT: MVT::i64, Action: Legal);
1857 }
1858 if (Subtarget.hasV66Ops()) {
1859 setOperationAction(Op: ISD::FADD, VT: MVT::f64, Action: Legal);
1860 setOperationAction(Op: ISD::FSUB, VT: MVT::f64, Action: Legal);
1861 }
1862 if (Subtarget.hasV67Ops()) {
1863 setOperationAction(Op: ISD::FMINIMUMNUM, VT: MVT::f64, Action: Legal);
1864 setOperationAction(Op: ISD::FMAXIMUMNUM, VT: MVT::f64, Action: Legal);
1865 setOperationAction(Op: ISD::FMUL, VT: MVT::f64, Action: Legal);
1866 }
1867
1868 setTargetDAGCombine(ISD::OR);
1869 setTargetDAGCombine(ISD::TRUNCATE);
1870 setTargetDAGCombine(ISD::VSELECT);
1871
1872 if (Subtarget.useHVXOps())
1873 initializeHVXLowering();
1874
1875 computeRegisterProperties(TRI: &HRI);
1876}
1877
1878bool
1879HexagonTargetLowering::validateConstPtrAlignment(SDValue Ptr, Align NeedAlign,
1880 const SDLoc &dl, SelectionDAG &DAG) const {
1881 auto *CA = dyn_cast<ConstantSDNode>(Val&: Ptr);
1882 if (!CA)
1883 return true;
1884 unsigned Addr = CA->getZExtValue();
1885 Align HaveAlign =
1886 Addr != 0 ? Align(1ull << llvm::countr_zero(Val: Addr)) : NeedAlign;
1887 if (HaveAlign >= NeedAlign)
1888 return true;
1889
1890 static int DK_MisalignedTrap = llvm::getNextAvailablePluginDiagnosticKind();
1891
1892 struct DiagnosticInfoMisalignedTrap : public DiagnosticInfo {
1893 DiagnosticInfoMisalignedTrap(StringRef M)
1894 : DiagnosticInfo(DK_MisalignedTrap, DS_Remark), Msg(M) {}
1895 void print(DiagnosticPrinter &DP) const override {
1896 DP << Msg;
1897 }
1898 static bool classof(const DiagnosticInfo *DI) {
1899 return DI->getKind() == DK_MisalignedTrap;
1900 }
1901 StringRef Msg;
1902 };
1903
1904 std::string ErrMsg;
1905 raw_string_ostream O(ErrMsg);
1906 O << "Misaligned constant address: " << format_hex(N: Addr, Width: 10)
1907 << " has alignment " << HaveAlign.value()
1908 << ", but the memory access requires " << NeedAlign.value();
1909 if (DebugLoc DL = dl.getDebugLoc())
1910 DL.print(OS&: O << ", at ");
1911 O << ". The instruction has been replaced with a trap.";
1912
1913 DAG.getContext()->diagnose(DI: DiagnosticInfoMisalignedTrap(O.str()));
1914 return false;
1915}
1916
1917SDValue
1918HexagonTargetLowering::replaceMemWithUndef(SDValue Op, SelectionDAG &DAG)
1919 const {
1920 const SDLoc &dl(Op);
1921 auto *LS = cast<LSBaseSDNode>(Val: Op.getNode());
1922 assert(!LS->isIndexed() && "Not expecting indexed ops on constant address");
1923
1924 SDValue Chain = LS->getChain();
1925 SDValue Trap = DAG.getNode(Opcode: ISD::TRAP, DL: dl, VT: MVT::Other, Operand: Chain);
1926 if (LS->getOpcode() == ISD::LOAD)
1927 return DAG.getMergeValues(Ops: {DAG.getUNDEF(VT: ty(Op)), Trap}, dl);
1928 return Trap;
1929}
1930
1931// Bit-reverse Load Intrinsic: Check if the instruction is a bit reverse load
1932// intrinsic.
1933static bool isBrevLdIntrinsic(const Value *Inst) {
1934 unsigned ID = cast<IntrinsicInst>(Val: Inst)->getIntrinsicID();
1935 return (ID == Intrinsic::hexagon_L2_loadrd_pbr ||
1936 ID == Intrinsic::hexagon_L2_loadri_pbr ||
1937 ID == Intrinsic::hexagon_L2_loadrh_pbr ||
1938 ID == Intrinsic::hexagon_L2_loadruh_pbr ||
1939 ID == Intrinsic::hexagon_L2_loadrb_pbr ||
1940 ID == Intrinsic::hexagon_L2_loadrub_pbr);
1941}
1942
1943// Bit-reverse Load Intrinsic :Crawl up and figure out the object from previous
1944// instruction. So far we only handle bitcast, extract value and bit reverse
1945// load intrinsic instructions. Should we handle CGEP ?
1946static Value *getBrevLdObject(Value *V) {
1947 if (Operator::getOpcode(V) == Instruction::ExtractValue ||
1948 Operator::getOpcode(V) == Instruction::BitCast)
1949 V = cast<Operator>(Val: V)->getOperand(i: 0);
1950 else if (isa<IntrinsicInst>(Val: V) && isBrevLdIntrinsic(Inst: V))
1951 V = cast<Instruction>(Val: V)->getOperand(i: 0);
1952 return V;
1953}
1954
1955// Bit-reverse Load Intrinsic: For a PHI Node return either an incoming edge or
1956// a back edge. If the back edge comes from the intrinsic itself, the incoming
1957// edge is returned.
1958static Value *returnEdge(const PHINode *PN, Value *IntrBaseVal) {
1959 const BasicBlock *Parent = PN->getParent();
1960 int Idx = -1;
1961 for (unsigned i = 0, e = PN->getNumIncomingValues(); i < e; ++i) {
1962 BasicBlock *Blk = PN->getIncomingBlock(i);
1963 // Determine if the back edge is originated from intrinsic.
1964 if (Blk == Parent) {
1965 Value *BackEdgeVal = PN->getIncomingValue(i);
1966 Value *BaseVal;
1967 // Loop over till we return the same Value or we hit the IntrBaseVal.
1968 do {
1969 BaseVal = BackEdgeVal;
1970 BackEdgeVal = getBrevLdObject(V: BackEdgeVal);
1971 } while ((BaseVal != BackEdgeVal) && (IntrBaseVal != BackEdgeVal));
1972 // If the getBrevLdObject returns IntrBaseVal, we should return the
1973 // incoming edge.
1974 if (IntrBaseVal == BackEdgeVal)
1975 continue;
1976 Idx = i;
1977 break;
1978 } else // Set the node to incoming edge.
1979 Idx = i;
1980 }
1981 assert(Idx >= 0 && "Unexpected index to incoming argument in PHI");
1982 return PN->getIncomingValue(i: Idx);
1983}
1984
1985// Bit-reverse Load Intrinsic: Figure out the underlying object the base
1986// pointer points to, for the bit-reverse load intrinsic. Setting this to
1987// memoperand might help alias analysis to figure out the dependencies.
1988static Value *getUnderLyingObjectForBrevLdIntr(Value *V) {
1989 Value *IntrBaseVal = V;
1990 Value *BaseVal;
1991 // Loop over till we return the same Value, implies we either figure out
1992 // the object or we hit a PHI
1993 do {
1994 BaseVal = V;
1995 V = getBrevLdObject(V);
1996 } while (BaseVal != V);
1997
1998 // Identify the object from PHINode.
1999 if (const PHINode *PN = dyn_cast<PHINode>(Val: V))
2000 return returnEdge(PN, IntrBaseVal);
2001 // For non PHI nodes, the object is the last value returned by getBrevLdObject
2002 else
2003 return V;
2004}
2005
2006/// Given an intrinsic, checks if on the target the intrinsic will need to map
2007/// to a MemIntrinsicNode (touches memory). If this is the case, it stores
2008/// the intrinsic information into the Infos vector.
2009void HexagonTargetLowering::getTgtMemIntrinsic(
2010 SmallVectorImpl<IntrinsicInfo> &Infos, const CallBase &I,
2011 MachineFunction &MF, unsigned Intrinsic) const {
2012 IntrinsicInfo Info;
2013 switch (Intrinsic) {
2014 case Intrinsic::hexagon_L2_loadrd_pbr:
2015 case Intrinsic::hexagon_L2_loadri_pbr:
2016 case Intrinsic::hexagon_L2_loadrh_pbr:
2017 case Intrinsic::hexagon_L2_loadruh_pbr:
2018 case Intrinsic::hexagon_L2_loadrb_pbr:
2019 case Intrinsic::hexagon_L2_loadrub_pbr: {
2020 Info.opc = ISD::INTRINSIC_W_CHAIN;
2021 auto &DL = I.getDataLayout();
2022 auto &Cont = I.getCalledFunction()->getParent()->getContext();
2023 // The intrinsic function call is of the form { ElTy, i8* }
2024 // @llvm.hexagon.L2.loadXX.pbr(i8*, i32). The pointer and memory access type
2025 // should be derived from ElTy.
2026 Type *ElTy = I.getCalledFunction()->getReturnType()->getStructElementType(N: 0);
2027 Info.memVT = MVT::getVT(Ty: ElTy);
2028 llvm::Value *BasePtrVal = I.getOperand(i_nocapture: 0);
2029 Info.ptrVal = getUnderLyingObjectForBrevLdIntr(V: BasePtrVal);
2030 // The offset value comes through Modifier register. For now, assume the
2031 // offset is 0.
2032 Info.offset = 0;
2033 Info.align = DL.getABITypeAlign(Ty: Info.memVT.getTypeForEVT(Context&: Cont));
2034 Info.flags = MachineMemOperand::MOLoad;
2035 Infos.push_back(Elt: Info);
2036 return;
2037 }
2038 case Intrinsic::hexagon_V6_vgathermw:
2039 case Intrinsic::hexagon_V6_vgathermw_128B:
2040 case Intrinsic::hexagon_V6_vgathermh:
2041 case Intrinsic::hexagon_V6_vgathermh_128B:
2042 case Intrinsic::hexagon_V6_vgathermhw:
2043 case Intrinsic::hexagon_V6_vgathermhw_128B:
2044 case Intrinsic::hexagon_V6_vgathermwq:
2045 case Intrinsic::hexagon_V6_vgathermwq_128B:
2046 case Intrinsic::hexagon_V6_vgathermhq:
2047 case Intrinsic::hexagon_V6_vgathermhq_128B:
2048 case Intrinsic::hexagon_V6_vgathermhwq:
2049 case Intrinsic::hexagon_V6_vgathermhwq_128B:
2050 case Intrinsic::hexagon_V6_vgather_vscattermh:
2051 case Intrinsic::hexagon_V6_vgather_vscattermh_128B: {
2052 const Module &M = *I.getParent()->getParent()->getParent();
2053 Info.opc = ISD::INTRINSIC_W_CHAIN;
2054 Type *VecTy = I.getArgOperand(i: I.arg_size() - 1)->getType();
2055 assert(VecTy->isVectorTy() && "Expected vector operand for vgather");
2056 Info.memVT = MVT::getVT(Ty: VecTy);
2057 Info.ptrVal = I.getArgOperand(i: 0);
2058 Info.offset = 0;
2059 Info.align =
2060 MaybeAlign(M.getDataLayout().getTypeAllocSizeInBits(Ty: VecTy) / 8);
2061 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
2062 MachineMemOperand::MOVolatile;
2063 Infos.push_back(Elt: Info);
2064 return;
2065 }
2066 default:
2067 break;
2068 }
2069}
2070
2071bool HexagonTargetLowering::hasBitTest(SDValue X, SDValue Y) const {
2072 return X.getValueType().isScalarInteger(); // 'tstbit'
2073}
2074
2075bool HexagonTargetLowering::isTruncateFree(Type *Ty1, Type *Ty2) const {
2076 return isTruncateFree(VT1: EVT::getEVT(Ty: Ty1), VT2: EVT::getEVT(Ty: Ty2));
2077}
2078
2079bool HexagonTargetLowering::isTruncateFree(EVT VT1, EVT VT2) const {
2080 if (!VT1.isSimple() || !VT2.isSimple())
2081 return false;
2082 return VT1.getSimpleVT() == MVT::i64 && VT2.getSimpleVT() == MVT::i32;
2083}
2084
2085bool HexagonTargetLowering::isFMAFasterThanFMulAndFAdd(
2086 const MachineFunction &MF, EVT VT) const {
2087 return isOperationLegalOrCustom(Op: ISD::FMA, VT);
2088}
2089
2090// Should we expand the build vector with shuffles?
2091bool HexagonTargetLowering::shouldExpandBuildVectorWithShuffles(EVT VT,
2092 unsigned DefinedValues) const {
2093 return false;
2094}
2095
2096bool HexagonTargetLowering::isExtractSubvectorCheap(EVT ResVT, EVT SrcVT,
2097 unsigned Index) const {
2098 assert(ResVT.getVectorElementType() == SrcVT.getVectorElementType());
2099 if (!ResVT.isSimple() || !SrcVT.isSimple())
2100 return false;
2101
2102 MVT ResTy = ResVT.getSimpleVT(), SrcTy = SrcVT.getSimpleVT();
2103 if (ResTy.getVectorElementType() != MVT::i1)
2104 return true;
2105
2106 // Non-HVX bool vectors are relatively cheap.
2107 return SrcTy.getVectorNumElements() <= 8;
2108}
2109
2110bool HexagonTargetLowering::isTargetCanonicalConstantNode(SDValue Op) const {
2111 return Op.getOpcode() == ISD::CONCAT_VECTORS ||
2112 TargetLowering::isTargetCanonicalConstantNode(Op);
2113}
2114
2115bool HexagonTargetLowering::isShuffleMaskLegal(ArrayRef<int> Mask,
2116 EVT VT) const {
2117 return true;
2118}
2119
2120TargetLoweringBase::LegalizeTypeAction
2121HexagonTargetLowering::getPreferredVectorAction(MVT VT) const {
2122 unsigned VecLen = VT.getVectorMinNumElements();
2123 MVT ElemTy = VT.getVectorElementType();
2124
2125 if (VecLen == 1 || VT.isScalableVector())
2126 return TargetLoweringBase::TypeScalarizeVector;
2127
2128 if (Subtarget.useHVXOps()) {
2129 unsigned Action = getPreferredHvxVectorAction(VecTy: VT);
2130 if (Action != ~0u)
2131 return static_cast<TargetLoweringBase::LegalizeTypeAction>(Action);
2132 }
2133
2134 // Always widen (remaining) vectors of i1.
2135 if (ElemTy == MVT::i1)
2136 return TargetLoweringBase::TypeWidenVector;
2137 // Widen non-power-of-2 vectors. Such types cannot be split right now,
2138 // and computeRegisterProperties will override "split" with "widen",
2139 // which can cause other issues.
2140 if (!isPowerOf2_32(Value: VecLen))
2141 return TargetLoweringBase::TypeWidenVector;
2142
2143 return TargetLoweringBase::TypeSplitVector;
2144}
2145
2146TargetLoweringBase::LegalizeAction
2147HexagonTargetLowering::getCustomOperationAction(SDNode &Op) const {
2148 if (Subtarget.useHVXOps()) {
2149 unsigned Action = getCustomHvxOperationAction(Op);
2150 if (Action != ~0u)
2151 return static_cast<TargetLoweringBase::LegalizeAction>(Action);
2152 }
2153 return TargetLoweringBase::Legal;
2154}
2155
2156std::pair<SDValue, int>
2157HexagonTargetLowering::getBaseAndOffset(SDValue Addr) const {
2158 if (Addr.getOpcode() == ISD::ADD) {
2159 SDValue Op1 = Addr.getOperand(i: 1);
2160 if (auto *CN = dyn_cast<const ConstantSDNode>(Val: Op1.getNode()))
2161 return { Addr.getOperand(i: 0), CN->getSExtValue() };
2162 }
2163 return { Addr, 0 };
2164}
2165
2166// Lower a vector shuffle (V1, V2, V3). V1 and V2 are the two vectors
2167// to select data from, V3 is the permutation.
2168SDValue
2169HexagonTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG)
2170 const {
2171 const auto *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
2172 ArrayRef<int> AM = SVN->getMask();
2173 assert(AM.size() <= 8 && "Unexpected shuffle mask");
2174 unsigned VecLen = AM.size();
2175
2176 MVT VecTy = ty(Op);
2177 assert(!Subtarget.isHVXVectorType(VecTy, true) &&
2178 "HVX shuffles should be legal");
2179 assert(VecTy.getSizeInBits() <= 64 && "Unexpected vector length");
2180
2181 SDValue Op0 = Op.getOperand(i: 0);
2182 SDValue Op1 = Op.getOperand(i: 1);
2183 const SDLoc &dl(Op);
2184
2185 // If the inputs are not the same as the output, bail. This is not an
2186 // error situation, but complicates the handling and the default expansion
2187 // (into BUILD_VECTOR) should be adequate.
2188 if (ty(Op: Op0) != VecTy || ty(Op: Op1) != VecTy)
2189 return SDValue();
2190
2191 // Normalize the mask so that the first non-negative index comes from
2192 // the first operand.
2193 SmallVector<int, 8> Mask(AM);
2194 unsigned F = llvm::find_if(Range&: AM, P: [](int M) { return M >= 0; }) - AM.data();
2195 if (F == AM.size())
2196 return DAG.getUNDEF(VT: VecTy);
2197 if (AM[F] >= int(VecLen)) {
2198 ShuffleVectorSDNode::commuteMask(Mask);
2199 std::swap(a&: Op0, b&: Op1);
2200 }
2201
2202 // Express the shuffle mask in terms of bytes.
2203 SmallVector<int,8> ByteMask;
2204 unsigned ElemBytes = VecTy.getVectorElementType().getSizeInBits() / 8;
2205 for (int M : Mask) {
2206 if (M < 0) {
2207 for (unsigned j = 0; j != ElemBytes; ++j)
2208 ByteMask.push_back(Elt: -1);
2209 } else {
2210 for (unsigned j = 0; j != ElemBytes; ++j)
2211 ByteMask.push_back(Elt: M*ElemBytes + j);
2212 }
2213 }
2214 assert(ByteMask.size() <= 8);
2215
2216 // All non-undef (non-negative) indexes are well within [0..127], so they
2217 // fit in a single byte. Build two 64-bit words:
2218 // - MaskIdx where each byte is the corresponding index (for non-negative
2219 // indexes), and 0xFF for negative indexes, and
2220 // - MaskUnd that has 0xFF for each negative index.
2221 uint64_t MaskIdx = 0;
2222 uint64_t MaskUnd = 0;
2223 for (unsigned i = 0, e = ByteMask.size(); i != e; ++i) {
2224 unsigned S = 8*i;
2225 uint64_t M = ByteMask[i] & 0xFF;
2226 if (M == 0xFF)
2227 MaskUnd |= M << S;
2228 MaskIdx |= M << S;
2229 }
2230
2231 if (ByteMask.size() == 4) {
2232 // Identity.
2233 if (MaskIdx == (0x03020100 | MaskUnd))
2234 return Op0;
2235 // Byte swap.
2236 if (MaskIdx == (0x00010203 | MaskUnd)) {
2237 SDValue T0 = DAG.getBitcast(VT: MVT::i32, V: Op0);
2238 SDValue T1 = DAG.getNode(Opcode: ISD::BSWAP, DL: dl, VT: MVT::i32, Operand: T0);
2239 return DAG.getBitcast(VT: VecTy, V: T1);
2240 }
2241
2242 // Byte packs.
2243 SDValue Concat10 =
2244 getCombine(Hi: Op1, Lo: Op0, dl, ResTy: typeJoin(Tys: {ty(Op: Op1), ty(Op: Op0)}), DAG);
2245 if (MaskIdx == (0x06040200 | MaskUnd))
2246 return getInstr(MachineOpc: Hexagon::S2_vtrunehb, dl, Ty: VecTy, Ops: {Concat10}, DAG);
2247 if (MaskIdx == (0x07050301 | MaskUnd))
2248 return getInstr(MachineOpc: Hexagon::S2_vtrunohb, dl, Ty: VecTy, Ops: {Concat10}, DAG);
2249
2250 SDValue Concat01 =
2251 getCombine(Hi: Op0, Lo: Op1, dl, ResTy: typeJoin(Tys: {ty(Op: Op0), ty(Op: Op1)}), DAG);
2252 if (MaskIdx == (0x02000604 | MaskUnd))
2253 return getInstr(MachineOpc: Hexagon::S2_vtrunehb, dl, Ty: VecTy, Ops: {Concat01}, DAG);
2254 if (MaskIdx == (0x03010705 | MaskUnd))
2255 return getInstr(MachineOpc: Hexagon::S2_vtrunohb, dl, Ty: VecTy, Ops: {Concat01}, DAG);
2256 }
2257
2258 if (ByteMask.size() == 8) {
2259 // Identity.
2260 if (MaskIdx == (0x0706050403020100ull | MaskUnd))
2261 return Op0;
2262 // Byte swap.
2263 if (MaskIdx == (0x0001020304050607ull | MaskUnd)) {
2264 SDValue T0 = DAG.getBitcast(VT: MVT::i64, V: Op0);
2265 SDValue T1 = DAG.getNode(Opcode: ISD::BSWAP, DL: dl, VT: MVT::i64, Operand: T0);
2266 return DAG.getBitcast(VT: VecTy, V: T1);
2267 }
2268
2269 // Halfword picks.
2270 if (MaskIdx == (0x0d0c050409080100ull | MaskUnd))
2271 return getInstr(MachineOpc: Hexagon::S2_shuffeh, dl, Ty: VecTy, Ops: {Op1, Op0}, DAG);
2272 if (MaskIdx == (0x0f0e07060b0a0302ull | MaskUnd))
2273 return getInstr(MachineOpc: Hexagon::S2_shuffoh, dl, Ty: VecTy, Ops: {Op1, Op0}, DAG);
2274 if (MaskIdx == (0x0d0c090805040100ull | MaskUnd))
2275 return getInstr(MachineOpc: Hexagon::S2_vtrunewh, dl, Ty: VecTy, Ops: {Op1, Op0}, DAG);
2276 if (MaskIdx == (0x0f0e0b0a07060302ull | MaskUnd))
2277 return getInstr(MachineOpc: Hexagon::S2_vtrunowh, dl, Ty: VecTy, Ops: {Op1, Op0}, DAG);
2278 if (MaskIdx == (0x0706030205040100ull | MaskUnd)) {
2279 VectorPair P = opSplit(Vec: Op0, dl, DAG);
2280 return getInstr(MachineOpc: Hexagon::S2_packhl, dl, Ty: VecTy, Ops: {P.second, P.first}, DAG);
2281 }
2282
2283 // Byte packs.
2284 if (MaskIdx == (0x0e060c040a020800ull | MaskUnd))
2285 return getInstr(MachineOpc: Hexagon::S2_shuffeb, dl, Ty: VecTy, Ops: {Op1, Op0}, DAG);
2286 if (MaskIdx == (0x0f070d050b030901ull | MaskUnd))
2287 return getInstr(MachineOpc: Hexagon::S2_shuffob, dl, Ty: VecTy, Ops: {Op1, Op0}, DAG);
2288 }
2289
2290 return SDValue();
2291}
2292
2293SDValue
2294HexagonTargetLowering::getSplatValue(SDValue Op, SelectionDAG &DAG) const {
2295 switch (Op.getOpcode()) {
2296 case ISD::BUILD_VECTOR:
2297 if (SDValue S = cast<BuildVectorSDNode>(Val&: Op)->getSplatValue())
2298 return S;
2299 break;
2300 case ISD::SPLAT_VECTOR:
2301 return Op.getOperand(i: 0);
2302 }
2303 return SDValue();
2304}
2305
2306// Create a Hexagon-specific node for shifting a vector by an integer.
2307SDValue
2308HexagonTargetLowering::getVectorShiftByInt(SDValue Op, SelectionDAG &DAG)
2309 const {
2310 unsigned NewOpc;
2311 switch (Op.getOpcode()) {
2312 case ISD::SHL:
2313 NewOpc = HexagonISD::VASL;
2314 break;
2315 case ISD::SRA:
2316 NewOpc = HexagonISD::VASR;
2317 break;
2318 case ISD::SRL:
2319 NewOpc = HexagonISD::VLSR;
2320 break;
2321 default:
2322 llvm_unreachable("Unexpected shift opcode");
2323 }
2324 if (SDValue Sp = getSplatValue(Op: Op.getOperand(i: 1), DAG)) {
2325 const SDLoc dl(Op);
2326 // Canonicalize shift amount to i32 as required.
2327 SDValue Sh = Sp;
2328 if (Sh.getValueType() != MVT::i32)
2329 Sh = DAG.getZExtOrTrunc(Op: Sh, DL: dl, VT: MVT::i32);
2330
2331 assert(Sh.getValueType() == MVT::i32 &&
2332 "Hexagon vector shift-by-int must use i32 shift operand");
2333 return DAG.getNode(Opcode: NewOpc, DL: dl, VT: ty(Op), N1: Op.getOperand(i: 0), N2: Sh);
2334 }
2335
2336 return SDValue();
2337}
2338
2339SDValue
2340HexagonTargetLowering::LowerVECTOR_SHIFT(SDValue Op, SelectionDAG &DAG) const {
2341 const SDLoc &dl(Op);
2342
2343 // First try to convert the shift (by vector) to a shift by a scalar.
2344 // If we first split the shift, the shift amount will become 'extract
2345 // subvector', and will no longer be recognized as scalar.
2346 SDValue Res = Op;
2347 if (SDValue S = getVectorShiftByInt(Op, DAG))
2348 Res = S;
2349
2350 unsigned Opc = Res.getOpcode();
2351 switch (Opc) {
2352 case HexagonISD::VASR:
2353 case HexagonISD::VLSR:
2354 case HexagonISD::VASL:
2355 break;
2356 default:
2357 // No instructions for shifts by non-scalars.
2358 return SDValue();
2359 }
2360
2361 MVT ResTy = ty(Op: Res);
2362 if (ResTy.getVectorElementType() != MVT::i8)
2363 return Res;
2364
2365 // For shifts of i8, extend the inputs to i16, then truncate back to i8.
2366 assert(ResTy.getVectorElementType() == MVT::i8);
2367 SDValue Val = Res.getOperand(i: 0), Amt = Res.getOperand(i: 1);
2368
2369 auto ShiftPartI8 = [&dl, &DAG, this](unsigned Opc, SDValue V, SDValue A) {
2370 MVT Ty = ty(Op: V);
2371 MVT ExtTy = MVT::getVectorVT(VT: MVT::i16, NumElements: Ty.getVectorNumElements());
2372 SDValue ExtV = Opc == HexagonISD::VASR ? DAG.getSExtOrTrunc(Op: V, DL: dl, VT: ExtTy)
2373 : DAG.getZExtOrTrunc(Op: V, DL: dl, VT: ExtTy);
2374 SDValue ExtS = DAG.getNode(Opcode: Opc, DL: dl, VT: ExtTy, Ops: {ExtV, A});
2375 return DAG.getZExtOrTrunc(Op: ExtS, DL: dl, VT: Ty);
2376 };
2377
2378 if (ResTy.getSizeInBits() == 32)
2379 return ShiftPartI8(Opc, Val, Amt);
2380
2381 auto [LoV, HiV] = opSplit(Vec: Val, dl, DAG);
2382 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResTy,
2383 Ops: {ShiftPartI8(Opc, LoV, Amt), ShiftPartI8(Opc, HiV, Amt)});
2384}
2385
2386SDValue
2387HexagonTargetLowering::LowerROTL(SDValue Op, SelectionDAG &DAG) const {
2388 if (isa<ConstantSDNode>(Val: Op.getOperand(i: 1).getNode()))
2389 return Op;
2390 return SDValue();
2391}
2392
2393SDValue
2394HexagonTargetLowering::LowerBITCAST(SDValue Op, SelectionDAG &DAG) const {
2395 MVT ResTy = ty(Op);
2396 SDValue InpV = Op.getOperand(i: 0);
2397 MVT InpTy = ty(Op: InpV);
2398 assert(ResTy.getSizeInBits() == InpTy.getSizeInBits());
2399 const SDLoc &dl(Op);
2400
2401 // Handle conversion from i8 to v8i1.
2402 if (InpTy == MVT::i8) {
2403 if (ResTy == MVT::v8i1) {
2404 SDValue Sc = DAG.getBitcast(VT: tyScalar(Ty: InpTy), V: InpV);
2405 SDValue Ext = DAG.getZExtOrTrunc(Op: Sc, DL: dl, VT: MVT::i32);
2406 return getInstr(MachineOpc: Hexagon::C2_tfrrp, dl, Ty: ResTy, Ops: Ext, DAG);
2407 }
2408 return SDValue();
2409 }
2410
2411 return Op;
2412}
2413
2414bool
2415HexagonTargetLowering::getBuildVectorConstInts(ArrayRef<SDValue> Values,
2416 MVT VecTy, SelectionDAG &DAG,
2417 MutableArrayRef<ConstantInt*> Consts) const {
2418 MVT ElemTy = VecTy.getVectorElementType();
2419 unsigned ElemWidth = ElemTy.getSizeInBits();
2420 IntegerType *IntTy = IntegerType::get(C&: *DAG.getContext(), NumBits: ElemWidth);
2421 bool AllConst = true;
2422
2423 for (unsigned i = 0, e = Values.size(); i != e; ++i) {
2424 SDValue V = Values[i];
2425 if (V.isUndef()) {
2426 Consts[i] = ConstantInt::get(Ty: IntTy, V: 0);
2427 continue;
2428 }
2429 // Make sure to always cast to IntTy.
2430 if (auto *CN = dyn_cast<ConstantSDNode>(Val: V.getNode())) {
2431 const ConstantInt *CI = CN->getConstantIntValue();
2432 Consts[i] = cast<ConstantInt>(
2433 Val: ConstantInt::get(Ty: IntTy, V: CI->getValue().trunc(width: ElemWidth)));
2434 } else if (auto *CN = dyn_cast<ConstantFPSDNode>(Val: V.getNode())) {
2435 const ConstantFP *CF = CN->getConstantFPValue();
2436 APInt A = CF->getValueAPF().bitcastToAPInt();
2437 Consts[i] = ConstantInt::get(Ty: IntTy, V: A.getZExtValue());
2438 } else {
2439 AllConst = false;
2440 }
2441 }
2442 return AllConst;
2443}
2444
2445SDValue
2446HexagonTargetLowering::buildVector32(ArrayRef<SDValue> Elem, const SDLoc &dl,
2447 MVT VecTy, SelectionDAG &DAG) const {
2448 MVT ElemTy = VecTy.getVectorElementType();
2449 assert(VecTy.getVectorNumElements() == Elem.size());
2450
2451 SmallVector<ConstantInt*,4> Consts(Elem.size());
2452 bool AllConst = getBuildVectorConstInts(Values: Elem, VecTy, DAG, Consts);
2453
2454 unsigned First, Num = Elem.size();
2455 for (First = 0; First != Num; ++First) {
2456 if (!isUndef(Op: Elem[First]))
2457 break;
2458 }
2459 if (First == Num)
2460 return DAG.getUNDEF(VT: VecTy);
2461
2462 if (AllConst &&
2463 llvm::all_of(Range&: Consts, P: [](ConstantInt *CI) { return CI->isZero(); }))
2464 return getZero(dl, Ty: VecTy, DAG);
2465
2466 if (ElemTy == MVT::i16 || ElemTy == MVT::f16) {
2467 assert(Elem.size() == 2);
2468 if (AllConst) {
2469 // The 'Consts' array will have all values as integers regardless
2470 // of the vector element type.
2471 uint32_t V = (Consts[0]->getZExtValue() & 0xFFFF) |
2472 Consts[1]->getZExtValue() << 16;
2473 return DAG.getBitcast(VT: VecTy, V: DAG.getConstant(Val: V, DL: dl, VT: MVT::i32));
2474 }
2475 SDValue E0, E1;
2476 if (ElemTy == MVT::f16) {
2477 E0 = DAG.getZExtOrTrunc(Op: DAG.getBitcast(VT: MVT::i16, V: Elem[0]), DL: dl, VT: MVT::i32);
2478 E1 = DAG.getZExtOrTrunc(Op: DAG.getBitcast(VT: MVT::i16, V: Elem[1]), DL: dl, VT: MVT::i32);
2479 } else {
2480 E0 = Elem[0];
2481 E1 = Elem[1];
2482 }
2483 SDValue N = getInstr(MachineOpc: Hexagon::A2_combine_ll, dl, Ty: MVT::i32, Ops: {E1, E0}, DAG);
2484 return DAG.getBitcast(VT: VecTy, V: N);
2485 }
2486
2487 if (ElemTy == MVT::i8) {
2488 // First try generating a constant.
2489 if (AllConst) {
2490 uint32_t V = (Consts[0]->getZExtValue() & 0xFF) |
2491 (Consts[1]->getZExtValue() & 0xFF) << 8 |
2492 (Consts[2]->getZExtValue() & 0xFF) << 16 |
2493 Consts[3]->getZExtValue() << 24;
2494 return DAG.getBitcast(VT: MVT::v4i8, V: DAG.getConstant(Val: V, DL: dl, VT: MVT::i32));
2495 }
2496
2497 // Then try splat.
2498 bool IsSplat = true;
2499 for (unsigned i = First+1; i != Num; ++i) {
2500 if (Elem[i] == Elem[First] || isUndef(Op: Elem[i]))
2501 continue;
2502 IsSplat = false;
2503 break;
2504 }
2505 if (IsSplat) {
2506 // Legalize the operand of SPLAT_VECTOR.
2507 SDValue Ext = DAG.getZExtOrTrunc(Op: Elem[First], DL: dl, VT: MVT::i32);
2508 return DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: VecTy, Operand: Ext);
2509 }
2510
2511 // Generate
2512 // (zxtb(Elem[0]) | (zxtb(Elem[1]) << 8)) |
2513 // (zxtb(Elem[2]) | (zxtb(Elem[3]) << 8)) << 16
2514 assert(Elem.size() == 4);
2515 SDValue Vs[4];
2516 for (unsigned i = 0; i != 4; ++i) {
2517 Vs[i] = DAG.getZExtOrTrunc(Op: Elem[i], DL: dl, VT: MVT::i32);
2518 Vs[i] = DAG.getZeroExtendInReg(Op: Vs[i], DL: dl, VT: MVT::i8);
2519 }
2520 SDValue S8 = DAG.getConstant(Val: 8, DL: dl, VT: MVT::i32);
2521 SDValue T0 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: MVT::i32, Ops: {Vs[1], S8});
2522 SDValue T1 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: MVT::i32, Ops: {Vs[3], S8});
2523 SDValue B0 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: MVT::i32, Ops: {Vs[0], T0});
2524 SDValue B1 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: MVT::i32, Ops: {Vs[2], T1});
2525
2526 SDValue R = getInstr(MachineOpc: Hexagon::A2_combine_ll, dl, Ty: MVT::i32, Ops: {B1, B0}, DAG);
2527 return DAG.getBitcast(VT: MVT::v4i8, V: R);
2528 }
2529
2530#ifndef NDEBUG
2531 dbgs() << "VecTy: " << VecTy << '\n';
2532#endif
2533 llvm_unreachable("Unexpected vector element type");
2534}
2535
2536SDValue
2537HexagonTargetLowering::buildVector64(ArrayRef<SDValue> Elem, const SDLoc &dl,
2538 MVT VecTy, SelectionDAG &DAG) const {
2539 MVT ElemTy = VecTy.getVectorElementType();
2540 assert(VecTy.getVectorNumElements() == Elem.size());
2541
2542 SmallVector<ConstantInt*,8> Consts(Elem.size());
2543 bool AllConst = getBuildVectorConstInts(Values: Elem, VecTy, DAG, Consts);
2544
2545 unsigned First, Num = Elem.size();
2546 for (First = 0; First != Num; ++First) {
2547 if (!isUndef(Op: Elem[First]))
2548 break;
2549 }
2550 if (First == Num)
2551 return DAG.getUNDEF(VT: VecTy);
2552
2553 if (AllConst &&
2554 llvm::all_of(Range&: Consts, P: [](ConstantInt *CI) { return CI->isZero(); }))
2555 return getZero(dl, Ty: VecTy, DAG);
2556
2557 // First try splat if possible.
2558 if (ElemTy == MVT::i16 || ElemTy == MVT::f16) {
2559 bool IsSplat = true;
2560 for (unsigned i = First+1; i != Num; ++i) {
2561 if (Elem[i] == Elem[First] || isUndef(Op: Elem[i]))
2562 continue;
2563 IsSplat = false;
2564 break;
2565 }
2566 if (IsSplat) {
2567 // Legalize the operand of SPLAT_VECTOR
2568 SDValue S = ElemTy == MVT::f16 ? DAG.getBitcast(VT: MVT::i16, V: Elem[First])
2569 : Elem[First];
2570 SDValue Ext = DAG.getZExtOrTrunc(Op: S, DL: dl, VT: MVT::i32);
2571 return DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: VecTy, Operand: Ext);
2572 }
2573 }
2574
2575 // Then try constant.
2576 if (AllConst) {
2577 uint64_t Val = 0;
2578 unsigned W = ElemTy.getSizeInBits();
2579 uint64_t Mask = (1ull << W) - 1;
2580 for (unsigned i = 0; i != Num; ++i)
2581 Val = (Val << W) | (Consts[Num-1-i]->getZExtValue() & Mask);
2582 SDValue V0 = DAG.getConstant(Val, DL: dl, VT: MVT::i64);
2583 return DAG.getBitcast(VT: VecTy, V: V0);
2584 }
2585
2586 // Build two 32-bit vectors and concatenate.
2587 MVT HalfTy = MVT::getVectorVT(VT: ElemTy, NumElements: Num/2);
2588 SDValue L = (ElemTy == MVT::i32)
2589 ? Elem[0]
2590 : buildVector32(Elem: Elem.take_front(N: Num/2), dl, VecTy: HalfTy, DAG);
2591 SDValue H = (ElemTy == MVT::i32)
2592 ? Elem[1]
2593 : buildVector32(Elem: Elem.drop_front(N: Num/2), dl, VecTy: HalfTy, DAG);
2594 return getCombine(Hi: H, Lo: L, dl, ResTy: VecTy, DAG);
2595}
2596
2597SDValue
2598HexagonTargetLowering::extractVector(SDValue VecV, SDValue IdxV,
2599 const SDLoc &dl, MVT ValTy, MVT ResTy,
2600 SelectionDAG &DAG) const {
2601 MVT VecTy = ty(Op: VecV);
2602 assert(!ValTy.isVector() ||
2603 VecTy.getVectorElementType() == ValTy.getVectorElementType());
2604 if (VecTy.getVectorElementType() == MVT::i1)
2605 return extractVectorPred(VecV, IdxV, dl, ValTy, ResTy, DAG);
2606
2607 unsigned VecWidth = VecTy.getSizeInBits();
2608 unsigned ValWidth = ValTy.getSizeInBits();
2609 unsigned ElemWidth = VecTy.getVectorElementType().getSizeInBits();
2610 assert((VecWidth % ElemWidth) == 0);
2611 assert(VecWidth == 32 || VecWidth == 64);
2612
2613 // Cast everything to scalar integer types.
2614 MVT ScalarTy = tyScalar(Ty: VecTy);
2615 VecV = DAG.getBitcast(VT: ScalarTy, V: VecV);
2616
2617 SDValue WidthV = DAG.getConstant(Val: ValWidth, DL: dl, VT: MVT::i32);
2618 SDValue ExtV;
2619
2620 if (auto *IdxN = dyn_cast<ConstantSDNode>(Val&: IdxV)) {
2621 unsigned Off = IdxN->getZExtValue() * ElemWidth;
2622 if (VecWidth == 64 && ValWidth == 32) {
2623 assert(Off == 0 || Off == 32);
2624 ExtV = Off == 0 ? LoHalf(V: VecV, DAG) : HiHalf(V: VecV, DAG);
2625 } else if (Off == 0 && (ValWidth % 8) == 0) {
2626 ExtV = DAG.getZeroExtendInReg(Op: VecV, DL: dl, VT: tyScalar(Ty: ValTy));
2627 } else {
2628 SDValue OffV = DAG.getConstant(Val: Off, DL: dl, VT: MVT::i32);
2629 // The return type of EXTRACTU must be the same as the type of the
2630 // input vector.
2631 ExtV = DAG.getNode(Opcode: HexagonISD::EXTRACTU, DL: dl, VT: ScalarTy,
2632 Ops: {VecV, WidthV, OffV});
2633 }
2634 } else {
2635 if (ty(Op: IdxV) != MVT::i32)
2636 IdxV = DAG.getZExtOrTrunc(Op: IdxV, DL: dl, VT: MVT::i32);
2637 SDValue OffV = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MVT::i32, N1: IdxV,
2638 N2: DAG.getConstant(Val: ElemWidth, DL: dl, VT: MVT::i32));
2639 ExtV = DAG.getNode(Opcode: HexagonISD::EXTRACTU, DL: dl, VT: ScalarTy,
2640 Ops: {VecV, WidthV, OffV});
2641 }
2642
2643 // Cast ExtV to the requested result type.
2644 ExtV = DAG.getZExtOrTrunc(Op: ExtV, DL: dl, VT: tyScalar(Ty: ResTy));
2645 ExtV = DAG.getBitcast(VT: ResTy, V: ExtV);
2646 return ExtV;
2647}
2648
2649SDValue
2650HexagonTargetLowering::extractVectorPred(SDValue VecV, SDValue IdxV,
2651 const SDLoc &dl, MVT ValTy, MVT ResTy,
2652 SelectionDAG &DAG) const {
2653 // Special case for v{8,4,2}i1 (the only boolean vectors legal in Hexagon
2654 // without any coprocessors).
2655 MVT VecTy = ty(Op: VecV);
2656 unsigned VecWidth = VecTy.getSizeInBits();
2657 unsigned ValWidth = ValTy.getSizeInBits();
2658 assert(VecWidth == VecTy.getVectorNumElements() &&
2659 "Vector elements should equal vector width size");
2660 assert(VecWidth == 8 || VecWidth == 4 || VecWidth == 2);
2661
2662 // Check if this is an extract of the lowest bit.
2663 if (isNullConstant(V: IdxV) && ValTy.getSizeInBits() == 1) {
2664 // Extracting the lowest bit is a no-op, but it changes the type,
2665 // so it must be kept as an operation to avoid errors related to
2666 // type mismatches.
2667 return DAG.getNode(Opcode: HexagonISD::TYPECAST, DL: dl, VT: MVT::i1, Operand: VecV);
2668 }
2669
2670 // If the value extracted is a single bit, use tstbit.
2671 if (ValWidth == 1) {
2672 SDValue A0 = getInstr(MachineOpc: Hexagon::C2_tfrpr, dl, Ty: MVT::i32, Ops: {VecV}, DAG);
2673 SDValue M0 = DAG.getConstant(Val: 8 / VecWidth, DL: dl, VT: MVT::i32);
2674 SDValue I0 = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MVT::i32, N1: IdxV, N2: M0);
2675 return DAG.getNode(Opcode: HexagonISD::TSTBIT, DL: dl, VT: MVT::i1, N1: A0, N2: I0);
2676 }
2677
2678 // Each bool vector (v2i1, v4i1, v8i1) always occupies 8 bits in
2679 // a predicate register. The elements of the vector are repeated
2680 // in the register (if necessary) so that the total number is 8.
2681 // The extracted subvector will need to be expanded in such a way.
2682 unsigned Scale = VecWidth / ValWidth;
2683
2684 // Generate (p2d VecV) >> 8*Idx to move the interesting bytes to
2685 // position 0.
2686 assert(ty(IdxV) == MVT::i32);
2687 unsigned VecRep = 8 / VecWidth;
2688 SDValue S0 = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MVT::i32, N1: IdxV,
2689 N2: DAG.getConstant(Val: 8*VecRep, DL: dl, VT: MVT::i32));
2690 SDValue T0 = DAG.getNode(Opcode: HexagonISD::P2D, DL: dl, VT: MVT::i64, Operand: VecV);
2691 SDValue T1 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i64, N1: T0, N2: S0);
2692 while (Scale > 1) {
2693 // The longest possible subvector is at most 32 bits, so it is always
2694 // contained in the low subregister.
2695 T1 = LoHalf(V: T1, DAG);
2696 T1 = expandPredicate(Vec32: T1, dl, DAG);
2697 Scale /= 2;
2698 }
2699
2700 return DAG.getNode(Opcode: HexagonISD::D2P, DL: dl, VT: ResTy, Operand: T1);
2701}
2702
2703SDValue
2704HexagonTargetLowering::insertVector(SDValue VecV, SDValue ValV, SDValue IdxV,
2705 const SDLoc &dl, MVT ValTy,
2706 SelectionDAG &DAG) const {
2707 MVT VecTy = ty(Op: VecV);
2708 if (VecTy.getVectorElementType() == MVT::i1)
2709 return insertVectorPred(VecV, ValV, IdxV, dl, ValTy, DAG);
2710
2711 unsigned VecWidth = VecTy.getSizeInBits();
2712 unsigned ValWidth = ValTy.getSizeInBits();
2713 assert(VecWidth == 32 || VecWidth == 64);
2714 assert((VecWidth % ValWidth) == 0);
2715
2716 // Cast everything to scalar integer types.
2717 MVT ScalarTy = MVT::getIntegerVT(BitWidth: VecWidth);
2718 // The actual type of ValV may be different than ValTy (which is related
2719 // to the vector type).
2720 unsigned VW = ty(Op: ValV).getSizeInBits();
2721 ValV = DAG.getBitcast(VT: MVT::getIntegerVT(BitWidth: VW), V: ValV);
2722 VecV = DAG.getBitcast(VT: ScalarTy, V: VecV);
2723 if (VW != VecWidth)
2724 ValV = DAG.getAnyExtOrTrunc(Op: ValV, DL: dl, VT: ScalarTy);
2725
2726 SDValue WidthV = DAG.getConstant(Val: ValWidth, DL: dl, VT: MVT::i32);
2727 SDValue InsV;
2728
2729 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: IdxV)) {
2730 unsigned W = C->getZExtValue() * ValWidth;
2731 SDValue OffV = DAG.getConstant(Val: W, DL: dl, VT: MVT::i32);
2732 InsV = DAG.getNode(Opcode: HexagonISD::INSERT, DL: dl, VT: ScalarTy,
2733 Ops: {VecV, ValV, WidthV, OffV});
2734 } else {
2735 if (ty(Op: IdxV) != MVT::i32)
2736 IdxV = DAG.getZExtOrTrunc(Op: IdxV, DL: dl, VT: MVT::i32);
2737 SDValue OffV = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MVT::i32, N1: IdxV, N2: WidthV);
2738 InsV = DAG.getNode(Opcode: HexagonISD::INSERT, DL: dl, VT: ScalarTy,
2739 Ops: {VecV, ValV, WidthV, OffV});
2740 }
2741
2742 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: VecTy, Operand: InsV);
2743}
2744
2745SDValue
2746HexagonTargetLowering::insertVectorPred(SDValue VecV, SDValue ValV,
2747 SDValue IdxV, const SDLoc &dl,
2748 MVT ValTy, SelectionDAG &DAG) const {
2749 MVT VecTy = ty(Op: VecV);
2750 unsigned VecLen = VecTy.getVectorNumElements();
2751
2752 if (ValTy == MVT::i1) {
2753 SDValue ToReg = getInstr(MachineOpc: Hexagon::C2_tfrpr, dl, Ty: MVT::i32, Ops: {VecV}, DAG);
2754 SDValue Ext = DAG.getSExtOrTrunc(Op: ValV, DL: dl, VT: MVT::i32);
2755 SDValue Width = DAG.getConstant(Val: 8 / VecLen, DL: dl, VT: MVT::i32);
2756 SDValue Idx = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MVT::i32, N1: IdxV, N2: Width);
2757 SDValue Ins =
2758 DAG.getNode(Opcode: HexagonISD::INSERT, DL: dl, VT: MVT::i32, Ops: {ToReg, Ext, Width, Idx});
2759 return getInstr(MachineOpc: Hexagon::C2_tfrrp, dl, Ty: VecTy, Ops: {Ins}, DAG);
2760 }
2761
2762 assert(ValTy.getVectorElementType() == MVT::i1);
2763 SDValue ValR = ValTy.isVector()
2764 ? DAG.getNode(Opcode: HexagonISD::P2D, DL: dl, VT: MVT::i64, Operand: ValV)
2765 : DAG.getSExtOrTrunc(Op: ValV, DL: dl, VT: MVT::i64);
2766
2767 unsigned Scale = VecLen / ValTy.getVectorNumElements();
2768 assert(Scale > 1);
2769
2770 for (unsigned R = Scale; R > 1; R /= 2) {
2771 ValR = contractPredicate(Vec64: ValR, dl, DAG);
2772 ValR = getCombine(Hi: DAG.getUNDEF(VT: MVT::i32), Lo: ValR, dl, ResTy: MVT::i64, DAG);
2773 }
2774
2775 SDValue Width = DAG.getConstant(Val: 64 / Scale, DL: dl, VT: MVT::i32);
2776 SDValue Idx = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MVT::i32, N1: IdxV, N2: Width);
2777 SDValue VecR = DAG.getNode(Opcode: HexagonISD::P2D, DL: dl, VT: MVT::i64, Operand: VecV);
2778 SDValue Ins =
2779 DAG.getNode(Opcode: HexagonISD::INSERT, DL: dl, VT: MVT::i64, Ops: {VecR, ValR, Width, Idx});
2780 return DAG.getNode(Opcode: HexagonISD::D2P, DL: dl, VT: VecTy, Operand: Ins);
2781}
2782
2783SDValue
2784HexagonTargetLowering::expandPredicate(SDValue Vec32, const SDLoc &dl,
2785 SelectionDAG &DAG) const {
2786 assert(ty(Vec32).getSizeInBits() == 32);
2787 if (isUndef(Op: Vec32))
2788 return DAG.getUNDEF(VT: MVT::i64);
2789 SDValue P = DAG.getBitcast(VT: MVT::v4i8, V: Vec32);
2790 SDValue X = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: MVT::v4i16, Operand: P);
2791 return DAG.getBitcast(VT: MVT::i64, V: X);
2792}
2793
2794SDValue
2795HexagonTargetLowering::contractPredicate(SDValue Vec64, const SDLoc &dl,
2796 SelectionDAG &DAG) const {
2797 assert(ty(Vec64).getSizeInBits() == 64);
2798 if (isUndef(Op: Vec64))
2799 return DAG.getUNDEF(VT: MVT::i32);
2800 // Collect even bytes:
2801 SDValue A = DAG.getBitcast(VT: MVT::v8i8, V: Vec64);
2802 SDValue S = DAG.getVectorShuffle(VT: MVT::v8i8, dl, N1: A, N2: DAG.getUNDEF(VT: MVT::v8i8),
2803 Mask: {0, 2, 4, 6, 1, 3, 5, 7});
2804 return extractVector(VecV: S, IdxV: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32), dl, ValTy: MVT::v4i8,
2805 ResTy: MVT::i32, DAG);
2806}
2807
2808SDValue
2809HexagonTargetLowering::getZero(const SDLoc &dl, MVT Ty, SelectionDAG &DAG)
2810 const {
2811 if (Ty.isVector()) {
2812 unsigned W = Ty.getSizeInBits();
2813 if (W <= 64)
2814 return DAG.getBitcast(VT: Ty, V: DAG.getConstant(Val: 0, DL: dl, VT: MVT::getIntegerVT(BitWidth: W)));
2815 return DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: Ty, Operand: getZero(dl, Ty: MVT::i32, DAG));
2816 }
2817
2818 if (Ty.isInteger())
2819 return DAG.getConstant(Val: 0, DL: dl, VT: Ty);
2820 if (Ty.isFloatingPoint())
2821 return DAG.getConstantFP(Val: 0.0, DL: dl, VT: Ty);
2822 llvm_unreachable("Invalid type for zero");
2823}
2824
2825SDValue
2826HexagonTargetLowering::appendUndef(SDValue Val, MVT ResTy, SelectionDAG &DAG)
2827 const {
2828 MVT ValTy = ty(Op: Val);
2829 assert(ValTy.getVectorElementType() == ResTy.getVectorElementType());
2830
2831 unsigned ValLen = ValTy.getVectorNumElements();
2832 unsigned ResLen = ResTy.getVectorNumElements();
2833 if (ValLen == ResLen)
2834 return Val;
2835
2836 const SDLoc &dl(Val);
2837 assert(ValLen < ResLen);
2838 assert(ResLen % ValLen == 0);
2839
2840 SmallVector<SDValue, 4> Concats = {Val};
2841 for (unsigned i = 1, e = ResLen / ValLen; i < e; ++i)
2842 Concats.push_back(Elt: DAG.getUNDEF(VT: ValTy));
2843
2844 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResTy, Ops: Concats);
2845}
2846
2847SDValue
2848HexagonTargetLowering::getCombine(SDValue Hi, SDValue Lo, const SDLoc &dl,
2849 MVT ResTy, SelectionDAG &DAG) const {
2850 MVT ElemTy = ty(Op: Hi);
2851 assert(ElemTy == ty(Lo));
2852
2853 if (!ElemTy.isVector()) {
2854 assert(ElemTy.isScalarInteger());
2855 MVT PairTy = MVT::getIntegerVT(BitWidth: 2 * ElemTy.getSizeInBits());
2856 SDValue Pair = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: PairTy, N1: Lo, N2: Hi);
2857 return DAG.getBitcast(VT: ResTy, V: Pair);
2858 }
2859
2860 unsigned Width = ElemTy.getSizeInBits();
2861 MVT IntTy = MVT::getIntegerVT(BitWidth: Width);
2862 MVT PairTy = MVT::getIntegerVT(BitWidth: 2 * Width);
2863 SDValue Pair =
2864 DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: PairTy,
2865 Ops: {DAG.getBitcast(VT: IntTy, V: Lo), DAG.getBitcast(VT: IntTy, V: Hi)});
2866 return DAG.getBitcast(VT: ResTy, V: Pair);
2867}
2868
2869SDValue
2870HexagonTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
2871 MVT VecTy = ty(Op);
2872 unsigned BW = VecTy.getSizeInBits();
2873 const SDLoc &dl(Op);
2874 SmallVector<SDValue,8> Ops;
2875 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i)
2876 Ops.push_back(Elt: Op.getOperand(i));
2877
2878 if (BW == 32)
2879 return buildVector32(Elem: Ops, dl, VecTy, DAG);
2880 if (BW == 64)
2881 return buildVector64(Elem: Ops, dl, VecTy, DAG);
2882
2883 if (VecTy == MVT::v8i1 || VecTy == MVT::v4i1 || VecTy == MVT::v2i1) {
2884 // Check if this is a special case or all-0 or all-1.
2885 bool All0 = true, All1 = true;
2886 for (SDValue P : Ops) {
2887 auto *CN = dyn_cast<ConstantSDNode>(Val: P.getNode());
2888 if (CN == nullptr) {
2889 All0 = All1 = false;
2890 break;
2891 }
2892 uint32_t C = CN->getZExtValue();
2893 All0 &= (C == 0);
2894 All1 &= (C == 1);
2895 }
2896 if (All0)
2897 return DAG.getNode(Opcode: HexagonISD::PFALSE, DL: dl, VT: VecTy);
2898 if (All1)
2899 return DAG.getNode(Opcode: HexagonISD::PTRUE, DL: dl, VT: VecTy);
2900
2901 // For each i1 element in the resulting predicate register, put 1
2902 // shifted by the index of the element into a general-purpose register,
2903 // then or them together and transfer it back into a predicate register.
2904 SDValue Rs[8];
2905 SDValue Z = getZero(dl, Ty: MVT::i32, DAG);
2906 // Always produce 8 bits, repeat inputs if necessary.
2907 unsigned Rep = 8 / VecTy.getVectorNumElements();
2908 for (unsigned i = 0; i != 8; ++i) {
2909 SDValue S = DAG.getConstant(Val: 1ull << i, DL: dl, VT: MVT::i32);
2910 Rs[i] = DAG.getSelect(DL: dl, VT: MVT::i32, Cond: Ops[i/Rep], LHS: S, RHS: Z);
2911 }
2912 for (ArrayRef<SDValue> A(Rs); A.size() != 1; A = A.drop_back(N: A.size()/2)) {
2913 for (unsigned i = 0, e = A.size()/2; i != e; ++i)
2914 Rs[i] = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: MVT::i32, N1: Rs[2*i], N2: Rs[2*i+1]);
2915 }
2916 // Move the value directly to a predicate register.
2917 return getInstr(MachineOpc: Hexagon::C2_tfrrp, dl, Ty: VecTy, Ops: {Rs[0]}, DAG);
2918 }
2919
2920 return SDValue();
2921}
2922
2923SDValue
2924HexagonTargetLowering::LowerCONCAT_VECTORS(SDValue Op,
2925 SelectionDAG &DAG) const {
2926 MVT VecTy = ty(Op);
2927 const SDLoc &dl(Op);
2928 if (VecTy.getSizeInBits() == 64) {
2929 assert(Op.getNumOperands() == 2);
2930 return getCombine(Hi: Op.getOperand(i: 1), Lo: Op.getOperand(i: 0), dl, ResTy: VecTy, DAG);
2931 }
2932
2933 MVT ElemTy = VecTy.getVectorElementType();
2934 if (ElemTy == MVT::i1) {
2935 assert(VecTy == MVT::v2i1 || VecTy == MVT::v4i1 || VecTy == MVT::v8i1);
2936 MVT OpTy = ty(Op: Op.getOperand(i: 0));
2937 // Scale is how many times the operands need to be contracted to match
2938 // the representation in the target register.
2939 unsigned Scale = VecTy.getVectorNumElements() / OpTy.getVectorNumElements();
2940 assert(Scale == Op.getNumOperands() && Scale > 1);
2941
2942 // First, convert all bool vectors to integers, then generate pairwise
2943 // inserts to form values of doubled length. Up until there are only
2944 // two values left to concatenate, all of these values will fit in a
2945 // 32-bit integer, so keep them as i32 to use 32-bit inserts.
2946 SmallVector<SDValue,4> Words[2];
2947 unsigned IdxW = 0;
2948
2949 for (SDValue P : Op.getNode()->op_values()) {
2950 SDValue W = DAG.getNode(Opcode: HexagonISD::P2D, DL: dl, VT: MVT::i64, Operand: P);
2951 for (unsigned R = Scale; R > 1; R /= 2) {
2952 W = contractPredicate(Vec64: W, dl, DAG);
2953 W = getCombine(Hi: DAG.getUNDEF(VT: MVT::i32), Lo: W, dl, ResTy: MVT::i64, DAG);
2954 }
2955 W = LoHalf(V: W, DAG);
2956 Words[IdxW].push_back(Elt: W);
2957 }
2958
2959 while (Scale > 2) {
2960 SDValue WidthV = DAG.getConstant(Val: 64 / Scale, DL: dl, VT: MVT::i32);
2961 Words[IdxW ^ 1].clear();
2962
2963 for (unsigned i = 0, e = Words[IdxW].size(); i != e; i += 2) {
2964 SDValue W0 = Words[IdxW][i], W1 = Words[IdxW][i+1];
2965 // Insert W1 into W0 right next to the significant bits of W0.
2966 SDValue T = DAG.getNode(Opcode: HexagonISD::INSERT, DL: dl, VT: MVT::i32,
2967 Ops: {W0, W1, WidthV, WidthV});
2968 Words[IdxW ^ 1].push_back(Elt: T);
2969 }
2970 IdxW ^= 1;
2971 Scale /= 2;
2972 }
2973
2974 // At this point there should only be two words left, and Scale should be 2.
2975 assert(Scale == 2 && Words[IdxW].size() == 2);
2976
2977 SDValue WW = getCombine(Hi: Words[IdxW][1], Lo: Words[IdxW][0], dl, ResTy: MVT::i64, DAG);
2978 return DAG.getNode(Opcode: HexagonISD::D2P, DL: dl, VT: VecTy, Operand: WW);
2979 }
2980
2981 return SDValue();
2982}
2983
2984SDValue
2985HexagonTargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
2986 SelectionDAG &DAG) const {
2987 SDValue Vec = Op.getOperand(i: 0);
2988 MVT ElemTy = ty(Op: Vec).getVectorElementType();
2989 return extractVector(VecV: Vec, IdxV: Op.getOperand(i: 1), dl: SDLoc(Op), ValTy: ElemTy, ResTy: ty(Op), DAG);
2990}
2991
2992SDValue
2993HexagonTargetLowering::LowerEXTRACT_SUBVECTOR(SDValue Op,
2994 SelectionDAG &DAG) const {
2995 return extractVector(VecV: Op.getOperand(i: 0), IdxV: Op.getOperand(i: 1), dl: SDLoc(Op),
2996 ValTy: ty(Op), ResTy: ty(Op), DAG);
2997}
2998
2999SDValue
3000HexagonTargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
3001 SelectionDAG &DAG) const {
3002 return insertVector(VecV: Op.getOperand(i: 0), ValV: Op.getOperand(i: 1), IdxV: Op.getOperand(i: 2),
3003 dl: SDLoc(Op), ValTy: ty(Op).getVectorElementType(), DAG);
3004}
3005
3006SDValue
3007HexagonTargetLowering::LowerINSERT_SUBVECTOR(SDValue Op,
3008 SelectionDAG &DAG) const {
3009 SDValue ValV = Op.getOperand(i: 1);
3010 return insertVector(VecV: Op.getOperand(i: 0), ValV, IdxV: Op.getOperand(i: 2),
3011 dl: SDLoc(Op), ValTy: ty(Op: ValV), DAG);
3012}
3013
3014bool
3015HexagonTargetLowering::allowTruncateForTailCall(Type *Ty1, Type *Ty2) const {
3016 // Assuming the caller does not have either a signext or zeroext modifier, and
3017 // only one value is accepted, any reasonable truncation is allowed.
3018 if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy())
3019 return false;
3020
3021 // FIXME: in principle up to 64-bit could be made safe, but it would be very
3022 // fragile at the moment: any support for multiple value returns would be
3023 // liable to disallow tail calls involving i64 -> iN truncation in many cases.
3024 return Ty1->getPrimitiveSizeInBits() <= 32;
3025}
3026
3027SDValue
3028HexagonTargetLowering::LowerLoad(SDValue Op, SelectionDAG &DAG) const {
3029 MVT Ty = ty(Op);
3030 const SDLoc &dl(Op);
3031 LoadSDNode *LN = cast<LoadSDNode>(Val: Op.getNode());
3032 MVT MemTy = LN->getMemoryVT().getSimpleVT();
3033 ISD::LoadExtType ET = LN->getExtensionType();
3034
3035 bool LoadPred = MemTy == MVT::v2i1 || MemTy == MVT::v4i1 || MemTy == MVT::v8i1;
3036 if (LoadPred) {
3037 SDValue NL = DAG.getLoad(
3038 AM: LN->getAddressingMode(), ExtType: ISD::ZEXTLOAD, VT: MVT::i32, dl, Chain: LN->getChain(),
3039 Ptr: LN->getBasePtr(), Offset: LN->getOffset(), PtrInfo: LN->getPointerInfo(),
3040 /*MemoryVT*/ MemVT: MVT::i8, Alignment: LN->getAlign(), MMOFlags: LN->getMemOperand()->getFlags(),
3041 AAInfo: LN->getAAInfo(), Ranges: LN->getRanges());
3042 LN = cast<LoadSDNode>(Val: NL.getNode());
3043 }
3044
3045 Align ClaimAlign = LN->getAlign();
3046 if (!validateConstPtrAlignment(Ptr: LN->getBasePtr(), NeedAlign: ClaimAlign, dl, DAG))
3047 return replaceMemWithUndef(Op, DAG);
3048
3049 // Call LowerUnalignedLoad for all loads, it recognizes loads that
3050 // don't need extra aligning.
3051 SDValue LU = LowerUnalignedLoad(Op: SDValue(LN, 0), DAG);
3052 if (LoadPred) {
3053 SDValue TP = getInstr(MachineOpc: Hexagon::C2_tfrrp, dl, Ty: MemTy, Ops: {LU}, DAG);
3054 if (ET == ISD::SEXTLOAD) {
3055 TP = DAG.getSExtOrTrunc(Op: TP, DL: dl, VT: Ty);
3056 } else if (ET != ISD::NON_EXTLOAD) {
3057 TP = DAG.getZExtOrTrunc(Op: TP, DL: dl, VT: Ty);
3058 }
3059 SDValue Ch = cast<LoadSDNode>(Val: LU.getNode())->getChain();
3060 return DAG.getMergeValues(Ops: {TP, Ch}, dl);
3061 }
3062 return LU;
3063}
3064
3065SDValue
3066HexagonTargetLowering::LowerStore(SDValue Op, SelectionDAG &DAG) const {
3067 const SDLoc &dl(Op);
3068 StoreSDNode *SN = cast<StoreSDNode>(Val: Op.getNode());
3069 SDValue Val = SN->getValue();
3070 MVT Ty = ty(Op: Val);
3071
3072 if (Ty == MVT::v2i1 || Ty == MVT::v4i1 || Ty == MVT::v8i1) {
3073 // Store the exact predicate (all bits).
3074 SDValue TR = getInstr(MachineOpc: Hexagon::C2_tfrpr, dl, Ty: MVT::i32, Ops: {Val}, DAG);
3075 SDValue NS = DAG.getTruncStore(Chain: SN->getChain(), dl, Val: TR, Ptr: SN->getBasePtr(),
3076 SVT: MVT::i8, MMO: SN->getMemOperand());
3077 if (SN->isIndexed()) {
3078 NS = DAG.getIndexedStore(OrigStore: NS, dl, Base: SN->getBasePtr(), Offset: SN->getOffset(),
3079 AM: SN->getAddressingMode());
3080 }
3081 SN = cast<StoreSDNode>(Val: NS.getNode());
3082 }
3083
3084 Align ClaimAlign = SN->getAlign();
3085 if (!validateConstPtrAlignment(Ptr: SN->getBasePtr(), NeedAlign: ClaimAlign, dl, DAG))
3086 return replaceMemWithUndef(Op, DAG);
3087
3088 MVT StoreTy = SN->getMemoryVT().getSimpleVT();
3089 Align NeedAlign = Subtarget.getTypeAlignment(Ty: StoreTy);
3090 if (ClaimAlign < NeedAlign)
3091 return expandUnalignedStore(ST: SN, DAG);
3092 return SDValue(SN, 0);
3093}
3094
3095SDValue
3096HexagonTargetLowering::LowerUnalignedLoad(SDValue Op, SelectionDAG &DAG)
3097 const {
3098 LoadSDNode *LN = cast<LoadSDNode>(Val: Op.getNode());
3099 MVT LoadTy = ty(Op);
3100 unsigned NeedAlign = Subtarget.getTypeAlignment(Ty: LoadTy).value();
3101 unsigned HaveAlign = LN->getAlign().value();
3102 if (HaveAlign >= NeedAlign)
3103 return Op;
3104
3105 const SDLoc &dl(Op);
3106 const DataLayout &DL = DAG.getDataLayout();
3107 LLVMContext &Ctx = *DAG.getContext();
3108
3109 // If the load aligning is disabled or the load can be broken up into two
3110 // smaller legal loads, do the default (target-independent) expansion.
3111 bool DoDefault = false;
3112 // Handle it in the default way if this is an indexed load.
3113 if (!LN->isUnindexed())
3114 DoDefault = true;
3115
3116 if (!AlignLoads) {
3117 if (allowsMemoryAccessForAlignment(Context&: Ctx, DL, VT: LN->getMemoryVT(),
3118 MMO: *LN->getMemOperand()))
3119 return Op;
3120 DoDefault = true;
3121 }
3122 if (!DoDefault && (2 * HaveAlign) == NeedAlign) {
3123 // The PartTy is the equivalent of "getLoadableTypeOfSize(HaveAlign)".
3124 MVT PartTy = HaveAlign <= 8 ? MVT::getIntegerVT(BitWidth: 8 * HaveAlign)
3125 : MVT::getVectorVT(VT: MVT::i8, NumElements: HaveAlign);
3126 DoDefault =
3127 allowsMemoryAccessForAlignment(Context&: Ctx, DL, VT: PartTy, MMO: *LN->getMemOperand());
3128 }
3129 if (DoDefault) {
3130 std::pair<SDValue, SDValue> P = expandUnalignedLoad(LD: LN, DAG);
3131 return DAG.getMergeValues(Ops: {P.first, P.second}, dl);
3132 }
3133
3134 // The code below generates two loads, both aligned as NeedAlign, and
3135 // with the distance of NeedAlign between them. For that to cover the
3136 // bits that need to be loaded (and without overlapping), the size of
3137 // the loads should be equal to NeedAlign. This is true for all loadable
3138 // types, but add an assertion in case something changes in the future.
3139 assert(LoadTy.getSizeInBits() == 8*NeedAlign);
3140
3141 unsigned LoadLen = NeedAlign;
3142 SDValue Base = LN->getBasePtr();
3143 SDValue Chain = LN->getChain();
3144 auto BO = getBaseAndOffset(Addr: Base);
3145 unsigned BaseOpc = BO.first.getOpcode();
3146 if (BaseOpc == HexagonISD::VALIGNADDR && BO.second % LoadLen == 0)
3147 return Op;
3148
3149 if (BO.second % LoadLen != 0) {
3150 BO.first = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::i32, N1: BO.first,
3151 N2: DAG.getConstant(Val: BO.second % LoadLen, DL: dl, VT: MVT::i32));
3152 BO.second -= BO.second % LoadLen;
3153 }
3154 SDValue BaseNoOff = (BaseOpc != HexagonISD::VALIGNADDR)
3155 ? DAG.getNode(Opcode: HexagonISD::VALIGNADDR, DL: dl, VT: MVT::i32, N1: BO.first,
3156 N2: DAG.getConstant(Val: NeedAlign, DL: dl, VT: MVT::i32))
3157 : BO.first;
3158 SDValue Base0 =
3159 DAG.getMemBasePlusOffset(Base: BaseNoOff, Offset: TypeSize::getFixed(ExactSize: BO.second), DL: dl);
3160 SDValue Base1 = DAG.getMemBasePlusOffset(
3161 Base: BaseNoOff, Offset: TypeSize::getFixed(ExactSize: BO.second + LoadLen), DL: dl);
3162
3163 MachineMemOperand *WideMMO = nullptr;
3164 if (MachineMemOperand *MMO = LN->getMemOperand()) {
3165 MachineFunction &MF = DAG.getMachineFunction();
3166 WideMMO = MF.getMachineMemOperand(
3167 PtrInfo: MMO->getPointerInfo(), F: MMO->getFlags(), Size: 2 * LoadLen, BaseAlignment: Align(LoadLen),
3168 AAInfo: MMO->getAAInfo(), Ranges: MMO->getRanges(), SSID: MMO->getSyncScopeID(),
3169 Ordering: MMO->getSuccessOrdering(), FailureOrdering: MMO->getFailureOrdering());
3170 }
3171
3172 SDValue Load0 = DAG.getLoad(VT: LoadTy, dl, Chain, Ptr: Base0, MMO: WideMMO);
3173 SDValue Load1 = DAG.getLoad(VT: LoadTy, dl, Chain, Ptr: Base1, MMO: WideMMO);
3174
3175 SDValue Aligned = DAG.getNode(Opcode: HexagonISD::VALIGN, DL: dl, VT: LoadTy,
3176 Ops: {Load1, Load0, BaseNoOff.getOperand(i: 0)});
3177 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other,
3178 N1: Load0.getValue(R: 1), N2: Load1.getValue(R: 1));
3179 SDValue M = DAG.getMergeValues(Ops: {Aligned, NewChain}, dl);
3180 return M;
3181}
3182
3183SDValue
3184HexagonTargetLowering::LowerUAddSubO(SDValue Op, SelectionDAG &DAG) const {
3185 SDValue X = Op.getOperand(i: 0), Y = Op.getOperand(i: 1);
3186 auto *CY = dyn_cast<ConstantSDNode>(Val&: Y);
3187 if (!CY)
3188 return SDValue();
3189
3190 const SDLoc &dl(Op);
3191 SDVTList VTs = Op.getNode()->getVTList();
3192 assert(VTs.NumVTs == 2);
3193 assert(VTs.VTs[1] == MVT::i1);
3194 unsigned Opc = Op.getOpcode();
3195
3196 if (CY) {
3197 uint64_t VY = CY->getZExtValue();
3198 assert(VY != 0 && "This should have been folded");
3199 // X +/- 1
3200 if (VY != 1)
3201 return SDValue();
3202
3203 if (Opc == ISD::UADDO) {
3204 SDValue Op = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: VTs.VTs[0], Ops: {X, Y});
3205 SDValue Ov = DAG.getSetCC(DL: dl, VT: MVT::i1, LHS: Op, RHS: getZero(dl, Ty: ty(Op), DAG),
3206 Cond: ISD::SETEQ);
3207 return DAG.getMergeValues(Ops: {Op, Ov}, dl);
3208 }
3209 if (Opc == ISD::USUBO) {
3210 SDValue Op = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: VTs.VTs[0], Ops: {X, Y});
3211 SDValue Ov = DAG.getSetCC(DL: dl, VT: MVT::i1, LHS: Op,
3212 RHS: DAG.getAllOnesConstant(DL: dl, VT: ty(Op)), Cond: ISD::SETEQ);
3213 return DAG.getMergeValues(Ops: {Op, Ov}, dl);
3214 }
3215 }
3216
3217 return SDValue();
3218}
3219
3220SDValue HexagonTargetLowering::LowerUAddSubOCarry(SDValue Op,
3221 SelectionDAG &DAG) const {
3222 const SDLoc &dl(Op);
3223 unsigned Opc = Op.getOpcode();
3224 SDValue X = Op.getOperand(i: 0), Y = Op.getOperand(i: 1), C = Op.getOperand(i: 2);
3225
3226 if (Opc == ISD::UADDO_CARRY)
3227 return DAG.getNode(Opcode: HexagonISD::ADDC, DL: dl, VTList: Op.getNode()->getVTList(),
3228 Ops: { X, Y, C });
3229
3230 EVT CarryTy = C.getValueType();
3231 SDValue SubC = DAG.getNode(Opcode: HexagonISD::SUBC, DL: dl, VTList: Op.getNode()->getVTList(),
3232 Ops: { X, Y, DAG.getLogicalNOT(DL: dl, Val: C, VT: CarryTy) });
3233 SDValue Out[] = { SubC.getValue(R: 0),
3234 DAG.getLogicalNOT(DL: dl, Val: SubC.getValue(R: 1), VT: CarryTy) };
3235 return DAG.getMergeValues(Ops: Out, dl);
3236}
3237
3238SDValue
3239HexagonTargetLowering::LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const {
3240 SDValue Chain = Op.getOperand(i: 0);
3241 SDValue Offset = Op.getOperand(i: 1);
3242 SDValue Handler = Op.getOperand(i: 2);
3243 SDLoc dl(Op);
3244 auto PtrVT = getPointerTy(DL: DAG.getDataLayout());
3245
3246 // Mark function as containing a call to EH_RETURN.
3247 HexagonMachineFunctionInfo *FuncInfo =
3248 DAG.getMachineFunction().getInfo<HexagonMachineFunctionInfo>();
3249 FuncInfo->setHasEHReturn();
3250
3251 unsigned OffsetReg = Hexagon::R28;
3252
3253 SDValue StoreAddr =
3254 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: DAG.getRegister(Reg: Hexagon::R30, VT: PtrVT),
3255 N2: DAG.getIntPtrConstant(Val: 4, DL: dl));
3256 Chain = DAG.getStore(Chain, dl, Val: Handler, Ptr: StoreAddr, PtrInfo: MachinePointerInfo());
3257 Chain = DAG.getCopyToReg(Chain, dl, Reg: OffsetReg, N: Offset);
3258
3259 // Not needed we already use it as explicit input to EH_RETURN.
3260 // MF.getRegInfo().addLiveOut(OffsetReg);
3261
3262 return DAG.getNode(Opcode: HexagonISD::EH_RETURN, DL: dl, VT: MVT::Other, Operand: Chain);
3263}
3264
3265SDValue
3266HexagonTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
3267 unsigned Opc = Op.getOpcode();
3268 // Handle INLINEASM first.
3269 if (Opc == ISD::INLINEASM || Opc == ISD::INLINEASM_BR)
3270 return LowerINLINEASM(Op, DAG);
3271
3272 if (isHvxOperation(N: Op.getNode(), DAG)) {
3273 // If HVX lowering returns nothing, try the default lowering.
3274 if (SDValue V = LowerHvxOperation(Op, DAG))
3275 return V;
3276 }
3277
3278 switch (Opc) {
3279 default:
3280#ifndef NDEBUG
3281 Op.getNode()->dumpr(&DAG);
3282#endif
3283 llvm_unreachable("Should not custom lower this!");
3284
3285 case ISD::FDIV:
3286 return LowerFDIV(Op, DAG);
3287 case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG);
3288 case ISD::INSERT_SUBVECTOR: return LowerINSERT_SUBVECTOR(Op, DAG);
3289 case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG);
3290 case ISD::EXTRACT_SUBVECTOR: return LowerEXTRACT_SUBVECTOR(Op, DAG);
3291 case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG);
3292 case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG);
3293 case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG);
3294 case ISD::BITCAST: return LowerBITCAST(Op, DAG);
3295 case ISD::LOAD: return LowerLoad(Op, DAG);
3296 case ISD::STORE: return LowerStore(Op, DAG);
3297 case ISD::UADDO:
3298 case ISD::USUBO: return LowerUAddSubO(Op, DAG);
3299 case ISD::UADDO_CARRY:
3300 case ISD::USUBO_CARRY: return LowerUAddSubOCarry(Op, DAG);
3301 case ISD::SRA:
3302 case ISD::SHL:
3303 case ISD::SRL: return LowerVECTOR_SHIFT(Op, DAG);
3304 case ISD::ROTL: return LowerROTL(Op, DAG);
3305 case ISD::ConstantPool: return LowerConstantPool(Op, DAG);
3306 case ISD::JumpTable: return LowerJumpTable(Op, DAG);
3307 case ISD::EH_RETURN: return LowerEH_RETURN(Op, DAG);
3308 case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG);
3309 case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG);
3310 case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG);
3311 case ISD::ATOMIC_FENCE: return LowerATOMIC_FENCE(Op, DAG);
3312 case ISD::GlobalAddress: return LowerGLOBALADDRESS(Op, DAG);
3313 case ISD::BlockAddress: return LowerBlockAddress(Op, DAG);
3314 case ISD::GLOBAL_OFFSET_TABLE: return LowerGLOBAL_OFFSET_TABLE(Op, DAG);
3315 case ISD::VACOPY: return LowerVACOPY(Op, DAG);
3316 case ISD::VASTART: return LowerVASTART(Op, DAG);
3317 case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG);
3318 case ISD::SETCC: return LowerSETCC(Op, DAG);
3319 case ISD::VSELECT: return LowerVSELECT(Op, DAG);
3320 case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG);
3321 case ISD::INTRINSIC_VOID: return LowerINTRINSIC_VOID(Op, DAG);
3322 case ISD::PREFETCH:
3323 return LowerPREFETCH(Op, DAG);
3324 break;
3325 }
3326
3327 return SDValue();
3328}
3329
3330void
3331HexagonTargetLowering::LowerOperationWrapper(SDNode *N,
3332 SmallVectorImpl<SDValue> &Results,
3333 SelectionDAG &DAG) const {
3334 if (isHvxOperation(N, DAG)) {
3335 LowerHvxOperationWrapper(N, Results, DAG);
3336 if (!Results.empty())
3337 return;
3338 }
3339
3340 SDValue Op(N, 0);
3341 unsigned Opc = N->getOpcode();
3342
3343 switch (Opc) {
3344 case HexagonISD::SSAT:
3345 case HexagonISD::USAT:
3346 Results.push_back(Elt: opJoin(Ops: SplitVectorOp(Op, DAG), dl: SDLoc(Op), DAG));
3347 break;
3348 case ISD::STORE:
3349 // We are only custom-lowering stores to verify the alignment of the
3350 // address if it is a compile-time constant. Since a store can be
3351 // modified during type-legalization (the value being stored may need
3352 // legalization), return empty Results here to indicate that we don't
3353 // really make any changes in the custom lowering.
3354 return;
3355 default:
3356 TargetLowering::LowerOperationWrapper(N, Results, DAG);
3357 break;
3358 }
3359}
3360
3361void
3362HexagonTargetLowering::ReplaceNodeResults(SDNode *N,
3363 SmallVectorImpl<SDValue> &Results,
3364 SelectionDAG &DAG) const {
3365 if (isHvxOperation(N, DAG)) {
3366 ReplaceHvxNodeResults(N, Results, DAG);
3367 if (!Results.empty())
3368 return;
3369 }
3370
3371 const SDLoc &dl(N);
3372 switch (N->getOpcode()) {
3373 case ISD::SRL:
3374 case ISD::SRA:
3375 case ISD::SHL:
3376 return;
3377 case ISD::BITCAST:
3378 // Handle a bitcast from v8i1 to i8.
3379 if (N->getValueType(ResNo: 0) == MVT::i8) {
3380 if (N->getOperand(Num: 0).getValueType() == MVT::v8i1) {
3381 SDValue P = getInstr(MachineOpc: Hexagon::C2_tfrpr, dl, Ty: MVT::i32,
3382 Ops: N->getOperand(Num: 0), DAG);
3383 SDValue T = DAG.getAnyExtOrTrunc(Op: P, DL: dl, VT: MVT::i8);
3384 Results.push_back(Elt: T);
3385 }
3386 }
3387 break;
3388 }
3389}
3390
3391SDValue
3392HexagonTargetLowering::PerformDAGCombine(SDNode *N,
3393 DAGCombinerInfo &DCI) const {
3394 SDValue Op(N, 0);
3395 const SDLoc &dl(Op);
3396 unsigned Opc = Op.getOpcode();
3397
3398 // Combining transformations applicable for arbitrary vector sizes.
3399 if (DCI.isBeforeLegalizeOps()) {
3400 switch (Opc) {
3401 case ISD::VECREDUCE_ADD:
3402 if (SDValue V = splitVecReduceAdd(N, DAG&: DCI.DAG))
3403 return V;
3404 if (SDValue V = expandVecReduceAdd(N, DAG&: DCI.DAG))
3405 return V;
3406 return SDValue();
3407 case ISD::PARTIAL_REDUCE_SMLA:
3408 case ISD::PARTIAL_REDUCE_UMLA:
3409 case ISD::PARTIAL_REDUCE_SUMLA:
3410 if (SDValue V = splitExtendingPartialReduceMLA(N, DAG&: DCI.DAG))
3411 return V;
3412 return SDValue();
3413 }
3414 } else {
3415 switch (Opc) {
3416 case ISD::VSELECT: {
3417 // (vselect (xor x, ptrue), v0, v1) -> (vselect x, v1, v0)
3418 SDValue Cond = Op.getOperand(i: 0);
3419 if (Cond->getOpcode() == ISD::XOR) {
3420 SDValue C0 = Cond.getOperand(i: 0), C1 = Cond.getOperand(i: 1);
3421 if (C1->getOpcode() == HexagonISD::PTRUE) {
3422 SDValue VSel = DCI.DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: ty(Op), N1: C0,
3423 N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 1));
3424 return VSel;
3425 }
3426 }
3427 return SDValue();
3428 }
3429 }
3430 }
3431
3432 if (isHvxOperation(N, DAG&: DCI.DAG)) {
3433 if (SDValue V = PerformHvxDAGCombine(N, DCI))
3434 return V;
3435 return SDValue();
3436 }
3437
3438 if (Opc == ISD::TRUNCATE) {
3439 SDValue Op0 = Op.getOperand(i: 0);
3440 // fold (truncate (build pair x, y)) -> (truncate x) or x
3441 if (Op0.getOpcode() == ISD::BUILD_PAIR) {
3442 EVT TruncTy = Op.getValueType();
3443 SDValue Elem0 = Op0.getOperand(i: 0);
3444 // if we match the low element of the pair, just return it.
3445 if (Elem0.getValueType() == TruncTy)
3446 return Elem0;
3447 // otherwise, if the low part is still too large, apply the truncate.
3448 if (Elem0.getValueType().bitsGT(VT: TruncTy))
3449 return DCI.DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: TruncTy, Operand: Elem0);
3450 }
3451 }
3452
3453 if (DCI.isBeforeLegalizeOps())
3454 return SDValue();
3455
3456 switch (Opc) {
3457 case HexagonISD::P2D: {
3458 SDValue P = Op.getOperand(i: 0);
3459 switch (P.getOpcode()) {
3460 case HexagonISD::PTRUE:
3461 return DCI.DAG.getAllOnesConstant(DL: dl, VT: ty(Op));
3462 case HexagonISD::PFALSE:
3463 return getZero(dl, Ty: ty(Op), DAG&: DCI.DAG);
3464 default:
3465 break;
3466 }
3467 break;
3468 }
3469 case ISD::TRUNCATE: {
3470 SDValue Op0 = Op.getOperand(i: 0);
3471 // fold (truncate (build pair x, y)) -> (truncate x) or x
3472 if (Op0.getOpcode() == ISD::BUILD_PAIR) {
3473 MVT TruncTy = ty(Op);
3474 SDValue Elem0 = Op0.getOperand(i: 0);
3475 // if we match the low element of the pair, just return it.
3476 if (ty(Op: Elem0) == TruncTy)
3477 return Elem0;
3478 // otherwise, if the low part is still too large, apply the truncate.
3479 if (ty(Op: Elem0).bitsGT(VT: TruncTy))
3480 return DCI.DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: TruncTy, Operand: Elem0);
3481 }
3482 break;
3483 }
3484 case ISD::OR: {
3485 // fold (or (shl xx, s), (zext y)) -> (COMBINE (shl xx, s-32), y)
3486 // if s >= 32
3487 auto fold0 = [&, this](SDValue Op) {
3488 if (ty(Op) != MVT::i64)
3489 return SDValue();
3490 SDValue Shl = Op.getOperand(i: 0);
3491 SDValue Zxt = Op.getOperand(i: 1);
3492 if (Shl.getOpcode() != ISD::SHL)
3493 std::swap(a&: Shl, b&: Zxt);
3494
3495 if (Shl.getOpcode() != ISD::SHL || Zxt.getOpcode() != ISD::ZERO_EXTEND)
3496 return SDValue();
3497
3498 SDValue Z = Zxt.getOperand(i: 0);
3499 auto *Amt = dyn_cast<ConstantSDNode>(Val: Shl.getOperand(i: 1));
3500 if (Amt && Amt->getZExtValue() >= 32 && ty(Op: Z).getSizeInBits() <= 32) {
3501 unsigned A = Amt->getZExtValue();
3502 SDValue S = Shl.getOperand(i: 0);
3503 SDValue T0 = DCI.DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: ty(Op: S), N1: S,
3504 N2: DCI.DAG.getConstant(Val: A - 32, DL: dl, VT: MVT::i32));
3505 SDValue T1 = DCI.DAG.getZExtOrTrunc(Op: T0, DL: dl, VT: MVT::i32);
3506 SDValue T2 = DCI.DAG.getZExtOrTrunc(Op: Z, DL: dl, VT: MVT::i32);
3507 return DCI.DAG.getNode(Opcode: HexagonISD::COMBINE, DL: dl, VT: MVT::i64, Ops: {T1, T2});
3508 }
3509 return SDValue();
3510 };
3511
3512 if (SDValue R = fold0(Op))
3513 return R;
3514 break;
3515 }
3516 }
3517
3518 return SDValue();
3519}
3520
3521/// Returns relocation base for the given PIC jumptable.
3522SDValue
3523HexagonTargetLowering::getPICJumpTableRelocBase(SDValue Table,
3524 SelectionDAG &DAG) const {
3525 int Idx = cast<JumpTableSDNode>(Val&: Table)->getIndex();
3526 EVT VT = Table.getValueType();
3527 SDValue T = DAG.getTargetJumpTable(JTI: Idx, VT, TargetFlags: HexagonII::MO_PCREL);
3528 return DAG.getNode(Opcode: HexagonISD::AT_PCREL, DL: SDLoc(Table), VT, Operand: T);
3529}
3530
3531//===----------------------------------------------------------------------===//
3532// Inline Assembly Support
3533//===----------------------------------------------------------------------===//
3534
3535TargetLowering::ConstraintType
3536HexagonTargetLowering::getConstraintType(StringRef Constraint) const {
3537 if (Constraint.size() == 1) {
3538 switch (Constraint[0]) {
3539 case 'q':
3540 case 'v':
3541 if (Subtarget.useHVXOps())
3542 return C_RegisterClass;
3543 break;
3544 case 'a':
3545 return C_RegisterClass;
3546 default:
3547 break;
3548 }
3549 }
3550 return TargetLowering::getConstraintType(Constraint);
3551}
3552
3553std::pair<unsigned, const TargetRegisterClass*>
3554HexagonTargetLowering::getRegForInlineAsmConstraint(
3555 const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const {
3556
3557 if (Constraint.size() == 1) {
3558 switch (Constraint[0]) {
3559 case 'r': // R0-R31
3560 switch (VT.SimpleTy) {
3561 default:
3562 return {0u, nullptr};
3563 case MVT::i1:
3564 case MVT::i8:
3565 case MVT::i16:
3566 case MVT::i32:
3567 case MVT::f32:
3568 return {0u, &Hexagon::IntRegsRegClass};
3569 case MVT::i64:
3570 case MVT::f64:
3571 return {0u, &Hexagon::DoubleRegsRegClass};
3572 }
3573 break;
3574 case 'a': // M0-M1
3575 if (VT != MVT::i32)
3576 return {0u, nullptr};
3577 return {0u, &Hexagon::ModRegsRegClass};
3578 case 'q': // q0-q3
3579 switch (VT.getSizeInBits()) {
3580 default:
3581 return {0u, nullptr};
3582 case 64:
3583 case 128:
3584 return {0u, &Hexagon::HvxQRRegClass};
3585 }
3586 break;
3587 case 'v': // V0-V31
3588 switch (VT.getSizeInBits()) {
3589 default:
3590 return {0u, nullptr};
3591 case 512:
3592 return {0u, &Hexagon::HvxVRRegClass};
3593 case 1024:
3594 if (Subtarget.hasV60Ops() && Subtarget.useHVX128BOps())
3595 return {0u, &Hexagon::HvxVRRegClass};
3596 return {0u, &Hexagon::HvxWRRegClass};
3597 case 2048:
3598 return {0u, &Hexagon::HvxWRRegClass};
3599 }
3600 break;
3601 default:
3602 return {0u, nullptr};
3603 }
3604 }
3605
3606 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
3607}
3608
3609/// isFPImmLegal - Returns true if the target can instruction select the
3610/// specified FP immediate natively. If false, the legalizer will
3611/// materialize the FP immediate as a load from a constant pool.
3612bool HexagonTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
3613 bool ForCodeSize) const {
3614 return true;
3615}
3616
3617/// Returns true if it is beneficial to convert a load of a constant
3618/// to just the constant itself.
3619bool HexagonTargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
3620 Type *Ty) const {
3621 if (!ConstantLoadsToImm)
3622 return false;
3623
3624 assert(Ty->isIntegerTy());
3625 unsigned BitSize = Ty->getPrimitiveSizeInBits();
3626 return (BitSize > 0 && BitSize <= 64);
3627}
3628
3629/// isLegalAddressingMode - Return true if the addressing mode represented by
3630/// AM is legal for this target, for a load/store of the specified type.
3631bool HexagonTargetLowering::isLegalAddressingMode(const DataLayout &DL,
3632 const AddrMode &AM, Type *Ty,
3633 unsigned AS, Instruction *I) const {
3634 if (Ty->isSized()) {
3635 // When LSR detects uses of the same base address to access different
3636 // types (e.g. unions), it will assume a conservative type for these
3637 // uses:
3638 // LSR Use: Kind=Address of void in addrspace(4294967295), ...
3639 // The type Ty passed here would then be "void". Skip the alignment
3640 // checks, but do not return false right away, since that confuses
3641 // LSR into crashing.
3642 Align A = DL.getABITypeAlign(Ty);
3643 // The base offset must be a multiple of the alignment.
3644 if (!isAligned(Lhs: A, SizeInBytes: AM.BaseOffs))
3645 return false;
3646 // The shifted offset must fit in 11 bits.
3647 if (!isInt<11>(x: AM.BaseOffs >> Log2(A)))
3648 return false;
3649 }
3650
3651 // No global is ever allowed as a base.
3652 if (AM.BaseGV)
3653 return false;
3654
3655 int Scale = AM.Scale;
3656 if (Scale < 0)
3657 Scale = -Scale;
3658 switch (Scale) {
3659 case 0: // No scale reg, "r+i", "r", or just "i".
3660 break;
3661 default: // No scaled addressing mode.
3662 return false;
3663 }
3664 return true;
3665}
3666
3667/// Return true if folding a constant offset with the given GlobalAddress is
3668/// legal. It is frequently not legal in PIC relocation models.
3669bool HexagonTargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA)
3670 const {
3671 return HTM.getRelocationModel() == Reloc::Static;
3672}
3673
3674/// isLegalICmpImmediate - Return true if the specified immediate is legal
3675/// icmp immediate, that is the target has icmp instructions which can compare
3676/// a register against the immediate without having to materialize the
3677/// immediate into a register.
3678bool HexagonTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
3679 return Imm >= -512 && Imm <= 511;
3680}
3681
3682/// IsEligibleForTailCallOptimization - Check whether the call is eligible
3683/// for tail call optimization. Targets which want to do tail call
3684/// optimization should implement this function.
3685bool HexagonTargetLowering::IsEligibleForTailCallOptimization(
3686 SDValue Callee,
3687 CallingConv::ID CalleeCC,
3688 bool IsVarArg,
3689 bool IsCalleeStructRet,
3690 bool IsCallerStructRet,
3691 const SmallVectorImpl<ISD::OutputArg> &Outs,
3692 const SmallVectorImpl<SDValue> &OutVals,
3693 const SmallVectorImpl<ISD::InputArg> &Ins,
3694 SelectionDAG& DAG) const {
3695 const Function &CallerF = DAG.getMachineFunction().getFunction();
3696 CallingConv::ID CallerCC = CallerF.getCallingConv();
3697 bool CCMatch = CallerCC == CalleeCC;
3698
3699 // ***************************************************************************
3700 // Look for obvious safe cases to perform tail call optimization that do not
3701 // require ABI changes.
3702 // ***************************************************************************
3703
3704 // If this is a tail call via a function pointer, then don't do it!
3705 if (!isa<GlobalAddressSDNode>(Val: Callee) &&
3706 !isa<ExternalSymbolSDNode>(Val: Callee)) {
3707 return false;
3708 }
3709
3710 // Do not optimize if the calling conventions do not match and the conventions
3711 // used are not C or Fast.
3712 if (!CCMatch) {
3713 bool R = (CallerCC == CallingConv::C || CallerCC == CallingConv::Fast);
3714 bool E = (CalleeCC == CallingConv::C || CalleeCC == CallingConv::Fast);
3715 // If R & E, then ok.
3716 if (!R || !E)
3717 return false;
3718 }
3719
3720 // Do not tail call optimize vararg calls.
3721 if (IsVarArg)
3722 return false;
3723
3724 // Also avoid tail call optimization if either caller or callee uses struct
3725 // return semantics.
3726 if (IsCalleeStructRet || IsCallerStructRet)
3727 return false;
3728
3729 // In addition to the cases above, we also disable Tail Call Optimization if
3730 // the calling convention code that at least one outgoing argument needs to
3731 // go on the stack. We cannot check that here because at this point that
3732 // information is not available.
3733 return true;
3734}
3735
3736/// Returns the target specific optimal type for load and store operations as
3737/// a result of memset, memcpy, and memmove lowering.
3738///
3739/// If DstAlign is zero that means it's safe to destination alignment can
3740/// satisfy any constraint. Similarly if SrcAlign is zero it means there isn't
3741/// a need to check it against alignment requirement, probably because the
3742/// source does not need to be loaded. If 'IsMemset' is true, that means it's
3743/// expanding a memset. If 'ZeroMemset' is true, that means it's a memset of
3744/// zero. 'MemcpyStrSrc' indicates whether the memcpy source is constant so it
3745/// does not need to be loaded. It returns EVT::Other if the type should be
3746/// determined using generic target-independent logic.
3747EVT HexagonTargetLowering::getOptimalMemOpType(
3748 LLVMContext &Context, const MemOp &Op,
3749 const AttributeList &FuncAttributes) const {
3750 if (Op.size() >= 8 && Op.isAligned(AlignCheck: Align(8)))
3751 return MVT::i64;
3752 if (Op.size() >= 4 && Op.isAligned(AlignCheck: Align(4)))
3753 return MVT::i32;
3754 if (Op.size() >= 2 && Op.isAligned(AlignCheck: Align(2)))
3755 return MVT::i16;
3756 return MVT::Other;
3757}
3758
3759// The helpers below are versions of llvm::getShuffleReduction and
3760// llvm::getOrderedReduction, adapted to use during DAG passes and simplified as
3761// follows:
3762// - ICmp and FCmp are not handled;
3763// - in every step in getShuffleReduction, the input is split into halves (not
3764// pairwise).
3765
3766static SDValue getOrderedReduction(SDValue Vec, unsigned Op,
3767 SelectionDAG &DAG) {
3768 assert(Op != Instruction::ICmp && Op != Instruction::FCmp);
3769
3770 EVT VT = Vec.getValueType();
3771 EVT EltT = VT.getVectorElementType();
3772 unsigned VF = VT.getVectorNumElements();
3773 assert(VF > 0 &&
3774 "Reduction emission only supported for non-zero length vectors!");
3775
3776 SDLoc DL(Vec);
3777 SDValue Result = DAG.getExtractVectorElt(DL, VT: EltT, Vec, Idx: 0);
3778 for (unsigned ExtractIdx = 1; ExtractIdx < VF; ++ExtractIdx) {
3779 SDValue Ext = DAG.getExtractVectorElt(DL, VT: EltT, Vec, Idx: ExtractIdx);
3780 Result = DAG.getNode(Opcode: Op, DL, VT: EltT, Ops: {Result, Ext});
3781 }
3782
3783 return Result;
3784}
3785
3786static SDValue getShuffleReduction(SDValue Vec, unsigned Op,
3787 SelectionDAG &DAG) {
3788 assert(Op != Instruction::ICmp && Op != Instruction::FCmp);
3789
3790 EVT VT = Vec.getValueType();
3791 unsigned VF = VT.getVectorNumElements();
3792 if (VF == 0)
3793 llvm_unreachable("Vector must be non-zero length");
3794 // VF is a power of 2 so we can emit the reduction using log2(VF) shuffles
3795 // and vector ops, reducing the set of values being computed by half each
3796 // round.
3797 assert(isPowerOf2_32(VF) &&
3798 "Reduction emission only supported for pow2 vectors!");
3799
3800 SDLoc DL(Vec);
3801 // TODO: Is it correct to create double-vector shuffle and fill 3/4 of it with
3802 // undefs?
3803 SmallVector<int, 32> ShuffleMask(VF);
3804 for (unsigned i = VF; i > 1; i >>= 1) {
3805 // Move the upper half of the vector to the lower half.
3806 for (unsigned j = 0; j != i / 2; ++j)
3807 ShuffleMask[j] = i / 2 + j;
3808 // Fill the rest of the mask with undef.
3809 std::fill(first: &ShuffleMask[i / 2], last: ShuffleMask.end(), value: -1);
3810
3811 SDValue Shuf =
3812 DAG.getVectorShuffle(VT, dl: DL, N1: Vec, N2: DAG.getUNDEF(VT), Mask: ShuffleMask);
3813
3814 Vec = DAG.getNode(Opcode: Op, DL, VT, Ops: {Vec, Shuf});
3815 }
3816 // The result is in the first element of the vector.
3817 return DAG.getExtractVectorElt(DL, VT: VT.getVectorElementType(), Vec, Idx: 0);
3818}
3819
3820SDValue HexagonTargetLowering::expandVecReduceAdd(SDNode *N,
3821 SelectionDAG &DAG) const {
3822 // Since we disabled automatic reduction expansion, generate log2 ladder code
3823 // if the vector is of a power-of-two length.
3824 SDValue Input = N->getOperand(Num: 0);
3825 if (isPowerOf2_32(Value: Input.getValueType().getVectorNumElements()))
3826 return getShuffleReduction(Vec: Input, Op: ISD::ADD, DAG);
3827 // Otherwise, reduction will be scalarized.
3828 return getOrderedReduction(Vec: Input, Op: ISD::ADD, DAG);
3829}
3830
3831bool HexagonTargetLowering::allowsMemoryAccess(
3832 LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace,
3833 Align Alignment, MachineMemOperand::Flags Flags, unsigned *Fast) const {
3834 if (!VT.isSimple())
3835 return false;
3836 MVT SVT = VT.getSimpleVT();
3837 if (Subtarget.isHVXVectorType(VecTy: SVT, IncludeBool: true))
3838 return allowsHvxMemoryAccess(VecTy: SVT, Flags, Fast);
3839 return TargetLoweringBase::allowsMemoryAccess(
3840 Context, DL, VT, AddrSpace, Alignment, Flags, Fast);
3841}
3842
3843bool HexagonTargetLowering::allowsMisalignedMemoryAccesses(
3844 EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags,
3845 unsigned *Fast) const {
3846 if (!VT.isSimple())
3847 return false;
3848 MVT SVT = VT.getSimpleVT();
3849 if (Subtarget.isHVXVectorType(VecTy: SVT, IncludeBool: true))
3850 return allowsHvxMisalignedMemoryAccesses(VecTy: SVT, Flags, Fast);
3851 if (Fast)
3852 *Fast = 0;
3853 return false;
3854}
3855
3856std::pair<const TargetRegisterClass*, uint8_t>
3857HexagonTargetLowering::findRepresentativeClass(const TargetRegisterInfo *TRI,
3858 MVT VT) const {
3859 if (Subtarget.isHVXVectorType(VecTy: VT, IncludeBool: true)) {
3860 unsigned BitWidth = VT.getSizeInBits();
3861 unsigned VecWidth = Subtarget.getVectorLength() * 8;
3862
3863 if (VT.getVectorElementType() == MVT::i1)
3864 return std::make_pair(x: &Hexagon::HvxQRRegClass, y: 1);
3865 if (BitWidth == VecWidth)
3866 return std::make_pair(x: &Hexagon::HvxVRRegClass, y: 1);
3867 assert(BitWidth == 2 * VecWidth);
3868 return std::make_pair(x: &Hexagon::HvxWRRegClass, y: 1);
3869 }
3870
3871 return TargetLowering::findRepresentativeClass(TRI, VT);
3872}
3873
3874bool HexagonTargetLowering::shouldReduceLoadWidth(
3875 SDNode *Load, ISD::LoadExtType ExtTy, EVT NewVT,
3876 std::optional<unsigned> ByteOffset) const {
3877 // TODO: This may be worth removing. Check regression tests for diffs.
3878 if (!TargetLoweringBase::shouldReduceLoadWidth(Load, ExtTy, NewVT,
3879 ByteOffset))
3880 return false;
3881
3882 auto *L = cast<LoadSDNode>(Val: Load);
3883 std::pair<SDValue, int> BO = getBaseAndOffset(Addr: L->getBasePtr());
3884 // Small-data object, do not shrink.
3885 if (BO.first.getOpcode() == HexagonISD::CONST32_GP)
3886 return false;
3887 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Val&: BO.first)) {
3888 auto &HTM = static_cast<const HexagonTargetMachine &>(getTargetMachine());
3889 const auto *GO = dyn_cast_or_null<const GlobalObject>(Val: GA->getGlobal());
3890 return !GO || !HTM.getObjFileLowering()->isGlobalInSmallSection(GO, TM: HTM);
3891 }
3892 return true;
3893}
3894
3895void HexagonTargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI,
3896 SDNode *Node) const {
3897 AdjustHvxInstrPostInstrSelection(MI, Node);
3898}
3899
3900Value *HexagonTargetLowering::emitLoadLinked(IRBuilderBase &Builder,
3901 Type *ValueTy, Value *Addr,
3902 AtomicOrdering Ord) const {
3903 unsigned SZ = ValueTy->getPrimitiveSizeInBits();
3904 assert((SZ == 32 || SZ == 64) && "Only 32/64-bit atomic loads supported");
3905 Intrinsic::ID IntID = (SZ == 32) ? Intrinsic::hexagon_L2_loadw_locked
3906 : Intrinsic::hexagon_L4_loadd_locked;
3907
3908 Value *Call =
3909 Builder.CreateIntrinsic(ID: IntID, Args: Addr, /*FMFSource=*/nullptr, Name: "larx");
3910
3911 return Builder.CreateBitCast(V: Call, DestTy: ValueTy);
3912}
3913
3914/// Perform a store-conditional operation to Addr. Return the status of the
3915/// store. This should be 0 if the store succeeded, non-zero otherwise.
3916Value *HexagonTargetLowering::emitStoreConditional(IRBuilderBase &Builder,
3917 Value *Val, Value *Addr,
3918 AtomicOrdering Ord) const {
3919 BasicBlock *BB = Builder.GetInsertBlock();
3920 Module *M = BB->getParent()->getParent();
3921 Type *Ty = Val->getType();
3922 unsigned SZ = Ty->getPrimitiveSizeInBits();
3923
3924 Type *CastTy = Builder.getIntNTy(N: SZ);
3925 assert((SZ == 32 || SZ == 64) && "Only 32/64-bit atomic stores supported");
3926 Intrinsic::ID IntID = (SZ == 32) ? Intrinsic::hexagon_S2_storew_locked
3927 : Intrinsic::hexagon_S4_stored_locked;
3928
3929 Val = Builder.CreateBitCast(V: Val, DestTy: CastTy);
3930
3931 Value *Call = Builder.CreateIntrinsic(ID: IntID, Args: {Addr, Val},
3932 /*FMFSource=*/nullptr, Name: "stcx");
3933 Value *Cmp = Builder.CreateICmpEQ(LHS: Call, RHS: Builder.getInt32(C: 0), Name: "");
3934 Value *Ext = Builder.CreateZExt(V: Cmp, DestTy: Type::getInt32Ty(C&: M->getContext()));
3935 return Ext;
3936}
3937
3938TargetLowering::AtomicExpansionKind
3939HexagonTargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const {
3940 // Do not expand loads and stores that don't exceed 64 bits.
3941 return LI->getType()->getPrimitiveSizeInBits() > 64
3942 ? AtomicExpansionKind::LLOnly
3943 : AtomicExpansionKind::None;
3944}
3945
3946TargetLowering::AtomicExpansionKind
3947HexagonTargetLowering::shouldExpandAtomicStoreInIR(StoreInst *SI) const {
3948 // Do not expand loads and stores that don't exceed 64 bits.
3949 return SI->getValueOperand()->getType()->getPrimitiveSizeInBits() > 64
3950 ? AtomicExpansionKind::Expand
3951 : AtomicExpansionKind::None;
3952}
3953
3954TargetLowering::AtomicExpansionKind
3955HexagonTargetLowering::shouldExpandAtomicCmpXchgInIR(
3956 const AtomicCmpXchgInst *AI) const {
3957 return AtomicExpansionKind::LLSC;
3958}
3959
3960bool HexagonTargetLowering::isMaskAndCmp0FoldingBeneficial(
3961 const Instruction &AndI) const {
3962 // Only sink 'and' mask to cmp use block if it is masking a single bit since
3963 // this will fold the and/cmp/br into a single tstbit instruction.
3964 ConstantInt *Mask = dyn_cast<ConstantInt>(Val: AndI.getOperand(i: 1));
3965 if (!Mask)
3966 return false;
3967 return Mask->getValue().isPowerOf2();
3968}
3969
3970// Check if the result of the node is only used as a return value, as
3971// otherwise we can't perform a tail-call.
3972bool HexagonTargetLowering::isUsedByReturnOnly(SDNode *N,
3973 SDValue &Chain) const {
3974 if (N->getNumValues() != 1)
3975 return false;
3976 if (!N->hasNUsesOfValue(NUses: 1, Value: 0))
3977 return false;
3978
3979 SDNode *Copy = *N->user_begin();
3980
3981 if (Copy->getOpcode() == ISD::BITCAST) {
3982 return isUsedByReturnOnly(N: Copy, Chain);
3983 }
3984
3985 if (Copy->getOpcode() != ISD::CopyToReg) {
3986 return false;
3987 }
3988
3989 // If the ISD::CopyToReg has a glue operand, we conservatively assume it
3990 // isn't safe to perform a tail call.
3991 if (Copy->getOperand(Num: Copy->getNumOperands() - 1).getValueType() == MVT::Glue)
3992 return false;
3993
3994 // The copy must be used by a HexagonISD::RET_GLUE, and nothing else.
3995 bool HasRet = false;
3996 for (SDNode *Node : Copy->users()) {
3997 if (Node->getOpcode() != HexagonISD::RET_GLUE)
3998 return false;
3999 HasRet = true;
4000 }
4001 if (!HasRet)
4002 return false;
4003
4004 Chain = Copy->getOperand(Num: 0);
4005 return true;
4006}
4007