1//===-- HexagonISelLowering.cpp - Hexagon DAG Lowering Implementation -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the interfaces that Hexagon uses to lower LLVM code
10// into a selection DAG.
11//
12//===----------------------------------------------------------------------===//
13
14#include "HexagonISelLowering.h"
15#include "Hexagon.h"
16#include "HexagonMachineFunctionInfo.h"
17#include "HexagonRegisterInfo.h"
18#include "HexagonSubtarget.h"
19#include "HexagonTargetMachine.h"
20#include "HexagonTargetObjectFile.h"
21#include "llvm/ADT/APInt.h"
22#include "llvm/ADT/ArrayRef.h"
23#include "llvm/ADT/SmallVector.h"
24#include "llvm/ADT/StringSwitch.h"
25#include "llvm/CodeGen/CallingConvLower.h"
26#include "llvm/CodeGen/MachineFrameInfo.h"
27#include "llvm/CodeGen/MachineFunction.h"
28#include "llvm/CodeGen/MachineMemOperand.h"
29#include "llvm/CodeGen/MachineRegisterInfo.h"
30#include "llvm/CodeGen/SelectionDAG.h"
31#include "llvm/CodeGen/TargetCallingConv.h"
32#include "llvm/CodeGen/ValueTypes.h"
33#include "llvm/IR/BasicBlock.h"
34#include "llvm/IR/CallingConv.h"
35#include "llvm/IR/DataLayout.h"
36#include "llvm/IR/DerivedTypes.h"
37#include "llvm/IR/DiagnosticInfo.h"
38#include "llvm/IR/DiagnosticPrinter.h"
39#include "llvm/IR/Function.h"
40#include "llvm/IR/GlobalValue.h"
41#include "llvm/IR/IRBuilder.h"
42#include "llvm/IR/InlineAsm.h"
43#include "llvm/IR/Instructions.h"
44#include "llvm/IR/IntrinsicInst.h"
45#include "llvm/IR/Intrinsics.h"
46#include "llvm/IR/IntrinsicsHexagon.h"
47#include "llvm/IR/Module.h"
48#include "llvm/IR/Type.h"
49#include "llvm/IR/Value.h"
50#include "llvm/Support/Casting.h"
51#include "llvm/Support/CodeGen.h"
52#include "llvm/Support/CommandLine.h"
53#include "llvm/Support/Debug.h"
54#include "llvm/Support/ErrorHandling.h"
55#include "llvm/Support/MathExtras.h"
56#include "llvm/Support/raw_ostream.h"
57#include "llvm/Target/TargetMachine.h"
58#include <algorithm>
59#include <cassert>
60#include <cstdint>
61#include <limits>
62#include <utility>
63
64using namespace llvm;
65
66#define DEBUG_TYPE "hexagon-lowering"
67
68static cl::opt<bool> EmitJumpTables("hexagon-emit-jump-tables",
69 cl::init(Val: true), cl::Hidden,
70 cl::desc("Control jump table emission on Hexagon target"));
71
72static cl::opt<bool>
73 EnableHexSDNodeSched("enable-hexagon-sdnode-sched", cl::Hidden,
74 cl::desc("Enable Hexagon SDNode scheduling"));
75
76static cl::opt<int> MinimumJumpTables("minimum-jump-tables", cl::Hidden,
77 cl::init(Val: 5),
78 cl::desc("Set minimum jump tables"));
79
80static cl::opt<bool>
81 ConstantLoadsToImm("constant-loads-to-imm", cl::Hidden, cl::init(Val: true),
82 cl::desc("Convert constant loads to immediate values."));
83
84static cl::opt<bool> AlignLoads("hexagon-align-loads",
85 cl::Hidden, cl::init(Val: false),
86 cl::desc("Rewrite unaligned loads as a pair of aligned loads"));
87
88static cl::opt<bool>
89 DisableArgsMinAlignment("hexagon-disable-args-min-alignment", cl::Hidden,
90 cl::init(Val: false),
91 cl::desc("Disable minimum alignment of 1 for "
92 "arguments passed by value on stack"));
93
94// Implement calling convention for Hexagon.
95
96static bool CC_SkipOdd(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
97 CCValAssign::LocInfo &LocInfo,
98 ISD::ArgFlagsTy &ArgFlags, CCState &State) {
99 static const MCPhysReg ArgRegs[] = {
100 Hexagon::R0, Hexagon::R1, Hexagon::R2,
101 Hexagon::R3, Hexagon::R4, Hexagon::R5
102 };
103 const unsigned NumArgRegs = std::size(ArgRegs);
104 unsigned RegNum = State.getFirstUnallocated(Regs: ArgRegs);
105
106 // RegNum is an index into ArgRegs: skip a register if RegNum is odd.
107 if (RegNum != NumArgRegs && RegNum % 2 == 1)
108 State.AllocateReg(Reg: ArgRegs[RegNum]);
109
110 // Always return false here, as this function only makes sure that the first
111 // unallocated register has an even register number and does not actually
112 // allocate a register for the current argument.
113 return false;
114}
115
116#include "HexagonGenCallingConv.inc"
117
118unsigned HexagonTargetLowering::getVectorTypeBreakdownForCallingConv(
119 LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT,
120 unsigned &NumIntermediates, MVT &RegisterVT) const {
121
122 bool isBoolVector = VT.getVectorElementType() == MVT::i1;
123 bool isPowerOf2 = VT.isPow2VectorType();
124 unsigned NumElts = VT.getVectorNumElements();
125
126 // Split vectors of type vXi1 into (X/8) vectors of type v8i1,
127 // where X is divisible by 8.
128 if (isBoolVector && !Subtarget.useHVXOps() && isPowerOf2 && NumElts >= 8) {
129 RegisterVT = MVT::v8i8;
130 IntermediateVT = MVT::v8i1;
131 NumIntermediates = NumElts / 8;
132 return NumIntermediates;
133 }
134
135 // In HVX 64-byte mode, vectors of type vXi1 are split into (X / 64) vectors
136 // of type v64i1, provided that X is divisible by 64.
137 if (isBoolVector && Subtarget.useHVX64BOps() && isPowerOf2 && NumElts >= 64) {
138 RegisterVT = MVT::v64i8;
139 IntermediateVT = MVT::v64i1;
140 NumIntermediates = NumElts / 64;
141 return NumIntermediates;
142 }
143
144 // In HVX 128-byte mode, vectors of type vXi1 are split into (X / 128) vectors
145 // of type v128i1, provided that X is divisible by 128.
146 if (isBoolVector && Subtarget.useHVX128BOps() && isPowerOf2 &&
147 NumElts >= 128) {
148 RegisterVT = MVT::v128i8;
149 IntermediateVT = MVT::v128i1;
150 NumIntermediates = NumElts / 128;
151 return NumIntermediates;
152 }
153
154 return TargetLowering::getVectorTypeBreakdownForCallingConv(
155 Context, CC, VT, IntermediateVT, NumIntermediates, RegisterVT);
156}
157
158std::pair<MVT, unsigned>
159HexagonTargetLowering::handleMaskRegisterForCallingConv(
160 const HexagonSubtarget &Subtarget, EVT VT) const {
161 assert(VT.getVectorElementType() == MVT::i1);
162
163 const unsigned NumElems = VT.getVectorNumElements();
164
165 if (!VT.isPow2VectorType())
166 return {MVT::INVALID_SIMPLE_VALUE_TYPE, 0};
167
168 if (!Subtarget.useHVXOps() && NumElems >= 8)
169 return {MVT::v8i8, NumElems / 8};
170
171 if (Subtarget.useHVX64BOps() && NumElems >= 64)
172 return {MVT::v64i8, NumElems / 64};
173
174 if (Subtarget.useHVX128BOps() && NumElems >= 128)
175 return {MVT::v128i8, NumElems / 128};
176
177 return {MVT::INVALID_SIMPLE_VALUE_TYPE, 0};
178}
179
180MVT HexagonTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
181 CallingConv::ID CC,
182 EVT VT) const {
183
184 if (VT.isVector() && VT.getVectorElementType() == MVT::i1) {
185 auto [RegisterVT, NumRegisters] =
186 handleMaskRegisterForCallingConv(Subtarget, VT);
187 if (RegisterVT != MVT::INVALID_SIMPLE_VALUE_TYPE)
188 return RegisterVT;
189 }
190
191 return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
192}
193
194SDValue
195HexagonTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG)
196 const {
197 unsigned IntNo = Op.getConstantOperandVal(i: 0);
198 SDLoc dl(Op);
199 switch (IntNo) {
200 default:
201 return SDValue(); // Don't custom lower most intrinsics.
202 case Intrinsic::thread_pointer: {
203 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
204 return DAG.getNode(Opcode: HexagonISD::THREAD_POINTER, DL: dl, VT: PtrVT);
205 }
206 }
207}
208
209/// CreateCopyOfByValArgument - Make a copy of an aggregate at address specified
210/// by "Src" to address "Dst" of size "Size". Alignment information is
211/// specified by the specific parameter attribute. The copy will be passed as
212/// a byval function parameter. Sometimes what we are copying is the end of a
213/// larger object, the part that does not fit in registers.
214static SDValue CreateCopyOfByValArgument(SDValue Src, SDValue Dst,
215 SDValue Chain, ISD::ArgFlagsTy Flags,
216 SelectionDAG &DAG, const SDLoc &dl) {
217 SDValue SizeNode = DAG.getConstant(Val: Flags.getByValSize(), DL: dl, VT: MVT::i32);
218 return DAG.getMemcpy(
219 Chain, dl, Dst, Src, Size: SizeNode, Alignment: Flags.getNonZeroByValAlign(),
220 /*isVolatile=*/isVol: false, /*AlwaysInline=*/false,
221 /*CI=*/nullptr, OverrideTailCall: std::nullopt, DstPtrInfo: MachinePointerInfo(), SrcPtrInfo: MachinePointerInfo());
222}
223
224bool
225HexagonTargetLowering::CanLowerReturn(
226 CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg,
227 const SmallVectorImpl<ISD::OutputArg> &Outs,
228 LLVMContext &Context, const Type *RetTy) const {
229 SmallVector<CCValAssign, 16> RVLocs;
230 CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
231
232 if (MF.getSubtarget<HexagonSubtarget>().useHVXOps())
233 return CCInfo.CheckReturn(Outs, Fn: RetCC_Hexagon_HVX);
234 return CCInfo.CheckReturn(Outs, Fn: RetCC_Hexagon);
235}
236
237// LowerReturn - Lower ISD::RET. If a struct is larger than 8 bytes and is
238// passed by value, the function prototype is modified to return void and
239// the value is stored in memory pointed by a pointer passed by caller.
240SDValue
241HexagonTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
242 bool IsVarArg,
243 const SmallVectorImpl<ISD::OutputArg> &Outs,
244 const SmallVectorImpl<SDValue> &OutVals,
245 const SDLoc &dl, SelectionDAG &DAG) const {
246 // CCValAssign - represent the assignment of the return value to locations.
247 SmallVector<CCValAssign, 16> RVLocs;
248
249 // CCState - Info about the registers and stack slot.
250 CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
251 *DAG.getContext());
252
253 // Analyze return values of ISD::RET
254 if (Subtarget.useHVXOps())
255 CCInfo.AnalyzeReturn(Outs, Fn: RetCC_Hexagon_HVX);
256 else
257 CCInfo.AnalyzeReturn(Outs, Fn: RetCC_Hexagon);
258
259 SDValue Glue;
260 SmallVector<SDValue, 4> RetOps(1, Chain);
261
262 // Copy the result values into the output registers.
263 for (unsigned i = 0; i != RVLocs.size(); ++i) {
264 CCValAssign &VA = RVLocs[i];
265 SDValue Val = OutVals[i];
266
267 switch (VA.getLocInfo()) {
268 default:
269 // Loc info must be one of Full, BCvt, SExt, ZExt, or AExt.
270 llvm_unreachable("Unknown loc info!");
271 case CCValAssign::Full:
272 break;
273 case CCValAssign::BCvt:
274 Val = DAG.getBitcast(VT: VA.getLocVT(), V: Val);
275 break;
276 case CCValAssign::SExt:
277 Val = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: Val);
278 break;
279 case CCValAssign::ZExt:
280 Val = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: Val);
281 break;
282 case CCValAssign::AExt:
283 Val = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: Val);
284 break;
285 }
286
287 Chain = DAG.getCopyToReg(Chain, dl, Reg: VA.getLocReg(), N: Val, Glue);
288
289 // Guarantee that all emitted copies are stuck together with flags.
290 Glue = Chain.getValue(R: 1);
291 RetOps.push_back(Elt: DAG.getRegister(Reg: VA.getLocReg(), VT: VA.getLocVT()));
292 }
293
294 RetOps[0] = Chain; // Update chain.
295
296 // Add the glue if we have it.
297 if (Glue.getNode())
298 RetOps.push_back(Elt: Glue);
299
300 return DAG.getNode(Opcode: HexagonISD::RET_GLUE, DL: dl, VT: MVT::Other, Ops: RetOps);
301}
302
303bool HexagonTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
304 // If either no tail call or told not to tail call at all, don't.
305 return CI->isTailCall();
306}
307
308Register HexagonTargetLowering::getRegisterByName(
309 const char* RegName, LLT VT, const MachineFunction &) const {
310 // Just support r19, the linux kernel uses it.
311 Register Reg = StringSwitch<Register>(RegName)
312 .Case(S: "r0", Value: Hexagon::R0)
313 .Case(S: "r1", Value: Hexagon::R1)
314 .Case(S: "r2", Value: Hexagon::R2)
315 .Case(S: "r3", Value: Hexagon::R3)
316 .Case(S: "r4", Value: Hexagon::R4)
317 .Case(S: "r5", Value: Hexagon::R5)
318 .Case(S: "r6", Value: Hexagon::R6)
319 .Case(S: "r7", Value: Hexagon::R7)
320 .Case(S: "r8", Value: Hexagon::R8)
321 .Case(S: "r9", Value: Hexagon::R9)
322 .Case(S: "r10", Value: Hexagon::R10)
323 .Case(S: "r11", Value: Hexagon::R11)
324 .Case(S: "r12", Value: Hexagon::R12)
325 .Case(S: "r13", Value: Hexagon::R13)
326 .Case(S: "r14", Value: Hexagon::R14)
327 .Case(S: "r15", Value: Hexagon::R15)
328 .Case(S: "r16", Value: Hexagon::R16)
329 .Case(S: "r17", Value: Hexagon::R17)
330 .Case(S: "r18", Value: Hexagon::R18)
331 .Case(S: "r19", Value: Hexagon::R19)
332 .Case(S: "r20", Value: Hexagon::R20)
333 .Case(S: "r21", Value: Hexagon::R21)
334 .Case(S: "r22", Value: Hexagon::R22)
335 .Case(S: "r23", Value: Hexagon::R23)
336 .Case(S: "r24", Value: Hexagon::R24)
337 .Case(S: "r25", Value: Hexagon::R25)
338 .Case(S: "r26", Value: Hexagon::R26)
339 .Case(S: "r27", Value: Hexagon::R27)
340 .Case(S: "r28", Value: Hexagon::R28)
341 .Case(S: "r29", Value: Hexagon::R29)
342 .Case(S: "r30", Value: Hexagon::R30)
343 .Case(S: "r31", Value: Hexagon::R31)
344 .Case(S: "r1:0", Value: Hexagon::D0)
345 .Case(S: "r3:2", Value: Hexagon::D1)
346 .Case(S: "r5:4", Value: Hexagon::D2)
347 .Case(S: "r7:6", Value: Hexagon::D3)
348 .Case(S: "r9:8", Value: Hexagon::D4)
349 .Case(S: "r11:10", Value: Hexagon::D5)
350 .Case(S: "r13:12", Value: Hexagon::D6)
351 .Case(S: "r15:14", Value: Hexagon::D7)
352 .Case(S: "r17:16", Value: Hexagon::D8)
353 .Case(S: "r19:18", Value: Hexagon::D9)
354 .Case(S: "r21:20", Value: Hexagon::D10)
355 .Case(S: "r23:22", Value: Hexagon::D11)
356 .Case(S: "r25:24", Value: Hexagon::D12)
357 .Case(S: "r27:26", Value: Hexagon::D13)
358 .Case(S: "r29:28", Value: Hexagon::D14)
359 .Case(S: "r31:30", Value: Hexagon::D15)
360 .Case(S: "sp", Value: Hexagon::R29)
361 .Case(S: "fp", Value: Hexagon::R30)
362 .Case(S: "lr", Value: Hexagon::R31)
363 .Case(S: "p0", Value: Hexagon::P0)
364 .Case(S: "p1", Value: Hexagon::P1)
365 .Case(S: "p2", Value: Hexagon::P2)
366 .Case(S: "p3", Value: Hexagon::P3)
367 .Case(S: "sa0", Value: Hexagon::SA0)
368 .Case(S: "lc0", Value: Hexagon::LC0)
369 .Case(S: "sa1", Value: Hexagon::SA1)
370 .Case(S: "lc1", Value: Hexagon::LC1)
371 .Case(S: "m0", Value: Hexagon::M0)
372 .Case(S: "m1", Value: Hexagon::M1)
373 .Case(S: "usr", Value: Hexagon::USR)
374 .Case(S: "ugp", Value: Hexagon::UGP)
375 .Case(S: "cs0", Value: Hexagon::CS0)
376 .Case(S: "cs1", Value: Hexagon::CS1)
377 .Default(Value: Register());
378 return Reg;
379}
380
381/// LowerCallResult - Lower the result values of an ISD::CALL into the
382/// appropriate copies out of appropriate physical registers. This assumes that
383/// Chain/Glue are the input chain/glue to use, and that TheCall is the call
384/// being lowered. Returns a SDNode with the same number of values as the
385/// ISD::CALL.
386SDValue HexagonTargetLowering::LowerCallResult(
387 SDValue Chain, SDValue Glue, CallingConv::ID CallConv, bool IsVarArg,
388 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
389 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals,
390 const SmallVectorImpl<SDValue> &OutVals, SDValue Callee) const {
391 // Assign locations to each value returned by this call.
392 SmallVector<CCValAssign, 16> RVLocs;
393
394 CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
395 *DAG.getContext());
396
397 if (Subtarget.useHVXOps())
398 CCInfo.AnalyzeCallResult(Ins, Fn: RetCC_Hexagon_HVX);
399 else
400 CCInfo.AnalyzeCallResult(Ins, Fn: RetCC_Hexagon);
401
402 // Copy all of the result registers out of their specified physreg.
403 for (unsigned i = 0; i != RVLocs.size(); ++i) {
404 SDValue RetVal;
405 if (RVLocs[i].getValVT() == MVT::i1) {
406 // Return values of type MVT::i1 require special handling. The reason
407 // is that MVT::i1 is associated with the PredRegs register class, but
408 // values of that type are still returned in R0. Generate an explicit
409 // copy into a predicate register from R0, and treat the value of the
410 // predicate register as the call result.
411 auto &MRI = DAG.getMachineFunction().getRegInfo();
412 SDValue FR0 = DAG.getCopyFromReg(Chain, dl, Reg: RVLocs[i].getLocReg(),
413 VT: MVT::i32, Glue);
414 // FR0 = (Value, Chain, Glue)
415 Register PredR = MRI.createVirtualRegister(RegClass: &Hexagon::PredRegsRegClass);
416 SDValue TPR = DAG.getCopyToReg(Chain: FR0.getValue(R: 1), dl, Reg: PredR,
417 N: FR0.getValue(R: 0), Glue: FR0.getValue(R: 2));
418 // TPR = (Chain, Glue)
419 // Don't glue this CopyFromReg, because it copies from a virtual
420 // register. If it is glued to the call, InstrEmitter will add it
421 // as an implicit def to the call (EmitMachineNode).
422 RetVal = DAG.getCopyFromReg(Chain: TPR.getValue(R: 0), dl, Reg: PredR, VT: MVT::i1);
423 Glue = TPR.getValue(R: 1);
424 Chain = TPR.getValue(R: 0);
425 } else {
426 RetVal = DAG.getCopyFromReg(Chain, dl, Reg: RVLocs[i].getLocReg(),
427 VT: RVLocs[i].getValVT(), Glue);
428 Glue = RetVal.getValue(R: 2);
429 Chain = RetVal.getValue(R: 1);
430 }
431 InVals.push_back(Elt: RetVal.getValue(R: 0));
432 }
433
434 return Chain;
435}
436
437/// LowerCall - Functions arguments are copied from virtual regs to
438/// (physical regs)/(stack frame), CALLSEQ_START and CALLSEQ_END are emitted.
439SDValue
440HexagonTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
441 SmallVectorImpl<SDValue> &InVals) const {
442 SelectionDAG &DAG = CLI.DAG;
443 SDLoc &dl = CLI.DL;
444 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
445 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
446 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
447 SDValue Chain = CLI.Chain;
448 SDValue Callee = CLI.Callee;
449 CallingConv::ID CallConv = CLI.CallConv;
450 bool IsVarArg = CLI.IsVarArg;
451 bool DoesNotReturn = CLI.DoesNotReturn;
452
453 bool IsStructRet = Outs.empty() ? false : Outs[0].Flags.isSRet();
454 MachineFunction &MF = DAG.getMachineFunction();
455 MachineFrameInfo &MFI = MF.getFrameInfo();
456 auto PtrVT = getPointerTy(DL: MF.getDataLayout());
457
458 if (GlobalAddressSDNode *GAN = dyn_cast<GlobalAddressSDNode>(Val&: Callee))
459 Callee = DAG.getTargetGlobalAddress(GV: GAN->getGlobal(), DL: dl, VT: MVT::i32);
460
461 // Linux ABI treats var-arg calls the same way as regular ones.
462 bool TreatAsVarArg = !Subtarget.isEnvironmentMusl() && IsVarArg;
463
464 // Analyze operands of the call, assigning locations to each operand.
465 SmallVector<CCValAssign, 16> ArgLocs;
466 CCState CCInfo(CallConv, TreatAsVarArg, MF, ArgLocs, *DAG.getContext());
467
468 if (Subtarget.useHVXOps())
469 CCInfo.AnalyzeCallOperands(Outs, Fn: CC_Hexagon_HVX);
470 else if (DisableArgsMinAlignment)
471 CCInfo.AnalyzeCallOperands(Outs, Fn: CC_Hexagon_Legacy);
472 else
473 CCInfo.AnalyzeCallOperands(Outs, Fn: CC_Hexagon);
474
475 if (CLI.IsTailCall) {
476 bool StructAttrFlag = MF.getFunction().hasStructRetAttr();
477 CLI.IsTailCall = IsEligibleForTailCallOptimization(Callee, CalleeCC: CallConv,
478 isVarArg: IsVarArg, isCalleeStructRet: IsStructRet, isCallerStructRet: StructAttrFlag, Outs,
479 OutVals, Ins, DAG);
480 for (const CCValAssign &VA : ArgLocs) {
481 if (VA.isMemLoc()) {
482 CLI.IsTailCall = false;
483 break;
484 }
485 }
486 LLVM_DEBUG(dbgs() << (CLI.IsTailCall ? "Eligible for Tail Call\n"
487 : "Argument must be passed on stack. "
488 "Not eligible for Tail Call\n"));
489 }
490 // Get a count of how many bytes are to be pushed on the stack.
491 unsigned NumBytes = CCInfo.getStackSize();
492 SmallVector<std::pair<unsigned, SDValue>, 16> RegsToPass;
493 SmallVector<SDValue, 8> MemOpChains;
494
495 const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo();
496 SDValue StackPtr =
497 DAG.getCopyFromReg(Chain, dl, Reg: HRI.getStackRegister(), VT: PtrVT);
498
499 bool NeedsArgAlign = false;
500 Align LargestAlignSeen;
501 // Walk the register/memloc assignments, inserting copies/loads.
502 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
503 CCValAssign &VA = ArgLocs[i];
504 SDValue Arg = OutVals[i];
505 ISD::ArgFlagsTy Flags = Outs[i].Flags;
506 // Record if we need > 8 byte alignment on an argument.
507 bool ArgAlign = Subtarget.isHVXVectorType(VecTy: VA.getValVT());
508 NeedsArgAlign |= ArgAlign;
509
510 // Promote the value if needed.
511 switch (VA.getLocInfo()) {
512 default:
513 // Loc info must be one of Full, BCvt, SExt, ZExt, or AExt.
514 llvm_unreachable("Unknown loc info!");
515 case CCValAssign::Full:
516 break;
517 case CCValAssign::BCvt:
518 Arg = DAG.getBitcast(VT: VA.getLocVT(), V: Arg);
519 break;
520 case CCValAssign::SExt:
521 Arg = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: Arg);
522 break;
523 case CCValAssign::ZExt:
524 Arg = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: Arg);
525 break;
526 case CCValAssign::AExt:
527 Arg = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: Arg);
528 break;
529 }
530
531 if (VA.isMemLoc()) {
532 unsigned LocMemOffset = VA.getLocMemOffset();
533 SDValue MemAddr = DAG.getConstant(Val: LocMemOffset, DL: dl,
534 VT: StackPtr.getValueType());
535 MemAddr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::i32, N1: StackPtr, N2: MemAddr);
536 if (ArgAlign)
537 LargestAlignSeen = std::max(
538 a: LargestAlignSeen, b: Align(VA.getLocVT().getStoreSizeInBits() / 8));
539 if (Flags.isByVal()) {
540 // The argument is a struct passed by value. According to LLVM, "Arg"
541 // is a pointer.
542 MemOpChains.push_back(Elt: CreateCopyOfByValArgument(Src: Arg, Dst: MemAddr, Chain,
543 Flags, DAG, dl));
544 } else {
545 MachinePointerInfo LocPI = MachinePointerInfo::getStack(
546 MF&: DAG.getMachineFunction(), Offset: LocMemOffset);
547 SDValue S = DAG.getStore(Chain, dl, Val: Arg, Ptr: MemAddr, PtrInfo: LocPI);
548 MemOpChains.push_back(Elt: S);
549 }
550 continue;
551 }
552
553 // Arguments that can be passed on register must be kept at RegsToPass
554 // vector.
555 if (VA.isRegLoc())
556 RegsToPass.push_back(Elt: std::make_pair(x: VA.getLocReg(), y&: Arg));
557 }
558
559 if (NeedsArgAlign && Subtarget.hasV60Ops()) {
560 LLVM_DEBUG(dbgs() << "Function needs byte stack align due to call args\n");
561 Align VecAlign = HRI.getSpillAlign(RC: Hexagon::HvxVRRegClass);
562 LargestAlignSeen = std::max(a: LargestAlignSeen, b: VecAlign);
563 MFI.ensureMaxAlignment(Alignment: LargestAlignSeen);
564 }
565 // Transform all store nodes into one single node because all store
566 // nodes are independent of each other.
567 if (!MemOpChains.empty())
568 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: MemOpChains);
569
570 SDValue Glue;
571 if (!CLI.IsTailCall) {
572 Chain = DAG.getCALLSEQ_START(Chain, InSize: NumBytes, OutSize: 0, DL: dl);
573 Glue = Chain.getValue(R: 1);
574 }
575
576 // Build a sequence of copy-to-reg nodes chained together with token
577 // chain and flag operands which copy the outgoing args into registers.
578 // The Glue is necessary since all emitted instructions must be
579 // stuck together.
580 if (!CLI.IsTailCall) {
581 for (const auto &R : RegsToPass) {
582 Chain = DAG.getCopyToReg(Chain, dl, Reg: R.first, N: R.second, Glue);
583 Glue = Chain.getValue(R: 1);
584 }
585 } else {
586 // For tail calls lower the arguments to the 'real' stack slot.
587 //
588 // Force all the incoming stack arguments to be loaded from the stack
589 // before any new outgoing arguments are stored to the stack, because the
590 // outgoing stack slots may alias the incoming argument stack slots, and
591 // the alias isn't otherwise explicit. This is slightly more conservative
592 // than necessary, because it means that each store effectively depends
593 // on every argument instead of just those arguments it would clobber.
594 //
595 // Do not flag preceding copytoreg stuff together with the following stuff.
596 Glue = SDValue();
597 for (const auto &R : RegsToPass) {
598 Chain = DAG.getCopyToReg(Chain, dl, Reg: R.first, N: R.second, Glue);
599 Glue = Chain.getValue(R: 1);
600 }
601 Glue = SDValue();
602 }
603
604 bool LongCalls = MF.getSubtarget<HexagonSubtarget>().useLongCalls();
605 unsigned Flags = LongCalls ? HexagonII::HMOTF_ConstExtended : 0;
606
607 // If the callee is a GlobalAddress/ExternalSymbol node (quite common, every
608 // direct call is) turn it into a TargetGlobalAddress/TargetExternalSymbol
609 // node so that legalize doesn't hack it.
610 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Val&: Callee)) {
611 Callee = DAG.getTargetGlobalAddress(GV: G->getGlobal(), DL: dl, VT: PtrVT, offset: 0, TargetFlags: Flags);
612 } else if (ExternalSymbolSDNode *S =
613 dyn_cast<ExternalSymbolSDNode>(Val&: Callee)) {
614 Callee = DAG.getTargetExternalSymbol(Sym: S->getSymbol(), VT: PtrVT, TargetFlags: Flags);
615 }
616
617 // Returns a chain & a flag for retval copy to use.
618 SmallVector<SDValue, 8> Ops;
619 Ops.push_back(Elt: Chain);
620 Ops.push_back(Elt: Callee);
621
622 // Add argument registers to the end of the list so that they are
623 // known live into the call.
624 for (const auto &R : RegsToPass)
625 Ops.push_back(Elt: DAG.getRegister(Reg: R.first, VT: R.second.getValueType()));
626
627 const uint32_t *Mask = HRI.getCallPreservedMask(MF, CallConv);
628 assert(Mask && "Missing call preserved mask for calling convention");
629 Ops.push_back(Elt: DAG.getRegisterMask(RegMask: Mask));
630
631 if (Glue.getNode())
632 Ops.push_back(Elt: Glue);
633
634 if (CLI.IsTailCall) {
635 MFI.setHasTailCall();
636 return DAG.getNode(Opcode: HexagonISD::TC_RETURN, DL: dl, VT: MVT::Other, Ops);
637 }
638
639 // Set this here because we need to know this for "hasFP" in frame lowering.
640 // The target-independent code calls getFrameRegister before setting it, and
641 // getFrameRegister uses hasFP to determine whether the function has FP.
642 MFI.setHasCalls(true);
643
644 unsigned OpCode = DoesNotReturn ? HexagonISD::CALLnr : HexagonISD::CALL;
645 Chain = DAG.getNode(Opcode: OpCode, DL: dl, ResultTys: {MVT::Other, MVT::Glue}, Ops);
646 Glue = Chain.getValue(R: 1);
647
648 // Create the CALLSEQ_END node.
649 Chain = DAG.getCALLSEQ_END(Chain, Size1: NumBytes, Size2: 0, Glue, DL: dl);
650 Glue = Chain.getValue(R: 1);
651
652 // Handle result values, copying them out of physregs into vregs that we
653 // return.
654 return LowerCallResult(Chain, Glue, CallConv, IsVarArg, Ins, dl, DAG,
655 InVals, OutVals, Callee);
656}
657
658/// Returns true by value, base pointer and offset pointer and addressing
659/// mode by reference if this node can be combined with a load / store to
660/// form a post-indexed load / store.
661bool HexagonTargetLowering::getPostIndexedAddressParts(SDNode *N, SDNode *Op,
662 SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM,
663 SelectionDAG &DAG) const {
664 LSBaseSDNode *LSN = dyn_cast<LSBaseSDNode>(Val: N);
665 if (!LSN)
666 return false;
667 EVT VT = LSN->getMemoryVT();
668 if (!VT.isSimple())
669 return false;
670 bool IsLegalType = VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32 ||
671 VT == MVT::i64 || VT == MVT::f32 || VT == MVT::f64 ||
672 VT == MVT::v2i16 || VT == MVT::v2i32 || VT == MVT::v4i8 ||
673 VT == MVT::v4i16 || VT == MVT::v8i8 ||
674 Subtarget.isHVXVectorType(VecTy: VT.getSimpleVT());
675 if (!IsLegalType)
676 return false;
677
678 if (Op->getOpcode() != ISD::ADD)
679 return false;
680 Base = Op->getOperand(Num: 0);
681 Offset = Op->getOperand(Num: 1);
682 if (!isa<ConstantSDNode>(Val: Offset.getNode()))
683 return false;
684 AM = ISD::POST_INC;
685
686 int32_t V = cast<ConstantSDNode>(Val: Offset.getNode())->getSExtValue();
687 return Subtarget.getInstrInfo()->isValidAutoIncImm(VT, Offset: V);
688}
689
690SDValue HexagonTargetLowering::LowerFDIV(SDValue Op, SelectionDAG &DAG) const {
691 if (DAG.getMachineFunction().getFunction().hasOptSize())
692 return SDValue();
693 else
694 return Op;
695}
696
697SDValue
698HexagonTargetLowering::LowerINLINEASM(SDValue Op, SelectionDAG &DAG) const {
699 MachineFunction &MF = DAG.getMachineFunction();
700 auto &HMFI = *MF.getInfo<HexagonMachineFunctionInfo>();
701 const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo();
702 unsigned LR = HRI.getRARegister();
703
704 if ((Op.getOpcode() != ISD::INLINEASM &&
705 Op.getOpcode() != ISD::INLINEASM_BR) || HMFI.hasClobberLR())
706 return Op;
707
708 unsigned NumOps = Op.getNumOperands();
709 if (Op.getOperand(i: NumOps-1).getValueType() == MVT::Glue)
710 --NumOps; // Ignore the flag operand.
711
712 for (unsigned i = InlineAsm::Op_FirstOperand; i != NumOps;) {
713 const InlineAsm::Flag Flags(Op.getConstantOperandVal(i));
714 unsigned NumVals = Flags.getNumOperandRegisters();
715 ++i; // Skip the ID value.
716
717 switch (Flags.getKind()) {
718 default:
719 llvm_unreachable("Bad flags!");
720 case InlineAsm::Kind::RegUse:
721 case InlineAsm::Kind::Imm:
722 case InlineAsm::Kind::Mem:
723 i += NumVals;
724 break;
725 case InlineAsm::Kind::Clobber:
726 case InlineAsm::Kind::RegDef:
727 case InlineAsm::Kind::RegDefEarlyClobber: {
728 for (; NumVals; --NumVals, ++i) {
729 Register Reg = cast<RegisterSDNode>(Val: Op.getOperand(i))->getReg();
730 if (Reg != LR)
731 continue;
732 HMFI.setHasClobberLR(true);
733 return Op;
734 }
735 break;
736 }
737 }
738 }
739
740 return Op;
741}
742
743// Need to transform ISD::PREFETCH into something that doesn't inherit
744// all of the properties of ISD::PREFETCH, specifically SDNPMayLoad and
745// SDNPMayStore.
746SDValue HexagonTargetLowering::LowerPREFETCH(SDValue Op,
747 SelectionDAG &DAG) const {
748 SDValue Chain = Op.getOperand(i: 0);
749 SDValue Addr = Op.getOperand(i: 1);
750 // Lower it to DCFETCH($reg, #0). A "pat" will try to merge the offset in,
751 // if the "reg" is fed by an "add".
752 SDLoc DL(Op);
753 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
754 return DAG.getNode(Opcode: HexagonISD::DCFETCH, DL, VT: MVT::Other, N1: Chain, N2: Addr, N3: Zero);
755}
756
757SDValue HexagonTargetLowering::LowerINTRINSIC_VOID(SDValue Op,
758 SelectionDAG &DAG) const {
759 SDValue Chain = Op.getOperand(i: 0);
760 unsigned IntNo = Op.getConstantOperandVal(i: 1);
761 // Lower the hexagon_prefetch builtin to DCFETCH, as above.
762 if (IntNo == Intrinsic::hexagon_prefetch) {
763 SDValue Addr = Op.getOperand(i: 2);
764 SDLoc DL(Op);
765 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
766 return DAG.getNode(Opcode: HexagonISD::DCFETCH, DL, VT: MVT::Other, N1: Chain, N2: Addr, N3: Zero);
767 }
768 return SDValue();
769}
770
771SDValue
772HexagonTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
773 SelectionDAG &DAG) const {
774 SDValue Chain = Op.getOperand(i: 0);
775 SDValue Size = Op.getOperand(i: 1);
776 SDValue Align = Op.getOperand(i: 2);
777 SDLoc dl(Op);
778
779 ConstantSDNode *AlignConst = dyn_cast<ConstantSDNode>(Val&: Align);
780 assert(AlignConst && "Non-constant Align in LowerDYNAMIC_STACKALLOC");
781
782 unsigned A = AlignConst->getSExtValue();
783 auto &HFI = *Subtarget.getFrameLowering();
784 // "Zero" means natural stack alignment.
785 if (A == 0)
786 A = HFI.getStackAlign().value();
787
788 LLVM_DEBUG({
789 dbgs () << __func__ << " Align: " << A << " Size: ";
790 Size.getNode()->dump(&DAG);
791 dbgs() << "\n";
792 });
793
794 SDValue AC = DAG.getConstant(Val: A, DL: dl, VT: MVT::i32);
795 SDVTList VTs = DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other);
796 SDValue AA = DAG.getNode(Opcode: HexagonISD::ALLOCA, DL: dl, VTList: VTs, N1: Chain, N2: Size, N3: AC);
797
798 DAG.ReplaceAllUsesOfValueWith(From: Op, To: AA);
799 return AA;
800}
801
802SDValue HexagonTargetLowering::LowerFormalArguments(
803 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
804 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
805 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
806 MachineFunction &MF = DAG.getMachineFunction();
807 MachineFrameInfo &MFI = MF.getFrameInfo();
808 MachineRegisterInfo &MRI = MF.getRegInfo();
809
810 // Linux ABI treats var-arg calls the same way as regular ones.
811 bool TreatAsVarArg = !Subtarget.isEnvironmentMusl() && IsVarArg;
812
813 // Assign locations to all of the incoming arguments.
814 SmallVector<CCValAssign, 16> ArgLocs;
815 CCState CCInfo(CallConv, TreatAsVarArg, MF, ArgLocs, *DAG.getContext());
816
817 if (Subtarget.useHVXOps())
818 CCInfo.AnalyzeFormalArguments(Ins, Fn: CC_Hexagon_HVX);
819 else if (DisableArgsMinAlignment)
820 CCInfo.AnalyzeFormalArguments(Ins, Fn: CC_Hexagon_Legacy);
821 else
822 CCInfo.AnalyzeFormalArguments(Ins, Fn: CC_Hexagon);
823
824 // For LLVM, in the case when returning a struct by value (>8byte),
825 // the first argument is a pointer that points to the location on caller's
826 // stack where the return value will be stored. For Hexagon, the location on
827 // caller's stack is passed only when the struct size is smaller than (and
828 // equal to) 8 bytes. If not, no address will be passed into callee and
829 // callee return the result directly through R0/R1.
830 auto NextSingleReg = [] (const TargetRegisterClass &RC, unsigned Reg) {
831 switch (RC.getID()) {
832 case Hexagon::IntRegsRegClassID:
833 return Reg - Hexagon::R0 + 1;
834 case Hexagon::DoubleRegsRegClassID:
835 return (Reg - Hexagon::D0 + 1) * 2;
836 case Hexagon::HvxVRRegClassID:
837 return Reg - Hexagon::V0 + 1;
838 case Hexagon::HvxWRRegClassID:
839 return (Reg - Hexagon::W0 + 1) * 2;
840 }
841 llvm_unreachable("Unexpected register class");
842 };
843
844 auto &HFL = const_cast<HexagonFrameLowering&>(*Subtarget.getFrameLowering());
845 auto &HMFI = *MF.getInfo<HexagonMachineFunctionInfo>();
846 HFL.FirstVarArgSavedReg = 0;
847 HMFI.setFirstNamedArgFrameIndex(-int(MFI.getNumFixedObjects()));
848
849 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
850 CCValAssign &VA = ArgLocs[i];
851 ISD::ArgFlagsTy Flags = Ins[i].Flags;
852 bool ByVal = Flags.isByVal();
853
854 // Arguments passed in registers:
855 // 1. 32- and 64-bit values and HVX vectors are passed directly,
856 // 2. Large structs are passed via an address, and the address is
857 // passed in a register.
858 if (VA.isRegLoc() && ByVal && Flags.getByValSize() <= 8)
859 llvm_unreachable("ByValSize must be bigger than 8 bytes");
860
861 bool InReg = VA.isRegLoc() &&
862 (!ByVal || (ByVal && Flags.getByValSize() > 8));
863
864 if (InReg) {
865 MVT RegVT = VA.getLocVT();
866 if (VA.getLocInfo() == CCValAssign::BCvt)
867 RegVT = VA.getValVT();
868
869 const TargetRegisterClass *RC = getRegClassFor(VT: RegVT);
870 Register VReg = MRI.createVirtualRegister(RegClass: RC);
871 SDValue Copy = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: RegVT);
872
873 // Treat values of type MVT::i1 specially: they are passed in
874 // registers of type i32, but they need to remain as values of
875 // type i1 for consistency of the argument lowering.
876 if (VA.getValVT() == MVT::i1) {
877 assert(RegVT.getSizeInBits() <= 32);
878 SDValue T = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: RegVT,
879 N1: Copy, N2: DAG.getConstant(Val: 1, DL: dl, VT: RegVT));
880 Copy = DAG.getSetCC(DL: dl, VT: MVT::i1, LHS: T, RHS: DAG.getConstant(Val: 0, DL: dl, VT: RegVT),
881 Cond: ISD::SETNE);
882 } else {
883#ifndef NDEBUG
884 unsigned RegSize = RegVT.getSizeInBits();
885 assert(RegSize == 32 || RegSize == 64 ||
886 Subtarget.isHVXVectorType(RegVT));
887#endif
888 }
889 InVals.push_back(Elt: Copy);
890 MRI.addLiveIn(Reg: VA.getLocReg(), vreg: VReg);
891 HFL.FirstVarArgSavedReg = NextSingleReg(*RC, VA.getLocReg());
892 } else {
893 assert(VA.isMemLoc() && "Argument should be passed in memory");
894
895 // If it's a byval parameter, then we need to compute the
896 // "real" size, not the size of the pointer.
897 unsigned ObjSize = Flags.isByVal()
898 ? Flags.getByValSize()
899 : VA.getLocVT().getStoreSizeInBits() / 8;
900
901 // Create the frame index object for this incoming parameter.
902 int Offset = HEXAGON_LRFP_SIZE + VA.getLocMemOffset();
903 int FI = MFI.CreateFixedObject(Size: ObjSize, SPOffset: Offset, IsImmutable: true);
904 SDValue FIN = DAG.getFrameIndex(FI, VT: MVT::i32);
905
906 if (Flags.isByVal()) {
907 // If it's a pass-by-value aggregate, then do not dereference the stack
908 // location. Instead, we should generate a reference to the stack
909 // location.
910 InVals.push_back(Elt: FIN);
911 } else {
912 SDValue L = DAG.getLoad(VT: VA.getValVT(), dl, Chain, Ptr: FIN,
913 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI, Offset: 0));
914 InVals.push_back(Elt: L);
915 }
916 }
917 }
918
919 if (IsVarArg && Subtarget.isEnvironmentMusl()) {
920 for (int i = HFL.FirstVarArgSavedReg; i < 6; i++)
921 MRI.addLiveIn(Reg: Hexagon::R0+i);
922 }
923
924 if (IsVarArg && Subtarget.isEnvironmentMusl()) {
925 HMFI.setFirstNamedArgFrameIndex(HMFI.getFirstNamedArgFrameIndex() - 1);
926 HMFI.setLastNamedArgFrameIndex(-int(MFI.getNumFixedObjects()));
927
928 // Create Frame index for the start of register saved area.
929 int NumVarArgRegs = 6 - HFL.FirstVarArgSavedReg;
930 bool RequiresPadding = (NumVarArgRegs & 1);
931 int RegSaveAreaSizePlusPadding = RequiresPadding
932 ? (NumVarArgRegs + 1) * 4
933 : NumVarArgRegs * 4;
934
935 if (RegSaveAreaSizePlusPadding > 0) {
936 // The offset to saved register area should be 8 byte aligned.
937 int RegAreaStart = HEXAGON_LRFP_SIZE + CCInfo.getStackSize();
938 if (!(RegAreaStart % 8))
939 RegAreaStart = (RegAreaStart + 7) & -8;
940
941 int RegSaveAreaFrameIndex =
942 MFI.CreateFixedObject(Size: RegSaveAreaSizePlusPadding, SPOffset: RegAreaStart, IsImmutable: true);
943 HMFI.setRegSavedAreaStartFrameIndex(RegSaveAreaFrameIndex);
944
945 // This will point to the next argument passed via stack.
946 int Offset = RegAreaStart + RegSaveAreaSizePlusPadding;
947 int FI = MFI.CreateFixedObject(Hexagon_PointerSize, SPOffset: Offset, IsImmutable: true);
948 HMFI.setVarArgsFrameIndex(FI);
949 } else {
950 // This will point to the next argument passed via stack, when
951 // there is no saved register area.
952 int Offset = HEXAGON_LRFP_SIZE + CCInfo.getStackSize();
953 int FI = MFI.CreateFixedObject(Hexagon_PointerSize, SPOffset: Offset, IsImmutable: true);
954 HMFI.setRegSavedAreaStartFrameIndex(FI);
955 HMFI.setVarArgsFrameIndex(FI);
956 }
957 }
958
959
960 if (IsVarArg && !Subtarget.isEnvironmentMusl()) {
961 // This will point to the next argument passed via stack.
962 int Offset = HEXAGON_LRFP_SIZE + CCInfo.getStackSize();
963 int FI = MFI.CreateFixedObject(Hexagon_PointerSize, SPOffset: Offset, IsImmutable: true);
964 HMFI.setVarArgsFrameIndex(FI);
965 }
966
967 return Chain;
968}
969
970SDValue
971HexagonTargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {
972 // VASTART stores the address of the VarArgsFrameIndex slot into the
973 // memory location argument.
974 MachineFunction &MF = DAG.getMachineFunction();
975 HexagonMachineFunctionInfo *QFI = MF.getInfo<HexagonMachineFunctionInfo>();
976 SDValue Addr = DAG.getFrameIndex(FI: QFI->getVarArgsFrameIndex(), VT: MVT::i32);
977 const Value *SV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 2))->getValue();
978
979 if (!Subtarget.isEnvironmentMusl()) {
980 return DAG.getStore(Chain: Op.getOperand(i: 0), dl: SDLoc(Op), Val: Addr, Ptr: Op.getOperand(i: 1),
981 PtrInfo: MachinePointerInfo(SV));
982 }
983 auto &FuncInfo = *MF.getInfo<HexagonMachineFunctionInfo>();
984 auto &HFL = *Subtarget.getFrameLowering();
985 SDLoc DL(Op);
986 SmallVector<SDValue, 8> MemOps;
987
988 // Get frame index of va_list.
989 SDValue FIN = Op.getOperand(i: 1);
990
991 // If first Vararg register is odd, add 4 bytes to start of
992 // saved register area to point to the first register location.
993 // This is because the saved register area has to be 8 byte aligned.
994 // In case of an odd start register, there will be 4 bytes of padding in
995 // the beginning of saved register area. If all registers area used up,
996 // the following condition will handle it correctly.
997 SDValue SavedRegAreaStartFrameIndex =
998 DAG.getFrameIndex(FI: FuncInfo.getRegSavedAreaStartFrameIndex(), VT: MVT::i32);
999
1000 auto PtrVT = getPointerTy(DL: DAG.getDataLayout());
1001
1002 if (HFL.FirstVarArgSavedReg & 1)
1003 SavedRegAreaStartFrameIndex =
1004 DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT,
1005 N1: DAG.getFrameIndex(FI: FuncInfo.getRegSavedAreaStartFrameIndex(),
1006 VT: MVT::i32),
1007 N2: DAG.getIntPtrConstant(Val: 4, DL));
1008
1009 // Store the saved register area start pointer.
1010 SDValue Store =
1011 DAG.getStore(Chain: Op.getOperand(i: 0), dl: DL,
1012 Val: SavedRegAreaStartFrameIndex,
1013 Ptr: FIN, PtrInfo: MachinePointerInfo(SV));
1014 MemOps.push_back(Elt: Store);
1015
1016 // Store saved register area end pointer.
1017 FIN = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT,
1018 N1: FIN, N2: DAG.getIntPtrConstant(Val: 4, DL));
1019 Store = DAG.getStore(Chain: Op.getOperand(i: 0), dl: DL,
1020 Val: DAG.getFrameIndex(FI: FuncInfo.getVarArgsFrameIndex(),
1021 VT: PtrVT),
1022 Ptr: FIN, PtrInfo: MachinePointerInfo(SV, 4));
1023 MemOps.push_back(Elt: Store);
1024
1025 // Store overflow area pointer.
1026 FIN = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT,
1027 N1: FIN, N2: DAG.getIntPtrConstant(Val: 4, DL));
1028 Store = DAG.getStore(Chain: Op.getOperand(i: 0), dl: DL,
1029 Val: DAG.getFrameIndex(FI: FuncInfo.getVarArgsFrameIndex(),
1030 VT: PtrVT),
1031 Ptr: FIN, PtrInfo: MachinePointerInfo(SV, 8));
1032 MemOps.push_back(Elt: Store);
1033
1034 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: MemOps);
1035}
1036
1037SDValue
1038HexagonTargetLowering::LowerVACOPY(SDValue Op, SelectionDAG &DAG) const {
1039 // Assert that the linux ABI is enabled for the current compilation.
1040 assert(Subtarget.isEnvironmentMusl() && "Linux ABI should be enabled");
1041 SDValue Chain = Op.getOperand(i: 0);
1042 SDValue DestPtr = Op.getOperand(i: 1);
1043 SDValue SrcPtr = Op.getOperand(i: 2);
1044 const Value *DestSV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 3))->getValue();
1045 const Value *SrcSV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 4))->getValue();
1046 SDLoc DL(Op);
1047 // Size of the va_list is 12 bytes as it has 3 pointers. Therefore,
1048 // we need to memcopy 12 bytes from va_list to another similar list.
1049 return DAG.getMemcpy(
1050 Chain, dl: DL, Dst: DestPtr, Src: SrcPtr, Size: DAG.getIntPtrConstant(Val: 12, DL), Alignment: Align(4),
1051 /*isVolatile*/ isVol: false, AlwaysInline: false, /*CI=*/nullptr, OverrideTailCall: std::nullopt,
1052 DstPtrInfo: MachinePointerInfo(DestSV), SrcPtrInfo: MachinePointerInfo(SrcSV));
1053}
1054
1055SDValue HexagonTargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
1056 const SDLoc &dl(Op);
1057 SDValue LHS = Op.getOperand(i: 0);
1058 SDValue RHS = Op.getOperand(i: 1);
1059 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 2))->get();
1060 MVT ResTy = ty(Op);
1061 MVT OpTy = ty(Op: LHS);
1062
1063 if (OpTy == MVT::v2i16 || OpTy == MVT::v4i8) {
1064 MVT ElemTy = OpTy.getVectorElementType();
1065 assert(ElemTy.isScalarInteger());
1066 MVT WideTy = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: 2*ElemTy.getSizeInBits()),
1067 NumElements: OpTy.getVectorNumElements());
1068 return DAG.getSetCC(DL: dl, VT: ResTy,
1069 LHS: DAG.getSExtOrTrunc(Op: LHS, DL: SDLoc(LHS), VT: WideTy),
1070 RHS: DAG.getSExtOrTrunc(Op: RHS, DL: SDLoc(RHS), VT: WideTy), Cond: CC);
1071 }
1072
1073 // Treat all other vector types as legal.
1074 if (ResTy.isVector())
1075 return Op;
1076
1077 // Comparisons of short integers should use sign-extend, not zero-extend,
1078 // since we can represent small negative values in the compare instructions.
1079 // The LLVM default is to use zero-extend arbitrarily in these cases.
1080 auto isSExtFree = [this](SDValue N) {
1081 switch (N.getOpcode()) {
1082 case ISD::TRUNCATE: {
1083 // A sign-extend of a truncate of a sign-extend is free.
1084 SDValue Op = N.getOperand(i: 0);
1085 if (Op.getOpcode() != ISD::AssertSext)
1086 return false;
1087 EVT OrigTy = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
1088 unsigned ThisBW = ty(Op: N).getSizeInBits();
1089 unsigned OrigBW = OrigTy.getSizeInBits();
1090 // The type that was sign-extended to get the AssertSext must be
1091 // narrower than the type of N (so that N has still the same value
1092 // as the original).
1093 return ThisBW >= OrigBW;
1094 }
1095 case ISD::LOAD:
1096 // We have sign-extended loads.
1097 return true;
1098 }
1099 return false;
1100 };
1101
1102 if (OpTy == MVT::i8 || OpTy == MVT::i16) {
1103 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: RHS);
1104 bool IsNegative = C && C->getAPIntValue().isNegative();
1105 if (IsNegative || isSExtFree(LHS) || isSExtFree(RHS))
1106 return DAG.getSetCC(DL: dl, VT: ResTy,
1107 LHS: DAG.getSExtOrTrunc(Op: LHS, DL: SDLoc(LHS), VT: MVT::i32),
1108 RHS: DAG.getSExtOrTrunc(Op: RHS, DL: SDLoc(RHS), VT: MVT::i32), Cond: CC);
1109 }
1110
1111 return SDValue();
1112}
1113
1114SDValue
1115HexagonTargetLowering::LowerVSELECT(SDValue Op, SelectionDAG &DAG) const {
1116 SDValue PredOp = Op.getOperand(i: 0);
1117 SDValue Op1 = Op.getOperand(i: 1), Op2 = Op.getOperand(i: 2);
1118 MVT OpTy = ty(Op: Op1);
1119 const SDLoc &dl(Op);
1120
1121 if (OpTy == MVT::v2i16 || OpTy == MVT::v4i8) {
1122 MVT ElemTy = OpTy.getVectorElementType();
1123 assert(ElemTy.isScalarInteger());
1124 MVT WideTy = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: 2*ElemTy.getSizeInBits()),
1125 NumElements: OpTy.getVectorNumElements());
1126 // Generate (trunc (select (_, sext, sext))).
1127 return DAG.getSExtOrTrunc(
1128 Op: DAG.getSelect(DL: dl, VT: WideTy, Cond: PredOp,
1129 LHS: DAG.getSExtOrTrunc(Op: Op1, DL: dl, VT: WideTy),
1130 RHS: DAG.getSExtOrTrunc(Op: Op2, DL: dl, VT: WideTy)),
1131 DL: dl, VT: OpTy);
1132 }
1133
1134 return SDValue();
1135}
1136
1137SDValue
1138HexagonTargetLowering::LowerConstantPool(SDValue Op, SelectionDAG &DAG) const {
1139 EVT ValTy = Op.getValueType();
1140 ConstantPoolSDNode *CPN = cast<ConstantPoolSDNode>(Val&: Op);
1141 Constant *CVal = nullptr;
1142 bool isVTi1Type = false;
1143 if (auto *CV = dyn_cast<ConstantVector>(Val: CPN->getConstVal())) {
1144 if (cast<VectorType>(Val: CV->getType())->getElementType()->isIntegerTy(Bitwidth: 1)) {
1145 IRBuilder<> IRB(CV->getContext());
1146 SmallVector<Constant*, 128> NewConst;
1147 unsigned VecLen = CV->getNumOperands();
1148 assert(isPowerOf2_32(VecLen) &&
1149 "conversion only supported for pow2 VectorSize");
1150 for (unsigned i = 0; i < VecLen; ++i)
1151 NewConst.push_back(Elt: IRB.getInt8(C: CV->getOperand(i_nocapture: i)->isNullValue()));
1152
1153 CVal = ConstantVector::get(V: NewConst);
1154 isVTi1Type = true;
1155 }
1156 }
1157 Align Alignment = CPN->getAlign();
1158 bool IsPositionIndependent = isPositionIndependent();
1159 unsigned char TF = IsPositionIndependent ? HexagonII::MO_PCREL : 0;
1160
1161 unsigned Offset = 0;
1162 SDValue T;
1163 if (CPN->isMachineConstantPoolEntry())
1164 T = DAG.getTargetConstantPool(C: CPN->getMachineCPVal(), VT: ValTy, Align: Alignment,
1165 Offset, TargetFlags: TF);
1166 else if (isVTi1Type)
1167 T = DAG.getTargetConstantPool(C: CVal, VT: ValTy, Align: Alignment, Offset, TargetFlags: TF);
1168 else
1169 T = DAG.getTargetConstantPool(C: CPN->getConstVal(), VT: ValTy, Align: Alignment, Offset,
1170 TargetFlags: TF);
1171
1172 assert(cast<ConstantPoolSDNode>(T)->getTargetFlags() == TF &&
1173 "Inconsistent target flag encountered");
1174
1175 if (IsPositionIndependent)
1176 return DAG.getNode(Opcode: HexagonISD::AT_PCREL, DL: SDLoc(Op), VT: ValTy, Operand: T);
1177 return DAG.getNode(Opcode: HexagonISD::CP, DL: SDLoc(Op), VT: ValTy, Operand: T);
1178}
1179
1180SDValue
1181HexagonTargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const {
1182 EVT VT = Op.getValueType();
1183 int Idx = cast<JumpTableSDNode>(Val&: Op)->getIndex();
1184 if (isPositionIndependent()) {
1185 SDValue T = DAG.getTargetJumpTable(JTI: Idx, VT, TargetFlags: HexagonII::MO_PCREL);
1186 return DAG.getNode(Opcode: HexagonISD::AT_PCREL, DL: SDLoc(Op), VT, Operand: T);
1187 }
1188
1189 SDValue T = DAG.getTargetJumpTable(JTI: Idx, VT);
1190 return DAG.getNode(Opcode: HexagonISD::JT, DL: SDLoc(Op), VT, Operand: T);
1191}
1192
1193SDValue
1194HexagonTargetLowering::LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const {
1195 const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo();
1196 MachineFunction &MF = DAG.getMachineFunction();
1197 MachineFrameInfo &MFI = MF.getFrameInfo();
1198 MFI.setReturnAddressIsTaken(true);
1199
1200 EVT VT = Op.getValueType();
1201 SDLoc dl(Op);
1202 unsigned Depth = Op.getConstantOperandVal(i: 0);
1203 if (Depth) {
1204 SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
1205 SDValue Offset = DAG.getConstant(Val: 4, DL: dl, VT: MVT::i32);
1206 return DAG.getLoad(VT, dl, Chain: DAG.getEntryNode(),
1207 Ptr: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: FrameAddr, N2: Offset),
1208 PtrInfo: MachinePointerInfo());
1209 }
1210
1211 // Return LR, which contains the return address. Mark it an implicit live-in.
1212 Register Reg = MF.addLiveIn(PReg: HRI.getRARegister(), RC: getRegClassFor(VT: MVT::i32));
1213 return DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl, Reg, VT);
1214}
1215
1216SDValue
1217HexagonTargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const {
1218 const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo();
1219 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
1220 MFI.setFrameAddressIsTaken(true);
1221
1222 EVT VT = Op.getValueType();
1223 SDLoc dl(Op);
1224 unsigned Depth = Op.getConstantOperandVal(i: 0);
1225 SDValue FrameAddr = DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl,
1226 Reg: HRI.getFrameRegister(), VT);
1227 while (Depth--)
1228 FrameAddr = DAG.getLoad(VT, dl, Chain: DAG.getEntryNode(), Ptr: FrameAddr,
1229 PtrInfo: MachinePointerInfo());
1230 return FrameAddr;
1231}
1232
1233SDValue
1234HexagonTargetLowering::LowerATOMIC_FENCE(SDValue Op, SelectionDAG& DAG) const {
1235 SDLoc dl(Op);
1236 return DAG.getNode(Opcode: HexagonISD::BARRIER, DL: dl, VT: MVT::Other, Operand: Op.getOperand(i: 0));
1237}
1238
1239SDValue
1240HexagonTargetLowering::LowerGLOBALADDRESS(SDValue Op, SelectionDAG &DAG) const {
1241 SDLoc dl(Op);
1242 auto *GAN = cast<GlobalAddressSDNode>(Val&: Op);
1243 auto PtrVT = getPointerTy(DL: DAG.getDataLayout());
1244 auto *GV = GAN->getGlobal();
1245 int64_t Offset = GAN->getOffset();
1246
1247 auto &HLOF = *HTM.getObjFileLowering();
1248 Reloc::Model RM = HTM.getRelocationModel();
1249
1250 if (RM == Reloc::Static) {
1251 SDValue GA = DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: Offset);
1252 const GlobalObject *GO = GV->getAliaseeObject();
1253 if (GO && Subtarget.useSmallData() && HLOF.isGlobalInSmallSection(GO, TM: HTM))
1254 return DAG.getNode(Opcode: HexagonISD::CONST32_GP, DL: dl, VT: PtrVT, Operand: GA);
1255 return DAG.getNode(Opcode: HexagonISD::CONST32, DL: dl, VT: PtrVT, Operand: GA);
1256 }
1257
1258 bool UsePCRel = getTargetMachine().shouldAssumeDSOLocal(GV);
1259 if (UsePCRel) {
1260 SDValue GA = DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: Offset,
1261 TargetFlags: HexagonII::MO_PCREL);
1262 return DAG.getNode(Opcode: HexagonISD::AT_PCREL, DL: dl, VT: PtrVT, Operand: GA);
1263 }
1264
1265 // Use GOT index.
1266 SDValue GOT = DAG.getGLOBAL_OFFSET_TABLE(VT: PtrVT);
1267 SDValue GA = DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: 0, TargetFlags: HexagonII::MO_GOT);
1268 SDValue Off = DAG.getConstant(Val: Offset, DL: dl, VT: MVT::i32);
1269 return DAG.getNode(Opcode: HexagonISD::AT_GOT, DL: dl, VT: PtrVT, N1: GOT, N2: GA, N3: Off);
1270}
1271
1272// Specifies that for loads and stores VT can be promoted to PromotedLdStVT.
1273SDValue
1274HexagonTargetLowering::LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const {
1275 const BlockAddress *BA = cast<BlockAddressSDNode>(Val&: Op)->getBlockAddress();
1276 SDLoc dl(Op);
1277 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
1278
1279 Reloc::Model RM = HTM.getRelocationModel();
1280 if (RM == Reloc::Static) {
1281 SDValue A = DAG.getTargetBlockAddress(BA, VT: PtrVT);
1282 return DAG.getNode(Opcode: HexagonISD::CONST32_GP, DL: dl, VT: PtrVT, Operand: A);
1283 }
1284
1285 SDValue A = DAG.getTargetBlockAddress(BA, VT: PtrVT, Offset: 0, TargetFlags: HexagonII::MO_PCREL);
1286 return DAG.getNode(Opcode: HexagonISD::AT_PCREL, DL: dl, VT: PtrVT, Operand: A);
1287}
1288
1289SDValue
1290HexagonTargetLowering::LowerGLOBAL_OFFSET_TABLE(SDValue Op, SelectionDAG &DAG)
1291 const {
1292 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
1293 SDValue GOTSym = DAG.getTargetExternalSymbol(HEXAGON_GOT_SYM_NAME, VT: PtrVT,
1294 TargetFlags: HexagonII::MO_PCREL);
1295 return DAG.getNode(Opcode: HexagonISD::AT_PCREL, DL: SDLoc(Op), VT: PtrVT, Operand: GOTSym);
1296}
1297
1298SDValue
1299HexagonTargetLowering::GetDynamicTLSAddr(SelectionDAG &DAG, SDValue Chain,
1300 GlobalAddressSDNode *GA, SDValue Glue, EVT PtrVT, unsigned ReturnReg,
1301 unsigned char OperandFlags) const {
1302 MachineFunction &MF = DAG.getMachineFunction();
1303 MachineFrameInfo &MFI = MF.getFrameInfo();
1304 SDVTList NodeTys = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
1305 SDLoc dl(GA);
1306 SDValue TGA = DAG.getTargetGlobalAddress(GV: GA->getGlobal(), DL: dl,
1307 VT: GA->getValueType(ResNo: 0),
1308 offset: GA->getOffset(),
1309 TargetFlags: OperandFlags);
1310 // Create Operands for the call.The Operands should have the following:
1311 // 1. Chain SDValue
1312 // 2. Callee which in this case is the Global address value.
1313 // 3. Registers live into the call.In this case its R0, as we
1314 // have just one argument to be passed.
1315 // 4. Glue.
1316 // Note: The order is important.
1317
1318 const auto &HRI = *Subtarget.getRegisterInfo();
1319 const uint32_t *Mask = HRI.getCallPreservedMask(MF, CallingConv::C);
1320 assert(Mask && "Missing call preserved mask for calling convention");
1321 SDValue Ops[] = { Chain, TGA, DAG.getRegister(Reg: Hexagon::R0, VT: PtrVT),
1322 DAG.getRegisterMask(RegMask: Mask), Glue };
1323 Chain = DAG.getNode(Opcode: HexagonISD::CALL, DL: dl, VTList: NodeTys, Ops);
1324
1325 // Inform MFI that function has calls.
1326 MFI.setAdjustsStack(true);
1327
1328 Glue = Chain.getValue(R: 1);
1329 return DAG.getCopyFromReg(Chain, dl, Reg: ReturnReg, VT: PtrVT, Glue);
1330}
1331
1332//
1333// Lower using the initial executable model for TLS addresses
1334//
1335SDValue
1336HexagonTargetLowering::LowerToTLSInitialExecModel(GlobalAddressSDNode *GA,
1337 SelectionDAG &DAG) const {
1338 SDLoc dl(GA);
1339 int64_t Offset = GA->getOffset();
1340 auto PtrVT = getPointerTy(DL: DAG.getDataLayout());
1341
1342 // Get the thread pointer.
1343 SDValue TP = DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl, Reg: Hexagon::UGP, VT: PtrVT);
1344
1345 bool IsPositionIndependent = isPositionIndependent();
1346 unsigned char TF =
1347 IsPositionIndependent ? HexagonII::MO_IEGOT : HexagonII::MO_IE;
1348
1349 // First generate the TLS symbol address
1350 SDValue TGA = DAG.getTargetGlobalAddress(GV: GA->getGlobal(), DL: dl, VT: PtrVT,
1351 offset: Offset, TargetFlags: TF);
1352
1353 SDValue Sym = DAG.getNode(Opcode: HexagonISD::CONST32, DL: dl, VT: PtrVT, Operand: TGA);
1354
1355 if (IsPositionIndependent) {
1356 // Generate the GOT pointer in case of position independent code
1357 SDValue GOT = LowerGLOBAL_OFFSET_TABLE(Op: Sym, DAG);
1358
1359 // Add the TLS Symbol address to GOT pointer.This gives
1360 // GOT relative relocation for the symbol.
1361 Sym = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: GOT, N2: Sym);
1362 }
1363
1364 // Load the offset value for TLS symbol.This offset is relative to
1365 // thread pointer.
1366 SDValue LoadOffset =
1367 DAG.getLoad(VT: PtrVT, dl, Chain: DAG.getEntryNode(), Ptr: Sym, PtrInfo: MachinePointerInfo());
1368
1369 // Address of the thread local variable is the add of thread
1370 // pointer and the offset of the variable.
1371 return DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: TP, N2: LoadOffset);
1372}
1373
1374//
1375// Lower using the local executable model for TLS addresses
1376//
1377SDValue
1378HexagonTargetLowering::LowerToTLSLocalExecModel(GlobalAddressSDNode *GA,
1379 SelectionDAG &DAG) const {
1380 SDLoc dl(GA);
1381 int64_t Offset = GA->getOffset();
1382 auto PtrVT = getPointerTy(DL: DAG.getDataLayout());
1383
1384 // Get the thread pointer.
1385 SDValue TP = DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl, Reg: Hexagon::UGP, VT: PtrVT);
1386 // Generate the TLS symbol address
1387 SDValue TGA = DAG.getTargetGlobalAddress(GV: GA->getGlobal(), DL: dl, VT: PtrVT, offset: Offset,
1388 TargetFlags: HexagonII::MO_TPREL);
1389 SDValue Sym = DAG.getNode(Opcode: HexagonISD::CONST32, DL: dl, VT: PtrVT, Operand: TGA);
1390
1391 // Address of the thread local variable is the add of thread
1392 // pointer and the offset of the variable.
1393 return DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: TP, N2: Sym);
1394}
1395
1396//
1397// Lower using the general dynamic model for TLS addresses
1398//
1399SDValue
1400HexagonTargetLowering::LowerToTLSGeneralDynamicModel(GlobalAddressSDNode *GA,
1401 SelectionDAG &DAG) const {
1402 SDLoc dl(GA);
1403 int64_t Offset = GA->getOffset();
1404 auto PtrVT = getPointerTy(DL: DAG.getDataLayout());
1405
1406 // First generate the TLS symbol address
1407 SDValue TGA = DAG.getTargetGlobalAddress(GV: GA->getGlobal(), DL: dl, VT: PtrVT, offset: Offset,
1408 TargetFlags: HexagonII::MO_GDGOT);
1409
1410 // Then, generate the GOT pointer
1411 SDValue GOT = LowerGLOBAL_OFFSET_TABLE(Op: TGA, DAG);
1412
1413 // Add the TLS symbol and the GOT pointer
1414 SDValue Sym = DAG.getNode(Opcode: HexagonISD::CONST32, DL: dl, VT: PtrVT, Operand: TGA);
1415 SDValue Chain = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: GOT, N2: Sym);
1416
1417 // Copy over the argument to R0
1418 SDValue InGlue;
1419 Chain = DAG.getCopyToReg(Chain: DAG.getEntryNode(), dl, Reg: Hexagon::R0, N: Chain, Glue: InGlue);
1420 InGlue = Chain.getValue(R: 1);
1421
1422 unsigned Flags = DAG.getSubtarget<HexagonSubtarget>().useLongCalls()
1423 ? HexagonII::MO_GDPLT | HexagonII::HMOTF_ConstExtended
1424 : HexagonII::MO_GDPLT;
1425
1426 return GetDynamicTLSAddr(DAG, Chain, GA, Glue: InGlue, PtrVT,
1427 ReturnReg: Hexagon::R0, OperandFlags: Flags);
1428}
1429
1430//
1431// Lower TLS addresses.
1432//
1433// For now for dynamic models, we only support the general dynamic model.
1434//
1435SDValue
1436HexagonTargetLowering::LowerGlobalTLSAddress(SDValue Op,
1437 SelectionDAG &DAG) const {
1438 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Val&: Op);
1439
1440 switch (HTM.getTLSModel(GV: GA->getGlobal())) {
1441 case TLSModel::GeneralDynamic:
1442 case TLSModel::LocalDynamic:
1443 return LowerToTLSGeneralDynamicModel(GA, DAG);
1444 case TLSModel::InitialExec:
1445 return LowerToTLSInitialExecModel(GA, DAG);
1446 case TLSModel::LocalExec:
1447 return LowerToTLSLocalExecModel(GA, DAG);
1448 }
1449 llvm_unreachable("Bogus TLS model");
1450}
1451
1452//===----------------------------------------------------------------------===//
1453// TargetLowering Implementation
1454//===----------------------------------------------------------------------===//
1455
1456HexagonTargetLowering::HexagonTargetLowering(const TargetMachine &TM,
1457 const HexagonSubtarget &ST)
1458 : TargetLowering(TM, ST),
1459 HTM(static_cast<const HexagonTargetMachine &>(TM)), Subtarget(ST) {
1460 auto &HRI = *Subtarget.getRegisterInfo();
1461
1462 setPrefLoopAlignment(Align(16));
1463 setMinFunctionAlignment(Align(4));
1464 setPrefFunctionAlignment(Align(16));
1465 setStackPointerRegisterToSaveRestore(HRI.getStackRegister());
1466 setBooleanContents(TargetLoweringBase::UndefinedBooleanContent);
1467 setBooleanVectorContents(TargetLoweringBase::UndefinedBooleanContent);
1468
1469 setMaxAtomicSizeInBitsSupported(64);
1470 setMinCmpXchgSizeInBits(32);
1471
1472 if (EnableHexSDNodeSched)
1473 setSchedulingPreference(Sched::VLIW);
1474 else
1475 setSchedulingPreference(Sched::Source);
1476
1477 // Limits for inline expansion of memcpy/memmove
1478 MaxStoresPerMemcpy = 6;
1479 MaxStoresPerMemcpyOptSize = 4;
1480 MaxStoresPerMemmove = 6;
1481 MaxStoresPerMemmoveOptSize = 4;
1482 MaxStoresPerMemset = 8;
1483 MaxStoresPerMemsetOptSize = 4;
1484
1485 setTargetDAGCombine(ISD::VECREDUCE_ADD);
1486
1487 //
1488 // Set up register classes.
1489 //
1490
1491 addRegisterClass(VT: MVT::i1, RC: &Hexagon::PredRegsRegClass);
1492 addRegisterClass(VT: MVT::v2i1, RC: &Hexagon::PredRegsRegClass); // bbbbaaaa
1493 addRegisterClass(VT: MVT::v4i1, RC: &Hexagon::PredRegsRegClass); // ddccbbaa
1494 addRegisterClass(VT: MVT::v8i1, RC: &Hexagon::PredRegsRegClass); // hgfedcba
1495 addRegisterClass(VT: MVT::i32, RC: &Hexagon::IntRegsRegClass);
1496 addRegisterClass(VT: MVT::v2i16, RC: &Hexagon::IntRegsRegClass);
1497 addRegisterClass(VT: MVT::v4i8, RC: &Hexagon::IntRegsRegClass);
1498 addRegisterClass(VT: MVT::i64, RC: &Hexagon::DoubleRegsRegClass);
1499 addRegisterClass(VT: MVT::v8i8, RC: &Hexagon::DoubleRegsRegClass);
1500 addRegisterClass(VT: MVT::v4i16, RC: &Hexagon::DoubleRegsRegClass);
1501 addRegisterClass(VT: MVT::v2i32, RC: &Hexagon::DoubleRegsRegClass);
1502
1503 addRegisterClass(VT: MVT::f32, RC: &Hexagon::IntRegsRegClass);
1504 addRegisterClass(VT: MVT::f64, RC: &Hexagon::DoubleRegsRegClass);
1505
1506 //
1507 // Handling of scalar operations.
1508 //
1509 // All operations default to "legal", except:
1510 // - indexed loads and stores (pre-/post-incremented),
1511 // - ANY_EXTEND_VECTOR_INREG, ATOMIC_CMP_SWAP_WITH_SUCCESS, CONCAT_VECTORS,
1512 // ConstantFP, FCEIL, FCOPYSIGN, FEXP, FEXP2, FFLOOR, FGETSIGN,
1513 // FLOG, FLOG2, FLOG10, FMAXIMUMNUM, FMINIMUMNUM, FNEARBYINT, FRINT, FROUND,
1514 // TRAP, FTRUNC, PREFETCH, SIGN_EXTEND_VECTOR_INREG,
1515 // ZERO_EXTEND_VECTOR_INREG,
1516 // which default to "expand" for at least one type.
1517
1518 // Misc operations.
1519 setOperationAction(Op: ISD::ConstantFP, VT: MVT::f32, Action: Legal);
1520 setOperationAction(Op: ISD::ConstantFP, VT: MVT::f64, Action: Legal);
1521 setOperationAction(Op: ISD::TRAP, VT: MVT::Other, Action: Legal);
1522 setOperationAction(Op: ISD::DEBUGTRAP, VT: MVT::Other, Action: Legal);
1523 setOperationAction(Op: ISD::ConstantPool, VT: MVT::i32, Action: Custom);
1524 setOperationAction(Op: ISD::JumpTable, VT: MVT::i32, Action: Custom);
1525 setOperationAction(Op: ISD::BUILD_PAIR, VT: MVT::i64, Action: Expand);
1526 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i1, Action: Expand);
1527 setOperationAction(Op: ISD::INLINEASM, VT: MVT::Other, Action: Custom);
1528 setOperationAction(Op: ISD::INLINEASM_BR, VT: MVT::Other, Action: Custom);
1529 setOperationAction(Op: ISD::PREFETCH, VT: MVT::Other, Action: Custom);
1530 setOperationAction(Op: ISD::READCYCLECOUNTER, VT: MVT::i64, Action: Legal);
1531 setOperationAction(Op: ISD::READSTEADYCOUNTER, VT: MVT::i64, Action: Legal);
1532 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::Other, Action: Custom);
1533 setOperationAction(Op: ISD::INTRINSIC_VOID, VT: MVT::Other, Action: Custom);
1534 setOperationAction(Op: ISD::EH_RETURN, VT: MVT::Other, Action: Custom);
1535 setOperationAction(Op: ISD::GLOBAL_OFFSET_TABLE, VT: MVT::i32, Action: Custom);
1536 setOperationAction(Op: ISD::GlobalTLSAddress, VT: MVT::i32, Action: Custom);
1537 setOperationAction(Op: ISD::ATOMIC_FENCE, VT: MVT::Other, Action: Custom);
1538
1539 // Custom legalize GlobalAddress nodes into CONST32.
1540 setOperationAction(Op: ISD::GlobalAddress, VT: MVT::i32, Action: Custom);
1541 setOperationAction(Op: ISD::GlobalAddress, VT: MVT::i8, Action: Custom);
1542 setOperationAction(Op: ISD::BlockAddress, VT: MVT::i32, Action: Custom);
1543
1544 // Hexagon needs to optimize cases with negative constants.
1545 setOperationAction(Op: ISD::SETCC, VT: MVT::i8, Action: Custom);
1546 setOperationAction(Op: ISD::SETCC, VT: MVT::i16, Action: Custom);
1547 setOperationAction(Op: ISD::SETCC, VT: MVT::v4i8, Action: Custom);
1548 setOperationAction(Op: ISD::SETCC, VT: MVT::v2i16, Action: Custom);
1549
1550 // VASTART needs to be custom lowered to use the VarArgsFrameIndex.
1551 setOperationAction(Op: ISD::VASTART, VT: MVT::Other, Action: Custom);
1552 setOperationAction(Op: ISD::VAEND, VT: MVT::Other, Action: Expand);
1553 setOperationAction(Op: ISD::VAARG, VT: MVT::Other, Action: Expand);
1554 if (Subtarget.isEnvironmentMusl())
1555 setOperationAction(Op: ISD::VACOPY, VT: MVT::Other, Action: Custom);
1556 else
1557 setOperationAction(Op: ISD::VACOPY, VT: MVT::Other, Action: Expand);
1558
1559 setOperationAction(Op: ISD::STACKSAVE, VT: MVT::Other, Action: Expand);
1560 setOperationAction(Op: ISD::STACKRESTORE, VT: MVT::Other, Action: Expand);
1561 setOperationAction(Op: ISD::DYNAMIC_STACKALLOC, VT: MVT::i32, Action: Custom);
1562
1563 if (EmitJumpTables)
1564 setMinimumJumpTableEntries(MinimumJumpTables);
1565 else
1566 setMinimumJumpTableEntries(std::numeric_limits<unsigned>::max());
1567 setOperationAction(Op: ISD::BR_JT, VT: MVT::Other, Action: Expand);
1568
1569 for (unsigned LegalIntOp :
1570 {ISD::ABS, ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX}) {
1571 setOperationAction(Op: LegalIntOp, VT: MVT::i32, Action: Legal);
1572 setOperationAction(Op: LegalIntOp, VT: MVT::i64, Action: Legal);
1573 }
1574
1575 // Hexagon has A4_addp_c and A4_subp_c that take and generate a carry bit,
1576 // but they only operate on i64.
1577 for (MVT VT : MVT::integer_valuetypes()) {
1578 setOperationAction(Op: ISD::UADDO, VT, Action: Custom);
1579 setOperationAction(Op: ISD::USUBO, VT, Action: Custom);
1580 setOperationAction(Op: ISD::SADDO, VT, Action: Expand);
1581 setOperationAction(Op: ISD::SSUBO, VT, Action: Expand);
1582 setOperationAction(Op: ISD::UADDO_CARRY, VT, Action: Expand);
1583 setOperationAction(Op: ISD::USUBO_CARRY, VT, Action: Expand);
1584 }
1585 setOperationAction(Op: ISD::UADDO_CARRY, VT: MVT::i64, Action: Custom);
1586 setOperationAction(Op: ISD::USUBO_CARRY, VT: MVT::i64, Action: Custom);
1587
1588 setOperationAction(Op: ISD::CTLZ, VT: MVT::i8, Action: Promote);
1589 setOperationAction(Op: ISD::CTLZ, VT: MVT::i16, Action: Promote);
1590 setOperationAction(Op: ISD::CTTZ, VT: MVT::i8, Action: Promote);
1591 setOperationAction(Op: ISD::CTTZ, VT: MVT::i16, Action: Promote);
1592
1593 // Popcount can count # of 1s in i64 but returns i32.
1594 setOperationAction(Op: ISD::CTPOP, VT: MVT::i8, Action: Promote);
1595 setOperationAction(Op: ISD::CTPOP, VT: MVT::i16, Action: Promote);
1596 setOperationAction(Op: ISD::CTPOP, VT: MVT::i32, Action: Promote);
1597 setOperationAction(Op: ISD::CTPOP, VT: MVT::i64, Action: Legal);
1598
1599 setOperationAction(Op: ISD::BITREVERSE, VT: MVT::i32, Action: Legal);
1600 setOperationAction(Op: ISD::BITREVERSE, VT: MVT::i64, Action: Legal);
1601 setOperationAction(Op: ISD::BSWAP, VT: MVT::i32, Action: Legal);
1602 setOperationAction(Op: ISD::BSWAP, VT: MVT::i64, Action: Legal);
1603
1604 setOperationAction(Op: ISD::FSHL, VT: MVT::i32, Action: Legal);
1605 setOperationAction(Op: ISD::FSHL, VT: MVT::i64, Action: Legal);
1606 setOperationAction(Op: ISD::FSHR, VT: MVT::i32, Action: Legal);
1607 setOperationAction(Op: ISD::FSHR, VT: MVT::i64, Action: Legal);
1608
1609 for (unsigned IntExpOp :
1610 {ISD::SDIV, ISD::UDIV, ISD::SREM, ISD::UREM,
1611 ISD::SDIVREM, ISD::UDIVREM, ISD::ROTL, ISD::ROTR,
1612 ISD::SHL_PARTS, ISD::SRA_PARTS, ISD::SRL_PARTS,
1613 ISD::SMUL_LOHI, ISD::UMUL_LOHI}) {
1614 for (MVT VT : MVT::integer_valuetypes())
1615 setOperationAction(Op: IntExpOp, VT, Action: Expand);
1616 }
1617 for (MVT VT : MVT::fp_valuetypes()) {
1618 for (unsigned FPExpOp : {ISD::FDIV, ISD::FSQRT, ISD::FSIN, ISD::FCOS,
1619 ISD::FSINCOS, ISD::FPOW, ISD::FCOPYSIGN})
1620 setOperationAction(Op: FPExpOp, VT, Action: Expand);
1621
1622 setOperationAction(Op: ISD::FREM, VT, Action: LibCall);
1623 }
1624
1625 // No extending loads from i32.
1626 for (MVT VT : MVT::integer_valuetypes()) {
1627 setLoadExtAction(ExtType: ISD::ZEXTLOAD, ValVT: VT, MemVT: MVT::i32, Action: Expand);
1628 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: VT, MemVT: MVT::i32, Action: Expand);
1629 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: MVT::i32, Action: Expand);
1630 }
1631 // Turn FP truncstore into trunc + store.
1632 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f32, Action: Expand);
1633 setTruncStoreAction(ValVT: MVT::f32, MemVT: MVT::bf16, Action: Expand);
1634 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::bf16, Action: Expand);
1635 // Turn FP extload into load/fpextend.
1636 for (MVT VT : MVT::fp_valuetypes())
1637 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: MVT::f32, Action: Expand);
1638
1639 // Expand BR_CC and SELECT_CC for all integer and fp types.
1640 for (MVT VT : MVT::integer_valuetypes()) {
1641 setOperationAction(Op: ISD::BR_CC, VT, Action: Expand);
1642 setOperationAction(Op: ISD::SELECT_CC, VT, Action: Expand);
1643 }
1644 for (MVT VT : MVT::fp_valuetypes()) {
1645 setOperationAction(Op: ISD::BR_CC, VT, Action: Expand);
1646 setOperationAction(Op: ISD::SELECT_CC, VT, Action: Expand);
1647 }
1648 setOperationAction(Op: ISD::BR_CC, VT: MVT::Other, Action: Expand);
1649
1650 //
1651 // Handling of vector operations.
1652 //
1653
1654 // Set the action for vector operations to "expand", then override it with
1655 // either "custom" or "legal" for specific cases.
1656 // clang-format off
1657 static const unsigned VectExpOps[] = {
1658 // Integer arithmetic:
1659 ISD::ADD, ISD::SUB, ISD::MUL, ISD::SDIV, ISD::UDIV,
1660 ISD::SREM, ISD::UREM, ISD::SDIVREM, ISD::UDIVREM, ISD::SADDO,
1661 ISD::UADDO, ISD::SSUBO, ISD::USUBO, ISD::SMUL_LOHI, ISD::UMUL_LOHI,
1662 // Logical/bit:
1663 ISD::AND, ISD::OR, ISD::XOR, ISD::ROTL, ISD::ROTR,
1664 ISD::CTPOP, ISD::CTLZ, ISD::CTTZ, ISD::BSWAP, ISD::BITREVERSE,
1665 // Floating point arithmetic/math functions:
1666 ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FMA, ISD::FDIV,
1667 ISD::FREM, ISD::FNEG, ISD::FABS, ISD::FSQRT, ISD::FSIN,
1668 ISD::FCOS, ISD::FPOW, ISD::FLOG, ISD::FLOG2,
1669 ISD::FLOG10, ISD::FEXP, ISD::FEXP2, ISD::FCEIL, ISD::FTRUNC,
1670 ISD::FRINT, ISD::FNEARBYINT, ISD::FROUND, ISD::FFLOOR,
1671 ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM,
1672 ISD::FSINCOS, ISD::FLDEXP,
1673 // Misc:
1674 ISD::BR_CC, ISD::SELECT_CC, ISD::ConstantPool,
1675 // Vector:
1676 ISD::BUILD_VECTOR, ISD::SCALAR_TO_VECTOR,
1677 ISD::EXTRACT_VECTOR_ELT, ISD::INSERT_VECTOR_ELT,
1678 ISD::EXTRACT_SUBVECTOR, ISD::INSERT_SUBVECTOR,
1679 ISD::CONCAT_VECTORS, ISD::VECTOR_SHUFFLE,
1680 ISD::SPLAT_VECTOR,
1681 };
1682 // clang-format on
1683
1684 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
1685 for (unsigned VectExpOp : VectExpOps)
1686 setOperationAction(Op: VectExpOp, VT, Action: Expand);
1687
1688 // Expand all extending loads and truncating stores:
1689 for (MVT TargetVT : MVT::fixedlen_vector_valuetypes()) {
1690 if (TargetVT == VT)
1691 continue;
1692 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: TargetVT, MemVT: VT, Action: Expand);
1693 setLoadExtAction(ExtType: ISD::ZEXTLOAD, ValVT: TargetVT, MemVT: VT, Action: Expand);
1694 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: TargetVT, MemVT: VT, Action: Expand);
1695 setTruncStoreAction(ValVT: VT, MemVT: TargetVT, Action: Expand);
1696 }
1697
1698 // Normalize all inputs to SELECT to be vectors of i32.
1699 if (VT.getVectorElementType() != MVT::i32) {
1700 MVT VT32 = MVT::getVectorVT(VT: MVT::i32, NumElements: VT.getSizeInBits()/32);
1701 setOperationAction(Op: ISD::SELECT, VT, Action: Promote);
1702 AddPromotedToType(Opc: ISD::SELECT, OrigVT: VT, DestVT: VT32);
1703 }
1704 setOperationAction(Op: ISD::SRA, VT, Action: Custom);
1705 setOperationAction(Op: ISD::SHL, VT, Action: Custom);
1706 setOperationAction(Op: ISD::SRL, VT, Action: Custom);
1707 }
1708
1709 setOperationAction(Op: ISD::SADDSAT, VT: MVT::i32, Action: Legal);
1710 setOperationAction(Op: ISD::SADDSAT, VT: MVT::i64, Action: Legal);
1711
1712 // Extending loads from (native) vectors of i8 into (native) vectors of i16
1713 // are legal.
1714 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v2i16, MemVT: MVT::v2i8, Action: Legal);
1715 setLoadExtAction(ExtType: ISD::ZEXTLOAD, ValVT: MVT::v2i16, MemVT: MVT::v2i8, Action: Legal);
1716 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: MVT::v2i16, MemVT: MVT::v2i8, Action: Legal);
1717 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v4i16, MemVT: MVT::v4i8, Action: Legal);
1718 setLoadExtAction(ExtType: ISD::ZEXTLOAD, ValVT: MVT::v4i16, MemVT: MVT::v4i8, Action: Legal);
1719 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: MVT::v4i16, MemVT: MVT::v4i8, Action: Legal);
1720
1721 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::v2i8, Action: Legal);
1722 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::v2i16, Action: Legal);
1723 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::v2i32, Action: Legal);
1724
1725 // Types natively supported:
1726 for (MVT NativeVT : {MVT::v8i1, MVT::v4i1, MVT::v2i1, MVT::v4i8,
1727 MVT::v8i8, MVT::v2i16, MVT::v4i16, MVT::v2i32}) {
1728 setOperationAction(Op: ISD::BUILD_VECTOR, VT: NativeVT, Action: Custom);
1729 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: NativeVT, Action: Custom);
1730 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: NativeVT, Action: Custom);
1731 setOperationAction(Op: ISD::EXTRACT_SUBVECTOR, VT: NativeVT, Action: Custom);
1732 setOperationAction(Op: ISD::INSERT_SUBVECTOR, VT: NativeVT, Action: Custom);
1733 setOperationAction(Op: ISD::CONCAT_VECTORS, VT: NativeVT, Action: Custom);
1734
1735 setOperationAction(Op: ISD::ADD, VT: NativeVT, Action: Legal);
1736 setOperationAction(Op: ISD::SUB, VT: NativeVT, Action: Legal);
1737 setOperationAction(Op: ISD::MUL, VT: NativeVT, Action: Legal);
1738 setOperationAction(Op: ISD::AND, VT: NativeVT, Action: Legal);
1739 setOperationAction(Op: ISD::OR, VT: NativeVT, Action: Legal);
1740 setOperationAction(Op: ISD::XOR, VT: NativeVT, Action: Legal);
1741
1742 if (NativeVT.getVectorElementType() != MVT::i1) {
1743 setOperationAction(Op: ISD::SPLAT_VECTOR, VT: NativeVT, Action: Legal);
1744 setOperationAction(Op: ISD::BSWAP, VT: NativeVT, Action: Legal);
1745 setOperationAction(Op: ISD::BITREVERSE, VT: NativeVT, Action: Legal);
1746 }
1747 }
1748
1749 for (MVT VT : {MVT::v8i8, MVT::v4i16, MVT::v2i32}) {
1750 setOperationAction(Op: ISD::SMIN, VT, Action: Legal);
1751 setOperationAction(Op: ISD::SMAX, VT, Action: Legal);
1752 setOperationAction(Op: ISD::UMIN, VT, Action: Legal);
1753 setOperationAction(Op: ISD::UMAX, VT, Action: Legal);
1754 }
1755
1756 // Custom lower unaligned loads.
1757 // Also, for both loads and stores, verify the alignment of the address
1758 // in case it is a compile-time constant. This is a usability feature to
1759 // provide a meaningful error message to users.
1760 for (MVT VT : {MVT::i16, MVT::i32, MVT::v4i8, MVT::i64, MVT::v8i8,
1761 MVT::v2i16, MVT::v4i16, MVT::v2i32}) {
1762 setOperationAction(Op: ISD::LOAD, VT, Action: Custom);
1763 setOperationAction(Op: ISD::STORE, VT, Action: Custom);
1764 }
1765
1766 // Custom-lower load/stores of boolean vectors.
1767 for (MVT VT : {MVT::v2i1, MVT::v4i1, MVT::v8i1}) {
1768 setOperationAction(Op: ISD::LOAD, VT, Action: Custom);
1769 setOperationAction(Op: ISD::STORE, VT, Action: Custom);
1770 }
1771
1772 // Normalize integer compares to EQ/GT/UGT
1773 for (MVT VT : {MVT::v2i16, MVT::v4i8, MVT::v8i8, MVT::v2i32, MVT::v4i16,
1774 MVT::v2i32}) {
1775 setCondCodeAction(CCs: ISD::SETNE, VT, Action: Expand);
1776 setCondCodeAction(CCs: ISD::SETLE, VT, Action: Expand);
1777 setCondCodeAction(CCs: ISD::SETGE, VT, Action: Expand);
1778 setCondCodeAction(CCs: ISD::SETLT, VT, Action: Expand);
1779 setCondCodeAction(CCs: ISD::SETULE, VT, Action: Expand);
1780 setCondCodeAction(CCs: ISD::SETUGE, VT, Action: Expand);
1781 setCondCodeAction(CCs: ISD::SETULT, VT, Action: Expand);
1782 }
1783
1784 // Normalize boolean compares to [U]LE/[U]LT
1785 for (MVT VT : {MVT::i1, MVT::v2i1, MVT::v4i1, MVT::v8i1}) {
1786 setCondCodeAction(CCs: ISD::SETGE, VT, Action: Expand);
1787 setCondCodeAction(CCs: ISD::SETGT, VT, Action: Expand);
1788 setCondCodeAction(CCs: ISD::SETUGE, VT, Action: Expand);
1789 setCondCodeAction(CCs: ISD::SETUGT, VT, Action: Expand);
1790 }
1791
1792 // Custom-lower bitcasts from i8 to v8i1.
1793 setOperationAction(Op: ISD::BITCAST, VT: MVT::i8, Action: Custom);
1794 setOperationAction(Op: ISD::SETCC, VT: MVT::v2i16, Action: Custom);
1795 setOperationAction(Op: ISD::VSELECT, VT: MVT::v4i8, Action: Custom);
1796 setOperationAction(Op: ISD::VSELECT, VT: MVT::v2i16, Action: Custom);
1797 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT: MVT::v4i8, Action: Custom);
1798 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT: MVT::v4i16, Action: Custom);
1799 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT: MVT::v8i8, Action: Custom);
1800
1801 // V5+.
1802 setOperationAction(Op: ISD::FMA, VT: MVT::f64, Action: Expand);
1803 setOperationAction(Op: ISD::FADD, VT: MVT::f64, Action: Expand);
1804 setOperationAction(Op: ISD::FSUB, VT: MVT::f64, Action: Expand);
1805 setOperationAction(Op: ISD::FMUL, VT: MVT::f64, Action: Expand);
1806 setOperationAction(Op: ISD::FDIV, VT: MVT::f32, Action: Custom);
1807
1808 setOperationAction(Op: ISD::FMINIMUMNUM, VT: MVT::f32, Action: Legal);
1809 setOperationAction(Op: ISD::FMAXIMUMNUM, VT: MVT::f32, Action: Legal);
1810
1811 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i1, Action: Promote);
1812 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i8, Action: Promote);
1813 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i16, Action: Promote);
1814 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::i1, Action: Promote);
1815 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::i8, Action: Promote);
1816 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::i16, Action: Promote);
1817 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::i1, Action: Promote);
1818 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::i8, Action: Promote);
1819 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::i16, Action: Promote);
1820 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i1, Action: Promote);
1821 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i8, Action: Promote);
1822 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i16, Action: Promote);
1823
1824 // Special handling for half-precision floating point conversions.
1825 // Lower half float conversions into library calls.
1826 setOperationAction(Op: ISD::FP16_TO_FP, VT: MVT::f32, Action: Expand);
1827 setOperationAction(Op: ISD::FP16_TO_FP, VT: MVT::f64, Action: Expand);
1828 setOperationAction(Op: ISD::FP_TO_FP16, VT: MVT::f32, Action: Expand);
1829 setOperationAction(Op: ISD::FP_TO_FP16, VT: MVT::f64, Action: Expand);
1830 setOperationAction(Op: ISD::BF16_TO_FP, VT: MVT::f32, Action: Expand);
1831 setOperationAction(Op: ISD::BF16_TO_FP, VT: MVT::f64, Action: Expand);
1832 setOperationAction(Op: ISD::FP_TO_BF16, VT: MVT::f64, Action: Expand);
1833
1834 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f32, MemVT: MVT::f16, Action: Expand);
1835 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::f16, Action: Expand);
1836 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f32, MemVT: MVT::bf16, Action: Expand);
1837 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::bf16, Action: Expand);
1838
1839 setTruncStoreAction(ValVT: MVT::f32, MemVT: MVT::f16, Action: Expand);
1840 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f16, Action: Expand);
1841
1842 // Handling of indexed loads/stores: default is "expand".
1843 //
1844 for (MVT VT : {MVT::i8, MVT::i16, MVT::i32, MVT::i64, MVT::f32, MVT::f64,
1845 MVT::v2i16, MVT::v2i32, MVT::v4i8, MVT::v4i16, MVT::v8i8}) {
1846 setIndexedLoadAction(IdxModes: ISD::POST_INC, VT, Action: Legal);
1847 setIndexedStoreAction(IdxModes: ISD::POST_INC, VT, Action: Legal);
1848 }
1849
1850 // Subtarget-specific operation actions.
1851 //
1852 if (Subtarget.hasV60Ops()) {
1853 setOperationAction(Op: ISD::ROTL, VT: MVT::i32, Action: Legal);
1854 setOperationAction(Op: ISD::ROTL, VT: MVT::i64, Action: Legal);
1855 setOperationAction(Op: ISD::ROTR, VT: MVT::i32, Action: Legal);
1856 setOperationAction(Op: ISD::ROTR, VT: MVT::i64, Action: Legal);
1857 }
1858 if (Subtarget.hasV66Ops()) {
1859 setOperationAction(Op: ISD::FADD, VT: MVT::f64, Action: Legal);
1860 setOperationAction(Op: ISD::FSUB, VT: MVT::f64, Action: Legal);
1861 }
1862 if (Subtarget.hasV67Ops()) {
1863 setOperationAction(Op: ISD::FMINIMUMNUM, VT: MVT::f64, Action: Legal);
1864 setOperationAction(Op: ISD::FMAXIMUMNUM, VT: MVT::f64, Action: Legal);
1865 setOperationAction(Op: ISD::FMUL, VT: MVT::f64, Action: Legal);
1866 }
1867
1868 setTargetDAGCombine(ISD::OR);
1869 setTargetDAGCombine(ISD::TRUNCATE);
1870 setTargetDAGCombine(ISD::VSELECT);
1871
1872 if (Subtarget.useHVXOps())
1873 initializeHVXLowering();
1874
1875 computeRegisterProperties(TRI: &HRI);
1876}
1877
1878bool
1879HexagonTargetLowering::validateConstPtrAlignment(SDValue Ptr, Align NeedAlign,
1880 const SDLoc &dl, SelectionDAG &DAG) const {
1881 auto *CA = dyn_cast<ConstantSDNode>(Val&: Ptr);
1882 if (!CA)
1883 return true;
1884 unsigned Addr = CA->getZExtValue();
1885 Align HaveAlign =
1886 Addr != 0 ? Align(1ull << llvm::countr_zero(Val: Addr)) : NeedAlign;
1887 if (HaveAlign >= NeedAlign)
1888 return true;
1889
1890 static int DK_MisalignedTrap = llvm::getNextAvailablePluginDiagnosticKind();
1891
1892 struct DiagnosticInfoMisalignedTrap : public DiagnosticInfo {
1893 DiagnosticInfoMisalignedTrap(StringRef M)
1894 : DiagnosticInfo(DK_MisalignedTrap, DS_Remark), Msg(M) {}
1895 void print(DiagnosticPrinter &DP) const override {
1896 DP << Msg;
1897 }
1898 static bool classof(const DiagnosticInfo *DI) {
1899 return DI->getKind() == DK_MisalignedTrap;
1900 }
1901 StringRef Msg;
1902 };
1903
1904 std::string ErrMsg;
1905 raw_string_ostream O(ErrMsg);
1906 O << "Misaligned constant address: " << format_hex(N: Addr, Width: 10)
1907 << " has alignment " << HaveAlign.value()
1908 << ", but the memory access requires " << NeedAlign.value();
1909 if (DebugLoc DL = dl.getDebugLoc())
1910 DL.print(OS&: O << ", at ");
1911 O << ". The instruction has been replaced with a trap.";
1912
1913 DAG.getContext()->diagnose(DI: DiagnosticInfoMisalignedTrap(O.str()));
1914 return false;
1915}
1916
1917SDValue
1918HexagonTargetLowering::replaceMemWithUndef(SDValue Op, SelectionDAG &DAG)
1919 const {
1920 const SDLoc &dl(Op);
1921 auto *LS = cast<LSBaseSDNode>(Val: Op.getNode());
1922 assert(!LS->isIndexed() && "Not expecting indexed ops on constant address");
1923
1924 SDValue Chain = LS->getChain();
1925 SDValue Trap = DAG.getNode(Opcode: ISD::TRAP, DL: dl, VT: MVT::Other, Operand: Chain);
1926 if (LS->getOpcode() == ISD::LOAD)
1927 return DAG.getMergeValues(Ops: {DAG.getUNDEF(VT: ty(Op)), Trap}, dl);
1928 return Trap;
1929}
1930
1931// Bit-reverse Load Intrinsic: Check if the instruction is a bit reverse load
1932// intrinsic.
1933static bool isBrevLdIntrinsic(const Value *Inst) {
1934 unsigned ID = cast<IntrinsicInst>(Val: Inst)->getIntrinsicID();
1935 return (ID == Intrinsic::hexagon_L2_loadrd_pbr ||
1936 ID == Intrinsic::hexagon_L2_loadri_pbr ||
1937 ID == Intrinsic::hexagon_L2_loadrh_pbr ||
1938 ID == Intrinsic::hexagon_L2_loadruh_pbr ||
1939 ID == Intrinsic::hexagon_L2_loadrb_pbr ||
1940 ID == Intrinsic::hexagon_L2_loadrub_pbr);
1941}
1942
1943// Bit-reverse Load Intrinsic :Crawl up and figure out the object from previous
1944// instruction. So far we only handle bitcast, extract value and bit reverse
1945// load intrinsic instructions. Should we handle CGEP ?
1946static Value *getBrevLdObject(Value *V) {
1947 if (Operator::getOpcode(V) == Instruction::ExtractValue ||
1948 Operator::getOpcode(V) == Instruction::BitCast)
1949 V = cast<Operator>(Val: V)->getOperand(i: 0);
1950 else if (isa<IntrinsicInst>(Val: V) && isBrevLdIntrinsic(Inst: V))
1951 V = cast<Instruction>(Val: V)->getOperand(i: 0);
1952 return V;
1953}
1954
1955// Bit-reverse Load Intrinsic: For a PHI Node return either an incoming edge or
1956// a back edge. If the back edge comes from the intrinsic itself, the incoming
1957// edge is returned.
1958static Value *returnEdge(const PHINode *PN, Value *IntrBaseVal) {
1959 const BasicBlock *Parent = PN->getParent();
1960 int Idx = -1;
1961 for (unsigned i = 0, e = PN->getNumIncomingValues(); i < e; ++i) {
1962 BasicBlock *Blk = PN->getIncomingBlock(i);
1963 // Determine if the back edge is originated from intrinsic.
1964 if (Blk == Parent) {
1965 Value *BackEdgeVal = PN->getIncomingValue(i);
1966 Value *BaseVal;
1967 // Loop over till we return the same Value or we hit the IntrBaseVal.
1968 do {
1969 BaseVal = BackEdgeVal;
1970 BackEdgeVal = getBrevLdObject(V: BackEdgeVal);
1971 } while ((BaseVal != BackEdgeVal) && (IntrBaseVal != BackEdgeVal));
1972 // If the getBrevLdObject returns IntrBaseVal, we should return the
1973 // incoming edge.
1974 if (IntrBaseVal == BackEdgeVal)
1975 continue;
1976 Idx = i;
1977 break;
1978 } else // Set the node to incoming edge.
1979 Idx = i;
1980 }
1981 assert(Idx >= 0 && "Unexpected index to incoming argument in PHI");
1982 return PN->getIncomingValue(i: Idx);
1983}
1984
1985// Bit-reverse Load Intrinsic: Figure out the underlying object the base
1986// pointer points to, for the bit-reverse load intrinsic. Setting this to
1987// memoperand might help alias analysis to figure out the dependencies.
1988static Value *getUnderLyingObjectForBrevLdIntr(Value *V) {
1989 Value *IntrBaseVal = V;
1990 Value *BaseVal;
1991 // Loop over till we return the same Value, implies we either figure out
1992 // the object or we hit a PHI
1993 do {
1994 BaseVal = V;
1995 V = getBrevLdObject(V);
1996 } while (BaseVal != V);
1997
1998 // Identify the object from PHINode.
1999 if (const PHINode *PN = dyn_cast<PHINode>(Val: V))
2000 return returnEdge(PN, IntrBaseVal);
2001 // For non PHI nodes, the object is the last value returned by getBrevLdObject
2002 else
2003 return V;
2004}
2005
2006/// Given an intrinsic, checks if on the target the intrinsic will need to map
2007/// to a MemIntrinsicNode (touches memory). If this is the case, it stores
2008/// the intrinsic information into the Infos vector.
2009void HexagonTargetLowering::getTgtMemIntrinsic(
2010 SmallVectorImpl<IntrinsicInfo> &Infos, const CallBase &I,
2011 MachineFunction &MF, unsigned Intrinsic) const {
2012 IntrinsicInfo Info;
2013 switch (Intrinsic) {
2014 case Intrinsic::hexagon_L2_loadrd_pbr:
2015 case Intrinsic::hexagon_L2_loadri_pbr:
2016 case Intrinsic::hexagon_L2_loadrh_pbr:
2017 case Intrinsic::hexagon_L2_loadruh_pbr:
2018 case Intrinsic::hexagon_L2_loadrb_pbr:
2019 case Intrinsic::hexagon_L2_loadrub_pbr: {
2020 Info.opc = ISD::INTRINSIC_W_CHAIN;
2021 auto &DL = I.getDataLayout();
2022 auto &Cont = I.getCalledFunction()->getParent()->getContext();
2023 // The intrinsic function call is of the form { ElTy, i8* }
2024 // @llvm.hexagon.L2.loadXX.pbr(i8*, i32). The pointer and memory access type
2025 // should be derived from ElTy.
2026 Type *ElTy = I.getCalledFunction()->getReturnType()->getStructElementType(N: 0);
2027 Info.memVT = MVT::getVT(Ty: ElTy);
2028 llvm::Value *BasePtrVal = I.getOperand(i_nocapture: 0);
2029 Info.ptrVal = getUnderLyingObjectForBrevLdIntr(V: BasePtrVal);
2030 // The offset value comes through Modifier register. For now, assume the
2031 // offset is 0.
2032 Info.offset = 0;
2033 Info.align = DL.getABITypeAlign(Ty: Info.memVT.getTypeForEVT(Context&: Cont));
2034 Info.flags = MachineMemOperand::MOLoad;
2035 Infos.push_back(Elt: Info);
2036 return;
2037 }
2038 case Intrinsic::hexagon_V6_vgathermw:
2039 case Intrinsic::hexagon_V6_vgathermw_128B:
2040 case Intrinsic::hexagon_V6_vgathermh:
2041 case Intrinsic::hexagon_V6_vgathermh_128B:
2042 case Intrinsic::hexagon_V6_vgathermhw:
2043 case Intrinsic::hexagon_V6_vgathermhw_128B:
2044 case Intrinsic::hexagon_V6_vgathermwq:
2045 case Intrinsic::hexagon_V6_vgathermwq_128B:
2046 case Intrinsic::hexagon_V6_vgathermhq:
2047 case Intrinsic::hexagon_V6_vgathermhq_128B:
2048 case Intrinsic::hexagon_V6_vgathermhwq:
2049 case Intrinsic::hexagon_V6_vgathermhwq_128B:
2050 case Intrinsic::hexagon_V6_vgather_vscattermh:
2051 case Intrinsic::hexagon_V6_vgather_vscattermh_128B: {
2052 const Module &M = *I.getParent()->getParent()->getParent();
2053 Info.opc = ISD::INTRINSIC_W_CHAIN;
2054 Type *VecTy = I.getArgOperand(i: 1)->getType();
2055 Info.memVT = MVT::getVT(Ty: VecTy);
2056 Info.ptrVal = I.getArgOperand(i: 0);
2057 Info.offset = 0;
2058 Info.align =
2059 MaybeAlign(M.getDataLayout().getTypeAllocSizeInBits(Ty: VecTy) / 8);
2060 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
2061 MachineMemOperand::MOVolatile;
2062 Infos.push_back(Elt: Info);
2063 return;
2064 }
2065 default:
2066 break;
2067 }
2068}
2069
2070bool HexagonTargetLowering::hasBitTest(SDValue X, SDValue Y) const {
2071 return X.getValueType().isScalarInteger(); // 'tstbit'
2072}
2073
2074bool HexagonTargetLowering::isTruncateFree(Type *Ty1, Type *Ty2) const {
2075 return isTruncateFree(VT1: EVT::getEVT(Ty: Ty1), VT2: EVT::getEVT(Ty: Ty2));
2076}
2077
2078bool HexagonTargetLowering::isTruncateFree(EVT VT1, EVT VT2) const {
2079 if (!VT1.isSimple() || !VT2.isSimple())
2080 return false;
2081 return VT1.getSimpleVT() == MVT::i64 && VT2.getSimpleVT() == MVT::i32;
2082}
2083
2084bool HexagonTargetLowering::isFMAFasterThanFMulAndFAdd(
2085 const MachineFunction &MF, EVT VT) const {
2086 return isOperationLegalOrCustom(Op: ISD::FMA, VT);
2087}
2088
2089// Should we expand the build vector with shuffles?
2090bool HexagonTargetLowering::shouldExpandBuildVectorWithShuffles(EVT VT,
2091 unsigned DefinedValues) const {
2092 return false;
2093}
2094
2095bool HexagonTargetLowering::isExtractSubvectorCheap(EVT ResVT, EVT SrcVT,
2096 unsigned Index) const {
2097 assert(ResVT.getVectorElementType() == SrcVT.getVectorElementType());
2098 if (!ResVT.isSimple() || !SrcVT.isSimple())
2099 return false;
2100
2101 MVT ResTy = ResVT.getSimpleVT(), SrcTy = SrcVT.getSimpleVT();
2102 if (ResTy.getVectorElementType() != MVT::i1)
2103 return true;
2104
2105 // Non-HVX bool vectors are relatively cheap.
2106 return SrcTy.getVectorNumElements() <= 8;
2107}
2108
2109bool HexagonTargetLowering::isTargetCanonicalConstantNode(SDValue Op) const {
2110 return Op.getOpcode() == ISD::CONCAT_VECTORS ||
2111 TargetLowering::isTargetCanonicalConstantNode(Op);
2112}
2113
2114bool HexagonTargetLowering::isShuffleMaskLegal(ArrayRef<int> Mask,
2115 EVT VT) const {
2116 return true;
2117}
2118
2119TargetLoweringBase::LegalizeTypeAction
2120HexagonTargetLowering::getPreferredVectorAction(MVT VT) const {
2121 unsigned VecLen = VT.getVectorMinNumElements();
2122 MVT ElemTy = VT.getVectorElementType();
2123
2124 if (VecLen == 1 || VT.isScalableVector())
2125 return TargetLoweringBase::TypeScalarizeVector;
2126
2127 if (Subtarget.useHVXOps()) {
2128 unsigned Action = getPreferredHvxVectorAction(VecTy: VT);
2129 if (Action != ~0u)
2130 return static_cast<TargetLoweringBase::LegalizeTypeAction>(Action);
2131 }
2132
2133 // Always widen (remaining) vectors of i1.
2134 if (ElemTy == MVT::i1)
2135 return TargetLoweringBase::TypeWidenVector;
2136 // Widen non-power-of-2 vectors. Such types cannot be split right now,
2137 // and computeRegisterProperties will override "split" with "widen",
2138 // which can cause other issues.
2139 if (!isPowerOf2_32(Value: VecLen))
2140 return TargetLoweringBase::TypeWidenVector;
2141
2142 return TargetLoweringBase::TypeSplitVector;
2143}
2144
2145TargetLoweringBase::LegalizeAction
2146HexagonTargetLowering::getCustomOperationAction(SDNode &Op) const {
2147 if (Subtarget.useHVXOps()) {
2148 unsigned Action = getCustomHvxOperationAction(Op);
2149 if (Action != ~0u)
2150 return static_cast<TargetLoweringBase::LegalizeAction>(Action);
2151 }
2152 return TargetLoweringBase::Legal;
2153}
2154
2155std::pair<SDValue, int>
2156HexagonTargetLowering::getBaseAndOffset(SDValue Addr) const {
2157 if (Addr.getOpcode() == ISD::ADD) {
2158 SDValue Op1 = Addr.getOperand(i: 1);
2159 if (auto *CN = dyn_cast<const ConstantSDNode>(Val: Op1.getNode()))
2160 return { Addr.getOperand(i: 0), CN->getSExtValue() };
2161 }
2162 return { Addr, 0 };
2163}
2164
2165// Lower a vector shuffle (V1, V2, V3). V1 and V2 are the two vectors
2166// to select data from, V3 is the permutation.
2167SDValue
2168HexagonTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG)
2169 const {
2170 const auto *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
2171 ArrayRef<int> AM = SVN->getMask();
2172 assert(AM.size() <= 8 && "Unexpected shuffle mask");
2173 unsigned VecLen = AM.size();
2174
2175 MVT VecTy = ty(Op);
2176 assert(!Subtarget.isHVXVectorType(VecTy, true) &&
2177 "HVX shuffles should be legal");
2178 assert(VecTy.getSizeInBits() <= 64 && "Unexpected vector length");
2179
2180 SDValue Op0 = Op.getOperand(i: 0);
2181 SDValue Op1 = Op.getOperand(i: 1);
2182 const SDLoc &dl(Op);
2183
2184 // If the inputs are not the same as the output, bail. This is not an
2185 // error situation, but complicates the handling and the default expansion
2186 // (into BUILD_VECTOR) should be adequate.
2187 if (ty(Op: Op0) != VecTy || ty(Op: Op1) != VecTy)
2188 return SDValue();
2189
2190 // Normalize the mask so that the first non-negative index comes from
2191 // the first operand.
2192 SmallVector<int, 8> Mask(AM);
2193 unsigned F = llvm::find_if(Range&: AM, P: [](int M) { return M >= 0; }) - AM.data();
2194 if (F == AM.size())
2195 return DAG.getUNDEF(VT: VecTy);
2196 if (AM[F] >= int(VecLen)) {
2197 ShuffleVectorSDNode::commuteMask(Mask);
2198 std::swap(a&: Op0, b&: Op1);
2199 }
2200
2201 // Express the shuffle mask in terms of bytes.
2202 SmallVector<int,8> ByteMask;
2203 unsigned ElemBytes = VecTy.getVectorElementType().getSizeInBits() / 8;
2204 for (int M : Mask) {
2205 if (M < 0) {
2206 for (unsigned j = 0; j != ElemBytes; ++j)
2207 ByteMask.push_back(Elt: -1);
2208 } else {
2209 for (unsigned j = 0; j != ElemBytes; ++j)
2210 ByteMask.push_back(Elt: M*ElemBytes + j);
2211 }
2212 }
2213 assert(ByteMask.size() <= 8);
2214
2215 // All non-undef (non-negative) indexes are well within [0..127], so they
2216 // fit in a single byte. Build two 64-bit words:
2217 // - MaskIdx where each byte is the corresponding index (for non-negative
2218 // indexes), and 0xFF for negative indexes, and
2219 // - MaskUnd that has 0xFF for each negative index.
2220 uint64_t MaskIdx = 0;
2221 uint64_t MaskUnd = 0;
2222 for (unsigned i = 0, e = ByteMask.size(); i != e; ++i) {
2223 unsigned S = 8*i;
2224 uint64_t M = ByteMask[i] & 0xFF;
2225 if (M == 0xFF)
2226 MaskUnd |= M << S;
2227 MaskIdx |= M << S;
2228 }
2229
2230 if (ByteMask.size() == 4) {
2231 // Identity.
2232 if (MaskIdx == (0x03020100 | MaskUnd))
2233 return Op0;
2234 // Byte swap.
2235 if (MaskIdx == (0x00010203 | MaskUnd)) {
2236 SDValue T0 = DAG.getBitcast(VT: MVT::i32, V: Op0);
2237 SDValue T1 = DAG.getNode(Opcode: ISD::BSWAP, DL: dl, VT: MVT::i32, Operand: T0);
2238 return DAG.getBitcast(VT: VecTy, V: T1);
2239 }
2240
2241 // Byte packs.
2242 SDValue Concat10 =
2243 getCombine(Hi: Op1, Lo: Op0, dl, ResTy: typeJoin(Tys: {ty(Op: Op1), ty(Op: Op0)}), DAG);
2244 if (MaskIdx == (0x06040200 | MaskUnd))
2245 return getInstr(MachineOpc: Hexagon::S2_vtrunehb, dl, Ty: VecTy, Ops: {Concat10}, DAG);
2246 if (MaskIdx == (0x07050301 | MaskUnd))
2247 return getInstr(MachineOpc: Hexagon::S2_vtrunohb, dl, Ty: VecTy, Ops: {Concat10}, DAG);
2248
2249 SDValue Concat01 =
2250 getCombine(Hi: Op0, Lo: Op1, dl, ResTy: typeJoin(Tys: {ty(Op: Op0), ty(Op: Op1)}), DAG);
2251 if (MaskIdx == (0x02000604 | MaskUnd))
2252 return getInstr(MachineOpc: Hexagon::S2_vtrunehb, dl, Ty: VecTy, Ops: {Concat01}, DAG);
2253 if (MaskIdx == (0x03010705 | MaskUnd))
2254 return getInstr(MachineOpc: Hexagon::S2_vtrunohb, dl, Ty: VecTy, Ops: {Concat01}, DAG);
2255 }
2256
2257 if (ByteMask.size() == 8) {
2258 // Identity.
2259 if (MaskIdx == (0x0706050403020100ull | MaskUnd))
2260 return Op0;
2261 // Byte swap.
2262 if (MaskIdx == (0x0001020304050607ull | MaskUnd)) {
2263 SDValue T0 = DAG.getBitcast(VT: MVT::i64, V: Op0);
2264 SDValue T1 = DAG.getNode(Opcode: ISD::BSWAP, DL: dl, VT: MVT::i64, Operand: T0);
2265 return DAG.getBitcast(VT: VecTy, V: T1);
2266 }
2267
2268 // Halfword picks.
2269 if (MaskIdx == (0x0d0c050409080100ull | MaskUnd))
2270 return getInstr(MachineOpc: Hexagon::S2_shuffeh, dl, Ty: VecTy, Ops: {Op1, Op0}, DAG);
2271 if (MaskIdx == (0x0f0e07060b0a0302ull | MaskUnd))
2272 return getInstr(MachineOpc: Hexagon::S2_shuffoh, dl, Ty: VecTy, Ops: {Op1, Op0}, DAG);
2273 if (MaskIdx == (0x0d0c090805040100ull | MaskUnd))
2274 return getInstr(MachineOpc: Hexagon::S2_vtrunewh, dl, Ty: VecTy, Ops: {Op1, Op0}, DAG);
2275 if (MaskIdx == (0x0f0e0b0a07060302ull | MaskUnd))
2276 return getInstr(MachineOpc: Hexagon::S2_vtrunowh, dl, Ty: VecTy, Ops: {Op1, Op0}, DAG);
2277 if (MaskIdx == (0x0706030205040100ull | MaskUnd)) {
2278 VectorPair P = opSplit(Vec: Op0, dl, DAG);
2279 return getInstr(MachineOpc: Hexagon::S2_packhl, dl, Ty: VecTy, Ops: {P.second, P.first}, DAG);
2280 }
2281
2282 // Byte packs.
2283 if (MaskIdx == (0x0e060c040a020800ull | MaskUnd))
2284 return getInstr(MachineOpc: Hexagon::S2_shuffeb, dl, Ty: VecTy, Ops: {Op1, Op0}, DAG);
2285 if (MaskIdx == (0x0f070d050b030901ull | MaskUnd))
2286 return getInstr(MachineOpc: Hexagon::S2_shuffob, dl, Ty: VecTy, Ops: {Op1, Op0}, DAG);
2287 }
2288
2289 return SDValue();
2290}
2291
2292SDValue
2293HexagonTargetLowering::getSplatValue(SDValue Op, SelectionDAG &DAG) const {
2294 switch (Op.getOpcode()) {
2295 case ISD::BUILD_VECTOR:
2296 if (SDValue S = cast<BuildVectorSDNode>(Val&: Op)->getSplatValue())
2297 return S;
2298 break;
2299 case ISD::SPLAT_VECTOR:
2300 return Op.getOperand(i: 0);
2301 }
2302 return SDValue();
2303}
2304
2305// Create a Hexagon-specific node for shifting a vector by an integer.
2306SDValue
2307HexagonTargetLowering::getVectorShiftByInt(SDValue Op, SelectionDAG &DAG)
2308 const {
2309 unsigned NewOpc;
2310 switch (Op.getOpcode()) {
2311 case ISD::SHL:
2312 NewOpc = HexagonISD::VASL;
2313 break;
2314 case ISD::SRA:
2315 NewOpc = HexagonISD::VASR;
2316 break;
2317 case ISD::SRL:
2318 NewOpc = HexagonISD::VLSR;
2319 break;
2320 default:
2321 llvm_unreachable("Unexpected shift opcode");
2322 }
2323 if (SDValue Sp = getSplatValue(Op: Op.getOperand(i: 1), DAG)) {
2324 const SDLoc dl(Op);
2325 // Canonicalize shift amount to i32 as required.
2326 SDValue Sh = Sp;
2327 if (Sh.getValueType() != MVT::i32)
2328 Sh = DAG.getZExtOrTrunc(Op: Sh, DL: dl, VT: MVT::i32);
2329
2330 assert(Sh.getValueType() == MVT::i32 &&
2331 "Hexagon vector shift-by-int must use i32 shift operand");
2332 return DAG.getNode(Opcode: NewOpc, DL: dl, VT: ty(Op), N1: Op.getOperand(i: 0), N2: Sh);
2333 }
2334
2335 return SDValue();
2336}
2337
2338SDValue
2339HexagonTargetLowering::LowerVECTOR_SHIFT(SDValue Op, SelectionDAG &DAG) const {
2340 const SDLoc &dl(Op);
2341
2342 // First try to convert the shift (by vector) to a shift by a scalar.
2343 // If we first split the shift, the shift amount will become 'extract
2344 // subvector', and will no longer be recognized as scalar.
2345 SDValue Res = Op;
2346 if (SDValue S = getVectorShiftByInt(Op, DAG))
2347 Res = S;
2348
2349 unsigned Opc = Res.getOpcode();
2350 switch (Opc) {
2351 case HexagonISD::VASR:
2352 case HexagonISD::VLSR:
2353 case HexagonISD::VASL:
2354 break;
2355 default:
2356 // No instructions for shifts by non-scalars.
2357 return SDValue();
2358 }
2359
2360 MVT ResTy = ty(Op: Res);
2361 if (ResTy.getVectorElementType() != MVT::i8)
2362 return Res;
2363
2364 // For shifts of i8, extend the inputs to i16, then truncate back to i8.
2365 assert(ResTy.getVectorElementType() == MVT::i8);
2366 SDValue Val = Res.getOperand(i: 0), Amt = Res.getOperand(i: 1);
2367
2368 auto ShiftPartI8 = [&dl, &DAG, this](unsigned Opc, SDValue V, SDValue A) {
2369 MVT Ty = ty(Op: V);
2370 MVT ExtTy = MVT::getVectorVT(VT: MVT::i16, NumElements: Ty.getVectorNumElements());
2371 SDValue ExtV = Opc == HexagonISD::VASR ? DAG.getSExtOrTrunc(Op: V, DL: dl, VT: ExtTy)
2372 : DAG.getZExtOrTrunc(Op: V, DL: dl, VT: ExtTy);
2373 SDValue ExtS = DAG.getNode(Opcode: Opc, DL: dl, VT: ExtTy, Ops: {ExtV, A});
2374 return DAG.getZExtOrTrunc(Op: ExtS, DL: dl, VT: Ty);
2375 };
2376
2377 if (ResTy.getSizeInBits() == 32)
2378 return ShiftPartI8(Opc, Val, Amt);
2379
2380 auto [LoV, HiV] = opSplit(Vec: Val, dl, DAG);
2381 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResTy,
2382 Ops: {ShiftPartI8(Opc, LoV, Amt), ShiftPartI8(Opc, HiV, Amt)});
2383}
2384
2385SDValue
2386HexagonTargetLowering::LowerROTL(SDValue Op, SelectionDAG &DAG) const {
2387 if (isa<ConstantSDNode>(Val: Op.getOperand(i: 1).getNode()))
2388 return Op;
2389 return SDValue();
2390}
2391
2392SDValue
2393HexagonTargetLowering::LowerBITCAST(SDValue Op, SelectionDAG &DAG) const {
2394 MVT ResTy = ty(Op);
2395 SDValue InpV = Op.getOperand(i: 0);
2396 MVT InpTy = ty(Op: InpV);
2397 assert(ResTy.getSizeInBits() == InpTy.getSizeInBits());
2398 const SDLoc &dl(Op);
2399
2400 // Handle conversion from i8 to v8i1.
2401 if (InpTy == MVT::i8) {
2402 if (ResTy == MVT::v8i1) {
2403 SDValue Sc = DAG.getBitcast(VT: tyScalar(Ty: InpTy), V: InpV);
2404 SDValue Ext = DAG.getZExtOrTrunc(Op: Sc, DL: dl, VT: MVT::i32);
2405 return getInstr(MachineOpc: Hexagon::C2_tfrrp, dl, Ty: ResTy, Ops: Ext, DAG);
2406 }
2407 return SDValue();
2408 }
2409
2410 return Op;
2411}
2412
2413bool
2414HexagonTargetLowering::getBuildVectorConstInts(ArrayRef<SDValue> Values,
2415 MVT VecTy, SelectionDAG &DAG,
2416 MutableArrayRef<ConstantInt*> Consts) const {
2417 MVT ElemTy = VecTy.getVectorElementType();
2418 unsigned ElemWidth = ElemTy.getSizeInBits();
2419 IntegerType *IntTy = IntegerType::get(C&: *DAG.getContext(), NumBits: ElemWidth);
2420 bool AllConst = true;
2421
2422 for (unsigned i = 0, e = Values.size(); i != e; ++i) {
2423 SDValue V = Values[i];
2424 if (V.isUndef()) {
2425 Consts[i] = ConstantInt::get(Ty: IntTy, V: 0);
2426 continue;
2427 }
2428 // Make sure to always cast to IntTy.
2429 if (auto *CN = dyn_cast<ConstantSDNode>(Val: V.getNode())) {
2430 const ConstantInt *CI = CN->getConstantIntValue();
2431 Consts[i] = cast<ConstantInt>(
2432 Val: ConstantInt::get(Ty: IntTy, V: CI->getValue().trunc(width: ElemWidth)));
2433 } else if (auto *CN = dyn_cast<ConstantFPSDNode>(Val: V.getNode())) {
2434 const ConstantFP *CF = CN->getConstantFPValue();
2435 APInt A = CF->getValueAPF().bitcastToAPInt();
2436 Consts[i] = ConstantInt::get(Ty: IntTy, V: A.getZExtValue());
2437 } else {
2438 AllConst = false;
2439 }
2440 }
2441 return AllConst;
2442}
2443
2444SDValue
2445HexagonTargetLowering::buildVector32(ArrayRef<SDValue> Elem, const SDLoc &dl,
2446 MVT VecTy, SelectionDAG &DAG) const {
2447 MVT ElemTy = VecTy.getVectorElementType();
2448 assert(VecTy.getVectorNumElements() == Elem.size());
2449
2450 SmallVector<ConstantInt*,4> Consts(Elem.size());
2451 bool AllConst = getBuildVectorConstInts(Values: Elem, VecTy, DAG, Consts);
2452
2453 unsigned First, Num = Elem.size();
2454 for (First = 0; First != Num; ++First) {
2455 if (!isUndef(Op: Elem[First]))
2456 break;
2457 }
2458 if (First == Num)
2459 return DAG.getUNDEF(VT: VecTy);
2460
2461 if (AllConst &&
2462 llvm::all_of(Range&: Consts, P: [](ConstantInt *CI) { return CI->isZero(); }))
2463 return getZero(dl, Ty: VecTy, DAG);
2464
2465 if (ElemTy == MVT::i16 || ElemTy == MVT::f16) {
2466 assert(Elem.size() == 2);
2467 if (AllConst) {
2468 // The 'Consts' array will have all values as integers regardless
2469 // of the vector element type.
2470 uint32_t V = (Consts[0]->getZExtValue() & 0xFFFF) |
2471 Consts[1]->getZExtValue() << 16;
2472 return DAG.getBitcast(VT: VecTy, V: DAG.getConstant(Val: V, DL: dl, VT: MVT::i32));
2473 }
2474 SDValue E0, E1;
2475 if (ElemTy == MVT::f16) {
2476 E0 = DAG.getZExtOrTrunc(Op: DAG.getBitcast(VT: MVT::i16, V: Elem[0]), DL: dl, VT: MVT::i32);
2477 E1 = DAG.getZExtOrTrunc(Op: DAG.getBitcast(VT: MVT::i16, V: Elem[1]), DL: dl, VT: MVT::i32);
2478 } else {
2479 E0 = Elem[0];
2480 E1 = Elem[1];
2481 }
2482 SDValue N = getInstr(MachineOpc: Hexagon::A2_combine_ll, dl, Ty: MVT::i32, Ops: {E1, E0}, DAG);
2483 return DAG.getBitcast(VT: VecTy, V: N);
2484 }
2485
2486 if (ElemTy == MVT::i8) {
2487 // First try generating a constant.
2488 if (AllConst) {
2489 uint32_t V = (Consts[0]->getZExtValue() & 0xFF) |
2490 (Consts[1]->getZExtValue() & 0xFF) << 8 |
2491 (Consts[2]->getZExtValue() & 0xFF) << 16 |
2492 Consts[3]->getZExtValue() << 24;
2493 return DAG.getBitcast(VT: MVT::v4i8, V: DAG.getConstant(Val: V, DL: dl, VT: MVT::i32));
2494 }
2495
2496 // Then try splat.
2497 bool IsSplat = true;
2498 for (unsigned i = First+1; i != Num; ++i) {
2499 if (Elem[i] == Elem[First] || isUndef(Op: Elem[i]))
2500 continue;
2501 IsSplat = false;
2502 break;
2503 }
2504 if (IsSplat) {
2505 // Legalize the operand of SPLAT_VECTOR.
2506 SDValue Ext = DAG.getZExtOrTrunc(Op: Elem[First], DL: dl, VT: MVT::i32);
2507 return DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: VecTy, Operand: Ext);
2508 }
2509
2510 // Generate
2511 // (zxtb(Elem[0]) | (zxtb(Elem[1]) << 8)) |
2512 // (zxtb(Elem[2]) | (zxtb(Elem[3]) << 8)) << 16
2513 assert(Elem.size() == 4);
2514 SDValue Vs[4];
2515 for (unsigned i = 0; i != 4; ++i) {
2516 Vs[i] = DAG.getZExtOrTrunc(Op: Elem[i], DL: dl, VT: MVT::i32);
2517 Vs[i] = DAG.getZeroExtendInReg(Op: Vs[i], DL: dl, VT: MVT::i8);
2518 }
2519 SDValue S8 = DAG.getConstant(Val: 8, DL: dl, VT: MVT::i32);
2520 SDValue T0 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: MVT::i32, Ops: {Vs[1], S8});
2521 SDValue T1 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: MVT::i32, Ops: {Vs[3], S8});
2522 SDValue B0 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: MVT::i32, Ops: {Vs[0], T0});
2523 SDValue B1 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: MVT::i32, Ops: {Vs[2], T1});
2524
2525 SDValue R = getInstr(MachineOpc: Hexagon::A2_combine_ll, dl, Ty: MVT::i32, Ops: {B1, B0}, DAG);
2526 return DAG.getBitcast(VT: MVT::v4i8, V: R);
2527 }
2528
2529#ifndef NDEBUG
2530 dbgs() << "VecTy: " << VecTy << '\n';
2531#endif
2532 llvm_unreachable("Unexpected vector element type");
2533}
2534
2535SDValue
2536HexagonTargetLowering::buildVector64(ArrayRef<SDValue> Elem, const SDLoc &dl,
2537 MVT VecTy, SelectionDAG &DAG) const {
2538 MVT ElemTy = VecTy.getVectorElementType();
2539 assert(VecTy.getVectorNumElements() == Elem.size());
2540
2541 SmallVector<ConstantInt*,8> Consts(Elem.size());
2542 bool AllConst = getBuildVectorConstInts(Values: Elem, VecTy, DAG, Consts);
2543
2544 unsigned First, Num = Elem.size();
2545 for (First = 0; First != Num; ++First) {
2546 if (!isUndef(Op: Elem[First]))
2547 break;
2548 }
2549 if (First == Num)
2550 return DAG.getUNDEF(VT: VecTy);
2551
2552 if (AllConst &&
2553 llvm::all_of(Range&: Consts, P: [](ConstantInt *CI) { return CI->isZero(); }))
2554 return getZero(dl, Ty: VecTy, DAG);
2555
2556 // First try splat if possible.
2557 if (ElemTy == MVT::i16 || ElemTy == MVT::f16) {
2558 bool IsSplat = true;
2559 for (unsigned i = First+1; i != Num; ++i) {
2560 if (Elem[i] == Elem[First] || isUndef(Op: Elem[i]))
2561 continue;
2562 IsSplat = false;
2563 break;
2564 }
2565 if (IsSplat) {
2566 // Legalize the operand of SPLAT_VECTOR
2567 SDValue S = ElemTy == MVT::f16 ? DAG.getBitcast(VT: MVT::i16, V: Elem[First])
2568 : Elem[First];
2569 SDValue Ext = DAG.getZExtOrTrunc(Op: S, DL: dl, VT: MVT::i32);
2570 return DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: VecTy, Operand: Ext);
2571 }
2572 }
2573
2574 // Then try constant.
2575 if (AllConst) {
2576 uint64_t Val = 0;
2577 unsigned W = ElemTy.getSizeInBits();
2578 uint64_t Mask = (1ull << W) - 1;
2579 for (unsigned i = 0; i != Num; ++i)
2580 Val = (Val << W) | (Consts[Num-1-i]->getZExtValue() & Mask);
2581 SDValue V0 = DAG.getConstant(Val, DL: dl, VT: MVT::i64);
2582 return DAG.getBitcast(VT: VecTy, V: V0);
2583 }
2584
2585 // Build two 32-bit vectors and concatenate.
2586 MVT HalfTy = MVT::getVectorVT(VT: ElemTy, NumElements: Num/2);
2587 SDValue L = (ElemTy == MVT::i32)
2588 ? Elem[0]
2589 : buildVector32(Elem: Elem.take_front(N: Num/2), dl, VecTy: HalfTy, DAG);
2590 SDValue H = (ElemTy == MVT::i32)
2591 ? Elem[1]
2592 : buildVector32(Elem: Elem.drop_front(N: Num/2), dl, VecTy: HalfTy, DAG);
2593 return getCombine(Hi: H, Lo: L, dl, ResTy: VecTy, DAG);
2594}
2595
2596SDValue
2597HexagonTargetLowering::extractVector(SDValue VecV, SDValue IdxV,
2598 const SDLoc &dl, MVT ValTy, MVT ResTy,
2599 SelectionDAG &DAG) const {
2600 MVT VecTy = ty(Op: VecV);
2601 assert(!ValTy.isVector() ||
2602 VecTy.getVectorElementType() == ValTy.getVectorElementType());
2603 if (VecTy.getVectorElementType() == MVT::i1)
2604 return extractVectorPred(VecV, IdxV, dl, ValTy, ResTy, DAG);
2605
2606 unsigned VecWidth = VecTy.getSizeInBits();
2607 unsigned ValWidth = ValTy.getSizeInBits();
2608 unsigned ElemWidth = VecTy.getVectorElementType().getSizeInBits();
2609 assert((VecWidth % ElemWidth) == 0);
2610 assert(VecWidth == 32 || VecWidth == 64);
2611
2612 // Cast everything to scalar integer types.
2613 MVT ScalarTy = tyScalar(Ty: VecTy);
2614 VecV = DAG.getBitcast(VT: ScalarTy, V: VecV);
2615
2616 SDValue WidthV = DAG.getConstant(Val: ValWidth, DL: dl, VT: MVT::i32);
2617 SDValue ExtV;
2618
2619 if (auto *IdxN = dyn_cast<ConstantSDNode>(Val&: IdxV)) {
2620 unsigned Off = IdxN->getZExtValue() * ElemWidth;
2621 if (VecWidth == 64 && ValWidth == 32) {
2622 assert(Off == 0 || Off == 32);
2623 ExtV = Off == 0 ? LoHalf(V: VecV, DAG) : HiHalf(V: VecV, DAG);
2624 } else if (Off == 0 && (ValWidth % 8) == 0) {
2625 ExtV = DAG.getZeroExtendInReg(Op: VecV, DL: dl, VT: tyScalar(Ty: ValTy));
2626 } else {
2627 SDValue OffV = DAG.getConstant(Val: Off, DL: dl, VT: MVT::i32);
2628 // The return type of EXTRACTU must be the same as the type of the
2629 // input vector.
2630 ExtV = DAG.getNode(Opcode: HexagonISD::EXTRACTU, DL: dl, VT: ScalarTy,
2631 Ops: {VecV, WidthV, OffV});
2632 }
2633 } else {
2634 if (ty(Op: IdxV) != MVT::i32)
2635 IdxV = DAG.getZExtOrTrunc(Op: IdxV, DL: dl, VT: MVT::i32);
2636 SDValue OffV = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MVT::i32, N1: IdxV,
2637 N2: DAG.getConstant(Val: ElemWidth, DL: dl, VT: MVT::i32));
2638 ExtV = DAG.getNode(Opcode: HexagonISD::EXTRACTU, DL: dl, VT: ScalarTy,
2639 Ops: {VecV, WidthV, OffV});
2640 }
2641
2642 // Cast ExtV to the requested result type.
2643 ExtV = DAG.getZExtOrTrunc(Op: ExtV, DL: dl, VT: tyScalar(Ty: ResTy));
2644 ExtV = DAG.getBitcast(VT: ResTy, V: ExtV);
2645 return ExtV;
2646}
2647
2648SDValue
2649HexagonTargetLowering::extractVectorPred(SDValue VecV, SDValue IdxV,
2650 const SDLoc &dl, MVT ValTy, MVT ResTy,
2651 SelectionDAG &DAG) const {
2652 // Special case for v{8,4,2}i1 (the only boolean vectors legal in Hexagon
2653 // without any coprocessors).
2654 MVT VecTy = ty(Op: VecV);
2655 unsigned VecWidth = VecTy.getSizeInBits();
2656 unsigned ValWidth = ValTy.getSizeInBits();
2657 assert(VecWidth == VecTy.getVectorNumElements() &&
2658 "Vector elements should equal vector width size");
2659 assert(VecWidth == 8 || VecWidth == 4 || VecWidth == 2);
2660
2661 // Check if this is an extract of the lowest bit.
2662 if (isNullConstant(V: IdxV) && ValTy.getSizeInBits() == 1) {
2663 // Extracting the lowest bit is a no-op, but it changes the type,
2664 // so it must be kept as an operation to avoid errors related to
2665 // type mismatches.
2666 return DAG.getNode(Opcode: HexagonISD::TYPECAST, DL: dl, VT: MVT::i1, Operand: VecV);
2667 }
2668
2669 // If the value extracted is a single bit, use tstbit.
2670 if (ValWidth == 1) {
2671 SDValue A0 = getInstr(MachineOpc: Hexagon::C2_tfrpr, dl, Ty: MVT::i32, Ops: {VecV}, DAG);
2672 SDValue M0 = DAG.getConstant(Val: 8 / VecWidth, DL: dl, VT: MVT::i32);
2673 SDValue I0 = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MVT::i32, N1: IdxV, N2: M0);
2674 return DAG.getNode(Opcode: HexagonISD::TSTBIT, DL: dl, VT: MVT::i1, N1: A0, N2: I0);
2675 }
2676
2677 // Each bool vector (v2i1, v4i1, v8i1) always occupies 8 bits in
2678 // a predicate register. The elements of the vector are repeated
2679 // in the register (if necessary) so that the total number is 8.
2680 // The extracted subvector will need to be expanded in such a way.
2681 unsigned Scale = VecWidth / ValWidth;
2682
2683 // Generate (p2d VecV) >> 8*Idx to move the interesting bytes to
2684 // position 0.
2685 assert(ty(IdxV) == MVT::i32);
2686 unsigned VecRep = 8 / VecWidth;
2687 SDValue S0 = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MVT::i32, N1: IdxV,
2688 N2: DAG.getConstant(Val: 8*VecRep, DL: dl, VT: MVT::i32));
2689 SDValue T0 = DAG.getNode(Opcode: HexagonISD::P2D, DL: dl, VT: MVT::i64, Operand: VecV);
2690 SDValue T1 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i64, N1: T0, N2: S0);
2691 while (Scale > 1) {
2692 // The longest possible subvector is at most 32 bits, so it is always
2693 // contained in the low subregister.
2694 T1 = LoHalf(V: T1, DAG);
2695 T1 = expandPredicate(Vec32: T1, dl, DAG);
2696 Scale /= 2;
2697 }
2698
2699 return DAG.getNode(Opcode: HexagonISD::D2P, DL: dl, VT: ResTy, Operand: T1);
2700}
2701
2702SDValue
2703HexagonTargetLowering::insertVector(SDValue VecV, SDValue ValV, SDValue IdxV,
2704 const SDLoc &dl, MVT ValTy,
2705 SelectionDAG &DAG) const {
2706 MVT VecTy = ty(Op: VecV);
2707 if (VecTy.getVectorElementType() == MVT::i1)
2708 return insertVectorPred(VecV, ValV, IdxV, dl, ValTy, DAG);
2709
2710 unsigned VecWidth = VecTy.getSizeInBits();
2711 unsigned ValWidth = ValTy.getSizeInBits();
2712 assert(VecWidth == 32 || VecWidth == 64);
2713 assert((VecWidth % ValWidth) == 0);
2714
2715 // Cast everything to scalar integer types.
2716 MVT ScalarTy = MVT::getIntegerVT(BitWidth: VecWidth);
2717 // The actual type of ValV may be different than ValTy (which is related
2718 // to the vector type).
2719 unsigned VW = ty(Op: ValV).getSizeInBits();
2720 ValV = DAG.getBitcast(VT: MVT::getIntegerVT(BitWidth: VW), V: ValV);
2721 VecV = DAG.getBitcast(VT: ScalarTy, V: VecV);
2722 if (VW != VecWidth)
2723 ValV = DAG.getAnyExtOrTrunc(Op: ValV, DL: dl, VT: ScalarTy);
2724
2725 SDValue WidthV = DAG.getConstant(Val: ValWidth, DL: dl, VT: MVT::i32);
2726 SDValue InsV;
2727
2728 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: IdxV)) {
2729 unsigned W = C->getZExtValue() * ValWidth;
2730 SDValue OffV = DAG.getConstant(Val: W, DL: dl, VT: MVT::i32);
2731 InsV = DAG.getNode(Opcode: HexagonISD::INSERT, DL: dl, VT: ScalarTy,
2732 Ops: {VecV, ValV, WidthV, OffV});
2733 } else {
2734 if (ty(Op: IdxV) != MVT::i32)
2735 IdxV = DAG.getZExtOrTrunc(Op: IdxV, DL: dl, VT: MVT::i32);
2736 SDValue OffV = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MVT::i32, N1: IdxV, N2: WidthV);
2737 InsV = DAG.getNode(Opcode: HexagonISD::INSERT, DL: dl, VT: ScalarTy,
2738 Ops: {VecV, ValV, WidthV, OffV});
2739 }
2740
2741 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: VecTy, Operand: InsV);
2742}
2743
2744SDValue
2745HexagonTargetLowering::insertVectorPred(SDValue VecV, SDValue ValV,
2746 SDValue IdxV, const SDLoc &dl,
2747 MVT ValTy, SelectionDAG &DAG) const {
2748 MVT VecTy = ty(Op: VecV);
2749 unsigned VecLen = VecTy.getVectorNumElements();
2750
2751 if (ValTy == MVT::i1) {
2752 SDValue ToReg = getInstr(MachineOpc: Hexagon::C2_tfrpr, dl, Ty: MVT::i32, Ops: {VecV}, DAG);
2753 SDValue Ext = DAG.getSExtOrTrunc(Op: ValV, DL: dl, VT: MVT::i32);
2754 SDValue Width = DAG.getConstant(Val: 8 / VecLen, DL: dl, VT: MVT::i32);
2755 SDValue Idx = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MVT::i32, N1: IdxV, N2: Width);
2756 SDValue Ins =
2757 DAG.getNode(Opcode: HexagonISD::INSERT, DL: dl, VT: MVT::i32, Ops: {ToReg, Ext, Width, Idx});
2758 return getInstr(MachineOpc: Hexagon::C2_tfrrp, dl, Ty: VecTy, Ops: {Ins}, DAG);
2759 }
2760
2761 assert(ValTy.getVectorElementType() == MVT::i1);
2762 SDValue ValR = ValTy.isVector()
2763 ? DAG.getNode(Opcode: HexagonISD::P2D, DL: dl, VT: MVT::i64, Operand: ValV)
2764 : DAG.getSExtOrTrunc(Op: ValV, DL: dl, VT: MVT::i64);
2765
2766 unsigned Scale = VecLen / ValTy.getVectorNumElements();
2767 assert(Scale > 1);
2768
2769 for (unsigned R = Scale; R > 1; R /= 2) {
2770 ValR = contractPredicate(Vec64: ValR, dl, DAG);
2771 ValR = getCombine(Hi: DAG.getUNDEF(VT: MVT::i32), Lo: ValR, dl, ResTy: MVT::i64, DAG);
2772 }
2773
2774 SDValue Width = DAG.getConstant(Val: 64 / Scale, DL: dl, VT: MVT::i32);
2775 SDValue Idx = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MVT::i32, N1: IdxV, N2: Width);
2776 SDValue VecR = DAG.getNode(Opcode: HexagonISD::P2D, DL: dl, VT: MVT::i64, Operand: VecV);
2777 SDValue Ins =
2778 DAG.getNode(Opcode: HexagonISD::INSERT, DL: dl, VT: MVT::i64, Ops: {VecR, ValR, Width, Idx});
2779 return DAG.getNode(Opcode: HexagonISD::D2P, DL: dl, VT: VecTy, Operand: Ins);
2780}
2781
2782SDValue
2783HexagonTargetLowering::expandPredicate(SDValue Vec32, const SDLoc &dl,
2784 SelectionDAG &DAG) const {
2785 assert(ty(Vec32).getSizeInBits() == 32);
2786 if (isUndef(Op: Vec32))
2787 return DAG.getUNDEF(VT: MVT::i64);
2788 SDValue P = DAG.getBitcast(VT: MVT::v4i8, V: Vec32);
2789 SDValue X = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: MVT::v4i16, Operand: P);
2790 return DAG.getBitcast(VT: MVT::i64, V: X);
2791}
2792
2793SDValue
2794HexagonTargetLowering::contractPredicate(SDValue Vec64, const SDLoc &dl,
2795 SelectionDAG &DAG) const {
2796 assert(ty(Vec64).getSizeInBits() == 64);
2797 if (isUndef(Op: Vec64))
2798 return DAG.getUNDEF(VT: MVT::i32);
2799 // Collect even bytes:
2800 SDValue A = DAG.getBitcast(VT: MVT::v8i8, V: Vec64);
2801 SDValue S = DAG.getVectorShuffle(VT: MVT::v8i8, dl, N1: A, N2: DAG.getUNDEF(VT: MVT::v8i8),
2802 Mask: {0, 2, 4, 6, 1, 3, 5, 7});
2803 return extractVector(VecV: S, IdxV: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32), dl, ValTy: MVT::v4i8,
2804 ResTy: MVT::i32, DAG);
2805}
2806
2807SDValue
2808HexagonTargetLowering::getZero(const SDLoc &dl, MVT Ty, SelectionDAG &DAG)
2809 const {
2810 if (Ty.isVector()) {
2811 unsigned W = Ty.getSizeInBits();
2812 if (W <= 64)
2813 return DAG.getBitcast(VT: Ty, V: DAG.getConstant(Val: 0, DL: dl, VT: MVT::getIntegerVT(BitWidth: W)));
2814 return DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: Ty, Operand: getZero(dl, Ty: MVT::i32, DAG));
2815 }
2816
2817 if (Ty.isInteger())
2818 return DAG.getConstant(Val: 0, DL: dl, VT: Ty);
2819 if (Ty.isFloatingPoint())
2820 return DAG.getConstantFP(Val: 0.0, DL: dl, VT: Ty);
2821 llvm_unreachable("Invalid type for zero");
2822}
2823
2824SDValue
2825HexagonTargetLowering::appendUndef(SDValue Val, MVT ResTy, SelectionDAG &DAG)
2826 const {
2827 MVT ValTy = ty(Op: Val);
2828 assert(ValTy.getVectorElementType() == ResTy.getVectorElementType());
2829
2830 unsigned ValLen = ValTy.getVectorNumElements();
2831 unsigned ResLen = ResTy.getVectorNumElements();
2832 if (ValLen == ResLen)
2833 return Val;
2834
2835 const SDLoc &dl(Val);
2836 assert(ValLen < ResLen);
2837 assert(ResLen % ValLen == 0);
2838
2839 SmallVector<SDValue, 4> Concats = {Val};
2840 for (unsigned i = 1, e = ResLen / ValLen; i < e; ++i)
2841 Concats.push_back(Elt: DAG.getUNDEF(VT: ValTy));
2842
2843 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResTy, Ops: Concats);
2844}
2845
2846SDValue
2847HexagonTargetLowering::getCombine(SDValue Hi, SDValue Lo, const SDLoc &dl,
2848 MVT ResTy, SelectionDAG &DAG) const {
2849 MVT ElemTy = ty(Op: Hi);
2850 assert(ElemTy == ty(Lo));
2851
2852 if (!ElemTy.isVector()) {
2853 assert(ElemTy.isScalarInteger());
2854 MVT PairTy = MVT::getIntegerVT(BitWidth: 2 * ElemTy.getSizeInBits());
2855 SDValue Pair = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: PairTy, N1: Lo, N2: Hi);
2856 return DAG.getBitcast(VT: ResTy, V: Pair);
2857 }
2858
2859 unsigned Width = ElemTy.getSizeInBits();
2860 MVT IntTy = MVT::getIntegerVT(BitWidth: Width);
2861 MVT PairTy = MVT::getIntegerVT(BitWidth: 2 * Width);
2862 SDValue Pair =
2863 DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: PairTy,
2864 Ops: {DAG.getBitcast(VT: IntTy, V: Lo), DAG.getBitcast(VT: IntTy, V: Hi)});
2865 return DAG.getBitcast(VT: ResTy, V: Pair);
2866}
2867
2868SDValue
2869HexagonTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
2870 MVT VecTy = ty(Op);
2871 unsigned BW = VecTy.getSizeInBits();
2872 const SDLoc &dl(Op);
2873 SmallVector<SDValue,8> Ops;
2874 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i)
2875 Ops.push_back(Elt: Op.getOperand(i));
2876
2877 if (BW == 32)
2878 return buildVector32(Elem: Ops, dl, VecTy, DAG);
2879 if (BW == 64)
2880 return buildVector64(Elem: Ops, dl, VecTy, DAG);
2881
2882 if (VecTy == MVT::v8i1 || VecTy == MVT::v4i1 || VecTy == MVT::v2i1) {
2883 // Check if this is a special case or all-0 or all-1.
2884 bool All0 = true, All1 = true;
2885 for (SDValue P : Ops) {
2886 auto *CN = dyn_cast<ConstantSDNode>(Val: P.getNode());
2887 if (CN == nullptr) {
2888 All0 = All1 = false;
2889 break;
2890 }
2891 uint32_t C = CN->getZExtValue();
2892 All0 &= (C == 0);
2893 All1 &= (C == 1);
2894 }
2895 if (All0)
2896 return DAG.getNode(Opcode: HexagonISD::PFALSE, DL: dl, VT: VecTy);
2897 if (All1)
2898 return DAG.getNode(Opcode: HexagonISD::PTRUE, DL: dl, VT: VecTy);
2899
2900 // For each i1 element in the resulting predicate register, put 1
2901 // shifted by the index of the element into a general-purpose register,
2902 // then or them together and transfer it back into a predicate register.
2903 SDValue Rs[8];
2904 SDValue Z = getZero(dl, Ty: MVT::i32, DAG);
2905 // Always produce 8 bits, repeat inputs if necessary.
2906 unsigned Rep = 8 / VecTy.getVectorNumElements();
2907 for (unsigned i = 0; i != 8; ++i) {
2908 SDValue S = DAG.getConstant(Val: 1ull << i, DL: dl, VT: MVT::i32);
2909 Rs[i] = DAG.getSelect(DL: dl, VT: MVT::i32, Cond: Ops[i/Rep], LHS: S, RHS: Z);
2910 }
2911 for (ArrayRef<SDValue> A(Rs); A.size() != 1; A = A.drop_back(N: A.size()/2)) {
2912 for (unsigned i = 0, e = A.size()/2; i != e; ++i)
2913 Rs[i] = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: MVT::i32, N1: Rs[2*i], N2: Rs[2*i+1]);
2914 }
2915 // Move the value directly to a predicate register.
2916 return getInstr(MachineOpc: Hexagon::C2_tfrrp, dl, Ty: VecTy, Ops: {Rs[0]}, DAG);
2917 }
2918
2919 return SDValue();
2920}
2921
2922SDValue
2923HexagonTargetLowering::LowerCONCAT_VECTORS(SDValue Op,
2924 SelectionDAG &DAG) const {
2925 MVT VecTy = ty(Op);
2926 const SDLoc &dl(Op);
2927 if (VecTy.getSizeInBits() == 64) {
2928 assert(Op.getNumOperands() == 2);
2929 return getCombine(Hi: Op.getOperand(i: 1), Lo: Op.getOperand(i: 0), dl, ResTy: VecTy, DAG);
2930 }
2931
2932 MVT ElemTy = VecTy.getVectorElementType();
2933 if (ElemTy == MVT::i1) {
2934 assert(VecTy == MVT::v2i1 || VecTy == MVT::v4i1 || VecTy == MVT::v8i1);
2935 MVT OpTy = ty(Op: Op.getOperand(i: 0));
2936 // Scale is how many times the operands need to be contracted to match
2937 // the representation in the target register.
2938 unsigned Scale = VecTy.getVectorNumElements() / OpTy.getVectorNumElements();
2939 assert(Scale == Op.getNumOperands() && Scale > 1);
2940
2941 // First, convert all bool vectors to integers, then generate pairwise
2942 // inserts to form values of doubled length. Up until there are only
2943 // two values left to concatenate, all of these values will fit in a
2944 // 32-bit integer, so keep them as i32 to use 32-bit inserts.
2945 SmallVector<SDValue,4> Words[2];
2946 unsigned IdxW = 0;
2947
2948 for (SDValue P : Op.getNode()->op_values()) {
2949 SDValue W = DAG.getNode(Opcode: HexagonISD::P2D, DL: dl, VT: MVT::i64, Operand: P);
2950 for (unsigned R = Scale; R > 1; R /= 2) {
2951 W = contractPredicate(Vec64: W, dl, DAG);
2952 W = getCombine(Hi: DAG.getUNDEF(VT: MVT::i32), Lo: W, dl, ResTy: MVT::i64, DAG);
2953 }
2954 W = LoHalf(V: W, DAG);
2955 Words[IdxW].push_back(Elt: W);
2956 }
2957
2958 while (Scale > 2) {
2959 SDValue WidthV = DAG.getConstant(Val: 64 / Scale, DL: dl, VT: MVT::i32);
2960 Words[IdxW ^ 1].clear();
2961
2962 for (unsigned i = 0, e = Words[IdxW].size(); i != e; i += 2) {
2963 SDValue W0 = Words[IdxW][i], W1 = Words[IdxW][i+1];
2964 // Insert W1 into W0 right next to the significant bits of W0.
2965 SDValue T = DAG.getNode(Opcode: HexagonISD::INSERT, DL: dl, VT: MVT::i32,
2966 Ops: {W0, W1, WidthV, WidthV});
2967 Words[IdxW ^ 1].push_back(Elt: T);
2968 }
2969 IdxW ^= 1;
2970 Scale /= 2;
2971 }
2972
2973 // At this point there should only be two words left, and Scale should be 2.
2974 assert(Scale == 2 && Words[IdxW].size() == 2);
2975
2976 SDValue WW = getCombine(Hi: Words[IdxW][1], Lo: Words[IdxW][0], dl, ResTy: MVT::i64, DAG);
2977 return DAG.getNode(Opcode: HexagonISD::D2P, DL: dl, VT: VecTy, Operand: WW);
2978 }
2979
2980 return SDValue();
2981}
2982
2983SDValue
2984HexagonTargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
2985 SelectionDAG &DAG) const {
2986 SDValue Vec = Op.getOperand(i: 0);
2987 MVT ElemTy = ty(Op: Vec).getVectorElementType();
2988 return extractVector(VecV: Vec, IdxV: Op.getOperand(i: 1), dl: SDLoc(Op), ValTy: ElemTy, ResTy: ty(Op), DAG);
2989}
2990
2991SDValue
2992HexagonTargetLowering::LowerEXTRACT_SUBVECTOR(SDValue Op,
2993 SelectionDAG &DAG) const {
2994 return extractVector(VecV: Op.getOperand(i: 0), IdxV: Op.getOperand(i: 1), dl: SDLoc(Op),
2995 ValTy: ty(Op), ResTy: ty(Op), DAG);
2996}
2997
2998SDValue
2999HexagonTargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
3000 SelectionDAG &DAG) const {
3001 return insertVector(VecV: Op.getOperand(i: 0), ValV: Op.getOperand(i: 1), IdxV: Op.getOperand(i: 2),
3002 dl: SDLoc(Op), ValTy: ty(Op).getVectorElementType(), DAG);
3003}
3004
3005SDValue
3006HexagonTargetLowering::LowerINSERT_SUBVECTOR(SDValue Op,
3007 SelectionDAG &DAG) const {
3008 SDValue ValV = Op.getOperand(i: 1);
3009 return insertVector(VecV: Op.getOperand(i: 0), ValV, IdxV: Op.getOperand(i: 2),
3010 dl: SDLoc(Op), ValTy: ty(Op: ValV), DAG);
3011}
3012
3013bool
3014HexagonTargetLowering::allowTruncateForTailCall(Type *Ty1, Type *Ty2) const {
3015 // Assuming the caller does not have either a signext or zeroext modifier, and
3016 // only one value is accepted, any reasonable truncation is allowed.
3017 if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy())
3018 return false;
3019
3020 // FIXME: in principle up to 64-bit could be made safe, but it would be very
3021 // fragile at the moment: any support for multiple value returns would be
3022 // liable to disallow tail calls involving i64 -> iN truncation in many cases.
3023 return Ty1->getPrimitiveSizeInBits() <= 32;
3024}
3025
3026SDValue
3027HexagonTargetLowering::LowerLoad(SDValue Op, SelectionDAG &DAG) const {
3028 MVT Ty = ty(Op);
3029 const SDLoc &dl(Op);
3030 LoadSDNode *LN = cast<LoadSDNode>(Val: Op.getNode());
3031 MVT MemTy = LN->getMemoryVT().getSimpleVT();
3032 ISD::LoadExtType ET = LN->getExtensionType();
3033
3034 bool LoadPred = MemTy == MVT::v2i1 || MemTy == MVT::v4i1 || MemTy == MVT::v8i1;
3035 if (LoadPred) {
3036 SDValue NL = DAG.getLoad(
3037 AM: LN->getAddressingMode(), ExtType: ISD::ZEXTLOAD, VT: MVT::i32, dl, Chain: LN->getChain(),
3038 Ptr: LN->getBasePtr(), Offset: LN->getOffset(), PtrInfo: LN->getPointerInfo(),
3039 /*MemoryVT*/ MemVT: MVT::i8, Alignment: LN->getAlign(), MMOFlags: LN->getMemOperand()->getFlags(),
3040 AAInfo: LN->getAAInfo(), Ranges: LN->getRanges());
3041 LN = cast<LoadSDNode>(Val: NL.getNode());
3042 }
3043
3044 Align ClaimAlign = LN->getAlign();
3045 if (!validateConstPtrAlignment(Ptr: LN->getBasePtr(), NeedAlign: ClaimAlign, dl, DAG))
3046 return replaceMemWithUndef(Op, DAG);
3047
3048 // Call LowerUnalignedLoad for all loads, it recognizes loads that
3049 // don't need extra aligning.
3050 SDValue LU = LowerUnalignedLoad(Op: SDValue(LN, 0), DAG);
3051 if (LoadPred) {
3052 SDValue TP = getInstr(MachineOpc: Hexagon::C2_tfrrp, dl, Ty: MemTy, Ops: {LU}, DAG);
3053 if (ET == ISD::SEXTLOAD) {
3054 TP = DAG.getSExtOrTrunc(Op: TP, DL: dl, VT: Ty);
3055 } else if (ET != ISD::NON_EXTLOAD) {
3056 TP = DAG.getZExtOrTrunc(Op: TP, DL: dl, VT: Ty);
3057 }
3058 SDValue Ch = cast<LoadSDNode>(Val: LU.getNode())->getChain();
3059 return DAG.getMergeValues(Ops: {TP, Ch}, dl);
3060 }
3061 return LU;
3062}
3063
3064SDValue
3065HexagonTargetLowering::LowerStore(SDValue Op, SelectionDAG &DAG) const {
3066 const SDLoc &dl(Op);
3067 StoreSDNode *SN = cast<StoreSDNode>(Val: Op.getNode());
3068 SDValue Val = SN->getValue();
3069 MVT Ty = ty(Op: Val);
3070
3071 if (Ty == MVT::v2i1 || Ty == MVT::v4i1 || Ty == MVT::v8i1) {
3072 // Store the exact predicate (all bits).
3073 SDValue TR = getInstr(MachineOpc: Hexagon::C2_tfrpr, dl, Ty: MVT::i32, Ops: {Val}, DAG);
3074 SDValue NS = DAG.getTruncStore(Chain: SN->getChain(), dl, Val: TR, Ptr: SN->getBasePtr(),
3075 SVT: MVT::i8, MMO: SN->getMemOperand());
3076 if (SN->isIndexed()) {
3077 NS = DAG.getIndexedStore(OrigStore: NS, dl, Base: SN->getBasePtr(), Offset: SN->getOffset(),
3078 AM: SN->getAddressingMode());
3079 }
3080 SN = cast<StoreSDNode>(Val: NS.getNode());
3081 }
3082
3083 Align ClaimAlign = SN->getAlign();
3084 if (!validateConstPtrAlignment(Ptr: SN->getBasePtr(), NeedAlign: ClaimAlign, dl, DAG))
3085 return replaceMemWithUndef(Op, DAG);
3086
3087 MVT StoreTy = SN->getMemoryVT().getSimpleVT();
3088 Align NeedAlign = Subtarget.getTypeAlignment(Ty: StoreTy);
3089 if (ClaimAlign < NeedAlign)
3090 return expandUnalignedStore(ST: SN, DAG);
3091 return SDValue(SN, 0);
3092}
3093
3094SDValue
3095HexagonTargetLowering::LowerUnalignedLoad(SDValue Op, SelectionDAG &DAG)
3096 const {
3097 LoadSDNode *LN = cast<LoadSDNode>(Val: Op.getNode());
3098 MVT LoadTy = ty(Op);
3099 unsigned NeedAlign = Subtarget.getTypeAlignment(Ty: LoadTy).value();
3100 unsigned HaveAlign = LN->getAlign().value();
3101 if (HaveAlign >= NeedAlign)
3102 return Op;
3103
3104 const SDLoc &dl(Op);
3105 const DataLayout &DL = DAG.getDataLayout();
3106 LLVMContext &Ctx = *DAG.getContext();
3107
3108 // If the load aligning is disabled or the load can be broken up into two
3109 // smaller legal loads, do the default (target-independent) expansion.
3110 bool DoDefault = false;
3111 // Handle it in the default way if this is an indexed load.
3112 if (!LN->isUnindexed())
3113 DoDefault = true;
3114
3115 if (!AlignLoads) {
3116 if (allowsMemoryAccessForAlignment(Context&: Ctx, DL, VT: LN->getMemoryVT(),
3117 MMO: *LN->getMemOperand()))
3118 return Op;
3119 DoDefault = true;
3120 }
3121 if (!DoDefault && (2 * HaveAlign) == NeedAlign) {
3122 // The PartTy is the equivalent of "getLoadableTypeOfSize(HaveAlign)".
3123 MVT PartTy = HaveAlign <= 8 ? MVT::getIntegerVT(BitWidth: 8 * HaveAlign)
3124 : MVT::getVectorVT(VT: MVT::i8, NumElements: HaveAlign);
3125 DoDefault =
3126 allowsMemoryAccessForAlignment(Context&: Ctx, DL, VT: PartTy, MMO: *LN->getMemOperand());
3127 }
3128 if (DoDefault) {
3129 std::pair<SDValue, SDValue> P = expandUnalignedLoad(LD: LN, DAG);
3130 return DAG.getMergeValues(Ops: {P.first, P.second}, dl);
3131 }
3132
3133 // The code below generates two loads, both aligned as NeedAlign, and
3134 // with the distance of NeedAlign between them. For that to cover the
3135 // bits that need to be loaded (and without overlapping), the size of
3136 // the loads should be equal to NeedAlign. This is true for all loadable
3137 // types, but add an assertion in case something changes in the future.
3138 assert(LoadTy.getSizeInBits() == 8*NeedAlign);
3139
3140 unsigned LoadLen = NeedAlign;
3141 SDValue Base = LN->getBasePtr();
3142 SDValue Chain = LN->getChain();
3143 auto BO = getBaseAndOffset(Addr: Base);
3144 unsigned BaseOpc = BO.first.getOpcode();
3145 if (BaseOpc == HexagonISD::VALIGNADDR && BO.second % LoadLen == 0)
3146 return Op;
3147
3148 if (BO.second % LoadLen != 0) {
3149 BO.first = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::i32, N1: BO.first,
3150 N2: DAG.getConstant(Val: BO.second % LoadLen, DL: dl, VT: MVT::i32));
3151 BO.second -= BO.second % LoadLen;
3152 }
3153 SDValue BaseNoOff = (BaseOpc != HexagonISD::VALIGNADDR)
3154 ? DAG.getNode(Opcode: HexagonISD::VALIGNADDR, DL: dl, VT: MVT::i32, N1: BO.first,
3155 N2: DAG.getConstant(Val: NeedAlign, DL: dl, VT: MVT::i32))
3156 : BO.first;
3157 SDValue Base0 =
3158 DAG.getMemBasePlusOffset(Base: BaseNoOff, Offset: TypeSize::getFixed(ExactSize: BO.second), DL: dl);
3159 SDValue Base1 = DAG.getMemBasePlusOffset(
3160 Base: BaseNoOff, Offset: TypeSize::getFixed(ExactSize: BO.second + LoadLen), DL: dl);
3161
3162 MachineMemOperand *WideMMO = nullptr;
3163 if (MachineMemOperand *MMO = LN->getMemOperand()) {
3164 MachineFunction &MF = DAG.getMachineFunction();
3165 WideMMO = MF.getMachineMemOperand(
3166 PtrInfo: MMO->getPointerInfo(), F: MMO->getFlags(), Size: 2 * LoadLen, BaseAlignment: Align(LoadLen),
3167 AAInfo: MMO->getAAInfo(), Ranges: MMO->getRanges(), SSID: MMO->getSyncScopeID(),
3168 Ordering: MMO->getSuccessOrdering(), FailureOrdering: MMO->getFailureOrdering());
3169 }
3170
3171 SDValue Load0 = DAG.getLoad(VT: LoadTy, dl, Chain, Ptr: Base0, MMO: WideMMO);
3172 SDValue Load1 = DAG.getLoad(VT: LoadTy, dl, Chain, Ptr: Base1, MMO: WideMMO);
3173
3174 SDValue Aligned = DAG.getNode(Opcode: HexagonISD::VALIGN, DL: dl, VT: LoadTy,
3175 Ops: {Load1, Load0, BaseNoOff.getOperand(i: 0)});
3176 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other,
3177 N1: Load0.getValue(R: 1), N2: Load1.getValue(R: 1));
3178 SDValue M = DAG.getMergeValues(Ops: {Aligned, NewChain}, dl);
3179 return M;
3180}
3181
3182SDValue
3183HexagonTargetLowering::LowerUAddSubO(SDValue Op, SelectionDAG &DAG) const {
3184 SDValue X = Op.getOperand(i: 0), Y = Op.getOperand(i: 1);
3185 auto *CY = dyn_cast<ConstantSDNode>(Val&: Y);
3186 if (!CY)
3187 return SDValue();
3188
3189 const SDLoc &dl(Op);
3190 SDVTList VTs = Op.getNode()->getVTList();
3191 assert(VTs.NumVTs == 2);
3192 assert(VTs.VTs[1] == MVT::i1);
3193 unsigned Opc = Op.getOpcode();
3194
3195 if (CY) {
3196 uint64_t VY = CY->getZExtValue();
3197 assert(VY != 0 && "This should have been folded");
3198 // X +/- 1
3199 if (VY != 1)
3200 return SDValue();
3201
3202 if (Opc == ISD::UADDO) {
3203 SDValue Op = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: VTs.VTs[0], Ops: {X, Y});
3204 SDValue Ov = DAG.getSetCC(DL: dl, VT: MVT::i1, LHS: Op, RHS: getZero(dl, Ty: ty(Op), DAG),
3205 Cond: ISD::SETEQ);
3206 return DAG.getMergeValues(Ops: {Op, Ov}, dl);
3207 }
3208 if (Opc == ISD::USUBO) {
3209 SDValue Op = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: VTs.VTs[0], Ops: {X, Y});
3210 SDValue Ov = DAG.getSetCC(DL: dl, VT: MVT::i1, LHS: Op,
3211 RHS: DAG.getAllOnesConstant(DL: dl, VT: ty(Op)), Cond: ISD::SETEQ);
3212 return DAG.getMergeValues(Ops: {Op, Ov}, dl);
3213 }
3214 }
3215
3216 return SDValue();
3217}
3218
3219SDValue HexagonTargetLowering::LowerUAddSubOCarry(SDValue Op,
3220 SelectionDAG &DAG) const {
3221 const SDLoc &dl(Op);
3222 unsigned Opc = Op.getOpcode();
3223 SDValue X = Op.getOperand(i: 0), Y = Op.getOperand(i: 1), C = Op.getOperand(i: 2);
3224
3225 if (Opc == ISD::UADDO_CARRY)
3226 return DAG.getNode(Opcode: HexagonISD::ADDC, DL: dl, VTList: Op.getNode()->getVTList(),
3227 Ops: { X, Y, C });
3228
3229 EVT CarryTy = C.getValueType();
3230 SDValue SubC = DAG.getNode(Opcode: HexagonISD::SUBC, DL: dl, VTList: Op.getNode()->getVTList(),
3231 Ops: { X, Y, DAG.getLogicalNOT(DL: dl, Val: C, VT: CarryTy) });
3232 SDValue Out[] = { SubC.getValue(R: 0),
3233 DAG.getLogicalNOT(DL: dl, Val: SubC.getValue(R: 1), VT: CarryTy) };
3234 return DAG.getMergeValues(Ops: Out, dl);
3235}
3236
3237SDValue
3238HexagonTargetLowering::LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const {
3239 SDValue Chain = Op.getOperand(i: 0);
3240 SDValue Offset = Op.getOperand(i: 1);
3241 SDValue Handler = Op.getOperand(i: 2);
3242 SDLoc dl(Op);
3243 auto PtrVT = getPointerTy(DL: DAG.getDataLayout());
3244
3245 // Mark function as containing a call to EH_RETURN.
3246 HexagonMachineFunctionInfo *FuncInfo =
3247 DAG.getMachineFunction().getInfo<HexagonMachineFunctionInfo>();
3248 FuncInfo->setHasEHReturn();
3249
3250 unsigned OffsetReg = Hexagon::R28;
3251
3252 SDValue StoreAddr =
3253 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: DAG.getRegister(Reg: Hexagon::R30, VT: PtrVT),
3254 N2: DAG.getIntPtrConstant(Val: 4, DL: dl));
3255 Chain = DAG.getStore(Chain, dl, Val: Handler, Ptr: StoreAddr, PtrInfo: MachinePointerInfo());
3256 Chain = DAG.getCopyToReg(Chain, dl, Reg: OffsetReg, N: Offset);
3257
3258 // Not needed we already use it as explicit input to EH_RETURN.
3259 // MF.getRegInfo().addLiveOut(OffsetReg);
3260
3261 return DAG.getNode(Opcode: HexagonISD::EH_RETURN, DL: dl, VT: MVT::Other, Operand: Chain);
3262}
3263
3264SDValue
3265HexagonTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
3266 unsigned Opc = Op.getOpcode();
3267 // Handle INLINEASM first.
3268 if (Opc == ISD::INLINEASM || Opc == ISD::INLINEASM_BR)
3269 return LowerINLINEASM(Op, DAG);
3270
3271 if (isHvxOperation(N: Op.getNode(), DAG)) {
3272 // If HVX lowering returns nothing, try the default lowering.
3273 if (SDValue V = LowerHvxOperation(Op, DAG))
3274 return V;
3275 }
3276
3277 switch (Opc) {
3278 default:
3279#ifndef NDEBUG
3280 Op.getNode()->dumpr(&DAG);
3281#endif
3282 llvm_unreachable("Should not custom lower this!");
3283
3284 case ISD::FDIV:
3285 return LowerFDIV(Op, DAG);
3286 case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG);
3287 case ISD::INSERT_SUBVECTOR: return LowerINSERT_SUBVECTOR(Op, DAG);
3288 case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG);
3289 case ISD::EXTRACT_SUBVECTOR: return LowerEXTRACT_SUBVECTOR(Op, DAG);
3290 case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG);
3291 case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG);
3292 case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG);
3293 case ISD::BITCAST: return LowerBITCAST(Op, DAG);
3294 case ISD::LOAD: return LowerLoad(Op, DAG);
3295 case ISD::STORE: return LowerStore(Op, DAG);
3296 case ISD::UADDO:
3297 case ISD::USUBO: return LowerUAddSubO(Op, DAG);
3298 case ISD::UADDO_CARRY:
3299 case ISD::USUBO_CARRY: return LowerUAddSubOCarry(Op, DAG);
3300 case ISD::SRA:
3301 case ISD::SHL:
3302 case ISD::SRL: return LowerVECTOR_SHIFT(Op, DAG);
3303 case ISD::ROTL: return LowerROTL(Op, DAG);
3304 case ISD::ConstantPool: return LowerConstantPool(Op, DAG);
3305 case ISD::JumpTable: return LowerJumpTable(Op, DAG);
3306 case ISD::EH_RETURN: return LowerEH_RETURN(Op, DAG);
3307 case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG);
3308 case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG);
3309 case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG);
3310 case ISD::ATOMIC_FENCE: return LowerATOMIC_FENCE(Op, DAG);
3311 case ISD::GlobalAddress: return LowerGLOBALADDRESS(Op, DAG);
3312 case ISD::BlockAddress: return LowerBlockAddress(Op, DAG);
3313 case ISD::GLOBAL_OFFSET_TABLE: return LowerGLOBAL_OFFSET_TABLE(Op, DAG);
3314 case ISD::VACOPY: return LowerVACOPY(Op, DAG);
3315 case ISD::VASTART: return LowerVASTART(Op, DAG);
3316 case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG);
3317 case ISD::SETCC: return LowerSETCC(Op, DAG);
3318 case ISD::VSELECT: return LowerVSELECT(Op, DAG);
3319 case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG);
3320 case ISD::INTRINSIC_VOID: return LowerINTRINSIC_VOID(Op, DAG);
3321 case ISD::PREFETCH:
3322 return LowerPREFETCH(Op, DAG);
3323 break;
3324 }
3325
3326 return SDValue();
3327}
3328
3329void
3330HexagonTargetLowering::LowerOperationWrapper(SDNode *N,
3331 SmallVectorImpl<SDValue> &Results,
3332 SelectionDAG &DAG) const {
3333 if (isHvxOperation(N, DAG)) {
3334 LowerHvxOperationWrapper(N, Results, DAG);
3335 if (!Results.empty())
3336 return;
3337 }
3338
3339 SDValue Op(N, 0);
3340 unsigned Opc = N->getOpcode();
3341
3342 switch (Opc) {
3343 case HexagonISD::SSAT:
3344 case HexagonISD::USAT:
3345 Results.push_back(Elt: opJoin(Ops: SplitVectorOp(Op, DAG), dl: SDLoc(Op), DAG));
3346 break;
3347 case ISD::STORE:
3348 // We are only custom-lowering stores to verify the alignment of the
3349 // address if it is a compile-time constant. Since a store can be
3350 // modified during type-legalization (the value being stored may need
3351 // legalization), return empty Results here to indicate that we don't
3352 // really make any changes in the custom lowering.
3353 return;
3354 default:
3355 TargetLowering::LowerOperationWrapper(N, Results, DAG);
3356 break;
3357 }
3358}
3359
3360void
3361HexagonTargetLowering::ReplaceNodeResults(SDNode *N,
3362 SmallVectorImpl<SDValue> &Results,
3363 SelectionDAG &DAG) const {
3364 if (isHvxOperation(N, DAG)) {
3365 ReplaceHvxNodeResults(N, Results, DAG);
3366 if (!Results.empty())
3367 return;
3368 }
3369
3370 const SDLoc &dl(N);
3371 switch (N->getOpcode()) {
3372 case ISD::SRL:
3373 case ISD::SRA:
3374 case ISD::SHL:
3375 return;
3376 case ISD::BITCAST:
3377 // Handle a bitcast from v8i1 to i8.
3378 if (N->getValueType(ResNo: 0) == MVT::i8) {
3379 if (N->getOperand(Num: 0).getValueType() == MVT::v8i1) {
3380 SDValue P = getInstr(MachineOpc: Hexagon::C2_tfrpr, dl, Ty: MVT::i32,
3381 Ops: N->getOperand(Num: 0), DAG);
3382 SDValue T = DAG.getAnyExtOrTrunc(Op: P, DL: dl, VT: MVT::i8);
3383 Results.push_back(Elt: T);
3384 }
3385 }
3386 break;
3387 }
3388}
3389
3390SDValue
3391HexagonTargetLowering::PerformDAGCombine(SDNode *N,
3392 DAGCombinerInfo &DCI) const {
3393 SDValue Op(N, 0);
3394 const SDLoc &dl(Op);
3395 unsigned Opc = Op.getOpcode();
3396
3397 // Combining transformations applicable for arbitrary vector sizes.
3398 if (DCI.isBeforeLegalizeOps()) {
3399 switch (Opc) {
3400 case ISD::VECREDUCE_ADD:
3401 if (SDValue V = splitVecReduceAdd(N, DAG&: DCI.DAG))
3402 return V;
3403 if (SDValue V = expandVecReduceAdd(N, DAG&: DCI.DAG))
3404 return V;
3405 return SDValue();
3406 case ISD::PARTIAL_REDUCE_SMLA:
3407 case ISD::PARTIAL_REDUCE_UMLA:
3408 case ISD::PARTIAL_REDUCE_SUMLA:
3409 if (SDValue V = splitExtendingPartialReduceMLA(N, DAG&: DCI.DAG))
3410 return V;
3411 return SDValue();
3412 }
3413 } else {
3414 switch (Opc) {
3415 case ISD::VSELECT: {
3416 // (vselect (xor x, ptrue), v0, v1) -> (vselect x, v1, v0)
3417 SDValue Cond = Op.getOperand(i: 0);
3418 if (Cond->getOpcode() == ISD::XOR) {
3419 SDValue C0 = Cond.getOperand(i: 0), C1 = Cond.getOperand(i: 1);
3420 if (C1->getOpcode() == HexagonISD::PTRUE) {
3421 SDValue VSel = DCI.DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: ty(Op), N1: C0,
3422 N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 1));
3423 return VSel;
3424 }
3425 }
3426 return SDValue();
3427 }
3428 }
3429 }
3430
3431 if (isHvxOperation(N, DAG&: DCI.DAG)) {
3432 if (SDValue V = PerformHvxDAGCombine(N, DCI))
3433 return V;
3434 return SDValue();
3435 }
3436
3437 if (Opc == ISD::TRUNCATE) {
3438 SDValue Op0 = Op.getOperand(i: 0);
3439 // fold (truncate (build pair x, y)) -> (truncate x) or x
3440 if (Op0.getOpcode() == ISD::BUILD_PAIR) {
3441 EVT TruncTy = Op.getValueType();
3442 SDValue Elem0 = Op0.getOperand(i: 0);
3443 // if we match the low element of the pair, just return it.
3444 if (Elem0.getValueType() == TruncTy)
3445 return Elem0;
3446 // otherwise, if the low part is still too large, apply the truncate.
3447 if (Elem0.getValueType().bitsGT(VT: TruncTy))
3448 return DCI.DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: TruncTy, Operand: Elem0);
3449 }
3450 }
3451
3452 if (DCI.isBeforeLegalizeOps())
3453 return SDValue();
3454
3455 switch (Opc) {
3456 case HexagonISD::P2D: {
3457 SDValue P = Op.getOperand(i: 0);
3458 switch (P.getOpcode()) {
3459 case HexagonISD::PTRUE:
3460 return DCI.DAG.getAllOnesConstant(DL: dl, VT: ty(Op));
3461 case HexagonISD::PFALSE:
3462 return getZero(dl, Ty: ty(Op), DAG&: DCI.DAG);
3463 default:
3464 break;
3465 }
3466 break;
3467 }
3468 case ISD::TRUNCATE: {
3469 SDValue Op0 = Op.getOperand(i: 0);
3470 // fold (truncate (build pair x, y)) -> (truncate x) or x
3471 if (Op0.getOpcode() == ISD::BUILD_PAIR) {
3472 MVT TruncTy = ty(Op);
3473 SDValue Elem0 = Op0.getOperand(i: 0);
3474 // if we match the low element of the pair, just return it.
3475 if (ty(Op: Elem0) == TruncTy)
3476 return Elem0;
3477 // otherwise, if the low part is still too large, apply the truncate.
3478 if (ty(Op: Elem0).bitsGT(VT: TruncTy))
3479 return DCI.DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: TruncTy, Operand: Elem0);
3480 }
3481 break;
3482 }
3483 case ISD::OR: {
3484 // fold (or (shl xx, s), (zext y)) -> (COMBINE (shl xx, s-32), y)
3485 // if s >= 32
3486 auto fold0 = [&, this](SDValue Op) {
3487 if (ty(Op) != MVT::i64)
3488 return SDValue();
3489 SDValue Shl = Op.getOperand(i: 0);
3490 SDValue Zxt = Op.getOperand(i: 1);
3491 if (Shl.getOpcode() != ISD::SHL)
3492 std::swap(a&: Shl, b&: Zxt);
3493
3494 if (Shl.getOpcode() != ISD::SHL || Zxt.getOpcode() != ISD::ZERO_EXTEND)
3495 return SDValue();
3496
3497 SDValue Z = Zxt.getOperand(i: 0);
3498 auto *Amt = dyn_cast<ConstantSDNode>(Val: Shl.getOperand(i: 1));
3499 if (Amt && Amt->getZExtValue() >= 32 && ty(Op: Z).getSizeInBits() <= 32) {
3500 unsigned A = Amt->getZExtValue();
3501 SDValue S = Shl.getOperand(i: 0);
3502 SDValue T0 = DCI.DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: ty(Op: S), N1: S,
3503 N2: DCI.DAG.getConstant(Val: A - 32, DL: dl, VT: MVT::i32));
3504 SDValue T1 = DCI.DAG.getZExtOrTrunc(Op: T0, DL: dl, VT: MVT::i32);
3505 SDValue T2 = DCI.DAG.getZExtOrTrunc(Op: Z, DL: dl, VT: MVT::i32);
3506 return DCI.DAG.getNode(Opcode: HexagonISD::COMBINE, DL: dl, VT: MVT::i64, Ops: {T1, T2});
3507 }
3508 return SDValue();
3509 };
3510
3511 if (SDValue R = fold0(Op))
3512 return R;
3513 break;
3514 }
3515 }
3516
3517 return SDValue();
3518}
3519
3520/// Returns relocation base for the given PIC jumptable.
3521SDValue
3522HexagonTargetLowering::getPICJumpTableRelocBase(SDValue Table,
3523 SelectionDAG &DAG) const {
3524 int Idx = cast<JumpTableSDNode>(Val&: Table)->getIndex();
3525 EVT VT = Table.getValueType();
3526 SDValue T = DAG.getTargetJumpTable(JTI: Idx, VT, TargetFlags: HexagonII::MO_PCREL);
3527 return DAG.getNode(Opcode: HexagonISD::AT_PCREL, DL: SDLoc(Table), VT, Operand: T);
3528}
3529
3530//===----------------------------------------------------------------------===//
3531// Inline Assembly Support
3532//===----------------------------------------------------------------------===//
3533
3534TargetLowering::ConstraintType
3535HexagonTargetLowering::getConstraintType(StringRef Constraint) const {
3536 if (Constraint.size() == 1) {
3537 switch (Constraint[0]) {
3538 case 'q':
3539 case 'v':
3540 if (Subtarget.useHVXOps())
3541 return C_RegisterClass;
3542 break;
3543 case 'a':
3544 return C_RegisterClass;
3545 default:
3546 break;
3547 }
3548 }
3549 return TargetLowering::getConstraintType(Constraint);
3550}
3551
3552std::pair<unsigned, const TargetRegisterClass*>
3553HexagonTargetLowering::getRegForInlineAsmConstraint(
3554 const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const {
3555
3556 if (Constraint.size() == 1) {
3557 switch (Constraint[0]) {
3558 case 'r': // R0-R31
3559 switch (VT.SimpleTy) {
3560 default:
3561 return {0u, nullptr};
3562 case MVT::i1:
3563 case MVT::i8:
3564 case MVT::i16:
3565 case MVT::i32:
3566 case MVT::f32:
3567 return {0u, &Hexagon::IntRegsRegClass};
3568 case MVT::i64:
3569 case MVT::f64:
3570 return {0u, &Hexagon::DoubleRegsRegClass};
3571 }
3572 break;
3573 case 'a': // M0-M1
3574 if (VT != MVT::i32)
3575 return {0u, nullptr};
3576 return {0u, &Hexagon::ModRegsRegClass};
3577 case 'q': // q0-q3
3578 switch (VT.getSizeInBits()) {
3579 default:
3580 return {0u, nullptr};
3581 case 64:
3582 case 128:
3583 return {0u, &Hexagon::HvxQRRegClass};
3584 }
3585 break;
3586 case 'v': // V0-V31
3587 switch (VT.getSizeInBits()) {
3588 default:
3589 return {0u, nullptr};
3590 case 512:
3591 return {0u, &Hexagon::HvxVRRegClass};
3592 case 1024:
3593 if (Subtarget.hasV60Ops() && Subtarget.useHVX128BOps())
3594 return {0u, &Hexagon::HvxVRRegClass};
3595 return {0u, &Hexagon::HvxWRRegClass};
3596 case 2048:
3597 return {0u, &Hexagon::HvxWRRegClass};
3598 }
3599 break;
3600 default:
3601 return {0u, nullptr};
3602 }
3603 }
3604
3605 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
3606}
3607
3608/// isFPImmLegal - Returns true if the target can instruction select the
3609/// specified FP immediate natively. If false, the legalizer will
3610/// materialize the FP immediate as a load from a constant pool.
3611bool HexagonTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
3612 bool ForCodeSize) const {
3613 return true;
3614}
3615
3616/// Returns true if it is beneficial to convert a load of a constant
3617/// to just the constant itself.
3618bool HexagonTargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
3619 Type *Ty) const {
3620 if (!ConstantLoadsToImm)
3621 return false;
3622
3623 assert(Ty->isIntegerTy());
3624 unsigned BitSize = Ty->getPrimitiveSizeInBits();
3625 return (BitSize > 0 && BitSize <= 64);
3626}
3627
3628/// isLegalAddressingMode - Return true if the addressing mode represented by
3629/// AM is legal for this target, for a load/store of the specified type.
3630bool HexagonTargetLowering::isLegalAddressingMode(const DataLayout &DL,
3631 const AddrMode &AM, Type *Ty,
3632 unsigned AS, Instruction *I) const {
3633 if (Ty->isSized()) {
3634 // When LSR detects uses of the same base address to access different
3635 // types (e.g. unions), it will assume a conservative type for these
3636 // uses:
3637 // LSR Use: Kind=Address of void in addrspace(4294967295), ...
3638 // The type Ty passed here would then be "void". Skip the alignment
3639 // checks, but do not return false right away, since that confuses
3640 // LSR into crashing.
3641 Align A = DL.getABITypeAlign(Ty);
3642 // The base offset must be a multiple of the alignment.
3643 if (!isAligned(Lhs: A, SizeInBytes: AM.BaseOffs))
3644 return false;
3645 // The shifted offset must fit in 11 bits.
3646 if (!isInt<11>(x: AM.BaseOffs >> Log2(A)))
3647 return false;
3648 }
3649
3650 // No global is ever allowed as a base.
3651 if (AM.BaseGV)
3652 return false;
3653
3654 int Scale = AM.Scale;
3655 if (Scale < 0)
3656 Scale = -Scale;
3657 switch (Scale) {
3658 case 0: // No scale reg, "r+i", "r", or just "i".
3659 break;
3660 default: // No scaled addressing mode.
3661 return false;
3662 }
3663 return true;
3664}
3665
3666/// Return true if folding a constant offset with the given GlobalAddress is
3667/// legal. It is frequently not legal in PIC relocation models.
3668bool HexagonTargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA)
3669 const {
3670 return HTM.getRelocationModel() == Reloc::Static;
3671}
3672
3673/// isLegalICmpImmediate - Return true if the specified immediate is legal
3674/// icmp immediate, that is the target has icmp instructions which can compare
3675/// a register against the immediate without having to materialize the
3676/// immediate into a register.
3677bool HexagonTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
3678 return Imm >= -512 && Imm <= 511;
3679}
3680
3681/// IsEligibleForTailCallOptimization - Check whether the call is eligible
3682/// for tail call optimization. Targets which want to do tail call
3683/// optimization should implement this function.
3684bool HexagonTargetLowering::IsEligibleForTailCallOptimization(
3685 SDValue Callee,
3686 CallingConv::ID CalleeCC,
3687 bool IsVarArg,
3688 bool IsCalleeStructRet,
3689 bool IsCallerStructRet,
3690 const SmallVectorImpl<ISD::OutputArg> &Outs,
3691 const SmallVectorImpl<SDValue> &OutVals,
3692 const SmallVectorImpl<ISD::InputArg> &Ins,
3693 SelectionDAG& DAG) const {
3694 const Function &CallerF = DAG.getMachineFunction().getFunction();
3695 CallingConv::ID CallerCC = CallerF.getCallingConv();
3696 bool CCMatch = CallerCC == CalleeCC;
3697
3698 // ***************************************************************************
3699 // Look for obvious safe cases to perform tail call optimization that do not
3700 // require ABI changes.
3701 // ***************************************************************************
3702
3703 // If this is a tail call via a function pointer, then don't do it!
3704 if (!isa<GlobalAddressSDNode>(Val: Callee) &&
3705 !isa<ExternalSymbolSDNode>(Val: Callee)) {
3706 return false;
3707 }
3708
3709 // Do not optimize if the calling conventions do not match and the conventions
3710 // used are not C or Fast.
3711 if (!CCMatch) {
3712 bool R = (CallerCC == CallingConv::C || CallerCC == CallingConv::Fast);
3713 bool E = (CalleeCC == CallingConv::C || CalleeCC == CallingConv::Fast);
3714 // If R & E, then ok.
3715 if (!R || !E)
3716 return false;
3717 }
3718
3719 // Do not tail call optimize vararg calls.
3720 if (IsVarArg)
3721 return false;
3722
3723 // Also avoid tail call optimization if either caller or callee uses struct
3724 // return semantics.
3725 if (IsCalleeStructRet || IsCallerStructRet)
3726 return false;
3727
3728 // In addition to the cases above, we also disable Tail Call Optimization if
3729 // the calling convention code that at least one outgoing argument needs to
3730 // go on the stack. We cannot check that here because at this point that
3731 // information is not available.
3732 return true;
3733}
3734
3735/// Returns the target specific optimal type for load and store operations as
3736/// a result of memset, memcpy, and memmove lowering.
3737///
3738/// If DstAlign is zero that means it's safe to destination alignment can
3739/// satisfy any constraint. Similarly if SrcAlign is zero it means there isn't
3740/// a need to check it against alignment requirement, probably because the
3741/// source does not need to be loaded. If 'IsMemset' is true, that means it's
3742/// expanding a memset. If 'ZeroMemset' is true, that means it's a memset of
3743/// zero. 'MemcpyStrSrc' indicates whether the memcpy source is constant so it
3744/// does not need to be loaded. It returns EVT::Other if the type should be
3745/// determined using generic target-independent logic.
3746EVT HexagonTargetLowering::getOptimalMemOpType(
3747 LLVMContext &Context, const MemOp &Op,
3748 const AttributeList &FuncAttributes) const {
3749 if (Op.size() >= 8 && Op.isAligned(AlignCheck: Align(8)))
3750 return MVT::i64;
3751 if (Op.size() >= 4 && Op.isAligned(AlignCheck: Align(4)))
3752 return MVT::i32;
3753 if (Op.size() >= 2 && Op.isAligned(AlignCheck: Align(2)))
3754 return MVT::i16;
3755 return MVT::Other;
3756}
3757
3758// The helpers below are versions of llvm::getShuffleReduction and
3759// llvm::getOrderedReduction, adapted to use during DAG passes and simplified as
3760// follows:
3761// - ICmp and FCmp are not handled;
3762// - in every step in getShuffleReduction, the input is split into halves (not
3763// pairwise).
3764
3765static SDValue getOrderedReduction(SDValue Vec, unsigned Op,
3766 SelectionDAG &DAG) {
3767 assert(Op != Instruction::ICmp && Op != Instruction::FCmp);
3768
3769 EVT VT = Vec.getValueType();
3770 EVT EltT = VT.getVectorElementType();
3771 unsigned VF = VT.getVectorNumElements();
3772 assert(VF > 0 &&
3773 "Reduction emission only supported for non-zero length vectors!");
3774
3775 SDLoc DL(Vec);
3776 SDValue Result = DAG.getExtractVectorElt(DL, VT: EltT, Vec, Idx: 0);
3777 for (unsigned ExtractIdx = 1; ExtractIdx < VF; ++ExtractIdx) {
3778 SDValue Ext = DAG.getExtractVectorElt(DL, VT: EltT, Vec, Idx: ExtractIdx);
3779 Result = DAG.getNode(Opcode: Op, DL, VT: EltT, Ops: {Result, Ext});
3780 }
3781
3782 return Result;
3783}
3784
3785static SDValue getShuffleReduction(SDValue Vec, unsigned Op,
3786 SelectionDAG &DAG) {
3787 assert(Op != Instruction::ICmp && Op != Instruction::FCmp);
3788
3789 EVT VT = Vec.getValueType();
3790 unsigned VF = VT.getVectorNumElements();
3791 if (VF == 0)
3792 llvm_unreachable("Vector must be non-zero length");
3793 // VF is a power of 2 so we can emit the reduction using log2(VF) shuffles
3794 // and vector ops, reducing the set of values being computed by half each
3795 // round.
3796 assert(isPowerOf2_32(VF) &&
3797 "Reduction emission only supported for pow2 vectors!");
3798
3799 SDLoc DL(Vec);
3800 // TODO: Is it correct to create double-vector shuffle and fill 3/4 of it with
3801 // undefs?
3802 SmallVector<int, 32> ShuffleMask(VF);
3803 for (unsigned i = VF; i > 1; i >>= 1) {
3804 // Move the upper half of the vector to the lower half.
3805 for (unsigned j = 0; j != i / 2; ++j)
3806 ShuffleMask[j] = i / 2 + j;
3807 // Fill the rest of the mask with undef.
3808 std::fill(first: &ShuffleMask[i / 2], last: ShuffleMask.end(), value: -1);
3809
3810 SDValue Shuf =
3811 DAG.getVectorShuffle(VT, dl: DL, N1: Vec, N2: DAG.getUNDEF(VT), Mask: ShuffleMask);
3812
3813 Vec = DAG.getNode(Opcode: Op, DL, VT, Ops: {Vec, Shuf});
3814 }
3815 // The result is in the first element of the vector.
3816 return DAG.getExtractVectorElt(DL, VT: VT.getVectorElementType(), Vec, Idx: 0);
3817}
3818
3819SDValue HexagonTargetLowering::expandVecReduceAdd(SDNode *N,
3820 SelectionDAG &DAG) const {
3821 // Since we disabled automatic reduction expansion, generate log2 ladder code
3822 // if the vector is of a power-of-two length.
3823 SDValue Input = N->getOperand(Num: 0);
3824 if (isPowerOf2_32(Value: Input.getValueType().getVectorNumElements()))
3825 return getShuffleReduction(Vec: Input, Op: ISD::ADD, DAG);
3826 // Otherwise, reduction will be scalarized.
3827 return getOrderedReduction(Vec: Input, Op: ISD::ADD, DAG);
3828}
3829
3830bool HexagonTargetLowering::allowsMemoryAccess(
3831 LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace,
3832 Align Alignment, MachineMemOperand::Flags Flags, unsigned *Fast) const {
3833 if (!VT.isSimple())
3834 return false;
3835 MVT SVT = VT.getSimpleVT();
3836 if (Subtarget.isHVXVectorType(VecTy: SVT, IncludeBool: true))
3837 return allowsHvxMemoryAccess(VecTy: SVT, Flags, Fast);
3838 return TargetLoweringBase::allowsMemoryAccess(
3839 Context, DL, VT, AddrSpace, Alignment, Flags, Fast);
3840}
3841
3842bool HexagonTargetLowering::allowsMisalignedMemoryAccesses(
3843 EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags,
3844 unsigned *Fast) const {
3845 if (!VT.isSimple())
3846 return false;
3847 MVT SVT = VT.getSimpleVT();
3848 if (Subtarget.isHVXVectorType(VecTy: SVT, IncludeBool: true))
3849 return allowsHvxMisalignedMemoryAccesses(VecTy: SVT, Flags, Fast);
3850 if (Fast)
3851 *Fast = 0;
3852 return false;
3853}
3854
3855std::pair<const TargetRegisterClass*, uint8_t>
3856HexagonTargetLowering::findRepresentativeClass(const TargetRegisterInfo *TRI,
3857 MVT VT) const {
3858 if (Subtarget.isHVXVectorType(VecTy: VT, IncludeBool: true)) {
3859 unsigned BitWidth = VT.getSizeInBits();
3860 unsigned VecWidth = Subtarget.getVectorLength() * 8;
3861
3862 if (VT.getVectorElementType() == MVT::i1)
3863 return std::make_pair(x: &Hexagon::HvxQRRegClass, y: 1);
3864 if (BitWidth == VecWidth)
3865 return std::make_pair(x: &Hexagon::HvxVRRegClass, y: 1);
3866 assert(BitWidth == 2 * VecWidth);
3867 return std::make_pair(x: &Hexagon::HvxWRRegClass, y: 1);
3868 }
3869
3870 return TargetLowering::findRepresentativeClass(TRI, VT);
3871}
3872
3873bool HexagonTargetLowering::shouldReduceLoadWidth(
3874 SDNode *Load, ISD::LoadExtType ExtTy, EVT NewVT,
3875 std::optional<unsigned> ByteOffset) const {
3876 // TODO: This may be worth removing. Check regression tests for diffs.
3877 if (!TargetLoweringBase::shouldReduceLoadWidth(Load, ExtTy, NewVT,
3878 ByteOffset))
3879 return false;
3880
3881 auto *L = cast<LoadSDNode>(Val: Load);
3882 std::pair<SDValue, int> BO = getBaseAndOffset(Addr: L->getBasePtr());
3883 // Small-data object, do not shrink.
3884 if (BO.first.getOpcode() == HexagonISD::CONST32_GP)
3885 return false;
3886 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Val&: BO.first)) {
3887 auto &HTM = static_cast<const HexagonTargetMachine &>(getTargetMachine());
3888 const auto *GO = dyn_cast_or_null<const GlobalObject>(Val: GA->getGlobal());
3889 return !GO || !HTM.getObjFileLowering()->isGlobalInSmallSection(GO, TM: HTM);
3890 }
3891 return true;
3892}
3893
3894void HexagonTargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI,
3895 SDNode *Node) const {
3896 AdjustHvxInstrPostInstrSelection(MI, Node);
3897}
3898
3899Value *HexagonTargetLowering::emitLoadLinked(IRBuilderBase &Builder,
3900 Type *ValueTy, Value *Addr,
3901 AtomicOrdering Ord) const {
3902 unsigned SZ = ValueTy->getPrimitiveSizeInBits();
3903 assert((SZ == 32 || SZ == 64) && "Only 32/64-bit atomic loads supported");
3904 Intrinsic::ID IntID = (SZ == 32) ? Intrinsic::hexagon_L2_loadw_locked
3905 : Intrinsic::hexagon_L4_loadd_locked;
3906
3907 Value *Call =
3908 Builder.CreateIntrinsic(ID: IntID, Args: Addr, /*FMFSource=*/nullptr, Name: "larx");
3909
3910 return Builder.CreateBitCast(V: Call, DestTy: ValueTy);
3911}
3912
3913/// Perform a store-conditional operation to Addr. Return the status of the
3914/// store. This should be 0 if the store succeeded, non-zero otherwise.
3915Value *HexagonTargetLowering::emitStoreConditional(IRBuilderBase &Builder,
3916 Value *Val, Value *Addr,
3917 AtomicOrdering Ord) const {
3918 BasicBlock *BB = Builder.GetInsertBlock();
3919 Module *M = BB->getParent()->getParent();
3920 Type *Ty = Val->getType();
3921 unsigned SZ = Ty->getPrimitiveSizeInBits();
3922
3923 Type *CastTy = Builder.getIntNTy(N: SZ);
3924 assert((SZ == 32 || SZ == 64) && "Only 32/64-bit atomic stores supported");
3925 Intrinsic::ID IntID = (SZ == 32) ? Intrinsic::hexagon_S2_storew_locked
3926 : Intrinsic::hexagon_S4_stored_locked;
3927
3928 Val = Builder.CreateBitCast(V: Val, DestTy: CastTy);
3929
3930 Value *Call = Builder.CreateIntrinsic(ID: IntID, Args: {Addr, Val},
3931 /*FMFSource=*/nullptr, Name: "stcx");
3932 Value *Cmp = Builder.CreateICmpEQ(LHS: Call, RHS: Builder.getInt32(C: 0), Name: "");
3933 Value *Ext = Builder.CreateZExt(V: Cmp, DestTy: Type::getInt32Ty(C&: M->getContext()));
3934 return Ext;
3935}
3936
3937TargetLowering::AtomicExpansionKind
3938HexagonTargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const {
3939 // Do not expand loads and stores that don't exceed 64 bits.
3940 return LI->getType()->getPrimitiveSizeInBits() > 64
3941 ? AtomicExpansionKind::LLOnly
3942 : AtomicExpansionKind::None;
3943}
3944
3945TargetLowering::AtomicExpansionKind
3946HexagonTargetLowering::shouldExpandAtomicStoreInIR(StoreInst *SI) const {
3947 // Do not expand loads and stores that don't exceed 64 bits.
3948 return SI->getValueOperand()->getType()->getPrimitiveSizeInBits() > 64
3949 ? AtomicExpansionKind::Expand
3950 : AtomicExpansionKind::None;
3951}
3952
3953TargetLowering::AtomicExpansionKind
3954HexagonTargetLowering::shouldExpandAtomicCmpXchgInIR(
3955 const AtomicCmpXchgInst *AI) const {
3956 return AtomicExpansionKind::LLSC;
3957}
3958
3959bool HexagonTargetLowering::isMaskAndCmp0FoldingBeneficial(
3960 const Instruction &AndI) const {
3961 // Only sink 'and' mask to cmp use block if it is masking a single bit since
3962 // this will fold the and/cmp/br into a single tstbit instruction.
3963 ConstantInt *Mask = dyn_cast<ConstantInt>(Val: AndI.getOperand(i: 1));
3964 if (!Mask)
3965 return false;
3966 return Mask->getValue().isPowerOf2();
3967}
3968
3969// Check if the result of the node is only used as a return value, as
3970// otherwise we can't perform a tail-call.
3971bool HexagonTargetLowering::isUsedByReturnOnly(SDNode *N,
3972 SDValue &Chain) const {
3973 if (N->getNumValues() != 1)
3974 return false;
3975 if (!N->hasNUsesOfValue(NUses: 1, Value: 0))
3976 return false;
3977
3978 SDNode *Copy = *N->user_begin();
3979
3980 if (Copy->getOpcode() == ISD::BITCAST) {
3981 return isUsedByReturnOnly(N: Copy, Chain);
3982 }
3983
3984 if (Copy->getOpcode() != ISD::CopyToReg) {
3985 return false;
3986 }
3987
3988 // If the ISD::CopyToReg has a glue operand, we conservatively assume it
3989 // isn't safe to perform a tail call.
3990 if (Copy->getOperand(Num: Copy->getNumOperands() - 1).getValueType() == MVT::Glue)
3991 return false;
3992
3993 // The copy must be used by a HexagonISD::RET_GLUE, and nothing else.
3994 bool HasRet = false;
3995 for (SDNode *Node : Copy->users()) {
3996 if (Node->getOpcode() != HexagonISD::RET_GLUE)
3997 return false;
3998 HasRet = true;
3999 }
4000 if (!HasRet)
4001 return false;
4002
4003 Chain = Copy->getOperand(Num: 0);
4004 return true;
4005}
4006