1//===-- HexagonISelLowering.cpp - Hexagon DAG Lowering Implementation -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the interfaces that Hexagon uses to lower LLVM code
10// into a selection DAG.
11//
12//===----------------------------------------------------------------------===//
13
14#include "HexagonISelLowering.h"
15#include "Hexagon.h"
16#include "HexagonMachineFunctionInfo.h"
17#include "HexagonRegisterInfo.h"
18#include "HexagonSubtarget.h"
19#include "HexagonTargetMachine.h"
20#include "HexagonTargetObjectFile.h"
21#include "llvm/ADT/APInt.h"
22#include "llvm/ADT/ArrayRef.h"
23#include "llvm/ADT/SmallVector.h"
24#include "llvm/ADT/StringSwitch.h"
25#include "llvm/CodeGen/CallingConvLower.h"
26#include "llvm/CodeGen/MachineFrameInfo.h"
27#include "llvm/CodeGen/MachineFunction.h"
28#include "llvm/CodeGen/MachineMemOperand.h"
29#include "llvm/CodeGen/MachineRegisterInfo.h"
30#include "llvm/CodeGen/SelectionDAG.h"
31#include "llvm/CodeGen/TargetCallingConv.h"
32#include "llvm/CodeGen/ValueTypes.h"
33#include "llvm/IR/BasicBlock.h"
34#include "llvm/IR/CallingConv.h"
35#include "llvm/IR/DataLayout.h"
36#include "llvm/IR/DerivedTypes.h"
37#include "llvm/IR/DiagnosticInfo.h"
38#include "llvm/IR/DiagnosticPrinter.h"
39#include "llvm/IR/Function.h"
40#include "llvm/IR/GlobalValue.h"
41#include "llvm/IR/IRBuilder.h"
42#include "llvm/IR/InlineAsm.h"
43#include "llvm/IR/Instructions.h"
44#include "llvm/IR/IntrinsicInst.h"
45#include "llvm/IR/Intrinsics.h"
46#include "llvm/IR/IntrinsicsHexagon.h"
47#include "llvm/IR/Module.h"
48#include "llvm/IR/Type.h"
49#include "llvm/IR/Value.h"
50#include "llvm/Support/Casting.h"
51#include "llvm/Support/CodeGen.h"
52#include "llvm/Support/CommandLine.h"
53#include "llvm/Support/Debug.h"
54#include "llvm/Support/ErrorHandling.h"
55#include "llvm/Support/MathExtras.h"
56#include "llvm/Support/raw_ostream.h"
57#include "llvm/Target/TargetMachine.h"
58#include <algorithm>
59#include <cassert>
60#include <cstdint>
61#include <limits>
62#include <utility>
63
64using namespace llvm;
65
66#define DEBUG_TYPE "hexagon-lowering"
67
68static cl::opt<bool> EmitJumpTables("hexagon-emit-jump-tables",
69 cl::init(Val: true), cl::Hidden,
70 cl::desc("Control jump table emission on Hexagon target"));
71
72static cl::opt<bool>
73 EnableHexSDNodeSched("enable-hexagon-sdnode-sched", cl::Hidden,
74 cl::desc("Enable Hexagon SDNode scheduling"));
75
76static cl::opt<int> MinimumJumpTables("minimum-jump-tables", cl::Hidden,
77 cl::init(Val: 5),
78 cl::desc("Set minimum jump tables"));
79
80static cl::opt<bool>
81 ConstantLoadsToImm("constant-loads-to-imm", cl::Hidden, cl::init(Val: true),
82 cl::desc("Convert constant loads to immediate values."));
83
84static cl::opt<bool> AlignLoads("hexagon-align-loads",
85 cl::Hidden, cl::init(Val: false),
86 cl::desc("Rewrite unaligned loads as a pair of aligned loads"));
87
88static cl::opt<bool>
89 DisableArgsMinAlignment("hexagon-disable-args-min-alignment", cl::Hidden,
90 cl::init(Val: false),
91 cl::desc("Disable minimum alignment of 1 for "
92 "arguments passed by value on stack"));
93
94// Implement calling convention for Hexagon.
95
96static bool CC_SkipOdd(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
97 CCValAssign::LocInfo &LocInfo,
98 ISD::ArgFlagsTy &ArgFlags, CCState &State) {
99 static const MCPhysReg ArgRegs[] = {
100 Hexagon::R0, Hexagon::R1, Hexagon::R2,
101 Hexagon::R3, Hexagon::R4, Hexagon::R5
102 };
103 const unsigned NumArgRegs = std::size(ArgRegs);
104 unsigned RegNum = State.getFirstUnallocated(Regs: ArgRegs);
105
106 // RegNum is an index into ArgRegs: skip a register if RegNum is odd.
107 if (RegNum != NumArgRegs && RegNum % 2 == 1)
108 State.AllocateReg(Reg: ArgRegs[RegNum]);
109
110 // Always return false here, as this function only makes sure that the first
111 // unallocated register has an even register number and does not actually
112 // allocate a register for the current argument.
113 return false;
114}
115
116// HLX vectors that do not fit in registers are passed on the VTCM stack.
117// CCState only tracks the scalar stack, so add a custom memory location here
118// and let assignVTCMStackOffsets set its offset.
119static bool CC_Hexagon_VTCM_Stack(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
120 CCValAssign::LocInfo &LocInfo,
121 ISD::ArgFlagsTy &ArgFlags, CCState &State) {
122 State.addLoc(V: CCValAssign::getCustomMem(ValNo, ValVT, Offset: 0, LocVT, HTP: LocInfo));
123 return true;
124}
125
126// Assign VTCM stack offsets to the HLX arguments, in argument order, and
127// return the size of the VTCM argument area.
128static unsigned assignVTCMStackOffsets(SmallVectorImpl<CCValAssign> &ArgLocs) {
129 unsigned Size = 0;
130 for (CCValAssign &VA : ArgLocs) {
131 if (!VA.isMemLoc() || !VA.needsCustom())
132 continue;
133 VA.convertToMem(Offset: Size);
134 Size += VA.getLocVT().getStoreSize();
135 }
136 return Size;
137}
138
139#define GET_CALLING_CONV_IMPL
140#include "HexagonGenCallingConv.inc"
141
142unsigned HexagonTargetLowering::getVectorTypeBreakdownForCallingConv(
143 LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT,
144 unsigned &NumIntermediates, MVT &RegisterVT) const {
145
146 bool isBoolVector = VT.getVectorElementType() == MVT::i1;
147 bool isPowerOf2 = VT.isPow2VectorType();
148 unsigned NumElts = VT.getVectorNumElements();
149
150 // Split vectors of type vXi1 into (X/8) vectors of type v8i1,
151 // where X is divisible by 8.
152 if (isBoolVector && !Subtarget.useHVXOps() && isPowerOf2 && NumElts >= 8) {
153 RegisterVT = MVT::v8i8;
154 IntermediateVT = MVT::v8i1;
155 NumIntermediates = NumElts / 8;
156 return NumIntermediates;
157 }
158
159 // In HVX 64-byte mode, vectors of type vXi1 are split into (X / 64) vectors
160 // of type v64i1, provided that X is divisible by 64.
161 if (isBoolVector && Subtarget.useHVX64BOps() && isPowerOf2 && NumElts >= 64) {
162 RegisterVT = MVT::v64i8;
163 IntermediateVT = MVT::v64i1;
164 NumIntermediates = NumElts / 64;
165 return NumIntermediates;
166 }
167
168 // In HVX 128-byte mode, vectors of type vXi1 are split into (X / 128) vectors
169 // of type v128i1, provided that X is divisible by 128.
170 if (isBoolVector && Subtarget.useHVX128BOps() && isPowerOf2 &&
171 NumElts >= 128) {
172 RegisterVT = MVT::v128i8;
173 IntermediateVT = MVT::v128i1;
174 NumIntermediates = NumElts / 128;
175 return NumIntermediates;
176 }
177
178 return TargetLowering::getVectorTypeBreakdownForCallingConv(
179 Context, CC, VT, IntermediateVT, NumIntermediates, RegisterVT);
180}
181
182std::pair<MVT, unsigned>
183HexagonTargetLowering::handleMaskRegisterForCallingConv(
184 const HexagonSubtarget &Subtarget, EVT VT) const {
185 assert(VT.getVectorElementType() == MVT::i1);
186
187 const unsigned NumElems = VT.getVectorNumElements();
188
189 if (!VT.isPow2VectorType())
190 return {MVT::INVALID_SIMPLE_VALUE_TYPE, 0};
191
192 if (!Subtarget.useHVXOps() && NumElems >= 8)
193 return {MVT::v8i8, NumElems / 8};
194
195 if (Subtarget.useHVX64BOps() && NumElems >= 64)
196 return {MVT::v64i8, NumElems / 64};
197
198 if (Subtarget.useHVX128BOps() && NumElems >= 128)
199 return {MVT::v128i8, NumElems / 128};
200
201 return {MVT::INVALID_SIMPLE_VALUE_TYPE, 0};
202}
203
204MVT HexagonTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
205 CallingConv::ID CC,
206 EVT VT) const {
207
208 if (VT.isVectorOf(EltVT: MVT::i1)) {
209 auto [RegisterVT, NumRegisters] =
210 handleMaskRegisterForCallingConv(Subtarget, VT);
211 if (RegisterVT != MVT::INVALID_SIMPLE_VALUE_TYPE)
212 return RegisterVT;
213 }
214
215 return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
216}
217
218SDValue
219HexagonTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG)
220 const {
221 unsigned IntNo = Op.getConstantOperandVal(i: 0);
222 SDLoc dl(Op);
223 switch (IntNo) {
224 default:
225 return SDValue(); // Don't custom lower most intrinsics.
226 case Intrinsic::thread_pointer: {
227 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
228 return DAG.getNode(Opcode: HexagonISD::THREAD_POINTER, DL: dl, VT: PtrVT);
229 }
230 }
231}
232
233/// CreateCopyOfByValArgument - Make a copy of an aggregate at address specified
234/// by "Src" to address "Dst" of size "Size". Alignment information is
235/// specified by the specific parameter attribute. The copy will be passed as
236/// a byval function parameter. Sometimes what we are copying is the end of a
237/// larger object, the part that does not fit in registers.
238static SDValue CreateCopyOfByValArgument(SDValue Src, SDValue Dst,
239 SDValue Chain, ISD::ArgFlagsTy Flags,
240 SelectionDAG &DAG, const SDLoc &dl) {
241 SDValue SizeNode = DAG.getConstant(Val: Flags.getByValSize(), DL: dl, VT: MVT::i32);
242 Align Alignment = Flags.getNonZeroByValAlign();
243 return DAG.getMemcpy(Chain, dl, Dst, Src, Size: SizeNode, DstAlign: Alignment, SrcAlign: Alignment,
244 /*isVolatile=*/isVol: false, /*AlwaysInline=*/false,
245 /*CI=*/nullptr, OverrideTailCall: std::nullopt, DstPtrInfo: MachinePointerInfo(),
246 SrcPtrInfo: MachinePointerInfo());
247}
248
249bool
250HexagonTargetLowering::CanLowerReturn(
251 CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg,
252 const SmallVectorImpl<ISD::OutputArg> &Outs,
253 LLVMContext &Context, const Type *RetTy) const {
254 SmallVector<CCValAssign, 16> RVLocs;
255 CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
256
257 return CCInfo.CheckReturn(Outs, Fn: RetCC_Hexagon);
258}
259
260// LowerReturn - Lower ISD::RET. If a struct is larger than 8 bytes and is
261// passed by value, the function prototype is modified to return void and
262// the value is stored in memory pointed by a pointer passed by caller.
263SDValue
264HexagonTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
265 bool IsVarArg,
266 const SmallVectorImpl<ISD::OutputArg> &Outs,
267 const SmallVectorImpl<SDValue> &OutVals,
268 const SDLoc &dl, SelectionDAG &DAG) const {
269 // CCValAssign - represent the assignment of the return value to locations.
270 SmallVector<CCValAssign, 16> RVLocs;
271
272 // CCState - Info about the registers and stack slot.
273 CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
274 *DAG.getContext());
275
276 // Analyze return values of ISD::RET
277 CCInfo.AnalyzeReturn(Outs, Fn: RetCC_Hexagon);
278
279 SDValue Glue;
280 SmallVector<SDValue, 4> RetOps(1, Chain);
281
282 // Copy the result values into the output registers.
283 for (unsigned i = 0; i != RVLocs.size(); ++i) {
284 CCValAssign &VA = RVLocs[i];
285 SDValue Val = OutVals[i];
286
287 switch (VA.getLocInfo()) {
288 default:
289 // Loc info must be one of Full, BCvt, SExt, ZExt, or AExt.
290 llvm_unreachable("Unknown loc info!");
291 case CCValAssign::Full:
292 break;
293 case CCValAssign::BCvt:
294 Val = DAG.getBitcast(VT: VA.getLocVT(), V: Val);
295 break;
296 case CCValAssign::SExt:
297 Val = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: Val);
298 break;
299 case CCValAssign::ZExt:
300 Val = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: Val);
301 break;
302 case CCValAssign::AExt:
303 Val = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: Val);
304 break;
305 }
306
307 Chain = DAG.getCopyToReg(Chain, dl, Reg: VA.getLocReg(), N: Val, Glue);
308
309 // Guarantee that all emitted copies are stuck together with flags.
310 Glue = Chain.getValue(R: 1);
311 RetOps.push_back(Elt: DAG.getRegister(Reg: VA.getLocReg(), VT: VA.getLocVT()));
312 }
313
314 RetOps[0] = Chain; // Update chain.
315
316 // Add the glue if we have it.
317 if (Glue.getNode())
318 RetOps.push_back(Elt: Glue);
319
320 return DAG.getNode(Opcode: HexagonISD::RET_GLUE, DL: dl, VT: MVT::Other, Ops: RetOps);
321}
322
323bool HexagonTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
324 // If either no tail call or told not to tail call at all, don't.
325 return CI->isTailCall();
326}
327
328Register HexagonTargetLowering::getRegisterByName(
329 const char* RegName, LLT VT, const MachineFunction &) const {
330 // Just support r19, the linux kernel uses it.
331 Register Reg = StringSwitch<Register>(RegName)
332 .Case(S: "r0", Value: Hexagon::R0)
333 .Case(S: "r1", Value: Hexagon::R1)
334 .Case(S: "r2", Value: Hexagon::R2)
335 .Case(S: "r3", Value: Hexagon::R3)
336 .Case(S: "r4", Value: Hexagon::R4)
337 .Case(S: "r5", Value: Hexagon::R5)
338 .Case(S: "r6", Value: Hexagon::R6)
339 .Case(S: "r7", Value: Hexagon::R7)
340 .Case(S: "r8", Value: Hexagon::R8)
341 .Case(S: "r9", Value: Hexagon::R9)
342 .Case(S: "r10", Value: Hexagon::R10)
343 .Case(S: "r11", Value: Hexagon::R11)
344 .Case(S: "r12", Value: Hexagon::R12)
345 .Case(S: "r13", Value: Hexagon::R13)
346 .Case(S: "r14", Value: Hexagon::R14)
347 .Case(S: "r15", Value: Hexagon::R15)
348 .Case(S: "r16", Value: Hexagon::R16)
349 .Case(S: "r17", Value: Hexagon::R17)
350 .Case(S: "r18", Value: Hexagon::R18)
351 .Case(S: "r19", Value: Hexagon::R19)
352 .Case(S: "r20", Value: Hexagon::R20)
353 .Case(S: "r21", Value: Hexagon::R21)
354 .Case(S: "r22", Value: Hexagon::R22)
355 .Case(S: "r23", Value: Hexagon::R23)
356 .Case(S: "r24", Value: Hexagon::R24)
357 .Case(S: "r25", Value: Hexagon::R25)
358 .Case(S: "r26", Value: Hexagon::R26)
359 .Case(S: "r27", Value: Hexagon::R27)
360 .Case(S: "r28", Value: Hexagon::R28)
361 .Case(S: "r29", Value: Hexagon::R29)
362 .Case(S: "r30", Value: Hexagon::R30)
363 .Case(S: "r31", Value: Hexagon::R31)
364 .Case(S: "r1:0", Value: Hexagon::D0)
365 .Case(S: "r3:2", Value: Hexagon::D1)
366 .Case(S: "r5:4", Value: Hexagon::D2)
367 .Case(S: "r7:6", Value: Hexagon::D3)
368 .Case(S: "r9:8", Value: Hexagon::D4)
369 .Case(S: "r11:10", Value: Hexagon::D5)
370 .Case(S: "r13:12", Value: Hexagon::D6)
371 .Case(S: "r15:14", Value: Hexagon::D7)
372 .Case(S: "r17:16", Value: Hexagon::D8)
373 .Case(S: "r19:18", Value: Hexagon::D9)
374 .Case(S: "r21:20", Value: Hexagon::D10)
375 .Case(S: "r23:22", Value: Hexagon::D11)
376 .Case(S: "r25:24", Value: Hexagon::D12)
377 .Case(S: "r27:26", Value: Hexagon::D13)
378 .Case(S: "r29:28", Value: Hexagon::D14)
379 .Case(S: "r31:30", Value: Hexagon::D15)
380 .Case(S: "sp", Value: Hexagon::R29)
381 .Case(S: "fp", Value: Hexagon::R30)
382 .Case(S: "lr", Value: Hexagon::R31)
383 .Case(S: "p0", Value: Hexagon::P0)
384 .Case(S: "p1", Value: Hexagon::P1)
385 .Case(S: "p2", Value: Hexagon::P2)
386 .Case(S: "p3", Value: Hexagon::P3)
387 .Case(S: "sa0", Value: Hexagon::SA0)
388 .Case(S: "lc0", Value: Hexagon::LC0)
389 .Case(S: "sa1", Value: Hexagon::SA1)
390 .Case(S: "lc1", Value: Hexagon::LC1)
391 .Case(S: "m0", Value: Hexagon::M0)
392 .Case(S: "m1", Value: Hexagon::M1)
393 .Case(S: "usr", Value: Hexagon::USR)
394 .Case(S: "ugp", Value: Hexagon::UGP)
395 .Case(S: "cs0", Value: Hexagon::CS0)
396 .Case(S: "cs1", Value: Hexagon::CS1)
397 .Default(Value: Register());
398 return Reg;
399}
400
401/// LowerCallResult - Lower the result values of an ISD::CALL into the
402/// appropriate copies out of appropriate physical registers. This assumes that
403/// Chain/Glue are the input chain/glue to use, and that TheCall is the call
404/// being lowered. Returns a SDNode with the same number of values as the
405/// ISD::CALL.
406SDValue HexagonTargetLowering::LowerCallResult(
407 SDValue Chain, SDValue Glue, CallingConv::ID CallConv, bool IsVarArg,
408 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
409 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals,
410 const SmallVectorImpl<SDValue> &OutVals, SDValue Callee) const {
411 // Assign locations to each value returned by this call.
412 SmallVector<CCValAssign, 16> RVLocs;
413
414 CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
415 *DAG.getContext());
416
417 CCInfo.AnalyzeCallResult(Ins, Fn: RetCC_Hexagon);
418
419 // Copy all of the result registers out of their specified physreg.
420 for (unsigned i = 0; i != RVLocs.size(); ++i) {
421 SDValue RetVal;
422 if (RVLocs[i].getValVT() == MVT::i1) {
423 // Return values of type MVT::i1 require special handling. The reason
424 // is that MVT::i1 is associated with the PredRegs register class, but
425 // values of that type are still returned in R0. Generate an explicit
426 // copy into a predicate register from R0, and treat the value of the
427 // predicate register as the call result.
428 auto &MRI = DAG.getMachineFunction().getRegInfo();
429 SDValue FR0 = DAG.getCopyFromReg(Chain, dl, Reg: RVLocs[i].getLocReg(),
430 VT: MVT::i32, Glue);
431 // FR0 = (Value, Chain, Glue)
432 Register PredR = MRI.createVirtualRegister(RegClass: &Hexagon::PredRegsRegClass);
433 SDValue TPR = DAG.getCopyToReg(Chain: FR0.getValue(R: 1), dl, Reg: PredR,
434 N: FR0.getValue(R: 0), Glue: FR0.getValue(R: 2));
435 // TPR = (Chain, Glue)
436 // Don't glue this CopyFromReg, because it copies from a virtual
437 // register. If it is glued to the call, InstrEmitter will add it
438 // as an implicit def to the call (EmitMachineNode).
439 RetVal = DAG.getCopyFromReg(Chain: TPR.getValue(R: 0), dl, Reg: PredR, VT: MVT::i1);
440 Glue = TPR.getValue(R: 1);
441 Chain = TPR.getValue(R: 0);
442 } else {
443 RetVal = DAG.getCopyFromReg(Chain, dl, Reg: RVLocs[i].getLocReg(),
444 VT: RVLocs[i].getValVT(), Glue);
445 Glue = RetVal.getValue(R: 2);
446 Chain = RetVal.getValue(R: 1);
447 }
448 InVals.push_back(Elt: RetVal.getValue(R: 0));
449 }
450
451 return Chain;
452}
453
454/// LowerCall - Functions arguments are copied from virtual regs to
455/// (physical regs)/(stack frame), CALLSEQ_START and CALLSEQ_END are emitted.
456SDValue
457HexagonTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
458 SmallVectorImpl<SDValue> &InVals) const {
459 SelectionDAG &DAG = CLI.DAG;
460 SDLoc &dl = CLI.DL;
461 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
462 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
463 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
464 SDValue Chain = CLI.Chain;
465 SDValue Callee = CLI.Callee;
466 CallingConv::ID CallConv = CLI.CallConv;
467 bool IsVarArg = CLI.IsVarArg;
468 bool DoesNotReturn = CLI.DoesNotReturn;
469
470 bool IsStructRet = Outs.empty() ? false : Outs[0].Flags.isSRet();
471 MachineFunction &MF = DAG.getMachineFunction();
472 MachineFrameInfo &MFI = MF.getFrameInfo();
473 auto PtrVT = getPointerTy(DL: MF.getDataLayout());
474
475 if (GlobalAddressSDNode *GAN = dyn_cast<GlobalAddressSDNode>(Val&: Callee))
476 Callee = DAG.getTargetGlobalAddress(GV: GAN->getGlobal(), DL: dl, VT: MVT::i32);
477
478 // Linux ABI treats var-arg calls the same way as regular ones.
479 bool TreatAsVarArg = !Subtarget.isEnvironmentMusl() && IsVarArg;
480
481 // Analyze operands of the call, assigning locations to each operand.
482 SmallVector<CCValAssign, 16> ArgLocs;
483 CCState CCInfo(CallConv, TreatAsVarArg, MF, ArgLocs, *DAG.getContext());
484
485 if (DisableArgsMinAlignment)
486 CCInfo.AnalyzeCallOperands(Outs, Fn: CC_Hexagon_Legacy);
487 else
488 CCInfo.AnalyzeCallOperands(Outs, Fn: CC_Hexagon);
489 unsigned NumVTCMBytes = assignVTCMStackOffsets(ArgLocs);
490
491 if (CLI.IsTailCall) {
492 bool StructAttrFlag = MF.getFunction().hasStructRetAttr();
493 CLI.IsTailCall = IsEligibleForTailCallOptimization(Callee, CalleeCC: CallConv,
494 isVarArg: IsVarArg, isCalleeStructRet: IsStructRet, isCallerStructRet: StructAttrFlag, Outs,
495 OutVals, Ins, DAG);
496 for (const CCValAssign &VA : ArgLocs) {
497 if (VA.isMemLoc()) {
498 CLI.IsTailCall = false;
499 break;
500 }
501 }
502 LLVM_DEBUG(dbgs() << (CLI.IsTailCall ? "Eligible for Tail Call\n"
503 : "Argument must be passed on stack. "
504 "Not eligible for Tail Call\n"));
505 }
506 // Get a count of how many bytes are to be pushed on the stack.
507 unsigned NumBytes = CCInfo.getStackSize();
508 SmallVector<std::pair<unsigned, SDValue>, 16> RegsToPass;
509 SmallVector<SDValue, 8> MemOpChains;
510
511 const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo();
512 SDValue StackPtr =
513 DAG.getCopyFromReg(Chain, dl, Reg: HRI.getStackRegister(), VT: PtrVT);
514 SDValue VTCMStackPtr;
515 if (NumVTCMBytes) {
516 VTCMStackPtr =
517 DAG.getCopyFromReg(Chain, dl, Reg: HRI.getVTCMStackRegister(), VT: PtrVT);
518 MF.getInfo<HexagonMachineFunctionInfo>()->reserveVTCMCallFrame(
519 Size: NumVTCMBytes);
520 }
521
522 bool NeedsArgAlign = false;
523 Align LargestAlignSeen;
524 // Walk the register/memloc assignments, inserting copies/loads.
525 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
526 CCValAssign &VA = ArgLocs[i];
527 SDValue Arg = OutVals[i];
528 ISD::ArgFlagsTy Flags = Outs[i].Flags;
529 // Record if we need > 8 byte alignment on an argument.
530 bool ArgAlign = Subtarget.isHVXVectorType(VecTy: VA.getValVT());
531 NeedsArgAlign |= ArgAlign;
532
533 // Promote the value if needed.
534 switch (VA.getLocInfo()) {
535 default:
536 // Loc info must be one of Full, BCvt, SExt, ZExt, or AExt.
537 llvm_unreachable("Unknown loc info!");
538 case CCValAssign::Full:
539 break;
540 case CCValAssign::BCvt:
541 Arg = DAG.getBitcast(VT: VA.getLocVT(), V: Arg);
542 break;
543 case CCValAssign::SExt:
544 Arg = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: Arg);
545 break;
546 case CCValAssign::ZExt:
547 Arg = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: Arg);
548 break;
549 case CCValAssign::AExt:
550 Arg = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: Arg);
551 break;
552 }
553
554 if (VA.isMemLoc()) {
555 unsigned LocMemOffset = VA.getLocMemOffset();
556 SDValue BasePtr = VA.needsCustom() ? VTCMStackPtr : StackPtr;
557 SDValue MemAddr =
558 DAG.getConstant(Val: LocMemOffset, DL: dl, VT: BasePtr.getValueType());
559 MemAddr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::i32, N1: BasePtr, N2: MemAddr);
560 if (ArgAlign)
561 LargestAlignSeen = std::max(
562 a: LargestAlignSeen, b: Align(VA.getLocVT().getStoreSizeInBits() / 8));
563 if (Flags.isByVal()) {
564 // The argument is a struct passed by value. According to LLVM, "Arg"
565 // is a pointer.
566 MemOpChains.push_back(Elt: CreateCopyOfByValArgument(Src: Arg, Dst: MemAddr, Chain,
567 Flags, DAG, dl));
568 } else {
569 MachinePointerInfo LocPI = MachinePointerInfo::getStack(
570 MF&: DAG.getMachineFunction(), Offset: LocMemOffset);
571 if (isHlxPairTy(Ty: VA.getLocVT())) {
572 splitHlxPairArgStore(DAG, dl, Chain, StoreVal: Arg, BaseAddr: MemAddr, PairTy: VA.getLocVT(),
573 PtrInfo: LocPI, MemOpChains);
574 } else {
575 SDValue S = DAG.getStore(Chain, dl, Val: Arg, Ptr: MemAddr, PtrInfo: LocPI);
576 MemOpChains.push_back(Elt: S);
577 }
578 }
579 continue;
580 }
581
582 // Arguments that can be passed on register must be kept at RegsToPass
583 // vector.
584 if (VA.isRegLoc())
585 RegsToPass.push_back(Elt: std::make_pair(x: VA.getLocReg(), y&: Arg));
586 }
587
588 if (NeedsArgAlign && Subtarget.hasV60Ops()) {
589 LLVM_DEBUG(dbgs() << "Function needs byte stack align due to call args\n");
590 Align VecAlign = HRI.getSpillAlign(RC: Hexagon::HvxVRRegClass);
591 LargestAlignSeen = std::max(a: LargestAlignSeen, b: VecAlign);
592 MFI.ensureMaxAlignment(Alignment: LargestAlignSeen);
593 }
594 // Transform all store nodes into one single node because all store
595 // nodes are independent of each other.
596 if (!MemOpChains.empty())
597 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: MemOpChains);
598
599 SDValue Glue;
600 if (!CLI.IsTailCall) {
601 Chain = DAG.getCALLSEQ_START(Chain, InSize: NumBytes, OutSize: 0, DL: dl);
602 Glue = Chain.getValue(R: 1);
603 }
604
605 // Build a sequence of copy-to-reg nodes chained together with token
606 // chain and flag operands which copy the outgoing args into registers.
607 // The Glue is necessary since all emitted instructions must be
608 // stuck together.
609 if (!CLI.IsTailCall) {
610 for (const auto &R : RegsToPass) {
611 Chain = DAG.getCopyToReg(Chain, dl, Reg: R.first, N: R.second, Glue);
612 Glue = Chain.getValue(R: 1);
613 }
614 } else {
615 // For tail calls lower the arguments to the 'real' stack slot.
616 //
617 // Force all the incoming stack arguments to be loaded from the stack
618 // before any new outgoing arguments are stored to the stack, because the
619 // outgoing stack slots may alias the incoming argument stack slots, and
620 // the alias isn't otherwise explicit. This is slightly more conservative
621 // than necessary, because it means that each store effectively depends
622 // on every argument instead of just those arguments it would clobber.
623 //
624 // Do not flag preceding copytoreg stuff together with the following stuff.
625 Glue = SDValue();
626 for (const auto &R : RegsToPass) {
627 Chain = DAG.getCopyToReg(Chain, dl, Reg: R.first, N: R.second, Glue);
628 Glue = Chain.getValue(R: 1);
629 }
630 Glue = SDValue();
631 }
632
633 bool LongCalls = MF.getSubtarget<HexagonSubtarget>().useLongCalls();
634 unsigned Flags = LongCalls ? HexagonII::HMOTF_ConstExtended : 0;
635
636 // If the callee is a GlobalAddress/ExternalSymbol node (quite common, every
637 // direct call is) turn it into a TargetGlobalAddress/TargetExternalSymbol
638 // node so that legalize doesn't hack it.
639 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Val&: Callee)) {
640 Callee = DAG.getTargetGlobalAddress(GV: G->getGlobal(), DL: dl, VT: PtrVT, offset: 0, TargetFlags: Flags);
641 } else if (ExternalSymbolSDNode *S =
642 dyn_cast<ExternalSymbolSDNode>(Val&: Callee)) {
643 Callee = DAG.getTargetExternalSymbol(Sym: S->getSymbol(), VT: PtrVT, TargetFlags: Flags);
644 }
645
646 // Returns a chain & a flag for retval copy to use.
647 SmallVector<SDValue, 8> Ops;
648 Ops.push_back(Elt: Chain);
649 Ops.push_back(Elt: Callee);
650
651 // Add argument registers to the end of the list so that they are
652 // known live into the call.
653 for (const auto &R : RegsToPass)
654 Ops.push_back(Elt: DAG.getRegister(Reg: R.first, VT: R.second.getValueType()));
655
656 // Callees that take or return HLX values preserve L12-L15.
657 bool HasHLXLocs =
658 any_of(Range&: ArgLocs,
659 P: [](const CCValAssign &VA) {
660 return VA.isRegLoc() &&
661 (Hexagon::HlxLRRegClass.contains(Reg: VA.getLocReg()) ||
662 Hexagon::HlxLLRRegClass.contains(Reg: VA.getLocReg()));
663 }) ||
664 any_of(Range&: Ins, P: [this](const ISD::InputArg &In) {
665 return Subtarget.isHLXVectorType(VecTy: In.VT);
666 });
667 const uint32_t *Mask = HasHLXLocs ? HRI.getHLXCallPreservedMask()
668 : HRI.getCallPreservedMask(MF, CallConv);
669 assert(Mask && "Missing call preserved mask for calling convention");
670 Ops.push_back(Elt: DAG.getRegisterMask(RegMask: Mask));
671
672 if (Glue.getNode())
673 Ops.push_back(Elt: Glue);
674
675 if (CLI.IsTailCall) {
676 MFI.setHasTailCall();
677 return DAG.getNode(Opcode: HexagonISD::TC_RETURN, DL: dl, VT: MVT::Other, Ops);
678 }
679
680 // Set this here because we need to know this for "hasFP" in frame lowering.
681 // The target-independent code calls getFrameRegister before setting it, and
682 // getFrameRegister uses hasFP to determine whether the function has FP.
683 MFI.setHasCalls(true);
684
685 unsigned OpCode = DoesNotReturn ? HexagonISD::CALLnr : HexagonISD::CALL;
686 Chain = DAG.getNode(Opcode: OpCode, DL: dl, ResultTys: {MVT::Other, MVT::Glue}, Ops);
687 if (CLI.CFIType)
688 Chain.getNode()->setCFIType(CLI.CFIType->getZExtValue());
689 Glue = Chain.getValue(R: 1);
690
691 // Create the CALLSEQ_END node.
692 Chain = DAG.getCALLSEQ_END(Chain, Size1: NumBytes, Size2: 0, Glue, DL: dl);
693 Glue = Chain.getValue(R: 1);
694
695 // Handle result values, copying them out of physregs into vregs that we
696 // return.
697 return LowerCallResult(Chain, Glue, CallConv, IsVarArg, Ins, dl, DAG,
698 InVals, OutVals, Callee);
699}
700
701/// Returns true by value, base pointer and offset pointer and addressing
702/// mode by reference if this node can be combined with a load / store to
703/// form a post-indexed load / store.
704bool HexagonTargetLowering::getPostIndexedAddressParts(SDNode *N, SDNode *Op,
705 SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM,
706 SelectionDAG &DAG) const {
707 LSBaseSDNode *LSN = dyn_cast<LSBaseSDNode>(Val: N);
708 if (!LSN)
709 return false;
710 EVT VT = LSN->getMemoryVT();
711 if (!VT.isSimple())
712 return false;
713 bool IsLegalType = VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32 ||
714 VT == MVT::i64 || VT == MVT::f32 || VT == MVT::f64 ||
715 VT == MVT::v2i16 || VT == MVT::v2i32 || VT == MVT::v4i8 ||
716 VT == MVT::v4i16 || VT == MVT::v8i8 ||
717 Subtarget.isHVXVectorType(VecTy: VT.getSimpleVT());
718 if (!IsLegalType)
719 return false;
720
721 if (Op->getOpcode() != ISD::ADD)
722 return false;
723 Base = Op->getOperand(Num: 0);
724 Offset = Op->getOperand(Num: 1);
725 if (!isa<ConstantSDNode>(Val: Offset.getNode()))
726 return false;
727 AM = ISD::POST_INC;
728
729 int32_t V = cast<ConstantSDNode>(Val: Offset.getNode())->getSExtValue();
730 return Subtarget.getInstrInfo()->isValidAutoIncImm(VT, Offset: V);
731}
732
733SDValue HexagonTargetLowering::LowerFDIV(SDValue Op, SelectionDAG &DAG) const {
734 if (DAG.getMachineFunction().getFunction().hasOptSize())
735 return SDValue();
736 else
737 return Op;
738}
739
740SDValue
741HexagonTargetLowering::LowerINLINEASM(SDValue Op, SelectionDAG &DAG) const {
742 MachineFunction &MF = DAG.getMachineFunction();
743 auto &HMFI = *MF.getInfo<HexagonMachineFunctionInfo>();
744 const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo();
745 unsigned LR = HRI.getRARegister();
746
747 if ((Op.getOpcode() != ISD::INLINEASM &&
748 Op.getOpcode() != ISD::INLINEASM_BR) || HMFI.hasClobberLR())
749 return Op;
750
751 unsigned NumOps = Op.getNumOperands();
752 if (Op.getOperand(i: NumOps-1).getValueType() == MVT::Glue)
753 --NumOps; // Ignore the flag operand.
754
755 for (unsigned i = InlineAsm::Op_FirstOperand; i != NumOps;) {
756 const InlineAsm::Flag Flags(Op.getConstantOperandVal(i));
757 unsigned NumVals = Flags.getNumOperandRegisters();
758 ++i; // Skip the ID value.
759
760 switch (Flags.getKind()) {
761 default:
762 llvm_unreachable("Bad flags!");
763 case InlineAsm::Kind::RegUse:
764 case InlineAsm::Kind::Imm:
765 case InlineAsm::Kind::Mem:
766 i += NumVals;
767 break;
768 case InlineAsm::Kind::Clobber:
769 case InlineAsm::Kind::RegDef:
770 case InlineAsm::Kind::RegDefEarlyClobber: {
771 for (; NumVals; --NumVals, ++i) {
772 Register Reg = cast<RegisterSDNode>(Val: Op.getOperand(i))->getReg();
773 if (Reg != LR)
774 continue;
775 HMFI.setHasClobberLR(true);
776 return Op;
777 }
778 break;
779 }
780 }
781 }
782
783 return Op;
784}
785
786// Need to transform ISD::PREFETCH into something that doesn't inherit
787// all of the properties of ISD::PREFETCH, specifically SDNPMayLoad and
788// SDNPMayStore.
789SDValue HexagonTargetLowering::LowerPREFETCH(SDValue Op,
790 SelectionDAG &DAG) const {
791 SDValue Chain = Op.getOperand(i: 0);
792 SDValue Addr = Op.getOperand(i: 1);
793 // Lower it to DCFETCH($reg, #0). A "pat" will try to merge the offset in,
794 // if the "reg" is fed by an "add".
795 SDLoc DL(Op);
796 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
797 return DAG.getNode(Opcode: HexagonISD::DCFETCH, DL, VT: MVT::Other, N1: Chain, N2: Addr, N3: Zero);
798}
799
800SDValue HexagonTargetLowering::LowerINTRINSIC_VOID(SDValue Op,
801 SelectionDAG &DAG) const {
802 SDValue Chain = Op.getOperand(i: 0);
803 unsigned IntNo = Op.getConstantOperandVal(i: 1);
804 // Lower the hexagon_prefetch builtin to DCFETCH, as above.
805 if (IntNo == Intrinsic::hexagon_prefetch) {
806 SDValue Addr = Op.getOperand(i: 2);
807 SDLoc DL(Op);
808 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
809 return DAG.getNode(Opcode: HexagonISD::DCFETCH, DL, VT: MVT::Other, N1: Chain, N2: Addr, N3: Zero);
810 }
811 return SDValue();
812}
813
814SDValue
815HexagonTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
816 SelectionDAG &DAG) const {
817 SDValue Chain = Op.getOperand(i: 0);
818 SDValue Size = Op.getOperand(i: 1);
819 SDValue Align = Op.getOperand(i: 2);
820 SDLoc dl(Op);
821
822 ConstantSDNode *AlignConst = dyn_cast<ConstantSDNode>(Val&: Align);
823 assert(AlignConst && "Non-constant Align in LowerDYNAMIC_STACKALLOC");
824
825 unsigned A = AlignConst->getSExtValue();
826 auto &HFI = *Subtarget.getFrameLowering();
827 // "Zero" means natural stack alignment.
828 if (A == 0)
829 A = HFI.getStackAlign().value();
830
831 LLVM_DEBUG({
832 dbgs () << __func__ << " Align: " << A << " Size: ";
833 Size.getNode()->dump(&DAG);
834 dbgs() << "\n";
835 });
836
837 SDValue AC = DAG.getConstant(Val: A, DL: dl, VT: MVT::i32);
838 SDVTList VTs = DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other);
839 SDValue AA = DAG.getNode(Opcode: HexagonISD::ALLOCA, DL: dl, VTList: VTs, N1: Chain, N2: Size, N3: AC);
840
841 DAG.ReplaceAllUsesOfValueWith(From: Op, To: AA);
842 return AA;
843}
844
845SDValue HexagonTargetLowering::LowerFMINFMAX(SDValue Op,
846 SelectionDAG &DAG) const {
847 EVT OpVT = Op.getValueType();
848 MVT SimpleVT = OpVT.getSimpleVT();
849 SDLoc DL(Op);
850
851 // Check if any of the inputs are NaN. If so, propagate the NaN
852 // to the output, otherwise return the maximum/minimum of the inputs.
853 // We can safely use ISD::FMINNUM/ISD::FMAXNUM to run
854 // Hexagon's F2_sfmin/F2_sfmax, when no operand is NaN.
855 // Note: We cannot directly compare nodes against NaN node to find NaNs,
856 // because comparing NaN with anything always returns False (except for !=0
857 // which always return True). To work around that, we compare input operands
858 // with themselves under ISD::SETUO, which only returns true if the operand is
859 // NaN.
860
861 SDValue Op1 = Op.getOperand(i: 0);
862 SDValue Op2 = Op.getOperand(i: 1);
863 SDValue isOp1NaN = DAG.getSetCC(DL, VT: MVT::i1, LHS: Op1, RHS: Op1, Cond: ISD::SETUO);
864 SDValue isOp2NaN = DAG.getSetCC(DL, VT: MVT::i1, LHS: Op2, RHS: Op2, Cond: ISD::SETUO);
865
866 switch (Op.getOpcode()) {
867 case ISD::FMAXIMUM: {
868 SDValue FmaxNode = DAG.getNode(Opcode: ISD::FMAXNUM, DL, VT: SimpleVT, N1: Op1, N2: Op2);
869 SDValue result =
870 DAG.getNode(Opcode: ISD::SELECT, DL, VT: SimpleVT, N1: isOp2NaN, N2: Op2, N3: FmaxNode);
871 return DAG.getNode(Opcode: ISD::SELECT, DL, VT: SimpleVT, N1: isOp1NaN, N2: Op1, N3: result);
872 }
873 case ISD::FMINIMUM: {
874 SDValue FminNode = DAG.getNode(Opcode: ISD::FMINNUM, DL, VT: SimpleVT, N1: Op1, N2: Op2);
875 SDValue result =
876 DAG.getNode(Opcode: ISD::SELECT, DL, VT: SimpleVT, N1: isOp2NaN, N2: Op2, N3: FminNode);
877 return DAG.getNode(Opcode: ISD::SELECT, DL, VT: SimpleVT, N1: isOp1NaN, N2: Op1, N3: result);
878 }
879 default:
880 llvm_unreachable("Invalid opcode for LowerFMINFMAX");
881 }
882}
883
884SDValue HexagonTargetLowering::LowerFormalArguments(
885 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
886 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
887 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
888 MachineFunction &MF = DAG.getMachineFunction();
889 MachineFrameInfo &MFI = MF.getFrameInfo();
890 MachineRegisterInfo &MRI = MF.getRegInfo();
891
892 // Linux ABI treats var-arg calls the same way as regular ones.
893 bool TreatAsVarArg = !Subtarget.isEnvironmentMusl() && IsVarArg;
894
895 // Assign locations to all of the incoming arguments.
896 SmallVector<CCValAssign, 16> ArgLocs;
897 CCState CCInfo(CallConv, TreatAsVarArg, MF, ArgLocs, *DAG.getContext());
898
899 if (DisableArgsMinAlignment)
900 CCInfo.AnalyzeFormalArguments(Ins, Fn: CC_Hexagon_Legacy);
901 else
902 CCInfo.AnalyzeFormalArguments(Ins, Fn: CC_Hexagon);
903 assignVTCMStackOffsets(ArgLocs);
904
905 // For LLVM, in the case when returning a struct by value (>8byte),
906 // the first argument is a pointer that points to the location on caller's
907 // stack where the return value will be stored. For Hexagon, the location on
908 // caller's stack is passed only when the struct size is smaller than (and
909 // equal to) 8 bytes. If not, no address will be passed into callee and
910 // callee return the result directly through R0/R1.
911 auto NextSingleReg = [] (const TargetRegisterClass &RC, unsigned Reg) {
912 switch (RC.getID()) {
913 case Hexagon::IntRegsRegClassID:
914 return Reg - Hexagon::R0 + 1;
915 case Hexagon::DoubleRegsRegClassID:
916 return (Reg - Hexagon::D0 + 1) * 2;
917 case Hexagon::HvxVRRegClassID:
918 return Reg - Hexagon::V0 + 1;
919 case Hexagon::HvxWRRegClassID:
920 return (Reg - Hexagon::W0 + 1) * 2;
921 case Hexagon::HlxLRRegClassID:
922 return Reg - Hexagon::L0 + 1;
923 case Hexagon::HlxLLRRegClassID:
924 return (Reg - Hexagon::LL0 + 1) * 2;
925 }
926 llvm_unreachable("Unexpected register class");
927 };
928
929 auto &HFL = const_cast<HexagonFrameLowering&>(*Subtarget.getFrameLowering());
930 auto &HMFI = *MF.getInfo<HexagonMachineFunctionInfo>();
931 HFL.FirstVarArgSavedReg = 0;
932 HMFI.setFirstNamedArgFrameIndex(-int(MFI.getNumFixedObjects()));
933
934 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
935 CCValAssign &VA = ArgLocs[i];
936 ISD::ArgFlagsTy Flags = Ins[i].Flags;
937 bool ByVal = Flags.isByVal();
938
939 // Arguments passed in registers:
940 // 1. 32- and 64-bit values and HVX/HLX vectors are passed directly,
941 // 2. Large structs are passed via an address, and the address is
942 // passed in a register.
943 if (VA.isRegLoc() && ByVal && Flags.getByValSize() <= 8)
944 llvm_unreachable("ByValSize must be bigger than 8 bytes");
945
946 bool InReg = VA.isRegLoc() &&
947 (!ByVal || (ByVal && Flags.getByValSize() > 8));
948
949 if (InReg) {
950 MVT RegVT = VA.getLocVT();
951 if (VA.getLocInfo() == CCValAssign::BCvt)
952 RegVT = VA.getValVT();
953
954 const TargetRegisterClass *RC = getRegClassFor(VT: RegVT);
955 Register VReg = MRI.createVirtualRegister(RegClass: RC);
956 SDValue Copy = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: RegVT);
957
958 // Treat values of type MVT::i1 specially: they are passed in
959 // registers of type i32, but they need to remain as values of
960 // type i1 for consistency of the argument lowering.
961 if (VA.getValVT() == MVT::i1) {
962 assert(RegVT.getSizeInBits() <= 32);
963 SDValue T = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: RegVT,
964 N1: Copy, N2: DAG.getConstant(Val: 1, DL: dl, VT: RegVT));
965 Copy = DAG.getSetCC(DL: dl, VT: MVT::i1, LHS: T, RHS: DAG.getConstant(Val: 0, DL: dl, VT: RegVT),
966 Cond: ISD::SETNE);
967 } else {
968#ifndef NDEBUG
969 unsigned RegSize = RegVT.getSizeInBits();
970 assert(RegSize == 32 || RegSize == 64 ||
971 Subtarget.isHVXVectorType(RegVT) ||
972 Subtarget.isHLXVectorType(RegVT));
973#endif
974 }
975 InVals.push_back(Elt: Copy);
976 MRI.addLiveIn(Reg: VA.getLocReg(), vreg: VReg);
977 HFL.FirstVarArgSavedReg = NextSingleReg(*RC, VA.getLocReg());
978 } else {
979 assert(VA.isMemLoc() && "Argument should be passed in memory");
980
981 // If it's a byval parameter, then we need to compute the
982 // "real" size, not the size of the pointer.
983 unsigned ObjSize = Flags.isByVal()
984 ? Flags.getByValSize()
985 : VA.getLocVT().getStoreSizeInBits() / 8;
986
987 // Create the frame index object for this incoming parameter. HLX
988 // arguments are in the caller's VTCM argument area, not on the scalar
989 // stack. Mark them NoAlloc so that the scalar frame layout skips them;
990 // computeVTCMStackOffsets moves every NoAlloc fixed object to the VTCM
991 // frame. Hexagon uses NoAlloc only for these objects.
992 int Offset = VA.needsCustom() ? VA.getLocMemOffset()
993 : HEXAGON_LRFP_SIZE + VA.getLocMemOffset();
994 int FI = MFI.CreateFixedObject(Size: ObjSize, SPOffset: Offset, IsImmutable: true);
995 if (VA.needsCustom())
996 MFI.setStackID(ObjectIdx: FI, ID: TargetStackID::NoAlloc);
997 SDValue FIN = DAG.getFrameIndex(FI, VT: MVT::i32);
998
999 if (Flags.isByVal()) {
1000 // If it's a pass-by-value aggregate, then do not dereference the stack
1001 // location. Instead, we should generate a reference to the stack
1002 // location.
1003 InVals.push_back(Elt: FIN);
1004 } else if (isHlxPairTy(Ty: VA.getValVT())) {
1005 InVals.push_back(
1006 Elt: splitHlxPairArgLoad(DAG, dl, Chain, BaseAddr: FIN, PairTy: VA.getValVT(),
1007 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI, Offset: 0)));
1008 } else {
1009 SDValue L = DAG.getLoad(VT: VA.getValVT(), dl, Chain, Ptr: FIN,
1010 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI, Offset: 0));
1011 InVals.push_back(Elt: L);
1012 }
1013 }
1014 }
1015
1016 if (IsVarArg && Subtarget.isEnvironmentMusl()) {
1017 for (int i = HFL.FirstVarArgSavedReg; i < 6; i++)
1018 MRI.addLiveIn(Reg: Hexagon::R0+i);
1019 }
1020
1021 if (IsVarArg && Subtarget.isEnvironmentMusl()) {
1022 HMFI.setFirstNamedArgFrameIndex(HMFI.getFirstNamedArgFrameIndex() - 1);
1023 HMFI.setLastNamedArgFrameIndex(-int(MFI.getNumFixedObjects()));
1024
1025 // Create Frame index for the start of register saved area.
1026 int NumVarArgRegs = 6 - HFL.FirstVarArgSavedReg;
1027 bool RequiresPadding = (NumVarArgRegs & 1);
1028 int RegSaveAreaSizePlusPadding = RequiresPadding
1029 ? (NumVarArgRegs + 1) * 4
1030 : NumVarArgRegs * 4;
1031
1032 if (RegSaveAreaSizePlusPadding > 0) {
1033 // The offset to saved register area should be 8 byte aligned.
1034 int RegAreaStart = HEXAGON_LRFP_SIZE + CCInfo.getStackSize();
1035 if (!(RegAreaStart % 8))
1036 RegAreaStart = (RegAreaStart + 7) & -8;
1037
1038 int RegSaveAreaFrameIndex =
1039 MFI.CreateFixedObject(Size: RegSaveAreaSizePlusPadding, SPOffset: RegAreaStart, IsImmutable: true);
1040 HMFI.setRegSavedAreaStartFrameIndex(RegSaveAreaFrameIndex);
1041
1042 // This will point to the next argument passed via stack.
1043 int Offset = RegAreaStart + RegSaveAreaSizePlusPadding;
1044 int FI = MFI.CreateFixedObject(Hexagon_PointerSize, SPOffset: Offset, IsImmutable: true);
1045 HMFI.setVarArgsFrameIndex(FI);
1046 } else {
1047 // This will point to the next argument passed via stack, when
1048 // there is no saved register area.
1049 int Offset = HEXAGON_LRFP_SIZE + CCInfo.getStackSize();
1050 int FI = MFI.CreateFixedObject(Hexagon_PointerSize, SPOffset: Offset, IsImmutable: true);
1051 HMFI.setRegSavedAreaStartFrameIndex(FI);
1052 HMFI.setVarArgsFrameIndex(FI);
1053 }
1054 }
1055
1056
1057 if (IsVarArg && !Subtarget.isEnvironmentMusl()) {
1058 // This will point to the next argument passed via stack.
1059 int Offset = HEXAGON_LRFP_SIZE + CCInfo.getStackSize();
1060 int FI = MFI.CreateFixedObject(Hexagon_PointerSize, SPOffset: Offset, IsImmutable: true);
1061 HMFI.setVarArgsFrameIndex(FI);
1062 }
1063
1064 return Chain;
1065}
1066
1067SDValue
1068HexagonTargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {
1069 // VASTART stores the address of the VarArgsFrameIndex slot into the
1070 // memory location argument.
1071 MachineFunction &MF = DAG.getMachineFunction();
1072 HexagonMachineFunctionInfo *QFI = MF.getInfo<HexagonMachineFunctionInfo>();
1073 SDValue Addr = DAG.getFrameIndex(FI: QFI->getVarArgsFrameIndex(), VT: MVT::i32);
1074 const Value *SV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 2))->getValue();
1075
1076 if (!Subtarget.isEnvironmentMusl()) {
1077 return DAG.getStore(Chain: Op.getOperand(i: 0), dl: SDLoc(Op), Val: Addr, Ptr: Op.getOperand(i: 1),
1078 PtrInfo: MachinePointerInfo(SV));
1079 }
1080 auto &FuncInfo = *MF.getInfo<HexagonMachineFunctionInfo>();
1081 auto &HFL = *Subtarget.getFrameLowering();
1082 SDLoc DL(Op);
1083 SmallVector<SDValue, 8> MemOps;
1084
1085 // Get frame index of va_list.
1086 SDValue FIN = Op.getOperand(i: 1);
1087
1088 // If first Vararg register is odd, add 4 bytes to start of
1089 // saved register area to point to the first register location.
1090 // This is because the saved register area has to be 8 byte aligned.
1091 // In case of an odd start register, there will be 4 bytes of padding in
1092 // the beginning of saved register area. If all registers area used up,
1093 // the following condition will handle it correctly.
1094 SDValue SavedRegAreaStartFrameIndex =
1095 DAG.getFrameIndex(FI: FuncInfo.getRegSavedAreaStartFrameIndex(), VT: MVT::i32);
1096
1097 auto PtrVT = getPointerTy(DL: DAG.getDataLayout());
1098
1099 if (HFL.FirstVarArgSavedReg & 1)
1100 SavedRegAreaStartFrameIndex =
1101 DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT,
1102 N1: DAG.getFrameIndex(FI: FuncInfo.getRegSavedAreaStartFrameIndex(),
1103 VT: MVT::i32),
1104 N2: DAG.getIntPtrConstant(Val: 4, DL));
1105
1106 // Store the saved register area start pointer.
1107 SDValue Store =
1108 DAG.getStore(Chain: Op.getOperand(i: 0), dl: DL,
1109 Val: SavedRegAreaStartFrameIndex,
1110 Ptr: FIN, PtrInfo: MachinePointerInfo(SV));
1111 MemOps.push_back(Elt: Store);
1112
1113 // Store saved register area end pointer.
1114 FIN = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT,
1115 N1: FIN, N2: DAG.getIntPtrConstant(Val: 4, DL));
1116 Store = DAG.getStore(Chain: Op.getOperand(i: 0), dl: DL,
1117 Val: DAG.getFrameIndex(FI: FuncInfo.getVarArgsFrameIndex(),
1118 VT: PtrVT),
1119 Ptr: FIN, PtrInfo: MachinePointerInfo(SV, 4));
1120 MemOps.push_back(Elt: Store);
1121
1122 // Store overflow area pointer.
1123 FIN = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT,
1124 N1: FIN, N2: DAG.getIntPtrConstant(Val: 4, DL));
1125 Store = DAG.getStore(Chain: Op.getOperand(i: 0), dl: DL,
1126 Val: DAG.getFrameIndex(FI: FuncInfo.getVarArgsFrameIndex(),
1127 VT: PtrVT),
1128 Ptr: FIN, PtrInfo: MachinePointerInfo(SV, 8));
1129 MemOps.push_back(Elt: Store);
1130
1131 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: MemOps);
1132}
1133
1134SDValue
1135HexagonTargetLowering::LowerVACOPY(SDValue Op, SelectionDAG &DAG) const {
1136 // Assert that the linux ABI is enabled for the current compilation.
1137 assert(Subtarget.isEnvironmentMusl() && "Linux ABI should be enabled");
1138 SDValue Chain = Op.getOperand(i: 0);
1139 SDValue DestPtr = Op.getOperand(i: 1);
1140 SDValue SrcPtr = Op.getOperand(i: 2);
1141 const Value *DestSV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 3))->getValue();
1142 const Value *SrcSV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 4))->getValue();
1143 SDLoc DL(Op);
1144 // Size of the va_list is 12 bytes as it has 3 pointers. Therefore,
1145 // we need to memcopy 12 bytes from va_list to another similar list.
1146 return DAG.getMemcpy(Chain, dl: DL, Dst: DestPtr, Src: SrcPtr,
1147 Size: DAG.getIntPtrConstant(Val: 12, DL), DstAlign: Align(4), SrcAlign: Align(4),
1148 /*isVolatile*/ isVol: false, AlwaysInline: false, /*CI=*/nullptr,
1149 OverrideTailCall: std::nullopt, DstPtrInfo: MachinePointerInfo(DestSV),
1150 SrcPtrInfo: MachinePointerInfo(SrcSV));
1151}
1152
1153SDValue HexagonTargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
1154 const SDLoc &dl(Op);
1155 SDValue LHS = Op.getOperand(i: 0);
1156 SDValue RHS = Op.getOperand(i: 1);
1157 ISD::CondCode CC = Op.getOperandCondCode(i: 2);
1158 MVT ResTy = ty(Op);
1159 MVT OpTy = ty(Op: LHS);
1160
1161 if (OpTy == MVT::v2i16 || OpTy == MVT::v4i8) {
1162 assert(OpTy.getVectorElementType().isScalarInteger());
1163 MVT WideTy = OpTy.widenIntegerElementType();
1164 return DAG.getSetCC(DL: dl, VT: ResTy,
1165 LHS: DAG.getSExtOrTrunc(Op: LHS, DL: SDLoc(LHS), VT: WideTy),
1166 RHS: DAG.getSExtOrTrunc(Op: RHS, DL: SDLoc(RHS), VT: WideTy), Cond: CC);
1167 }
1168
1169 // Treat all other vector types as legal.
1170 if (ResTy.isVector())
1171 return Op;
1172
1173 // Equality comparisons of short integers should use sign-extend, not
1174 // zero-extend, since we can represent small negative values in the compare
1175 // instructions. The LLVM default is to use zero-extend arbitrarily in
1176 // these cases.
1177 auto isSExtFree = [this](SDValue N) {
1178 switch (N.getOpcode()) {
1179 case ISD::TRUNCATE: {
1180 // A sign-extend of a truncate of a sign-extend is free.
1181 SDValue Op = N.getOperand(i: 0);
1182 if (Op.getOpcode() != ISD::AssertSext)
1183 return false;
1184 EVT OrigTy = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
1185 unsigned ThisBW = ty(Op: N).getSizeInBits();
1186 unsigned OrigBW = OrigTy.getSizeInBits();
1187 // The type that was sign-extended to get the AssertSext must be
1188 // narrower than the type of N (so that N has still the same value
1189 // as the original).
1190 return ThisBW >= OrigBW;
1191 }
1192 case ISD::LOAD:
1193 // We have sign-extended loads.
1194 return true;
1195 }
1196 return false;
1197 };
1198
1199 // Only do this for equality comparisons. Signed comparisons are already
1200 // sign-extended by the generic operand promotion, and for unsigned
1201 // comparisons a sign-extension is never profitable: it does not change the
1202 // result (sign-extension preserves the unsigned ordering of the values of
1203 // the narrower type), but it turns constants with the sign bit of the
1204 // narrower type set into large 32-bit values, which then have to be
1205 // materialized in a register or use a constant extender.
1206 if ((OpTy == MVT::i8 || OpTy == MVT::i16) && ISD::isIntEqualitySetCC(Code: CC)) {
1207 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: RHS);
1208 bool IsNegative = C && C->getAPIntValue().isNegative();
1209 if (IsNegative || isSExtFree(LHS) || isSExtFree(RHS))
1210 return DAG.getSetCC(DL: dl, VT: ResTy,
1211 LHS: DAG.getSExtOrTrunc(Op: LHS, DL: SDLoc(LHS), VT: MVT::i32),
1212 RHS: DAG.getSExtOrTrunc(Op: RHS, DL: SDLoc(RHS), VT: MVT::i32), Cond: CC);
1213 }
1214
1215 return SDValue();
1216}
1217
1218SDValue
1219HexagonTargetLowering::LowerVSELECT(SDValue Op, SelectionDAG &DAG) const {
1220 SDValue PredOp = Op.getOperand(i: 0);
1221 SDValue Op1 = Op.getOperand(i: 1), Op2 = Op.getOperand(i: 2);
1222 MVT OpTy = ty(Op: Op1);
1223 const SDLoc &dl(Op);
1224
1225 if (OpTy == MVT::v2i16 || OpTy == MVT::v4i8) {
1226 assert(OpTy.getVectorElementType().isScalarInteger());
1227 MVT WideTy = OpTy.widenIntegerElementType();
1228 // Generate (trunc (select (_, sext, sext))).
1229 return DAG.getSExtOrTrunc(
1230 Op: DAG.getSelect(DL: dl, VT: WideTy, Cond: PredOp,
1231 LHS: DAG.getSExtOrTrunc(Op: Op1, DL: dl, VT: WideTy),
1232 RHS: DAG.getSExtOrTrunc(Op: Op2, DL: dl, VT: WideTy)),
1233 DL: dl, VT: OpTy);
1234 }
1235
1236 return SDValue();
1237}
1238
1239SDValue
1240HexagonTargetLowering::LowerConstantPool(SDValue Op, SelectionDAG &DAG) const {
1241 EVT ValTy = Op.getValueType();
1242 ConstantPoolSDNode *CPN = cast<ConstantPoolSDNode>(Val&: Op);
1243 Constant *CVal = nullptr;
1244 bool isVTi1Type = false;
1245 if (auto *CV = dyn_cast<ConstantVector>(Val: CPN->getConstVal())) {
1246 if (cast<VectorType>(Val: CV->getType())->getElementType()->isIntegerTy(BitWidth: 1)) {
1247 SmallVector<Constant*, 128> NewConst;
1248 unsigned VecLen = CV->getNumOperands();
1249 assert(isPowerOf2_32(VecLen) &&
1250 "conversion only supported for pow2 VectorSize");
1251 for (unsigned i = 0; i < VecLen; ++i)
1252 NewConst.push_back(Elt: ConstantInt::get(Ty: Type::getInt8Ty(C&: CV->getContext()),
1253 V: CV->getOperand(i_nocapture: i)->isNullValue()));
1254
1255 CVal = ConstantVector::get(V: NewConst);
1256 isVTi1Type = true;
1257 }
1258 }
1259 Align Alignment = CPN->getAlign();
1260 bool IsPositionIndependent = isPositionIndependent();
1261 unsigned char TF = IsPositionIndependent ? HexagonII::MO_PCREL : 0;
1262
1263 unsigned Offset = 0;
1264 SDValue T;
1265 if (CPN->isMachineConstantPoolEntry())
1266 T = DAG.getTargetConstantPool(C: CPN->getMachineCPVal(), VT: ValTy, Align: Alignment,
1267 Offset, TargetFlags: TF);
1268 else if (isVTi1Type)
1269 T = DAG.getTargetConstantPool(C: CVal, VT: ValTy, Align: Alignment, Offset, TargetFlags: TF);
1270 else
1271 T = DAG.getTargetConstantPool(C: CPN->getConstVal(), VT: ValTy, Align: Alignment, Offset,
1272 TargetFlags: TF);
1273
1274 assert(cast<ConstantPoolSDNode>(T)->getTargetFlags() == TF &&
1275 "Inconsistent target flag encountered");
1276
1277 if (IsPositionIndependent)
1278 return DAG.getNode(Opcode: HexagonISD::AT_PCREL, DL: SDLoc(Op), VT: ValTy, Operand: T);
1279 return DAG.getNode(Opcode: HexagonISD::CP, DL: SDLoc(Op), VT: ValTy, Operand: T);
1280}
1281
1282SDValue
1283HexagonTargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const {
1284 EVT VT = Op.getValueType();
1285 int Idx = cast<JumpTableSDNode>(Val&: Op)->getIndex();
1286 if (isPositionIndependent()) {
1287 SDValue T = DAG.getTargetJumpTable(JTI: Idx, VT, TargetFlags: HexagonII::MO_PCREL);
1288 return DAG.getNode(Opcode: HexagonISD::AT_PCREL, DL: SDLoc(Op), VT, Operand: T);
1289 }
1290
1291 SDValue T = DAG.getTargetJumpTable(JTI: Idx, VT);
1292 return DAG.getNode(Opcode: HexagonISD::JT, DL: SDLoc(Op), VT, Operand: T);
1293}
1294
1295SDValue
1296HexagonTargetLowering::LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const {
1297 const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo();
1298 MachineFunction &MF = DAG.getMachineFunction();
1299 MachineFrameInfo &MFI = MF.getFrameInfo();
1300 MFI.setReturnAddressIsTaken(true);
1301
1302 EVT VT = Op.getValueType();
1303 SDLoc dl(Op);
1304 unsigned Depth = Op.getConstantOperandVal(i: 0);
1305 if (Depth) {
1306 SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
1307 SDValue Offset = DAG.getConstant(Val: 4, DL: dl, VT: MVT::i32);
1308 return DAG.getLoad(VT, dl, Chain: DAG.getEntryNode(),
1309 Ptr: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: FrameAddr, N2: Offset),
1310 PtrInfo: MachinePointerInfo());
1311 }
1312
1313 // Return LR, which contains the return address. Mark it an implicit live-in.
1314 Register Reg = MF.addLiveIn(PReg: HRI.getRARegister(), RC: getRegClassFor(VT: MVT::i32));
1315 return DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl, Reg, VT);
1316}
1317
1318SDValue
1319HexagonTargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const {
1320 const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo();
1321 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
1322 MFI.setFrameAddressIsTaken(true);
1323
1324 EVT VT = Op.getValueType();
1325 SDLoc dl(Op);
1326 unsigned Depth = Op.getConstantOperandVal(i: 0);
1327 SDValue FrameAddr = DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl,
1328 Reg: HRI.getFrameRegister(), VT);
1329 while (Depth--)
1330 FrameAddr = DAG.getLoad(VT, dl, Chain: DAG.getEntryNode(), Ptr: FrameAddr,
1331 PtrInfo: MachinePointerInfo());
1332 return FrameAddr;
1333}
1334
1335SDValue
1336HexagonTargetLowering::LowerATOMIC_FENCE(SDValue Op, SelectionDAG& DAG) const {
1337 SDLoc dl(Op);
1338 return DAG.getNode(Opcode: HexagonISD::BARRIER, DL: dl, VT: MVT::Other, Operand: Op.getOperand(i: 0));
1339}
1340
1341SDValue
1342HexagonTargetLowering::LowerGLOBALADDRESS(SDValue Op, SelectionDAG &DAG) const {
1343 SDLoc dl(Op);
1344 auto *GAN = cast<GlobalAddressSDNode>(Val&: Op);
1345 auto PtrVT = getPointerTy(DL: DAG.getDataLayout());
1346 auto *GV = GAN->getGlobal();
1347 int64_t Offset = GAN->getOffset();
1348
1349 auto &HLOF = *HTM.getObjFileLowering();
1350 Reloc::Model RM = HTM.getRelocationModel();
1351
1352 if (RM == Reloc::Static) {
1353 SDValue GA = DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: Offset);
1354 const GlobalObject *GO = GV->getAliaseeObject();
1355 if (GO && Subtarget.useSmallData() && HLOF.isGlobalInSmallSection(GO, TM: HTM))
1356 return DAG.getNode(Opcode: HexagonISD::CONST32_GP, DL: dl, VT: PtrVT, Operand: GA);
1357 return DAG.getNode(Opcode: HexagonISD::CONST32, DL: dl, VT: PtrVT, Operand: GA);
1358 }
1359
1360 bool UsePCRel = getTargetMachine().shouldAssumeDSOLocal(GV);
1361 if (UsePCRel) {
1362 SDValue GA = DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: Offset,
1363 TargetFlags: HexagonII::MO_PCREL);
1364 return DAG.getNode(Opcode: HexagonISD::AT_PCREL, DL: dl, VT: PtrVT, Operand: GA);
1365 }
1366
1367 // Use GOT index.
1368 SDValue GOT = DAG.getGLOBAL_OFFSET_TABLE(VT: PtrVT);
1369 SDValue GA = DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: 0, TargetFlags: HexagonII::MO_GOT);
1370 SDValue Off = DAG.getConstant(Val: Offset, DL: dl, VT: MVT::i32);
1371 return DAG.getNode(Opcode: HexagonISD::AT_GOT, DL: dl, VT: PtrVT, N1: GOT, N2: GA, N3: Off);
1372}
1373
1374// Specifies that for loads and stores VT can be promoted to PromotedLdStVT.
1375SDValue
1376HexagonTargetLowering::LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const {
1377 const BlockAddress *BA = cast<BlockAddressSDNode>(Val&: Op)->getBlockAddress();
1378 SDLoc dl(Op);
1379 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
1380
1381 Reloc::Model RM = HTM.getRelocationModel();
1382 if (RM == Reloc::Static) {
1383 SDValue A = DAG.getTargetBlockAddress(BA, VT: PtrVT);
1384 return DAG.getNode(Opcode: HexagonISD::CONST32_GP, DL: dl, VT: PtrVT, Operand: A);
1385 }
1386
1387 SDValue A = DAG.getTargetBlockAddress(BA, VT: PtrVT, Offset: 0, TargetFlags: HexagonII::MO_PCREL);
1388 return DAG.getNode(Opcode: HexagonISD::AT_PCREL, DL: dl, VT: PtrVT, Operand: A);
1389}
1390
1391SDValue
1392HexagonTargetLowering::LowerGLOBAL_OFFSET_TABLE(SDValue Op, SelectionDAG &DAG)
1393 const {
1394 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
1395 SDValue GOTSym = DAG.getTargetExternalSymbol(HEXAGON_GOT_SYM_NAME, VT: PtrVT,
1396 TargetFlags: HexagonII::MO_PCREL);
1397 return DAG.getNode(Opcode: HexagonISD::AT_PCREL, DL: SDLoc(Op), VT: PtrVT, Operand: GOTSym);
1398}
1399
1400SDValue
1401HexagonTargetLowering::GetDynamicTLSAddr(SelectionDAG &DAG, SDValue Chain,
1402 GlobalAddressSDNode *GA, SDValue Glue, EVT PtrVT, unsigned ReturnReg,
1403 unsigned char OperandFlags) const {
1404 MachineFunction &MF = DAG.getMachineFunction();
1405 MachineFrameInfo &MFI = MF.getFrameInfo();
1406 SDVTList NodeTys = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
1407 SDLoc dl(GA);
1408 SDValue TGA = DAG.getTargetGlobalAddress(GV: GA->getGlobal(), DL: dl,
1409 VT: GA->getValueType(ResNo: 0),
1410 offset: GA->getOffset(),
1411 TargetFlags: OperandFlags);
1412 // Create Operands for the call.The Operands should have the following:
1413 // 1. Chain SDValue
1414 // 2. Callee which in this case is the Global address value.
1415 // 3. Registers live into the call.In this case its R0, as we
1416 // have just one argument to be passed.
1417 // 4. Glue.
1418 // Note: The order is important.
1419
1420 const auto &HRI = *Subtarget.getRegisterInfo();
1421 const uint32_t *Mask = HRI.getCallPreservedMask(MF, CallingConv::C);
1422 assert(Mask && "Missing call preserved mask for calling convention");
1423 SDValue Ops[] = { Chain, TGA, DAG.getRegister(Reg: Hexagon::R0, VT: PtrVT),
1424 DAG.getRegisterMask(RegMask: Mask), Glue };
1425 Chain = DAG.getNode(Opcode: HexagonISD::CALL, DL: dl, VTList: NodeTys, Ops);
1426
1427 // Inform MFI that function has calls.
1428 MFI.setAdjustsStack(true);
1429
1430 Glue = Chain.getValue(R: 1);
1431 return DAG.getCopyFromReg(Chain, dl, Reg: ReturnReg, VT: PtrVT, Glue);
1432}
1433
1434//
1435// Lower using the initial executable model for TLS addresses
1436//
1437SDValue
1438HexagonTargetLowering::LowerToTLSInitialExecModel(GlobalAddressSDNode *GA,
1439 SelectionDAG &DAG) const {
1440 SDLoc dl(GA);
1441 int64_t Offset = GA->getOffset();
1442 auto PtrVT = getPointerTy(DL: DAG.getDataLayout());
1443
1444 // Get the thread pointer.
1445 SDValue TP = DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl, Reg: Hexagon::UGP, VT: PtrVT);
1446
1447 bool IsPositionIndependent = isPositionIndependent();
1448 unsigned char TF =
1449 IsPositionIndependent ? HexagonII::MO_IEGOT : HexagonII::MO_IE;
1450
1451 // First generate the TLS symbol address
1452 SDValue TGA = DAG.getTargetGlobalAddress(GV: GA->getGlobal(), DL: dl, VT: PtrVT,
1453 offset: Offset, TargetFlags: TF);
1454
1455 SDValue Sym = DAG.getNode(Opcode: HexagonISD::CONST32, DL: dl, VT: PtrVT, Operand: TGA);
1456
1457 if (IsPositionIndependent) {
1458 // Generate the GOT pointer in case of position independent code
1459 SDValue GOT = LowerGLOBAL_OFFSET_TABLE(Op: Sym, DAG);
1460
1461 // Add the TLS Symbol address to GOT pointer.This gives
1462 // GOT relative relocation for the symbol.
1463 Sym = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: GOT, N2: Sym);
1464 }
1465
1466 // Load the offset value for TLS symbol.This offset is relative to
1467 // thread pointer.
1468 SDValue LoadOffset =
1469 DAG.getLoad(VT: PtrVT, dl, Chain: DAG.getEntryNode(), Ptr: Sym, PtrInfo: MachinePointerInfo());
1470
1471 // Address of the thread local variable is the add of thread
1472 // pointer and the offset of the variable.
1473 return DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: TP, N2: LoadOffset);
1474}
1475
1476//
1477// Lower using the local executable model for TLS addresses
1478//
1479SDValue
1480HexagonTargetLowering::LowerToTLSLocalExecModel(GlobalAddressSDNode *GA,
1481 SelectionDAG &DAG) const {
1482 SDLoc dl(GA);
1483 int64_t Offset = GA->getOffset();
1484 auto PtrVT = getPointerTy(DL: DAG.getDataLayout());
1485
1486 // Get the thread pointer.
1487 SDValue TP = DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl, Reg: Hexagon::UGP, VT: PtrVT);
1488 // Generate the TLS symbol address
1489 SDValue TGA = DAG.getTargetGlobalAddress(GV: GA->getGlobal(), DL: dl, VT: PtrVT, offset: Offset,
1490 TargetFlags: HexagonII::MO_TPREL);
1491 SDValue Sym = DAG.getNode(Opcode: HexagonISD::CONST32, DL: dl, VT: PtrVT, Operand: TGA);
1492
1493 // Address of the thread local variable is the add of thread
1494 // pointer and the offset of the variable.
1495 return DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: TP, N2: Sym);
1496}
1497
1498//
1499// Lower using the general dynamic model for TLS addresses
1500//
1501SDValue
1502HexagonTargetLowering::LowerToTLSGeneralDynamicModel(GlobalAddressSDNode *GA,
1503 SelectionDAG &DAG) const {
1504 SDLoc dl(GA);
1505 int64_t Offset = GA->getOffset();
1506 auto PtrVT = getPointerTy(DL: DAG.getDataLayout());
1507
1508 // First generate the TLS symbol address
1509 SDValue TGA = DAG.getTargetGlobalAddress(GV: GA->getGlobal(), DL: dl, VT: PtrVT, offset: Offset,
1510 TargetFlags: HexagonII::MO_GDGOT);
1511
1512 // Then, generate the GOT pointer
1513 SDValue GOT = LowerGLOBAL_OFFSET_TABLE(Op: TGA, DAG);
1514
1515 // Add the TLS symbol and the GOT pointer
1516 SDValue Sym = DAG.getNode(Opcode: HexagonISD::CONST32, DL: dl, VT: PtrVT, Operand: TGA);
1517 SDValue Chain = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: GOT, N2: Sym);
1518
1519 // Copy over the argument to R0
1520 SDValue InGlue;
1521 Chain = DAG.getCopyToReg(Chain: DAG.getEntryNode(), dl, Reg: Hexagon::R0, N: Chain, Glue: InGlue);
1522 InGlue = Chain.getValue(R: 1);
1523
1524 unsigned Flags = DAG.getSubtarget<HexagonSubtarget>().useLongCalls()
1525 ? HexagonII::MO_GDPLT | HexagonII::HMOTF_ConstExtended
1526 : HexagonII::MO_GDPLT;
1527
1528 return GetDynamicTLSAddr(DAG, Chain, GA, Glue: InGlue, PtrVT,
1529 ReturnReg: Hexagon::R0, OperandFlags: Flags);
1530}
1531
1532//
1533// Lower TLS addresses.
1534//
1535// For now for dynamic models, we only support the general dynamic model.
1536//
1537SDValue
1538HexagonTargetLowering::LowerGlobalTLSAddress(SDValue Op,
1539 SelectionDAG &DAG) const {
1540 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Val&: Op);
1541
1542 switch (HTM.getTLSModel(GV: GA->getGlobal())) {
1543 case TLSModel::GeneralDynamic:
1544 case TLSModel::LocalDynamic:
1545 return LowerToTLSGeneralDynamicModel(GA, DAG);
1546 case TLSModel::InitialExec:
1547 return LowerToTLSInitialExecModel(GA, DAG);
1548 case TLSModel::LocalExec:
1549 return LowerToTLSLocalExecModel(GA, DAG);
1550 }
1551 llvm_unreachable("Bogus TLS model");
1552}
1553
1554//===----------------------------------------------------------------------===//
1555// TargetLowering Implementation
1556//===----------------------------------------------------------------------===//
1557
1558HexagonTargetLowering::HexagonTargetLowering(const TargetMachine &TM,
1559 const HexagonSubtarget &ST)
1560 : TargetLowering(TM, ST),
1561 HTM(static_cast<const HexagonTargetMachine &>(TM)), Subtarget(ST) {
1562 auto &HRI = *Subtarget.getRegisterInfo();
1563
1564 setPrefLoopAlignment(Align(16));
1565 setMinFunctionAlignment(Align(4));
1566 setPrefFunctionAlignment(Align(16));
1567 setStackPointerRegisterToSaveRestore(HRI.getStackRegister());
1568 setBooleanContents(TargetLoweringBase::UndefinedBooleanContent);
1569 setBooleanVectorContents(TargetLoweringBase::UndefinedBooleanContent);
1570
1571 setMaxAtomicSizeInBitsSupported(64);
1572 setMinCmpXchgSizeInBits(32);
1573
1574 if (EnableHexSDNodeSched)
1575 setSchedulingPreference(Sched::VLIW);
1576 else
1577 setSchedulingPreference(Sched::Source);
1578
1579 // Limits for inline expansion of memcpy/memmove
1580 MaxStoresPerMemcpy = 6;
1581 MaxStoresPerMemcpyOptSize = 4;
1582 MaxStoresPerMemmove = 6;
1583 MaxStoresPerMemmoveOptSize = 4;
1584 MaxStoresPerMemset = 8;
1585 MaxStoresPerMemsetOptSize = 4;
1586
1587 setTargetDAGCombine(ISD::VECREDUCE_ADD);
1588
1589 //
1590 // Set up register classes.
1591 //
1592
1593 addRegisterClass(VT: MVT::i1, RC: &Hexagon::PredRegsRegClass);
1594 addRegisterClass(VT: MVT::v2i1, RC: &Hexagon::PredRegsRegClass); // bbbbaaaa
1595 addRegisterClass(VT: MVT::v4i1, RC: &Hexagon::PredRegsRegClass); // ddccbbaa
1596 addRegisterClass(VT: MVT::v8i1, RC: &Hexagon::PredRegsRegClass); // hgfedcba
1597 addRegisterClass(VT: MVT::i32, RC: &Hexagon::IntRegsRegClass);
1598 addRegisterClass(VT: MVT::v2i16, RC: &Hexagon::IntRegsRegClass);
1599 addRegisterClass(VT: MVT::v4i8, RC: &Hexagon::IntRegsRegClass);
1600 addRegisterClass(VT: MVT::i64, RC: &Hexagon::DoubleRegsRegClass);
1601 addRegisterClass(VT: MVT::v8i8, RC: &Hexagon::DoubleRegsRegClass);
1602 addRegisterClass(VT: MVT::v4i16, RC: &Hexagon::DoubleRegsRegClass);
1603 addRegisterClass(VT: MVT::v2i32, RC: &Hexagon::DoubleRegsRegClass);
1604
1605 addRegisterClass(VT: MVT::f32, RC: &Hexagon::IntRegsRegClass);
1606 addRegisterClass(VT: MVT::f64, RC: &Hexagon::DoubleRegsRegClass);
1607
1608 //
1609 // Handling of scalar operations.
1610 //
1611 // All operations default to "legal", except:
1612 // - indexed loads and stores (pre-/post-incremented),
1613 // - ANY_EXTEND_VECTOR_INREG, ATOMIC_CMP_SWAP_WITH_SUCCESS, CONCAT_VECTORS,
1614 // ConstantFP, FCEIL, FCOPYSIGN, FEXP, FEXP2, FFLOOR, FGETSIGN,
1615 // FLOG, FLOG2, FLOG10, FMAXIMUMNUM, FMINIMUMNUM, FNEARBYINT, FRINT, FROUND,
1616 // TRAP, FTRUNC, PREFETCH, SIGN_EXTEND_VECTOR_INREG,
1617 // ZERO_EXTEND_VECTOR_INREG,
1618 // which default to "expand" for at least one type.
1619
1620 // Misc operations.
1621 setOperationAction(Op: ISD::ConstantFP, VT: MVT::f32, Action: Legal);
1622 setOperationAction(Op: ISD::ConstantFP, VT: MVT::f64, Action: Legal);
1623 setOperationAction(Op: ISD::TRAP, VT: MVT::Other, Action: Legal);
1624 setOperationAction(Op: ISD::DEBUGTRAP, VT: MVT::Other, Action: Legal);
1625 setOperationAction(Op: ISD::ConstantPool, VT: MVT::i32, Action: Custom);
1626 setOperationAction(Op: ISD::JumpTable, VT: MVT::i32, Action: Custom);
1627 setOperationAction(Op: ISD::BUILD_PAIR, VT: MVT::i64, Action: Expand);
1628 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i1, Action: Expand);
1629 setOperationAction(Op: ISD::INLINEASM, VT: MVT::Other, Action: Custom);
1630 setOperationAction(Op: ISD::INLINEASM_BR, VT: MVT::Other, Action: Custom);
1631 setOperationAction(Op: ISD::PREFETCH, VT: MVT::Other, Action: Custom);
1632 setOperationAction(Op: ISD::READCYCLECOUNTER, VT: MVT::i64, Action: Legal);
1633 setOperationAction(Op: ISD::READSTEADYCOUNTER, VT: MVT::i64, Action: Legal);
1634 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::Other, Action: Custom);
1635 setOperationAction(Op: ISD::INTRINSIC_VOID, VT: MVT::Other, Action: Custom);
1636 setOperationAction(Op: ISD::INTRINSIC_W_CHAIN, VT: MVT::Other, Action: Custom);
1637 setOperationAction(Op: ISD::EH_RETURN, VT: MVT::Other, Action: Custom);
1638 setOperationAction(Op: ISD::GLOBAL_OFFSET_TABLE, VT: MVT::i32, Action: Custom);
1639 setOperationAction(Op: ISD::GlobalTLSAddress, VT: MVT::i32, Action: Custom);
1640 setOperationAction(Op: ISD::ATOMIC_FENCE, VT: MVT::Other, Action: Custom);
1641
1642 // Custom legalize GlobalAddress nodes into CONST32.
1643 setOperationAction(Op: ISD::GlobalAddress, VT: MVT::i32, Action: Custom);
1644 setOperationAction(Op: ISD::GlobalAddress, VT: MVT::i8, Action: Custom);
1645 setOperationAction(Op: ISD::BlockAddress, VT: MVT::i32, Action: Custom);
1646
1647 // Hexagon needs to optimize cases with negative constants.
1648 setOperationAction(Op: ISD::SETCC, VT: MVT::i8, Action: Custom);
1649 setOperationAction(Op: ISD::SETCC, VT: MVT::i16, Action: Custom);
1650 setOperationAction(Op: ISD::SETCC, VT: MVT::v4i8, Action: Custom);
1651 setOperationAction(Op: ISD::SETCC, VT: MVT::v2i16, Action: Custom);
1652
1653 // VASTART needs to be custom lowered to use the VarArgsFrameIndex.
1654 setOperationAction(Op: ISD::VASTART, VT: MVT::Other, Action: Custom);
1655 setOperationAction(Op: ISD::VAEND, VT: MVT::Other, Action: Expand);
1656 setOperationAction(Op: ISD::VAARG, VT: MVT::Other, Action: Expand);
1657 if (Subtarget.isEnvironmentMusl())
1658 setOperationAction(Op: ISD::VACOPY, VT: MVT::Other, Action: Custom);
1659 else
1660 setOperationAction(Op: ISD::VACOPY, VT: MVT::Other, Action: Expand);
1661
1662 setOperationAction(Op: ISD::STACKSAVE, VT: MVT::Other, Action: Expand);
1663 setOperationAction(Op: ISD::STACKRESTORE, VT: MVT::Other, Action: Expand);
1664 setOperationAction(Op: ISD::DYNAMIC_STACKALLOC, VT: MVT::i32, Action: Custom);
1665
1666 if (EmitJumpTables)
1667 setMinimumJumpTableEntries(MinimumJumpTables);
1668 else
1669 setMinimumJumpTableEntries(std::numeric_limits<unsigned>::max());
1670 setOperationAction(Op: ISD::BR_JT, VT: MVT::Other, Action: Expand);
1671
1672 for (unsigned LegalIntOp :
1673 {ISD::ABS, ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX}) {
1674 setOperationAction(Op: LegalIntOp, VT: MVT::i32, Action: Legal);
1675 setOperationAction(Op: LegalIntOp, VT: MVT::i64, Action: Legal);
1676 }
1677
1678 // Hexagon has A4_addp_c and A4_subp_c that take and generate a carry bit,
1679 // but they only operate on i64.
1680 for (MVT VT : MVT::integer_valuetypes()) {
1681 setOperationAction(Op: ISD::UADDO, VT, Action: Custom);
1682 setOperationAction(Op: ISD::USUBO, VT, Action: Custom);
1683 setOperationAction(Op: ISD::SADDO, VT, Action: Expand);
1684 setOperationAction(Op: ISD::SSUBO, VT, Action: Expand);
1685 setOperationAction(Op: ISD::UADDO_CARRY, VT, Action: Expand);
1686 setOperationAction(Op: ISD::USUBO_CARRY, VT, Action: Expand);
1687 }
1688 setOperationAction(Op: ISD::UADDO_CARRY, VT: MVT::i64, Action: Custom);
1689 setOperationAction(Op: ISD::USUBO_CARRY, VT: MVT::i64, Action: Custom);
1690
1691 setOperationAction(Op: ISD::CTLZ, VT: MVT::i8, Action: Promote);
1692 setOperationAction(Op: ISD::CTLZ, VT: MVT::i16, Action: Promote);
1693 setOperationAction(Op: ISD::CTTZ, VT: MVT::i8, Action: Promote);
1694 setOperationAction(Op: ISD::CTTZ, VT: MVT::i16, Action: Promote);
1695
1696 // Popcount can count # of 1s in i64 but returns i32.
1697 setOperationAction(Op: ISD::CTPOP, VT: MVT::i8, Action: Promote);
1698 setOperationAction(Op: ISD::CTPOP, VT: MVT::i16, Action: Promote);
1699 setOperationAction(Op: ISD::CTPOP, VT: MVT::i32, Action: Promote);
1700 setOperationAction(Op: ISD::CTPOP, VT: MVT::i64, Action: Legal);
1701
1702 setOperationAction(Op: ISD::BITREVERSE, VT: MVT::i32, Action: Legal);
1703 setOperationAction(Op: ISD::BITREVERSE, VT: MVT::i64, Action: Legal);
1704 setOperationAction(Op: ISD::BSWAP, VT: MVT::i32, Action: Legal);
1705 setOperationAction(Op: ISD::BSWAP, VT: MVT::i64, Action: Legal);
1706
1707 setOperationAction(Op: ISD::FSHL, VT: MVT::i32, Action: Legal);
1708 setOperationAction(Op: ISD::FSHL, VT: MVT::i64, Action: Legal);
1709 setOperationAction(Op: ISD::FSHR, VT: MVT::i32, Action: Legal);
1710 setOperationAction(Op: ISD::FSHR, VT: MVT::i64, Action: Legal);
1711
1712 for (unsigned IntExpOp :
1713 {ISD::SDIV, ISD::UDIV, ISD::SREM, ISD::UREM,
1714 ISD::SDIVREM, ISD::UDIVREM, ISD::ROTL, ISD::ROTR,
1715 ISD::SHL_PARTS, ISD::SRA_PARTS, ISD::SRL_PARTS,
1716 ISD::SMUL_LOHI, ISD::UMUL_LOHI}) {
1717 for (MVT VT : MVT::integer_valuetypes())
1718 setOperationAction(Op: IntExpOp, VT, Action: Expand);
1719 }
1720 for (MVT VT : MVT::fp_valuetypes()) {
1721 for (unsigned FPExpOp : {ISD::FDIV, ISD::FSQRT, ISD::FSIN, ISD::FCOS,
1722 ISD::FSINCOS, ISD::FPOW, ISD::FCOPYSIGN})
1723 setOperationAction(Op: FPExpOp, VT, Action: Expand);
1724
1725 setOperationAction(Op: ISD::FREM, VT, Action: LibCall);
1726 }
1727
1728 // No extending loads from i32.
1729 for (MVT VT : MVT::integer_valuetypes()) {
1730 setLoadExtAction(ExtType: ISD::ZEXTLOAD, ValVT: VT, MemVT: MVT::i32, Action: Expand);
1731 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: VT, MemVT: MVT::i32, Action: Expand);
1732 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: MVT::i32, Action: Expand);
1733 }
1734 // Turn FP truncstore into trunc + store.
1735 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f32, Action: Expand);
1736 setTruncStoreAction(ValVT: MVT::f32, MemVT: MVT::bf16, Action: Expand);
1737 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::bf16, Action: Expand);
1738 // Turn FP extload into load/fpextend.
1739 for (MVT VT : MVT::fp_valuetypes())
1740 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: MVT::f32, Action: Expand);
1741
1742 // Expand BR_CC and SELECT_CC for all integer and fp types.
1743 for (MVT VT : MVT::integer_valuetypes()) {
1744 setOperationAction(Op: ISD::BR_CC, VT, Action: Expand);
1745 setOperationAction(Op: ISD::SELECT_CC, VT, Action: Expand);
1746 }
1747 for (MVT VT : MVT::fp_valuetypes()) {
1748 setOperationAction(Op: ISD::BR_CC, VT, Action: Expand);
1749 setOperationAction(Op: ISD::SELECT_CC, VT, Action: Expand);
1750 }
1751 setOperationAction(Op: ISD::BR_CC, VT: MVT::Other, Action: Expand);
1752
1753 //
1754 // Handling of vector operations.
1755 //
1756
1757 // Set the action for vector operations to "expand", then override it with
1758 // either "custom" or "legal" for specific cases.
1759 // clang-format off
1760 static const unsigned VectExpOps[] = {
1761 // Integer arithmetic:
1762 ISD::ADD, ISD::SUB, ISD::MUL, ISD::SDIV, ISD::UDIV,
1763 ISD::SREM, ISD::UREM, ISD::SDIVREM, ISD::UDIVREM, ISD::SADDO,
1764 ISD::UADDO, ISD::SSUBO, ISD::USUBO, ISD::SMUL_LOHI, ISD::UMUL_LOHI,
1765 // Logical/bit:
1766 ISD::AND, ISD::OR, ISD::XOR, ISD::ROTL, ISD::ROTR,
1767 ISD::CTPOP, ISD::CTLZ, ISD::CTTZ, ISD::BSWAP, ISD::BITREVERSE,
1768 // Floating point arithmetic/math functions:
1769 ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FMA, ISD::FDIV,
1770 ISD::FREM, ISD::FNEG, ISD::FABS, ISD::FSQRT, ISD::FSIN,
1771 ISD::FCOS, ISD::FPOW, ISD::FLOG, ISD::FLOG2,
1772 ISD::FLOG10, ISD::FEXP, ISD::FEXP2, ISD::FCEIL, ISD::FTRUNC,
1773 ISD::FRINT, ISD::FNEARBYINT, ISD::FROUND, ISD::FFLOOR,
1774 ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM,
1775 ISD::FSINCOS, ISD::FLDEXP,
1776 // Misc:
1777 ISD::BR_CC, ISD::SELECT_CC, ISD::ConstantPool,
1778 // Vector:
1779 ISD::BUILD_VECTOR, ISD::SCALAR_TO_VECTOR,
1780 ISD::EXTRACT_VECTOR_ELT, ISD::INSERT_VECTOR_ELT,
1781 ISD::EXTRACT_SUBVECTOR, ISD::INSERT_SUBVECTOR,
1782 ISD::CONCAT_VECTORS, ISD::VECTOR_SHUFFLE,
1783 ISD::SPLAT_VECTOR,
1784 };
1785 // clang-format on
1786
1787 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
1788 for (unsigned VectExpOp : VectExpOps)
1789 setOperationAction(Op: VectExpOp, VT, Action: Expand);
1790
1791 // Expand all extending loads and truncating stores:
1792 for (MVT TargetVT : MVT::fixedlen_vector_valuetypes()) {
1793 if (TargetVT == VT)
1794 continue;
1795 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: TargetVT, MemVT: VT, Action: Expand);
1796 setLoadExtAction(ExtType: ISD::ZEXTLOAD, ValVT: TargetVT, MemVT: VT, Action: Expand);
1797 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: TargetVT, MemVT: VT, Action: Expand);
1798 setTruncStoreAction(ValVT: VT, MemVT: TargetVT, Action: Expand);
1799 }
1800
1801 // Normalize all inputs to SELECT to be vectors of i32.
1802 if (VT.getVectorElementType() != MVT::i32) {
1803 MVT VT32 = MVT::getVectorVT(VT: MVT::i32, NumElements: VT.getSizeInBits()/32);
1804 setOperationAction(Op: ISD::SELECT, VT, Action: Promote);
1805 AddPromotedToType(Opc: ISD::SELECT, OrigVT: VT, DestVT: VT32);
1806 }
1807 setOperationAction(Op: ISD::SRA, VT, Action: Custom);
1808 setOperationAction(Op: ISD::SHL, VT, Action: Custom);
1809 setOperationAction(Op: ISD::SRL, VT, Action: Custom);
1810 }
1811
1812 setOperationAction(Op: ISD::SADDSAT, VT: MVT::i32, Action: Legal);
1813 setOperationAction(Op: ISD::SADDSAT, VT: MVT::i64, Action: Legal);
1814
1815 // Extending loads from (native) vectors of i8 into (native) vectors of i16
1816 // are legal.
1817 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v2i16, MemVT: MVT::v2i8, Action: Legal);
1818 setLoadExtAction(ExtType: ISD::ZEXTLOAD, ValVT: MVT::v2i16, MemVT: MVT::v2i8, Action: Legal);
1819 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: MVT::v2i16, MemVT: MVT::v2i8, Action: Legal);
1820 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v4i16, MemVT: MVT::v4i8, Action: Legal);
1821 setLoadExtAction(ExtType: ISD::ZEXTLOAD, ValVT: MVT::v4i16, MemVT: MVT::v4i8, Action: Legal);
1822 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: MVT::v4i16, MemVT: MVT::v4i8, Action: Legal);
1823
1824 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::v2i8, Action: Legal);
1825 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::v2i16, Action: Legal);
1826 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::v2i32, Action: Legal);
1827
1828 // Types natively supported:
1829 for (MVT NativeVT : {MVT::v8i1, MVT::v4i1, MVT::v2i1, MVT::v4i8,
1830 MVT::v8i8, MVT::v2i16, MVT::v4i16, MVT::v2i32}) {
1831 setOperationAction(Op: ISD::BUILD_VECTOR, VT: NativeVT, Action: Custom);
1832 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: NativeVT, Action: Custom);
1833 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: NativeVT, Action: Custom);
1834 setOperationAction(Op: ISD::EXTRACT_SUBVECTOR, VT: NativeVT, Action: Custom);
1835 setOperationAction(Op: ISD::INSERT_SUBVECTOR, VT: NativeVT, Action: Custom);
1836 setOperationAction(Op: ISD::CONCAT_VECTORS, VT: NativeVT, Action: Custom);
1837
1838 setOperationAction(Op: ISD::ADD, VT: NativeVT, Action: Legal);
1839 setOperationAction(Op: ISD::SUB, VT: NativeVT, Action: Legal);
1840 setOperationAction(Op: ISD::MUL, VT: NativeVT, Action: Legal);
1841 setOperationAction(Op: ISD::AND, VT: NativeVT, Action: Legal);
1842 setOperationAction(Op: ISD::OR, VT: NativeVT, Action: Legal);
1843 setOperationAction(Op: ISD::XOR, VT: NativeVT, Action: Legal);
1844
1845 if (NativeVT.getVectorElementType() != MVT::i1) {
1846 setOperationAction(Op: ISD::SPLAT_VECTOR, VT: NativeVT, Action: Legal);
1847 setOperationAction(Op: ISD::BSWAP, VT: NativeVT, Action: Legal);
1848 setOperationAction(Op: ISD::BITREVERSE, VT: NativeVT, Action: Legal);
1849 }
1850 }
1851
1852 for (MVT VT : {MVT::v8i8, MVT::v4i16, MVT::v2i32}) {
1853 setOperationAction(Op: ISD::SMIN, VT, Action: Legal);
1854 setOperationAction(Op: ISD::SMAX, VT, Action: Legal);
1855 setOperationAction(Op: ISD::UMIN, VT, Action: Legal);
1856 setOperationAction(Op: ISD::UMAX, VT, Action: Legal);
1857 }
1858
1859 // Custom lower unaligned loads.
1860 // Also, for both loads and stores, verify the alignment of the address
1861 // in case it is a compile-time constant. This is a usability feature to
1862 // provide a meaningful error message to users.
1863 for (MVT VT : {MVT::i16, MVT::i32, MVT::v4i8, MVT::i64, MVT::v8i8,
1864 MVT::v2i16, MVT::v4i16, MVT::v2i32}) {
1865 setOperationAction(Op: ISD::LOAD, VT, Action: Custom);
1866 setOperationAction(Op: ISD::STORE, VT, Action: Custom);
1867 }
1868
1869 // Custom-lower load/stores of boolean vectors.
1870 for (MVT VT : {MVT::v2i1, MVT::v4i1, MVT::v8i1}) {
1871 setOperationAction(Op: ISD::LOAD, VT, Action: Custom);
1872 setOperationAction(Op: ISD::STORE, VT, Action: Custom);
1873 }
1874
1875 // Normalize integer compares to EQ/GT/UGT
1876 for (MVT VT : {MVT::v2i16, MVT::v4i8, MVT::v8i8, MVT::v2i32, MVT::v4i16,
1877 MVT::v2i32}) {
1878 setCondCodeAction(CCs: ISD::SETNE, VT, Action: Expand);
1879 setCondCodeAction(CCs: ISD::SETLE, VT, Action: Expand);
1880 setCondCodeAction(CCs: ISD::SETGE, VT, Action: Expand);
1881 setCondCodeAction(CCs: ISD::SETLT, VT, Action: Expand);
1882 setCondCodeAction(CCs: ISD::SETULE, VT, Action: Expand);
1883 setCondCodeAction(CCs: ISD::SETUGE, VT, Action: Expand);
1884 setCondCodeAction(CCs: ISD::SETULT, VT, Action: Expand);
1885 }
1886
1887 // Normalize boolean compares to [U]LE/[U]LT
1888 for (MVT VT : {MVT::i1, MVT::v2i1, MVT::v4i1, MVT::v8i1}) {
1889 setCondCodeAction(CCs: ISD::SETGE, VT, Action: Expand);
1890 setCondCodeAction(CCs: ISD::SETGT, VT, Action: Expand);
1891 setCondCodeAction(CCs: ISD::SETUGE, VT, Action: Expand);
1892 setCondCodeAction(CCs: ISD::SETUGT, VT, Action: Expand);
1893 }
1894
1895 // Custom-lower bitcasts from i8 to v8i1.
1896 setOperationAction(Op: ISD::BITCAST, VT: MVT::i8, Action: Custom);
1897 setOperationAction(Op: ISD::SETCC, VT: MVT::v2i16, Action: Custom);
1898 setOperationAction(Op: ISD::VSELECT, VT: MVT::v4i8, Action: Custom);
1899 setOperationAction(Op: ISD::VSELECT, VT: MVT::v2i16, Action: Custom);
1900 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT: MVT::v4i8, Action: Custom);
1901 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT: MVT::v4i16, Action: Custom);
1902 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT: MVT::v8i8, Action: Custom);
1903
1904 // V5+.
1905 setOperationAction(Op: ISD::FMA, VT: MVT::f64, Action: Expand);
1906 setOperationAction(Op: ISD::FADD, VT: MVT::f64, Action: Expand);
1907 setOperationAction(Op: ISD::FSUB, VT: MVT::f64, Action: Expand);
1908 setOperationAction(Op: ISD::FMUL, VT: MVT::f64, Action: Expand);
1909 setOperationAction(Op: ISD::FDIV, VT: MVT::f32, Action: Custom);
1910
1911 setOperationAction(Op: ISD::FMINIMUMNUM, VT: MVT::f32, Action: Legal);
1912 setOperationAction(Op: ISD::FMAXIMUMNUM, VT: MVT::f32, Action: Legal);
1913 setOperationAction(Op: ISD::FMINNUM, VT: MVT::f32, Action: Legal);
1914 setOperationAction(Op: ISD::FMAXNUM, VT: MVT::f32, Action: Legal);
1915 setOperationAction(Op: ISD::FMAXIMUM, VT: MVT::f32, Action: Custom);
1916 setOperationAction(Op: ISD::FMAXIMUM, VT: MVT::f16, Action: Custom);
1917 setOperationAction(Op: ISD::FMINIMUM, VT: MVT::f32, Action: Custom);
1918 setOperationAction(Op: ISD::FMINIMUM, VT: MVT::f16, Action: Custom);
1919
1920 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i1, Action: Promote);
1921 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i8, Action: Promote);
1922 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i16, Action: Promote);
1923 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::i1, Action: Promote);
1924 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::i8, Action: Promote);
1925 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::i16, Action: Promote);
1926 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::i1, Action: Promote);
1927 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::i8, Action: Promote);
1928 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::i16, Action: Promote);
1929 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i1, Action: Promote);
1930 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i8, Action: Promote);
1931 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i16, Action: Promote);
1932
1933 // Special handling for half-precision floating point conversions.
1934 // Lower half float conversions into library calls.
1935 setOperationAction(Op: ISD::FP16_TO_FP, VT: MVT::f32, Action: Expand);
1936 setOperationAction(Op: ISD::FP16_TO_FP, VT: MVT::f64, Action: Expand);
1937 setOperationAction(Op: ISD::FP_TO_FP16, VT: MVT::f32, Action: Expand);
1938 setOperationAction(Op: ISD::FP_TO_FP16, VT: MVT::f64, Action: Expand);
1939 setOperationAction(Op: ISD::BF16_TO_FP, VT: MVT::f32, Action: Expand);
1940 setOperationAction(Op: ISD::BF16_TO_FP, VT: MVT::f64, Action: Expand);
1941 setOperationAction(Op: ISD::FP_TO_BF16, VT: MVT::f64, Action: Expand);
1942
1943 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f32, MemVT: MVT::f16, Action: Expand);
1944 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::f16, Action: Expand);
1945 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f32, MemVT: MVT::bf16, Action: Expand);
1946 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::bf16, Action: Expand);
1947
1948 setTruncStoreAction(ValVT: MVT::f32, MemVT: MVT::f16, Action: Expand);
1949 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f16, Action: Expand);
1950
1951 // Handling of indexed loads/stores: default is "expand".
1952 //
1953 for (MVT VT : {MVT::i8, MVT::i16, MVT::i32, MVT::i64, MVT::f32, MVT::f64,
1954 MVT::v2i16, MVT::v2i32, MVT::v4i8, MVT::v4i16, MVT::v8i8}) {
1955 setIndexedLoadAction(IdxModes: ISD::POST_INC, VT, Action: Legal);
1956 setIndexedStoreAction(IdxModes: ISD::POST_INC, VT, Action: Legal);
1957 }
1958
1959 // Subtarget-specific operation actions.
1960 //
1961 if (Subtarget.hasV60Ops()) {
1962 setOperationAction(Op: ISD::ROTL, VT: MVT::i32, Action: Legal);
1963 setOperationAction(Op: ISD::ROTL, VT: MVT::i64, Action: Legal);
1964 setOperationAction(Op: ISD::ROTR, VT: MVT::i32, Action: Legal);
1965 setOperationAction(Op: ISD::ROTR, VT: MVT::i64, Action: Legal);
1966 }
1967 if (Subtarget.hasV66Ops()) {
1968 setOperationAction(Op: ISD::FADD, VT: MVT::f64, Action: Legal);
1969 setOperationAction(Op: ISD::FSUB, VT: MVT::f64, Action: Legal);
1970 }
1971 if (Subtarget.hasV67Ops()) {
1972 setOperationAction(Op: ISD::FMINIMUMNUM, VT: MVT::f64, Action: Legal);
1973 setOperationAction(Op: ISD::FMAXIMUMNUM, VT: MVT::f64, Action: Legal);
1974 setOperationAction(Op: ISD::FMINNUM, VT: MVT::f64, Action: Legal);
1975 setOperationAction(Op: ISD::FMAXNUM, VT: MVT::f64, Action: Legal);
1976 setOperationAction(Op: ISD::FMUL, VT: MVT::f64, Action: Legal);
1977 }
1978
1979 setTargetDAGCombine(ISD::OR);
1980 setTargetDAGCombine(ISD::TRUNCATE);
1981 setTargetDAGCombine(ISD::VSELECT);
1982
1983 if (Subtarget.useHVXOps())
1984 initializeHVXLowering();
1985
1986 if (Subtarget.useHLXOps())
1987 initializeHLXLowering();
1988
1989 computeRegisterProperties(TRI: &HRI);
1990}
1991
1992bool
1993HexagonTargetLowering::validateConstPtrAlignment(SDValue Ptr, Align NeedAlign,
1994 const SDLoc &dl, SelectionDAG &DAG) const {
1995 auto *CA = dyn_cast<ConstantSDNode>(Val&: Ptr);
1996 if (!CA)
1997 return true;
1998 unsigned Addr = CA->getZExtValue();
1999 Align HaveAlign =
2000 Addr != 0 ? Align::fromLog2(Log2Value: llvm::countr_zero(Val: Addr)) : NeedAlign;
2001 if (HaveAlign >= NeedAlign)
2002 return true;
2003
2004 static int DK_MisalignedTrap = llvm::getNextAvailablePluginDiagnosticKind();
2005
2006 struct DiagnosticInfoMisalignedTrap : public DiagnosticInfo {
2007 DiagnosticInfoMisalignedTrap(StringRef M)
2008 : DiagnosticInfo(DK_MisalignedTrap, DS_Remark), Msg(M) {}
2009 void print(DiagnosticPrinter &DP) const override {
2010 DP << Msg;
2011 }
2012 static bool classof(const DiagnosticInfo *DI) {
2013 return DI->getKind() == DK_MisalignedTrap;
2014 }
2015 StringRef Msg;
2016 };
2017
2018 std::string ErrMsg;
2019 raw_string_ostream O(ErrMsg);
2020 O << "Misaligned constant address: " << format_hex(N: Addr, Width: 10)
2021 << " has alignment " << HaveAlign.value()
2022 << ", but the memory access requires " << NeedAlign.value();
2023 if (DebugLoc DL = dl.getDebugLoc())
2024 DL.print(OS&: O << ", at ");
2025 O << ". The instruction has been replaced with a trap.";
2026
2027 DAG.getContext()->diagnose(DI: DiagnosticInfoMisalignedTrap(O.str()));
2028 return false;
2029}
2030
2031SDValue
2032HexagonTargetLowering::replaceMemWithUndef(SDValue Op, SelectionDAG &DAG)
2033 const {
2034 const SDLoc &dl(Op);
2035 auto *LS = cast<LSBaseSDNode>(Val: Op.getNode());
2036 assert(!LS->isIndexed() && "Not expecting indexed ops on constant address");
2037
2038 SDValue Chain = LS->getChain();
2039 SDValue Trap = DAG.getNode(Opcode: ISD::TRAP, DL: dl, VT: MVT::Other, Operand: Chain);
2040 if (LS->getOpcode() == ISD::LOAD)
2041 return DAG.getMergeValues(Ops: {DAG.getUNDEF(VT: ty(Op)), Trap}, dl);
2042 return Trap;
2043}
2044
2045// Bit-reverse Load Intrinsic: Check if the instruction is a bit reverse load
2046// intrinsic.
2047static bool isBrevLdIntrinsic(const Value *Inst) {
2048 unsigned ID = cast<IntrinsicInst>(Val: Inst)->getIntrinsicID();
2049 return (ID == Intrinsic::hexagon_L2_loadrd_pbr ||
2050 ID == Intrinsic::hexagon_L2_loadri_pbr ||
2051 ID == Intrinsic::hexagon_L2_loadrh_pbr ||
2052 ID == Intrinsic::hexagon_L2_loadruh_pbr ||
2053 ID == Intrinsic::hexagon_L2_loadrb_pbr ||
2054 ID == Intrinsic::hexagon_L2_loadrub_pbr);
2055}
2056
2057// Bit-reverse Load Intrinsic :Crawl up and figure out the object from previous
2058// instruction. So far we only handle bitcast, extract value and bit reverse
2059// load intrinsic instructions. Should we handle CGEP ?
2060static Value *getBrevLdObject(Value *V) {
2061 if (Operator::getOpcode(V) == Instruction::ExtractValue ||
2062 Operator::getOpcode(V) == Instruction::BitCast)
2063 V = cast<Operator>(Val: V)->getOperand(i: 0);
2064 else if (isa<IntrinsicInst>(Val: V) && isBrevLdIntrinsic(Inst: V))
2065 V = cast<Instruction>(Val: V)->getOperand(i: 0);
2066 return V;
2067}
2068
2069// Bit-reverse Load Intrinsic: For a PHI Node return either an incoming edge or
2070// a back edge. If the back edge comes from the intrinsic itself, the incoming
2071// edge is returned.
2072static Value *returnEdge(const PHINode *PN, Value *IntrBaseVal) {
2073 const BasicBlock *Parent = PN->getParent();
2074 int Idx = -1;
2075 for (unsigned i = 0, e = PN->getNumIncomingValues(); i < e; ++i) {
2076 BasicBlock *Blk = PN->getIncomingBlock(i);
2077 // Determine if the back edge is originated from intrinsic.
2078 if (Blk == Parent) {
2079 Value *BackEdgeVal = PN->getIncomingValue(i);
2080 Value *BaseVal;
2081 // Loop over till we return the same Value or we hit the IntrBaseVal.
2082 do {
2083 BaseVal = BackEdgeVal;
2084 BackEdgeVal = getBrevLdObject(V: BackEdgeVal);
2085 } while ((BaseVal != BackEdgeVal) && (IntrBaseVal != BackEdgeVal));
2086 // If the getBrevLdObject returns IntrBaseVal, we should return the
2087 // incoming edge.
2088 if (IntrBaseVal == BackEdgeVal)
2089 continue;
2090 Idx = i;
2091 break;
2092 } else // Set the node to incoming edge.
2093 Idx = i;
2094 }
2095 assert(Idx >= 0 && "Unexpected index to incoming argument in PHI");
2096 return PN->getIncomingValue(i: Idx);
2097}
2098
2099// Bit-reverse Load Intrinsic: Figure out the underlying object the base
2100// pointer points to, for the bit-reverse load intrinsic. Setting this to
2101// memoperand might help alias analysis to figure out the dependencies.
2102// A bit-reverse load accesses the base pointer with its low 16 bits reversed,
2103// and post-increments the base pointer by the modifier value. For a chain of
2104// bit-reverse loads, the offset that a load accesses relative to the
2105// underlying object is the bit-reverse of the sum of the modifiers of the
2106// preceding loads in the chain. Offset is set to that value, and HasOffset is
2107// set to true, when the sum is known, that is when all of those modifiers are
2108// constants and the sum fits in 16 unsigned bits. Otherwise HasOffset is set
2109// to false and Offset is left unchanged.
2110static Value *getUnderLyingObjectForBrevLdIntr(Value *V, int &Offset,
2111 bool &HasOffset) {
2112 Value *IntrBaseVal = V;
2113 Value *BaseVal;
2114 int64_t Sum = 0;
2115 HasOffset = true;
2116 // Loop over till we return the same Value, implies we either figure out
2117 // the object or we hit a PHI
2118 do {
2119 BaseVal = V;
2120 V = getBrevLdObject(V);
2121 // Identify if this is part of a chain of bit-reverse loads, and accumulate
2122 // the modifier of the preceding load in the chain.
2123 if (HasOffset && BaseVal != V && isa<IntrinsicInst>(Val: V) &&
2124 isBrevLdIntrinsic(Inst: V)) {
2125 Value *Modifier = cast<IntrinsicInst>(Val: V)->getOperand(i_nocapture: 1);
2126 if (auto *CN = dyn_cast<ConstantInt>(Val: Modifier))
2127 Sum += CN->getSExtValue();
2128 else
2129 HasOffset = false;
2130 }
2131 } while (BaseVal != V);
2132
2133 // Only the low 16 bits of the base pointer take part in the bit-reverse. A
2134 // sum that does not fit in them would also change the remaining bits.
2135 if (HasOffset && Sum >= 0 && isUInt<16>(x: Sum))
2136 Offset = APInt(16, Sum).reverseBits().getZExtValue();
2137 else
2138 HasOffset = false;
2139
2140 // Identify the object from PHINode.
2141 if (const PHINode *PN = dyn_cast<PHINode>(Val: V))
2142 return returnEdge(PN, IntrBaseVal);
2143 // For non PHI nodes, the object is the last value returned by getBrevLdObject
2144 else
2145 return V;
2146}
2147
2148/// Infer IntrinsicInfo for an HLX memory intrinsic from its IR signature.
2149/// Loads return a vector type; stores have a void return and pass the vector
2150/// as an argument.
2151static void getHlxMemIntrinsicInfo(const CallBase &I,
2152 const HexagonSubtarget &ST,
2153 TargetLowering::IntrinsicInfo &Info) {
2154 Type *RetTy = I.getCalledFunction()->getReturnType();
2155 bool IsLoad = RetTy->isVectorTy();
2156 Type *VecTy = IsLoad ? RetTy : I.getArgOperand(i: 1)->getType();
2157 assert(VecTy->isVectorTy() && ST.isHLXVectorType(MVT::getVT(VecTy)) &&
2158 "Expected HLX vector type in HLX intrinsic");
2159
2160 Info.opc = IsLoad ? ISD::INTRINSIC_W_CHAIN : ISD::INTRINSIC_VOID;
2161 Info.memVT = MVT::getVT(Ty: VecTy);
2162 Info.ptrVal = I.getArgOperand(i: 0);
2163 Info.offset = 0;
2164 Info.align = Align(ST.getHLXVectorLength());
2165 Info.flags = IsLoad ? MachineMemOperand::MOLoad : MachineMemOperand::MOStore;
2166}
2167
2168/// Given an intrinsic, checks if on the target the intrinsic will need to map
2169/// to a MemIntrinsicNode (touches memory). If this is the case, it stores
2170/// the intrinsic information into the Infos vector.
2171void HexagonTargetLowering::getTgtMemIntrinsic(
2172 SmallVectorImpl<IntrinsicInfo> &Infos, const CallBase &I,
2173 MachineFunction &MF, unsigned Intrinsic) const {
2174 IntrinsicInfo Info;
2175 switch (Intrinsic) {
2176 case Intrinsic::hexagon_LX_L16b_tmp:
2177 case Intrinsic::hexagon_LX_LL16b_tmp:
2178 case Intrinsic::hexagon_LX_S16b_new:
2179 case Intrinsic::hexagon_LX_SS16b_new:
2180 getHlxMemIntrinsicInfo(I, ST: Subtarget, Info);
2181 Infos.push_back(Elt: Info);
2182 return;
2183 case Intrinsic::hexagon_L2_loadrd_pbr:
2184 case Intrinsic::hexagon_L2_loadri_pbr:
2185 case Intrinsic::hexagon_L2_loadrh_pbr:
2186 case Intrinsic::hexagon_L2_loadruh_pbr:
2187 case Intrinsic::hexagon_L2_loadrb_pbr:
2188 case Intrinsic::hexagon_L2_loadrub_pbr: {
2189 Info.opc = ISD::INTRINSIC_W_CHAIN;
2190 auto &DL = I.getDataLayout();
2191 auto &Cont = I.getCalledFunction()->getParent()->getContext();
2192 // The intrinsic function call is of the form { ElTy, i8* }
2193 // @llvm.hexagon.L2.loadXX.pbr(i8*, i32). The pointer and memory access type
2194 // should be derived from ElTy.
2195 Type *ElTy = I.getCalledFunction()->getReturnType()->getStructElementType(N: 0);
2196 Info.memVT = MVT::getVT(Ty: ElTy);
2197 llvm::Value *BasePtrVal = I.getOperand(i_nocapture: 0);
2198 // The offset value comes through the Modifier register. Determine the
2199 // offset that is going to be accessed relative to the underlying object.
2200 // If it cannot be determined, leave the pointer information out of the
2201 // memory operand, so that alias analysis stays conservative.
2202 bool HasOffset = false;
2203 Info.offset = 0;
2204 Value *UnderlyingObj =
2205 getUnderLyingObjectForBrevLdIntr(V: BasePtrVal, Offset&: Info.offset, HasOffset);
2206 // The underlying object is unknown if the base pointer could not be traced
2207 // back to a pointer value. Also, unless the object is aligned to 64K, the
2208 // low 16 bits of the base pointer are not known, and reversing them can
2209 // produce an address anywhere in the surrounding 64K region, possibly
2210 // outside of the object.
2211 if (!UnderlyingObj->getType()->isPointerTy() ||
2212 UnderlyingObj->getPointerAlignment(DL) < Align(65536))
2213 HasOffset = false;
2214 if (HasOffset)
2215 Info.ptrVal = UnderlyingObj;
2216 Info.align = DL.getABITypeAlign(Ty: Info.memVT.getTypeForEVT(Context&: Cont));
2217 Info.flags = MachineMemOperand::MOLoad;
2218 Infos.push_back(Elt: Info);
2219 return;
2220 }
2221 case Intrinsic::hexagon_V6_vgathermw:
2222 case Intrinsic::hexagon_V6_vgathermw_128B:
2223 case Intrinsic::hexagon_V6_vgathermh:
2224 case Intrinsic::hexagon_V6_vgathermh_128B:
2225 case Intrinsic::hexagon_V6_vgathermhw:
2226 case Intrinsic::hexagon_V6_vgathermhw_128B:
2227 case Intrinsic::hexagon_V6_vgathermwq:
2228 case Intrinsic::hexagon_V6_vgathermwq_128B:
2229 case Intrinsic::hexagon_V6_vgathermhq:
2230 case Intrinsic::hexagon_V6_vgathermhq_128B:
2231 case Intrinsic::hexagon_V6_vgathermhwq:
2232 case Intrinsic::hexagon_V6_vgathermhwq_128B:
2233 case Intrinsic::hexagon_V6_vgather_vscattermh:
2234 case Intrinsic::hexagon_V6_vgather_vscattermh_128B: {
2235 const Module &M = *I.getParent()->getParent()->getParent();
2236 Info.opc = ISD::INTRINSIC_W_CHAIN;
2237 Type *VecTy = I.getArgOperand(i: I.arg_size() - 1)->getType();
2238 assert(VecTy->isVectorTy() && "Expected vector operand for vgather");
2239 Info.memVT = MVT::getVT(Ty: VecTy);
2240 Info.ptrVal = I.getArgOperand(i: 0);
2241 Info.offset = 0;
2242 Info.align =
2243 MaybeAlign(M.getDataLayout().getTypeAllocSizeInBits(Ty: VecTy) / 8);
2244 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
2245 MachineMemOperand::MOVolatile;
2246 Infos.push_back(Elt: Info);
2247 return;
2248 }
2249 default:
2250 break;
2251 }
2252}
2253
2254bool HexagonTargetLowering::hasBitTest(SDValue X, SDValue Y) const {
2255 return X.getValueType().isScalarInteger(); // 'tstbit'
2256}
2257
2258bool HexagonTargetLowering::isTruncateFree(Type *Ty1, Type *Ty2) const {
2259 return isTruncateFree(VT1: EVT::getEVT(Ty: Ty1), VT2: EVT::getEVT(Ty: Ty2));
2260}
2261
2262bool HexagonTargetLowering::isTruncateFree(EVT VT1, EVT VT2) const {
2263 if (!VT1.isSimple() || !VT2.isSimple())
2264 return false;
2265 return VT1.getSimpleVT() == MVT::i64 && VT2.getSimpleVT() == MVT::i32;
2266}
2267
2268bool HexagonTargetLowering::isFMAFasterThanFMulAndFAdd(
2269 const MachineFunction &MF, EVT VT) const {
2270 return isOperationLegalOrCustom(Op: ISD::FMA, VT);
2271}
2272
2273// Should we expand the build vector with shuffles?
2274bool HexagonTargetLowering::shouldExpandBuildVectorWithShuffles(EVT VT,
2275 unsigned DefinedValues) const {
2276 return false;
2277}
2278
2279TargetLowering::ExtractSubvectorCost
2280HexagonTargetLowering::getExtractSubvectorCost(EVT ResVT, EVT SrcVT,
2281 unsigned Index) const {
2282 assert(ResVT.getVectorElementType() == SrcVT.getVectorElementType());
2283 if (!ResVT.isSimple() || !SrcVT.isSimple())
2284 return ExtractSubvectorCost::Expensive;
2285
2286 MVT ResTy = ResVT.getSimpleVT(), SrcTy = SrcVT.getSimpleVT();
2287 if (ResTy.getVectorElementType() != MVT::i1)
2288 return ExtractSubvectorCost::Free;
2289
2290 // Non-HVX bool vectors are relatively cheap.
2291 if (SrcTy.getVectorNumElements() <= 8)
2292 return ExtractSubvectorCost::Free;
2293 return ExtractSubvectorCost::Expensive;
2294}
2295
2296bool HexagonTargetLowering::isTargetCanonicalConstantNode(SDValue Op) const {
2297 return Op.getOpcode() == ISD::CONCAT_VECTORS ||
2298 TargetLowering::isTargetCanonicalConstantNode(Op);
2299}
2300
2301bool HexagonTargetLowering::isShuffleMaskLegal(ArrayRef<int> Mask,
2302 EVT VT) const {
2303 return true;
2304}
2305
2306TargetLoweringBase::LegalizeTypeAction
2307HexagonTargetLowering::getPreferredVectorAction(MVT VT) const {
2308 unsigned VecLen = VT.getVectorMinNumElements();
2309 MVT ElemTy = VT.getVectorElementType();
2310
2311 if (VecLen == 1 || VT.isScalableVector())
2312 return TargetLoweringBase::TypeScalarizeVector;
2313
2314 if (Subtarget.useHVXOps()) {
2315 unsigned Action = getPreferredHvxVectorAction(VecTy: VT);
2316 if (Action != ~0u)
2317 return static_cast<TargetLoweringBase::LegalizeTypeAction>(Action);
2318 }
2319
2320 // Always widen (remaining) vectors of i1.
2321 if (ElemTy == MVT::i1)
2322 return TargetLoweringBase::TypeWidenVector;
2323 // Widen non-power-of-2 vectors. Such types cannot be split right now,
2324 // and computeRegisterProperties will override "split" with "widen",
2325 // which can cause other issues.
2326 if (!isPowerOf2_32(Value: VecLen))
2327 return TargetLoweringBase::TypeWidenVector;
2328
2329 return TargetLoweringBase::TypeSplitVector;
2330}
2331
2332TargetLoweringBase::LegalizeAction
2333HexagonTargetLowering::getCustomOperationAction(SDNode &Op) const {
2334 if (Subtarget.useHVXOps()) {
2335 unsigned Action = getCustomHvxOperationAction(Op);
2336 if (Action != ~0u)
2337 return static_cast<TargetLoweringBase::LegalizeAction>(Action);
2338 }
2339 return TargetLoweringBase::Legal;
2340}
2341
2342std::pair<SDValue, int>
2343HexagonTargetLowering::getBaseAndOffset(SDValue Addr) const {
2344 if (Addr.getOpcode() == ISD::ADD) {
2345 SDValue Op1 = Addr.getOperand(i: 1);
2346 if (auto *CN = dyn_cast<const ConstantSDNode>(Val: Op1.getNode()))
2347 return { Addr.getOperand(i: 0), CN->getSExtValue() };
2348 }
2349 return { Addr, 0 };
2350}
2351
2352// Lower a vector shuffle (V1, V2, V3). V1 and V2 are the two vectors
2353// to select data from, V3 is the permutation.
2354SDValue
2355HexagonTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG)
2356 const {
2357 const auto *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
2358 ArrayRef<int> AM = SVN->getMask();
2359 assert(AM.size() <= 8 && "Unexpected shuffle mask");
2360 unsigned VecLen = AM.size();
2361
2362 MVT VecTy = ty(Op);
2363 assert(!Subtarget.isHVXVectorType(VecTy, true) &&
2364 "HVX shuffles should be legal");
2365 assert(VecTy.getSizeInBits() <= 64 && "Unexpected vector length");
2366
2367 SDValue Op0 = Op.getOperand(i: 0);
2368 SDValue Op1 = Op.getOperand(i: 1);
2369 const SDLoc &dl(Op);
2370
2371 // If the inputs are not the same as the output, bail. This is not an
2372 // error situation, but complicates the handling and the default expansion
2373 // (into BUILD_VECTOR) should be adequate.
2374 if (ty(Op: Op0) != VecTy || ty(Op: Op1) != VecTy)
2375 return SDValue();
2376
2377 // Normalize the mask so that the first non-negative index comes from
2378 // the first operand.
2379 SmallVector<int, 8> Mask(AM);
2380 unsigned F = llvm::find_if(Range&: AM, P: [](int M) { return M >= 0; }) - AM.data();
2381 if (F == AM.size())
2382 return DAG.getUNDEF(VT: VecTy);
2383 if (AM[F] >= int(VecLen)) {
2384 ShuffleVectorSDNode::commuteMask(Mask);
2385 std::swap(a&: Op0, b&: Op1);
2386 }
2387
2388 // Express the shuffle mask in terms of bytes.
2389 SmallVector<int,8> ByteMask;
2390 unsigned ElemBytes = VecTy.getVectorElementType().getSizeInBits() / 8;
2391 for (int M : Mask) {
2392 if (M < 0) {
2393 for (unsigned j = 0; j != ElemBytes; ++j)
2394 ByteMask.push_back(Elt: -1);
2395 } else {
2396 for (unsigned j = 0; j != ElemBytes; ++j)
2397 ByteMask.push_back(Elt: M*ElemBytes + j);
2398 }
2399 }
2400 assert(ByteMask.size() <= 8);
2401
2402 // All non-undef (non-negative) indexes are well within [0..127], so they
2403 // fit in a single byte. Build two 64-bit words:
2404 // - MaskIdx where each byte is the corresponding index (for non-negative
2405 // indexes), and 0xFF for negative indexes, and
2406 // - MaskUnd that has 0xFF for each negative index.
2407 uint64_t MaskIdx = 0;
2408 uint64_t MaskUnd = 0;
2409 for (unsigned i = 0, e = ByteMask.size(); i != e; ++i) {
2410 unsigned S = 8*i;
2411 uint64_t M = ByteMask[i] & 0xFF;
2412 if (M == 0xFF)
2413 MaskUnd |= M << S;
2414 MaskIdx |= M << S;
2415 }
2416
2417 if (ByteMask.size() == 4) {
2418 // Identity.
2419 if (MaskIdx == (0x03020100 | MaskUnd))
2420 return Op0;
2421 // Byte swap.
2422 if (MaskIdx == (0x00010203 | MaskUnd)) {
2423 SDValue T0 = DAG.getBitcast(VT: MVT::i32, V: Op0);
2424 SDValue T1 = DAG.getNode(Opcode: ISD::BSWAP, DL: dl, VT: MVT::i32, Operand: T0);
2425 return DAG.getBitcast(VT: VecTy, V: T1);
2426 }
2427
2428 // Byte packs.
2429 SDValue Concat10 =
2430 getCombine(Hi: Op1, Lo: Op0, dl, ResTy: typeJoin(Tys: {ty(Op: Op1), ty(Op: Op0)}), DAG);
2431 if (MaskIdx == (0x06040200 | MaskUnd))
2432 return getInstr(MachineOpc: Hexagon::S2_vtrunehb, dl, Ty: VecTy, Ops: {Concat10}, DAG);
2433 if (MaskIdx == (0x07050301 | MaskUnd))
2434 return getInstr(MachineOpc: Hexagon::S2_vtrunohb, dl, Ty: VecTy, Ops: {Concat10}, DAG);
2435
2436 SDValue Concat01 =
2437 getCombine(Hi: Op0, Lo: Op1, dl, ResTy: typeJoin(Tys: {ty(Op: Op0), ty(Op: Op1)}), DAG);
2438 if (MaskIdx == (0x02000604 | MaskUnd))
2439 return getInstr(MachineOpc: Hexagon::S2_vtrunehb, dl, Ty: VecTy, Ops: {Concat01}, DAG);
2440 if (MaskIdx == (0x03010705 | MaskUnd))
2441 return getInstr(MachineOpc: Hexagon::S2_vtrunohb, dl, Ty: VecTy, Ops: {Concat01}, DAG);
2442 }
2443
2444 if (ByteMask.size() == 8) {
2445 // Identity.
2446 if (MaskIdx == (0x0706050403020100ull | MaskUnd))
2447 return Op0;
2448 // Byte swap.
2449 if (MaskIdx == (0x0001020304050607ull | MaskUnd)) {
2450 SDValue T0 = DAG.getBitcast(VT: MVT::i64, V: Op0);
2451 SDValue T1 = DAG.getNode(Opcode: ISD::BSWAP, DL: dl, VT: MVT::i64, Operand: T0);
2452 return DAG.getBitcast(VT: VecTy, V: T1);
2453 }
2454
2455 // Halfword picks.
2456 if (MaskIdx == (0x0d0c050409080100ull | MaskUnd))
2457 return getInstr(MachineOpc: Hexagon::S2_shuffeh, dl, Ty: VecTy, Ops: {Op1, Op0}, DAG);
2458 if (MaskIdx == (0x0f0e07060b0a0302ull | MaskUnd))
2459 return getInstr(MachineOpc: Hexagon::S2_shuffoh, dl, Ty: VecTy, Ops: {Op1, Op0}, DAG);
2460 if (MaskIdx == (0x0d0c090805040100ull | MaskUnd))
2461 return getInstr(MachineOpc: Hexagon::S2_vtrunewh, dl, Ty: VecTy, Ops: {Op1, Op0}, DAG);
2462 if (MaskIdx == (0x0f0e0b0a07060302ull | MaskUnd))
2463 return getInstr(MachineOpc: Hexagon::S2_vtrunowh, dl, Ty: VecTy, Ops: {Op1, Op0}, DAG);
2464 if (MaskIdx == (0x0706030205040100ull | MaskUnd)) {
2465 VectorPair P = opSplit(Vec: Op0, dl, DAG);
2466 return getInstr(MachineOpc: Hexagon::S2_packhl, dl, Ty: VecTy, Ops: {P.second, P.first}, DAG);
2467 }
2468
2469 // Byte packs.
2470 if (MaskIdx == (0x0e060c040a020800ull | MaskUnd))
2471 return getInstr(MachineOpc: Hexagon::S2_shuffeb, dl, Ty: VecTy, Ops: {Op1, Op0}, DAG);
2472 if (MaskIdx == (0x0f070d050b030901ull | MaskUnd))
2473 return getInstr(MachineOpc: Hexagon::S2_shuffob, dl, Ty: VecTy, Ops: {Op1, Op0}, DAG);
2474 }
2475
2476 return SDValue();
2477}
2478
2479SDValue
2480HexagonTargetLowering::getSplatValue(SDValue Op, SelectionDAG &DAG) const {
2481 switch (Op.getOpcode()) {
2482 case ISD::BUILD_VECTOR:
2483 if (SDValue S = cast<BuildVectorSDNode>(Val&: Op)->getSplatValue())
2484 return S;
2485 break;
2486 case ISD::SPLAT_VECTOR:
2487 return Op.getOperand(i: 0);
2488 }
2489 return SDValue();
2490}
2491
2492// Create a Hexagon-specific node for shifting a vector by an integer.
2493SDValue
2494HexagonTargetLowering::getVectorShiftByInt(SDValue Op, SelectionDAG &DAG)
2495 const {
2496 unsigned NewOpc;
2497 switch (Op.getOpcode()) {
2498 case ISD::SHL:
2499 NewOpc = HexagonISD::VASL;
2500 break;
2501 case ISD::SRA:
2502 NewOpc = HexagonISD::VASR;
2503 break;
2504 case ISD::SRL:
2505 NewOpc = HexagonISD::VLSR;
2506 break;
2507 default:
2508 llvm_unreachable("Unexpected shift opcode");
2509 }
2510 if (SDValue Sp = getSplatValue(Op: Op.getOperand(i: 1), DAG)) {
2511 const SDLoc dl(Op);
2512 // Canonicalize shift amount to i32 as required.
2513 SDValue Sh = Sp;
2514 if (Sh.getValueType() != MVT::i32)
2515 Sh = DAG.getZExtOrTrunc(Op: Sh, DL: dl, VT: MVT::i32);
2516
2517 assert(Sh.getValueType() == MVT::i32 &&
2518 "Hexagon vector shift-by-int must use i32 shift operand");
2519 return DAG.getNode(Opcode: NewOpc, DL: dl, VT: ty(Op), N1: Op.getOperand(i: 0), N2: Sh);
2520 }
2521
2522 return SDValue();
2523}
2524
2525SDValue
2526HexagonTargetLowering::LowerVECTOR_SHIFT(SDValue Op, SelectionDAG &DAG) const {
2527 const SDLoc &dl(Op);
2528
2529 // First try to convert the shift (by vector) to a shift by a scalar.
2530 // If we first split the shift, the shift amount will become 'extract
2531 // subvector', and will no longer be recognized as scalar.
2532 SDValue Res = Op;
2533 if (SDValue S = getVectorShiftByInt(Op, DAG))
2534 Res = S;
2535
2536 unsigned Opc = Res.getOpcode();
2537 switch (Opc) {
2538 case HexagonISD::VASR:
2539 case HexagonISD::VLSR:
2540 case HexagonISD::VASL:
2541 break;
2542 default:
2543 // No instructions for shifts by non-scalars.
2544 return SDValue();
2545 }
2546
2547 MVT ResTy = ty(Op: Res);
2548 if (ResTy.getVectorElementType() != MVT::i8)
2549 return Res;
2550
2551 // For shifts of i8, extend the inputs to i16, then truncate back to i8.
2552 assert(ResTy.getVectorElementType() == MVT::i8);
2553 SDValue Val = Res.getOperand(i: 0), Amt = Res.getOperand(i: 1);
2554
2555 auto ShiftPartI8 = [&dl, &DAG, this](unsigned Opc, SDValue V, SDValue A) {
2556 MVT Ty = ty(Op: V);
2557 MVT ExtTy = MVT::getVectorVT(VT: MVT::i16, NumElements: Ty.getVectorNumElements());
2558 SDValue ExtV = Opc == HexagonISD::VASR ? DAG.getSExtOrTrunc(Op: V, DL: dl, VT: ExtTy)
2559 : DAG.getZExtOrTrunc(Op: V, DL: dl, VT: ExtTy);
2560 SDValue ExtS = DAG.getNode(Opcode: Opc, DL: dl, VT: ExtTy, Ops: {ExtV, A});
2561 return DAG.getZExtOrTrunc(Op: ExtS, DL: dl, VT: Ty);
2562 };
2563
2564 if (ResTy.getSizeInBits() == 32)
2565 return ShiftPartI8(Opc, Val, Amt);
2566
2567 auto [LoV, HiV] = opSplit(Vec: Val, dl, DAG);
2568 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResTy,
2569 Ops: {ShiftPartI8(Opc, LoV, Amt), ShiftPartI8(Opc, HiV, Amt)});
2570}
2571
2572SDValue
2573HexagonTargetLowering::LowerROTL(SDValue Op, SelectionDAG &DAG) const {
2574 if (isa<ConstantSDNode>(Val: Op.getOperand(i: 1).getNode()))
2575 return Op;
2576 return SDValue();
2577}
2578
2579SDValue
2580HexagonTargetLowering::LowerBITCAST(SDValue Op, SelectionDAG &DAG) const {
2581 MVT ResTy = ty(Op);
2582 SDValue InpV = Op.getOperand(i: 0);
2583 MVT InpTy = ty(Op: InpV);
2584 assert(ResTy.getSizeInBits() == InpTy.getSizeInBits());
2585 const SDLoc &dl(Op);
2586
2587 // Handle conversion from i8 to v8i1.
2588 if (InpTy == MVT::i8) {
2589 if (ResTy == MVT::v8i1) {
2590 SDValue Sc = DAG.getBitcast(VT: tyScalar(Ty: InpTy), V: InpV);
2591 SDValue Ext = DAG.getZExtOrTrunc(Op: Sc, DL: dl, VT: MVT::i32);
2592 return getInstr(MachineOpc: Hexagon::C2_tfrrp, dl, Ty: ResTy, Ops: Ext, DAG);
2593 }
2594 return SDValue();
2595 }
2596
2597 return Op;
2598}
2599
2600bool
2601HexagonTargetLowering::getBuildVectorConstInts(ArrayRef<SDValue> Values,
2602 MVT VecTy, SelectionDAG &DAG,
2603 MutableArrayRef<ConstantInt*> Consts) const {
2604 MVT ElemTy = VecTy.getVectorElementType();
2605 unsigned ElemWidth = ElemTy.getSizeInBits();
2606 IntegerType *IntTy = IntegerType::get(C&: *DAG.getContext(), NumBits: ElemWidth);
2607 bool AllConst = true;
2608
2609 for (unsigned i = 0, e = Values.size(); i != e; ++i) {
2610 SDValue V = Values[i];
2611 if (V.isUndef()) {
2612 Consts[i] = ConstantInt::get(Ty: IntTy, V: 0);
2613 continue;
2614 }
2615 // Make sure to always cast to IntTy.
2616 if (auto *CN = dyn_cast<ConstantSDNode>(Val: V.getNode())) {
2617 const ConstantInt *CI = CN->getConstantIntValue();
2618 Consts[i] = cast<ConstantInt>(
2619 Val: ConstantInt::get(Ty: IntTy, V: CI->getValue().trunc(width: ElemWidth)));
2620 } else if (auto *CN = dyn_cast<ConstantFPSDNode>(Val: V.getNode())) {
2621 const ConstantFP *CF = CN->getConstantFPValue();
2622 APInt A = CF->getValueAPF().bitcastToAPInt();
2623 Consts[i] = ConstantInt::get(Ty: IntTy, V: A.getZExtValue());
2624 } else {
2625 AllConst = false;
2626 }
2627 }
2628 return AllConst;
2629}
2630
2631SDValue
2632HexagonTargetLowering::buildVector32(ArrayRef<SDValue> Elem, const SDLoc &dl,
2633 MVT VecTy, SelectionDAG &DAG) const {
2634 MVT ElemTy = VecTy.getVectorElementType();
2635 assert(VecTy.getVectorNumElements() == Elem.size());
2636
2637 SmallVector<ConstantInt*,4> Consts(Elem.size());
2638 bool AllConst = getBuildVectorConstInts(Values: Elem, VecTy, DAG, Consts);
2639
2640 unsigned First, Num = Elem.size();
2641 for (First = 0; First != Num; ++First) {
2642 if (!isUndef(Op: Elem[First]))
2643 break;
2644 }
2645 if (First == Num)
2646 return DAG.getUNDEF(VT: VecTy);
2647
2648 if (AllConst &&
2649 llvm::all_of(Range&: Consts, P: [](ConstantInt *CI) { return CI->isZero(); }))
2650 return getZero(dl, Ty: VecTy, DAG);
2651
2652 if (ElemTy == MVT::i16 || ElemTy == MVT::f16) {
2653 assert(Elem.size() == 2);
2654 if (AllConst) {
2655 // The 'Consts' array will have all values as integers regardless
2656 // of the vector element type.
2657 uint32_t V = (Consts[0]->getZExtValue() & 0xFFFF) |
2658 Consts[1]->getZExtValue() << 16;
2659 return DAG.getBitcast(VT: VecTy, V: DAG.getConstant(Val: V, DL: dl, VT: MVT::i32));
2660 }
2661 SDValue E0, E1;
2662 if (ElemTy == MVT::f16) {
2663 E0 = DAG.getZExtOrTrunc(Op: DAG.getBitcast(VT: MVT::i16, V: Elem[0]), DL: dl, VT: MVT::i32);
2664 E1 = DAG.getZExtOrTrunc(Op: DAG.getBitcast(VT: MVT::i16, V: Elem[1]), DL: dl, VT: MVT::i32);
2665 } else {
2666 E0 = Elem[0];
2667 E1 = Elem[1];
2668 }
2669 SDValue N = getInstr(MachineOpc: Hexagon::A2_combine_ll, dl, Ty: MVT::i32, Ops: {E1, E0}, DAG);
2670 return DAG.getBitcast(VT: VecTy, V: N);
2671 }
2672
2673 if (ElemTy == MVT::i8) {
2674 // First try generating a constant.
2675 if (AllConst) {
2676 uint32_t V = (Consts[0]->getZExtValue() & 0xFF) |
2677 (Consts[1]->getZExtValue() & 0xFF) << 8 |
2678 (Consts[2]->getZExtValue() & 0xFF) << 16 |
2679 Consts[3]->getZExtValue() << 24;
2680 return DAG.getBitcast(VT: MVT::v4i8, V: DAG.getConstant(Val: V, DL: dl, VT: MVT::i32));
2681 }
2682
2683 // Then try splat.
2684 bool IsSplat = true;
2685 for (unsigned i = First+1; i != Num; ++i) {
2686 if (Elem[i] == Elem[First] || isUndef(Op: Elem[i]))
2687 continue;
2688 IsSplat = false;
2689 break;
2690 }
2691 if (IsSplat) {
2692 // Legalize the operand of SPLAT_VECTOR.
2693 SDValue Ext = DAG.getZExtOrTrunc(Op: Elem[First], DL: dl, VT: MVT::i32);
2694 return DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: VecTy, Operand: Ext);
2695 }
2696
2697 // Generate
2698 // (zxtb(Elem[0]) | (zxtb(Elem[1]) << 8)) |
2699 // (zxtb(Elem[2]) | (zxtb(Elem[3]) << 8)) << 16
2700 assert(Elem.size() == 4);
2701 SDValue Vs[4];
2702 for (unsigned i = 0; i != 4; ++i) {
2703 Vs[i] = DAG.getZExtOrTrunc(Op: Elem[i], DL: dl, VT: MVT::i32);
2704 Vs[i] = DAG.getZeroExtendInReg(Op: Vs[i], DL: dl, VT: MVT::i8);
2705 }
2706 SDValue S8 = DAG.getConstant(Val: 8, DL: dl, VT: MVT::i32);
2707 SDValue T0 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: MVT::i32, Ops: {Vs[1], S8});
2708 SDValue T1 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: MVT::i32, Ops: {Vs[3], S8});
2709 SDValue B0 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: MVT::i32, Ops: {Vs[0], T0});
2710 SDValue B1 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: MVT::i32, Ops: {Vs[2], T1});
2711
2712 SDValue R = getInstr(MachineOpc: Hexagon::A2_combine_ll, dl, Ty: MVT::i32, Ops: {B1, B0}, DAG);
2713 return DAG.getBitcast(VT: MVT::v4i8, V: R);
2714 }
2715
2716#ifndef NDEBUG
2717 dbgs() << "VecTy: " << VecTy << '\n';
2718#endif
2719 llvm_unreachable("Unexpected vector element type");
2720}
2721
2722SDValue
2723HexagonTargetLowering::buildVector64(ArrayRef<SDValue> Elem, const SDLoc &dl,
2724 MVT VecTy, SelectionDAG &DAG) const {
2725 MVT ElemTy = VecTy.getVectorElementType();
2726 assert(VecTy.getVectorNumElements() == Elem.size());
2727
2728 SmallVector<ConstantInt*,8> Consts(Elem.size());
2729 bool AllConst = getBuildVectorConstInts(Values: Elem, VecTy, DAG, Consts);
2730
2731 unsigned First, Num = Elem.size();
2732 for (First = 0; First != Num; ++First) {
2733 if (!isUndef(Op: Elem[First]))
2734 break;
2735 }
2736 if (First == Num)
2737 return DAG.getUNDEF(VT: VecTy);
2738
2739 if (AllConst &&
2740 llvm::all_of(Range&: Consts, P: [](ConstantInt *CI) { return CI->isZero(); }))
2741 return getZero(dl, Ty: VecTy, DAG);
2742
2743 // First try splat if possible.
2744 if (ElemTy == MVT::i16 || ElemTy == MVT::f16) {
2745 bool IsSplat = true;
2746 for (unsigned i = First+1; i != Num; ++i) {
2747 if (Elem[i] == Elem[First] || isUndef(Op: Elem[i]))
2748 continue;
2749 IsSplat = false;
2750 break;
2751 }
2752 if (IsSplat) {
2753 // Legalize the operand of SPLAT_VECTOR
2754 SDValue S = ElemTy == MVT::f16 ? DAG.getBitcast(VT: MVT::i16, V: Elem[First])
2755 : Elem[First];
2756 SDValue Ext = DAG.getZExtOrTrunc(Op: S, DL: dl, VT: MVT::i32);
2757 return DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: VecTy, Operand: Ext);
2758 }
2759 }
2760
2761 // Then try constant.
2762 if (AllConst) {
2763 uint64_t Val = 0;
2764 unsigned W = ElemTy.getSizeInBits();
2765 uint64_t Mask = (1ull << W) - 1;
2766 for (unsigned i = 0; i != Num; ++i)
2767 Val = (Val << W) | (Consts[Num-1-i]->getZExtValue() & Mask);
2768 SDValue V0 = DAG.getConstant(Val, DL: dl, VT: MVT::i64);
2769 return DAG.getBitcast(VT: VecTy, V: V0);
2770 }
2771
2772 // Build two 32-bit vectors and concatenate.
2773 MVT HalfTy = MVT::getVectorVT(VT: ElemTy, NumElements: Num/2);
2774 SDValue L = (ElemTy == MVT::i32)
2775 ? Elem[0]
2776 : buildVector32(Elem: Elem.take_front(N: Num/2), dl, VecTy: HalfTy, DAG);
2777 SDValue H = (ElemTy == MVT::i32)
2778 ? Elem[1]
2779 : buildVector32(Elem: Elem.drop_front(N: Num/2), dl, VecTy: HalfTy, DAG);
2780 return getCombine(Hi: H, Lo: L, dl, ResTy: VecTy, DAG);
2781}
2782
2783SDValue
2784HexagonTargetLowering::extractVector(SDValue VecV, SDValue IdxV,
2785 const SDLoc &dl, MVT ValTy, MVT ResTy,
2786 SelectionDAG &DAG) const {
2787 MVT VecTy = ty(Op: VecV);
2788 assert(!ValTy.isVector() ||
2789 VecTy.getVectorElementType() == ValTy.getVectorElementType());
2790 if (VecTy.getVectorElementType() == MVT::i1)
2791 return extractVectorPred(VecV, IdxV, dl, ValTy, ResTy, DAG);
2792
2793 unsigned VecWidth = VecTy.getSizeInBits();
2794 unsigned ValWidth = ValTy.getSizeInBits();
2795 unsigned ElemWidth = VecTy.getVectorElementType().getSizeInBits();
2796 assert((VecWidth % ElemWidth) == 0);
2797 assert(VecWidth == 32 || VecWidth == 64);
2798
2799 // Cast everything to scalar integer types.
2800 MVT ScalarTy = tyScalar(Ty: VecTy);
2801 VecV = DAG.getBitcast(VT: ScalarTy, V: VecV);
2802
2803 SDValue WidthV = DAG.getConstant(Val: ValWidth, DL: dl, VT: MVT::i32);
2804 SDValue ExtV;
2805
2806 if (auto *IdxN = dyn_cast<ConstantSDNode>(Val&: IdxV)) {
2807 unsigned Off = IdxN->getZExtValue() * ElemWidth;
2808 if (VecWidth == 64 && ValWidth == 32) {
2809 assert(Off == 0 || Off == 32);
2810 ExtV = Off == 0 ? LoHalf(V: VecV, DAG) : HiHalf(V: VecV, DAG);
2811 } else if (Off == 0 && (ValWidth % 8) == 0) {
2812 ExtV = DAG.getZeroExtendInReg(Op: VecV, DL: dl, VT: tyScalar(Ty: ValTy));
2813 } else {
2814 SDValue OffV = DAG.getConstant(Val: Off, DL: dl, VT: MVT::i32);
2815 // The return type of EXTRACTU must be the same as the type of the
2816 // input vector.
2817 ExtV = DAG.getNode(Opcode: HexagonISD::EXTRACTU, DL: dl, VT: ScalarTy,
2818 Ops: {VecV, WidthV, OffV});
2819 }
2820 } else {
2821 if (ty(Op: IdxV) != MVT::i32)
2822 IdxV = DAG.getZExtOrTrunc(Op: IdxV, DL: dl, VT: MVT::i32);
2823 SDValue OffV = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MVT::i32, N1: IdxV,
2824 N2: DAG.getConstant(Val: ElemWidth, DL: dl, VT: MVT::i32));
2825 ExtV = DAG.getNode(Opcode: HexagonISD::EXTRACTU, DL: dl, VT: ScalarTy,
2826 Ops: {VecV, WidthV, OffV});
2827 }
2828
2829 // Cast ExtV to the requested result type.
2830 ExtV = DAG.getZExtOrTrunc(Op: ExtV, DL: dl, VT: tyScalar(Ty: ResTy));
2831 ExtV = DAG.getBitcast(VT: ResTy, V: ExtV);
2832 return ExtV;
2833}
2834
2835SDValue
2836HexagonTargetLowering::extractVectorPred(SDValue VecV, SDValue IdxV,
2837 const SDLoc &dl, MVT ValTy, MVT ResTy,
2838 SelectionDAG &DAG) const {
2839 // Special case for v{8,4,2}i1 (the only boolean vectors legal in Hexagon
2840 // without any coprocessors).
2841 MVT VecTy = ty(Op: VecV);
2842 unsigned VecWidth = VecTy.getSizeInBits();
2843 unsigned ValWidth = ValTy.getSizeInBits();
2844 assert(VecWidth == VecTy.getVectorNumElements() &&
2845 "Vector elements should equal vector width size");
2846 assert(VecWidth == 8 || VecWidth == 4 || VecWidth == 2);
2847
2848 // Check if this is an extract of the lowest bit.
2849 if (isNullConstant(V: IdxV) && ValTy.getSizeInBits() == 1) {
2850 // Extracting the lowest bit is a no-op, but it changes the type,
2851 // so it must be kept as an operation to avoid errors related to
2852 // type mismatches.
2853 return DAG.getNode(Opcode: HexagonISD::TYPECAST, DL: dl, VT: MVT::i1, Operand: VecV);
2854 }
2855
2856 // If the value extracted is a single bit, use tstbit.
2857 if (ValWidth == 1) {
2858 SDValue A0 = getInstr(MachineOpc: Hexagon::C2_tfrpr, dl, Ty: MVT::i32, Ops: {VecV}, DAG);
2859 SDValue M0 = DAG.getConstant(Val: 8 / VecWidth, DL: dl, VT: MVT::i32);
2860 SDValue I0 = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MVT::i32, N1: IdxV, N2: M0);
2861 return DAG.getNode(Opcode: HexagonISD::TSTBIT, DL: dl, VT: MVT::i1, N1: A0, N2: I0);
2862 }
2863
2864 // Each bool vector (v2i1, v4i1, v8i1) always occupies 8 bits in
2865 // a predicate register. The elements of the vector are repeated
2866 // in the register (if necessary) so that the total number is 8.
2867 // The extracted subvector will need to be expanded in such a way.
2868 unsigned Scale = VecWidth / ValWidth;
2869
2870 // Generate (p2d VecV) >> 8*Idx to move the interesting bytes to
2871 // position 0.
2872 assert(ty(IdxV) == MVT::i32);
2873 unsigned VecRep = 8 / VecWidth;
2874 SDValue S0 = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MVT::i32, N1: IdxV,
2875 N2: DAG.getConstant(Val: 8*VecRep, DL: dl, VT: MVT::i32));
2876 SDValue T0 = DAG.getNode(Opcode: HexagonISD::P2D, DL: dl, VT: MVT::i64, Operand: VecV);
2877 SDValue T1 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i64, N1: T0, N2: S0);
2878 while (Scale > 1) {
2879 // The longest possible subvector is at most 32 bits, so it is always
2880 // contained in the low subregister.
2881 T1 = LoHalf(V: T1, DAG);
2882 T1 = expandPredicate(Vec32: T1, dl, DAG);
2883 Scale /= 2;
2884 }
2885
2886 return DAG.getNode(Opcode: HexagonISD::D2P, DL: dl, VT: ResTy, Operand: T1);
2887}
2888
2889SDValue
2890HexagonTargetLowering::insertVector(SDValue VecV, SDValue ValV, SDValue IdxV,
2891 const SDLoc &dl, MVT ValTy,
2892 SelectionDAG &DAG) const {
2893 MVT VecTy = ty(Op: VecV);
2894 if (VecTy.getVectorElementType() == MVT::i1)
2895 return insertVectorPred(VecV, ValV, IdxV, dl, ValTy, DAG);
2896
2897 unsigned VecWidth = VecTy.getSizeInBits();
2898 unsigned ValWidth = ValTy.getSizeInBits();
2899 assert(VecWidth == 32 || VecWidth == 64);
2900 assert((VecWidth % ValWidth) == 0);
2901
2902 // Cast everything to scalar integer types.
2903 MVT ScalarTy = MVT::getIntegerVT(BitWidth: VecWidth);
2904 // The actual type of ValV may be different than ValTy (which is related
2905 // to the vector type).
2906 unsigned VW = ty(Op: ValV).getSizeInBits();
2907 ValV = DAG.getBitcast(VT: MVT::getIntegerVT(BitWidth: VW), V: ValV);
2908 VecV = DAG.getBitcast(VT: ScalarTy, V: VecV);
2909 if (VW != VecWidth)
2910 ValV = DAG.getAnyExtOrTrunc(Op: ValV, DL: dl, VT: ScalarTy);
2911
2912 SDValue WidthV = DAG.getConstant(Val: ValWidth, DL: dl, VT: MVT::i32);
2913 SDValue InsV;
2914
2915 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: IdxV)) {
2916 unsigned W = C->getZExtValue() * ValWidth;
2917 SDValue OffV = DAG.getConstant(Val: W, DL: dl, VT: MVT::i32);
2918 InsV = DAG.getNode(Opcode: HexagonISD::INSERT, DL: dl, VT: ScalarTy,
2919 Ops: {VecV, ValV, WidthV, OffV});
2920 } else {
2921 if (ty(Op: IdxV) != MVT::i32)
2922 IdxV = DAG.getZExtOrTrunc(Op: IdxV, DL: dl, VT: MVT::i32);
2923 SDValue OffV = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MVT::i32, N1: IdxV, N2: WidthV);
2924 InsV = DAG.getNode(Opcode: HexagonISD::INSERT, DL: dl, VT: ScalarTy,
2925 Ops: {VecV, ValV, WidthV, OffV});
2926 }
2927
2928 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: VecTy, Operand: InsV);
2929}
2930
2931SDValue
2932HexagonTargetLowering::insertVectorPred(SDValue VecV, SDValue ValV,
2933 SDValue IdxV, const SDLoc &dl,
2934 MVT ValTy, SelectionDAG &DAG) const {
2935 MVT VecTy = ty(Op: VecV);
2936 unsigned VecLen = VecTy.getVectorNumElements();
2937
2938 if (ValTy == MVT::i1) {
2939 SDValue ToReg = getInstr(MachineOpc: Hexagon::C2_tfrpr, dl, Ty: MVT::i32, Ops: {VecV}, DAG);
2940 SDValue Ext = DAG.getSExtOrTrunc(Op: ValV, DL: dl, VT: MVT::i32);
2941 SDValue Width = DAG.getConstant(Val: 8 / VecLen, DL: dl, VT: MVT::i32);
2942 SDValue Idx = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MVT::i32, N1: IdxV, N2: Width);
2943 SDValue Ins =
2944 DAG.getNode(Opcode: HexagonISD::INSERT, DL: dl, VT: MVT::i32, Ops: {ToReg, Ext, Width, Idx});
2945 return getInstr(MachineOpc: Hexagon::C2_tfrrp, dl, Ty: VecTy, Ops: {Ins}, DAG);
2946 }
2947
2948 assert(ValTy.getVectorElementType() == MVT::i1);
2949 SDValue ValR = ValTy.isVector()
2950 ? DAG.getNode(Opcode: HexagonISD::P2D, DL: dl, VT: MVT::i64, Operand: ValV)
2951 : DAG.getSExtOrTrunc(Op: ValV, DL: dl, VT: MVT::i64);
2952
2953 unsigned Scale = VecLen / ValTy.getVectorNumElements();
2954 assert(Scale > 1);
2955
2956 for (unsigned R = Scale; R > 1; R /= 2) {
2957 ValR = contractPredicate(Vec64: ValR, dl, DAG);
2958 ValR = getCombine(Hi: DAG.getUNDEF(VT: MVT::i32), Lo: ValR, dl, ResTy: MVT::i64, DAG);
2959 }
2960
2961 SDValue Width = DAG.getConstant(Val: 64 / Scale, DL: dl, VT: MVT::i32);
2962 SDValue Idx = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MVT::i32, N1: IdxV, N2: Width);
2963 SDValue VecR = DAG.getNode(Opcode: HexagonISD::P2D, DL: dl, VT: MVT::i64, Operand: VecV);
2964 SDValue Ins =
2965 DAG.getNode(Opcode: HexagonISD::INSERT, DL: dl, VT: MVT::i64, Ops: {VecR, ValR, Width, Idx});
2966 return DAG.getNode(Opcode: HexagonISD::D2P, DL: dl, VT: VecTy, Operand: Ins);
2967}
2968
2969SDValue
2970HexagonTargetLowering::expandPredicate(SDValue Vec32, const SDLoc &dl,
2971 SelectionDAG &DAG) const {
2972 assert(ty(Vec32).getSizeInBits() == 32);
2973 if (isUndef(Op: Vec32))
2974 return DAG.getUNDEF(VT: MVT::i64);
2975 SDValue P = DAG.getBitcast(VT: MVT::v4i8, V: Vec32);
2976 SDValue X = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: MVT::v4i16, Operand: P);
2977 return DAG.getBitcast(VT: MVT::i64, V: X);
2978}
2979
2980SDValue
2981HexagonTargetLowering::contractPredicate(SDValue Vec64, const SDLoc &dl,
2982 SelectionDAG &DAG) const {
2983 assert(ty(Vec64).getSizeInBits() == 64);
2984 if (isUndef(Op: Vec64))
2985 return DAG.getUNDEF(VT: MVT::i32);
2986 // Collect even bytes:
2987 SDValue A = DAG.getBitcast(VT: MVT::v8i8, V: Vec64);
2988 SDValue S = DAG.getVectorShuffle(VT: MVT::v8i8, dl, N1: A, N2: DAG.getUNDEF(VT: MVT::v8i8),
2989 Mask: {0, 2, 4, 6, 1, 3, 5, 7});
2990 return extractVector(VecV: S, IdxV: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32), dl, ValTy: MVT::v4i8,
2991 ResTy: MVT::i32, DAG);
2992}
2993
2994SDValue
2995HexagonTargetLowering::getZero(const SDLoc &dl, MVT Ty, SelectionDAG &DAG)
2996 const {
2997 if (Ty.isVector()) {
2998 unsigned W = Ty.getSizeInBits();
2999 if (W <= 64)
3000 return DAG.getBitcast(VT: Ty, V: DAG.getConstant(Val: 0, DL: dl, VT: MVT::getIntegerVT(BitWidth: W)));
3001 return DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: Ty, Operand: getZero(dl, Ty: MVT::i32, DAG));
3002 }
3003
3004 if (Ty.isInteger())
3005 return DAG.getConstant(Val: 0, DL: dl, VT: Ty);
3006 if (Ty.isFloatingPoint())
3007 return DAG.getConstantFP(Val: 0.0, DL: dl, VT: Ty);
3008 llvm_unreachable("Invalid type for zero");
3009}
3010
3011SDValue
3012HexagonTargetLowering::appendUndef(SDValue Val, MVT ResTy, SelectionDAG &DAG)
3013 const {
3014 MVT ValTy = ty(Op: Val);
3015 assert(ValTy.getVectorElementType() == ResTy.getVectorElementType());
3016
3017 unsigned ValLen = ValTy.getVectorNumElements();
3018 unsigned ResLen = ResTy.getVectorNumElements();
3019 if (ValLen == ResLen)
3020 return Val;
3021
3022 const SDLoc &dl(Val);
3023 assert(ValLen < ResLen);
3024 assert(ResLen % ValLen == 0);
3025
3026 SmallVector<SDValue, 4> Concats = {Val};
3027 for (unsigned i = 1, e = ResLen / ValLen; i < e; ++i)
3028 Concats.push_back(Elt: DAG.getUNDEF(VT: ValTy));
3029
3030 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResTy, Ops: Concats);
3031}
3032
3033SDValue
3034HexagonTargetLowering::getCombine(SDValue Hi, SDValue Lo, const SDLoc &dl,
3035 MVT ResTy, SelectionDAG &DAG) const {
3036 MVT ElemTy = ty(Op: Hi);
3037 assert(ElemTy == ty(Lo));
3038
3039 if (!ElemTy.isVector()) {
3040 assert(ElemTy.isScalarInteger());
3041 MVT PairTy = ElemTy.widenIntegerElementType();
3042 SDValue Pair = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: PairTy, N1: Lo, N2: Hi);
3043 return DAG.getBitcast(VT: ResTy, V: Pair);
3044 }
3045
3046 unsigned Width = ElemTy.getSizeInBits();
3047 MVT IntTy = MVT::getIntegerVT(BitWidth: Width);
3048 SDValue Pair =
3049 DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: IntTy.widenIntegerElementType(),
3050 Ops: {DAG.getBitcast(VT: IntTy, V: Lo), DAG.getBitcast(VT: IntTy, V: Hi)});
3051 return DAG.getBitcast(VT: ResTy, V: Pair);
3052}
3053
3054SDValue
3055HexagonTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
3056 MVT VecTy = ty(Op);
3057 unsigned BW = VecTy.getSizeInBits();
3058 const SDLoc &dl(Op);
3059 SmallVector<SDValue,8> Ops;
3060 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i)
3061 Ops.push_back(Elt: Op.getOperand(i));
3062
3063 if (BW == 32)
3064 return buildVector32(Elem: Ops, dl, VecTy, DAG);
3065 if (BW == 64)
3066 return buildVector64(Elem: Ops, dl, VecTy, DAG);
3067
3068 if (VecTy == MVT::v8i1 || VecTy == MVT::v4i1 || VecTy == MVT::v2i1) {
3069 // Check if this is a special case or all-0 or all-1.
3070 bool All0 = true, All1 = true;
3071 for (SDValue P : Ops) {
3072 auto *CN = dyn_cast<ConstantSDNode>(Val: P.getNode());
3073 if (CN == nullptr) {
3074 All0 = All1 = false;
3075 break;
3076 }
3077 uint32_t C = CN->getZExtValue();
3078 All0 &= (C == 0);
3079 All1 &= (C == 1);
3080 }
3081 if (All0)
3082 return DAG.getNode(Opcode: HexagonISD::PFALSE, DL: dl, VT: VecTy);
3083 if (All1)
3084 return DAG.getNode(Opcode: HexagonISD::PTRUE, DL: dl, VT: VecTy);
3085
3086 // For each i1 element in the resulting predicate register, put 1
3087 // shifted by the index of the element into a general-purpose register,
3088 // then or them together and transfer it back into a predicate register.
3089 SDValue Rs[8];
3090 SDValue Z = getZero(dl, Ty: MVT::i32, DAG);
3091 // Always produce 8 bits, repeat inputs if necessary.
3092 unsigned Rep = 8 / VecTy.getVectorNumElements();
3093 for (unsigned i = 0; i != 8; ++i) {
3094 SDValue S = DAG.getConstant(Val: 1ull << i, DL: dl, VT: MVT::i32);
3095 Rs[i] = DAG.getSelect(DL: dl, VT: MVT::i32, Cond: Ops[i/Rep], LHS: S, RHS: Z);
3096 }
3097 for (ArrayRef<SDValue> A(Rs); A.size() != 1; A = A.drop_back(N: A.size()/2)) {
3098 for (unsigned i = 0, e = A.size()/2; i != e; ++i)
3099 Rs[i] = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: MVT::i32, N1: Rs[2*i], N2: Rs[2*i+1]);
3100 }
3101 // Move the value directly to a predicate register.
3102 return getInstr(MachineOpc: Hexagon::C2_tfrrp, dl, Ty: VecTy, Ops: {Rs[0]}, DAG);
3103 }
3104
3105 return SDValue();
3106}
3107
3108SDValue
3109HexagonTargetLowering::LowerCONCAT_VECTORS(SDValue Op,
3110 SelectionDAG &DAG) const {
3111 MVT VecTy = ty(Op);
3112 const SDLoc &dl(Op);
3113 if (VecTy.getSizeInBits() == 64) {
3114 assert(Op.getNumOperands() == 2);
3115 return getCombine(Hi: Op.getOperand(i: 1), Lo: Op.getOperand(i: 0), dl, ResTy: VecTy, DAG);
3116 }
3117
3118 MVT ElemTy = VecTy.getVectorElementType();
3119 if (ElemTy == MVT::i1) {
3120 assert(VecTy == MVT::v2i1 || VecTy == MVT::v4i1 || VecTy == MVT::v8i1);
3121 MVT OpTy = ty(Op: Op.getOperand(i: 0));
3122 // Scale is how many times the operands need to be contracted to match
3123 // the representation in the target register.
3124 unsigned Scale = VecTy.getVectorNumElements() / OpTy.getVectorNumElements();
3125 assert(Scale == Op.getNumOperands() && Scale > 1);
3126
3127 // First, convert all bool vectors to integers, then generate pairwise
3128 // inserts to form values of doubled length. Up until there are only
3129 // two values left to concatenate, all of these values will fit in a
3130 // 32-bit integer, so keep them as i32 to use 32-bit inserts.
3131 SmallVector<SDValue,4> Words[2];
3132 unsigned IdxW = 0;
3133
3134 for (SDValue P : Op.getNode()->op_values()) {
3135 SDValue W = DAG.getNode(Opcode: HexagonISD::P2D, DL: dl, VT: MVT::i64, Operand: P);
3136 for (unsigned R = Scale; R > 1; R /= 2) {
3137 W = contractPredicate(Vec64: W, dl, DAG);
3138 W = getCombine(Hi: DAG.getUNDEF(VT: MVT::i32), Lo: W, dl, ResTy: MVT::i64, DAG);
3139 }
3140 W = LoHalf(V: W, DAG);
3141 Words[IdxW].push_back(Elt: W);
3142 }
3143
3144 while (Scale > 2) {
3145 SDValue WidthV = DAG.getConstant(Val: 64 / Scale, DL: dl, VT: MVT::i32);
3146 Words[IdxW ^ 1].clear();
3147
3148 for (unsigned i = 0, e = Words[IdxW].size(); i != e; i += 2) {
3149 SDValue W0 = Words[IdxW][i], W1 = Words[IdxW][i+1];
3150 // Insert W1 into W0 right next to the significant bits of W0.
3151 SDValue T = DAG.getNode(Opcode: HexagonISD::INSERT, DL: dl, VT: MVT::i32,
3152 Ops: {W0, W1, WidthV, WidthV});
3153 Words[IdxW ^ 1].push_back(Elt: T);
3154 }
3155 IdxW ^= 1;
3156 Scale /= 2;
3157 }
3158
3159 // At this point there should only be two words left, and Scale should be 2.
3160 assert(Scale == 2 && Words[IdxW].size() == 2);
3161
3162 SDValue WW = getCombine(Hi: Words[IdxW][1], Lo: Words[IdxW][0], dl, ResTy: MVT::i64, DAG);
3163 return DAG.getNode(Opcode: HexagonISD::D2P, DL: dl, VT: VecTy, Operand: WW);
3164 }
3165
3166 return SDValue();
3167}
3168
3169SDValue
3170HexagonTargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
3171 SelectionDAG &DAG) const {
3172 SDValue Vec = Op.getOperand(i: 0);
3173 MVT ElemTy = ty(Op: Vec).getVectorElementType();
3174 return extractVector(VecV: Vec, IdxV: Op.getOperand(i: 1), dl: SDLoc(Op), ValTy: ElemTy, ResTy: ty(Op), DAG);
3175}
3176
3177SDValue
3178HexagonTargetLowering::LowerEXTRACT_SUBVECTOR(SDValue Op,
3179 SelectionDAG &DAG) const {
3180 return extractVector(VecV: Op.getOperand(i: 0), IdxV: Op.getOperand(i: 1), dl: SDLoc(Op),
3181 ValTy: ty(Op), ResTy: ty(Op), DAG);
3182}
3183
3184SDValue
3185HexagonTargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
3186 SelectionDAG &DAG) const {
3187 return insertVector(VecV: Op.getOperand(i: 0), ValV: Op.getOperand(i: 1), IdxV: Op.getOperand(i: 2),
3188 dl: SDLoc(Op), ValTy: ty(Op).getVectorElementType(), DAG);
3189}
3190
3191SDValue
3192HexagonTargetLowering::LowerINSERT_SUBVECTOR(SDValue Op,
3193 SelectionDAG &DAG) const {
3194 SDValue ValV = Op.getOperand(i: 1);
3195 return insertVector(VecV: Op.getOperand(i: 0), ValV, IdxV: Op.getOperand(i: 2),
3196 dl: SDLoc(Op), ValTy: ty(Op: ValV), DAG);
3197}
3198
3199bool
3200HexagonTargetLowering::allowTruncateForTailCall(Type *Ty1, Type *Ty2) const {
3201 // Assuming the caller does not have either a signext or zeroext modifier, and
3202 // only one value is accepted, any reasonable truncation is allowed.
3203 if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy())
3204 return false;
3205
3206 // FIXME: in principle up to 64-bit could be made safe, but it would be very
3207 // fragile at the moment: any support for multiple value returns would be
3208 // liable to disallow tail calls involving i64 -> iN truncation in many cases.
3209 return Ty1->getPrimitiveSizeInBits() <= 32;
3210}
3211
3212SDValue
3213HexagonTargetLowering::LowerLoad(SDValue Op, SelectionDAG &DAG) const {
3214 MVT Ty = ty(Op);
3215 const SDLoc &dl(Op);
3216 LoadSDNode *LN = cast<LoadSDNode>(Val: Op.getNode());
3217 MVT MemTy = LN->getMemoryVT().getSimpleVT();
3218 ISD::LoadExtType ET = LN->getExtensionType();
3219
3220 bool LoadPred = MemTy == MVT::v2i1 || MemTy == MVT::v4i1 || MemTy == MVT::v8i1;
3221 if (LoadPred) {
3222 SDValue NL = DAG.getLoad(
3223 AM: LN->getAddressingMode(), ExtType: ISD::ZEXTLOAD, VT: MVT::i32, dl, Chain: LN->getChain(),
3224 Ptr: LN->getBasePtr(), Offset: LN->getOffset(), PtrInfo: LN->getPointerInfo(),
3225 /*MemoryVT*/ MemVT: MVT::i8, Alignment: LN->getAlign(), MMOFlags: LN->getMemOperand()->getFlags(),
3226 Metadata: MMOMetadata(LN->getAAInfo(), LN->getRanges()));
3227 LN = cast<LoadSDNode>(Val: NL.getNode());
3228 }
3229
3230 Align ClaimAlign = LN->getAlign();
3231 if (!validateConstPtrAlignment(Ptr: LN->getBasePtr(), NeedAlign: ClaimAlign, dl, DAG))
3232 return replaceMemWithUndef(Op, DAG);
3233
3234 // Call LowerUnalignedLoad for all loads, it recognizes loads that
3235 // don't need extra aligning.
3236 SDValue LU = LowerUnalignedLoad(Op: SDValue(LN, 0), DAG);
3237 if (LoadPred) {
3238 SDValue TP = getInstr(MachineOpc: Hexagon::C2_tfrrp, dl, Ty: MemTy, Ops: {LU}, DAG);
3239 if (ET == ISD::SEXTLOAD) {
3240 TP = DAG.getSExtOrTrunc(Op: TP, DL: dl, VT: Ty);
3241 } else if (ET != ISD::NON_EXTLOAD) {
3242 TP = DAG.getZExtOrTrunc(Op: TP, DL: dl, VT: Ty);
3243 }
3244 SDValue Ch = cast<LoadSDNode>(Val: LU.getNode())->getChain();
3245 return DAG.getMergeValues(Ops: {TP, Ch}, dl);
3246 }
3247 return LU;
3248}
3249
3250SDValue
3251HexagonTargetLowering::LowerStore(SDValue Op, SelectionDAG &DAG) const {
3252 const SDLoc &dl(Op);
3253 StoreSDNode *SN = cast<StoreSDNode>(Val: Op.getNode());
3254 SDValue Val = SN->getValue();
3255 MVT Ty = ty(Op: Val);
3256
3257 if (Ty == MVT::v2i1 || Ty == MVT::v4i1 || Ty == MVT::v8i1) {
3258 // Store the exact predicate (all bits).
3259 SDValue TR = getInstr(MachineOpc: Hexagon::C2_tfrpr, dl, Ty: MVT::i32, Ops: {Val}, DAG);
3260 SDValue NS = DAG.getTruncStore(Chain: SN->getChain(), dl, Val: TR, Ptr: SN->getBasePtr(),
3261 SVT: MVT::i8, MMO: SN->getMemOperand());
3262 if (SN->isIndexed()) {
3263 NS = DAG.getIndexedStore(OrigStore: NS, dl, Base: SN->getBasePtr(), Offset: SN->getOffset(),
3264 AM: SN->getAddressingMode());
3265 }
3266 SN = cast<StoreSDNode>(Val: NS.getNode());
3267 }
3268
3269 Align ClaimAlign = SN->getAlign();
3270 if (!validateConstPtrAlignment(Ptr: SN->getBasePtr(), NeedAlign: ClaimAlign, dl, DAG))
3271 return replaceMemWithUndef(Op, DAG);
3272
3273 MVT StoreTy = SN->getMemoryVT().getSimpleVT();
3274 Align NeedAlign = Subtarget.getTypeAlignment(Ty: StoreTy);
3275 if (ClaimAlign < NeedAlign)
3276 return expandUnalignedStore(ST: SN, DAG);
3277 return SDValue(SN, 0);
3278}
3279
3280SDValue
3281HexagonTargetLowering::LowerUnalignedLoad(SDValue Op, SelectionDAG &DAG)
3282 const {
3283 LoadSDNode *LN = cast<LoadSDNode>(Val: Op.getNode());
3284 MVT LoadTy = ty(Op);
3285 unsigned NeedAlign = Subtarget.getTypeAlignment(Ty: LoadTy).value();
3286 unsigned HaveAlign = LN->getAlign().value();
3287 if (HaveAlign >= NeedAlign)
3288 return Op;
3289
3290 const SDLoc &dl(Op);
3291 const DataLayout &DL = DAG.getDataLayout();
3292 LLVMContext &Ctx = *DAG.getContext();
3293
3294 // If the load aligning is disabled or the load can be broken up into two
3295 // smaller legal loads, do the default (target-independent) expansion.
3296 bool DoDefault = false;
3297 // Handle it in the default way if this is an indexed load.
3298 if (!LN->isUnindexed())
3299 DoDefault = true;
3300
3301 if (!AlignLoads) {
3302 if (allowsMemoryAccessForAlignment(Context&: Ctx, DL, VT: LN->getMemoryVT(),
3303 MMO: *LN->getMemOperand()))
3304 return Op;
3305 DoDefault = true;
3306 }
3307 if (!DoDefault && (2 * HaveAlign) == NeedAlign) {
3308 // The PartTy is the equivalent of "getLoadableTypeOfSize(HaveAlign)".
3309 MVT PartTy = HaveAlign <= 8 ? MVT::getIntegerVT(BitWidth: 8 * HaveAlign)
3310 : MVT::getVectorVT(VT: MVT::i8, NumElements: HaveAlign);
3311 DoDefault =
3312 allowsMemoryAccessForAlignment(Context&: Ctx, DL, VT: PartTy, MMO: *LN->getMemOperand());
3313 }
3314 if (DoDefault) {
3315 std::pair<SDValue, SDValue> P = expandUnalignedLoad(LD: LN, DAG);
3316 return DAG.getMergeValues(Ops: {P.first, P.second}, dl);
3317 }
3318
3319 // The code below generates two loads, both aligned as NeedAlign, and
3320 // with the distance of NeedAlign between them. For that to cover the
3321 // bits that need to be loaded (and without overlapping), the size of
3322 // the loads should be equal to NeedAlign. This is true for all loadable
3323 // types, but add an assertion in case something changes in the future.
3324 assert(LoadTy.getSizeInBits() == 8*NeedAlign);
3325
3326 unsigned LoadLen = NeedAlign;
3327 SDValue Base = LN->getBasePtr();
3328 SDValue Chain = LN->getChain();
3329 auto BO = getBaseAndOffset(Addr: Base);
3330 unsigned BaseOpc = BO.first.getOpcode();
3331 if (BaseOpc == HexagonISD::VALIGNADDR && BO.second % LoadLen == 0)
3332 return Op;
3333
3334 if (BO.second % LoadLen != 0) {
3335 BO.first = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::i32, N1: BO.first,
3336 N2: DAG.getConstant(Val: BO.second % LoadLen, DL: dl, VT: MVT::i32));
3337 BO.second -= BO.second % LoadLen;
3338 }
3339 SDValue BaseNoOff = (BaseOpc != HexagonISD::VALIGNADDR)
3340 ? DAG.getNode(Opcode: HexagonISD::VALIGNADDR, DL: dl, VT: MVT::i32, N1: BO.first,
3341 N2: DAG.getConstant(Val: NeedAlign, DL: dl, VT: MVT::i32))
3342 : BO.first;
3343 SDValue Base0 =
3344 DAG.getMemBasePlusOffset(Base: BaseNoOff, Offset: TypeSize::getFixed(ExactSize: BO.second), DL: dl);
3345 SDValue Base1 = DAG.getMemBasePlusOffset(
3346 Base: BaseNoOff, Offset: TypeSize::getFixed(ExactSize: BO.second + LoadLen), DL: dl);
3347
3348 MachineMemOperand *WideMMO = nullptr;
3349 if (MachineMemOperand *MMO = LN->getMemOperand()) {
3350 MachineFunction &MF = DAG.getMachineFunction();
3351 WideMMO = MF.getMachineMemOperand(
3352 PtrInfo: MMO->getPointerInfo(), F: MMO->getFlags(), Size: 2 * LoadLen, BaseAlignment: Align(LoadLen),
3353 Metadata: MMOMetadata(MMO->getAAInfo(), MMO->getRanges()), SSID: MMO->getSyncScopeID(),
3354 Ordering: MMO->getSuccessOrdering(), FailureOrdering: MMO->getFailureOrdering());
3355 }
3356
3357 SDValue Load0 = DAG.getLoad(VT: LoadTy, dl, Chain, Ptr: Base0, MMO: WideMMO);
3358 SDValue Load1 = DAG.getLoad(VT: LoadTy, dl, Chain, Ptr: Base1, MMO: WideMMO);
3359
3360 SDValue Aligned = DAG.getNode(Opcode: HexagonISD::VALIGN, DL: dl, VT: LoadTy,
3361 Ops: {Load1, Load0, BaseNoOff.getOperand(i: 0)});
3362 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other,
3363 N1: Load0.getValue(R: 1), N2: Load1.getValue(R: 1));
3364 SDValue M = DAG.getMergeValues(Ops: {Aligned, NewChain}, dl);
3365 return M;
3366}
3367
3368SDValue
3369HexagonTargetLowering::LowerUAddSubO(SDValue Op, SelectionDAG &DAG) const {
3370 SDValue X = Op.getOperand(i: 0), Y = Op.getOperand(i: 1);
3371 auto *CY = dyn_cast<ConstantSDNode>(Val&: Y);
3372 if (!CY)
3373 return SDValue();
3374
3375 const SDLoc &dl(Op);
3376 SDVTList VTs = Op.getNode()->getVTList();
3377 assert(VTs.NumVTs == 2);
3378 assert(VTs.VTs[1] == MVT::i1);
3379 unsigned Opc = Op.getOpcode();
3380
3381 if (CY) {
3382 uint64_t VY = CY->getZExtValue();
3383 assert(VY != 0 && "This should have been folded");
3384 // X +/- 1
3385 if (VY != 1)
3386 return SDValue();
3387
3388 if (Opc == ISD::UADDO) {
3389 SDValue Op = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: VTs.VTs[0], Ops: {X, Y});
3390 SDValue Ov = DAG.getSetCC(DL: dl, VT: MVT::i1, LHS: Op, RHS: getZero(dl, Ty: ty(Op), DAG),
3391 Cond: ISD::SETEQ);
3392 return DAG.getMergeValues(Ops: {Op, Ov}, dl);
3393 }
3394 if (Opc == ISD::USUBO) {
3395 SDValue Op = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: VTs.VTs[0], Ops: {X, Y});
3396 SDValue Ov = DAG.getSetCC(DL: dl, VT: MVT::i1, LHS: Op,
3397 RHS: DAG.getAllOnesConstant(DL: dl, VT: ty(Op)), Cond: ISD::SETEQ);
3398 return DAG.getMergeValues(Ops: {Op, Ov}, dl);
3399 }
3400 }
3401
3402 return SDValue();
3403}
3404
3405SDValue HexagonTargetLowering::LowerUAddSubOCarry(SDValue Op,
3406 SelectionDAG &DAG) const {
3407 const SDLoc &dl(Op);
3408 unsigned Opc = Op.getOpcode();
3409 SDValue X = Op.getOperand(i: 0), Y = Op.getOperand(i: 1), C = Op.getOperand(i: 2);
3410
3411 if (Opc == ISD::UADDO_CARRY)
3412 return DAG.getNode(Opcode: HexagonISD::ADDC, DL: dl, VTList: Op.getNode()->getVTList(),
3413 Ops: { X, Y, C });
3414
3415 EVT CarryTy = C.getValueType();
3416 SDValue SubC = DAG.getNode(Opcode: HexagonISD::SUBC, DL: dl, VTList: Op.getNode()->getVTList(),
3417 Ops: { X, Y, DAG.getLogicalNOT(DL: dl, Val: C, VT: CarryTy) });
3418 SDValue Out[] = { SubC.getValue(R: 0),
3419 DAG.getLogicalNOT(DL: dl, Val: SubC.getValue(R: 1), VT: CarryTy) };
3420 return DAG.getMergeValues(Ops: Out, dl);
3421}
3422
3423SDValue
3424HexagonTargetLowering::LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const {
3425 SDValue Chain = Op.getOperand(i: 0);
3426 SDValue Offset = Op.getOperand(i: 1);
3427 SDValue Handler = Op.getOperand(i: 2);
3428 SDLoc dl(Op);
3429 auto PtrVT = getPointerTy(DL: DAG.getDataLayout());
3430
3431 // Mark function as containing a call to EH_RETURN.
3432 HexagonMachineFunctionInfo *FuncInfo =
3433 DAG.getMachineFunction().getInfo<HexagonMachineFunctionInfo>();
3434 FuncInfo->setHasEHReturn();
3435
3436 unsigned OffsetReg = Hexagon::R28;
3437
3438 SDValue StoreAddr =
3439 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: DAG.getRegister(Reg: Hexagon::R30, VT: PtrVT),
3440 N2: DAG.getIntPtrConstant(Val: 4, DL: dl));
3441 Chain = DAG.getStore(Chain, dl, Val: Handler, Ptr: StoreAddr, PtrInfo: MachinePointerInfo());
3442 Chain = DAG.getCopyToReg(Chain, dl, Reg: OffsetReg, N: Offset);
3443
3444 // Not needed we already use it as explicit input to EH_RETURN.
3445 // MF.getRegInfo().addLiveOut(OffsetReg);
3446
3447 return DAG.getNode(Opcode: HexagonISD::EH_RETURN, DL: dl, VT: MVT::Other, Operand: Chain);
3448}
3449
3450SDValue
3451HexagonTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
3452 unsigned Opc = Op.getOpcode();
3453 // Handle INLINEASM first.
3454 if (Opc == ISD::INLINEASM || Opc == ISD::INLINEASM_BR)
3455 return LowerINLINEASM(Op, DAG);
3456
3457 if (isHlxOperation(N: Op.getNode(), DAG)) {
3458 // If HLX lowering returns nothing, try the default lowering.
3459 if (SDValue V = LowerHlxOperation(Op, DAG))
3460 return V;
3461 } else if (isHvxOperation(N: Op.getNode(), DAG)) {
3462 // If HVX lowering returns nothing, try the default lowering.
3463 if (SDValue V = LowerHvxOperation(Op, DAG))
3464 return V;
3465 }
3466
3467 switch (Opc) {
3468 default:
3469#ifndef NDEBUG
3470 Op.getNode()->dumpr(&DAG);
3471#endif
3472 llvm_unreachable("Should not custom lower this!");
3473
3474 case ISD::FDIV:
3475 return LowerFDIV(Op, DAG);
3476 case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG);
3477 case ISD::INSERT_SUBVECTOR: return LowerINSERT_SUBVECTOR(Op, DAG);
3478 case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG);
3479 case ISD::EXTRACT_SUBVECTOR: return LowerEXTRACT_SUBVECTOR(Op, DAG);
3480 case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG);
3481 case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG);
3482 case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG);
3483 case ISD::BITCAST: return LowerBITCAST(Op, DAG);
3484 case ISD::LOAD: return LowerLoad(Op, DAG);
3485 case ISD::STORE: return LowerStore(Op, DAG);
3486 case ISD::UADDO:
3487 case ISD::USUBO: return LowerUAddSubO(Op, DAG);
3488 case ISD::UADDO_CARRY:
3489 case ISD::USUBO_CARRY: return LowerUAddSubOCarry(Op, DAG);
3490 case ISD::SRA:
3491 case ISD::SHL:
3492 case ISD::SRL: return LowerVECTOR_SHIFT(Op, DAG);
3493 case ISD::ROTL: return LowerROTL(Op, DAG);
3494 case ISD::ConstantPool: return LowerConstantPool(Op, DAG);
3495 case ISD::JumpTable: return LowerJumpTable(Op, DAG);
3496 case ISD::EH_RETURN: return LowerEH_RETURN(Op, DAG);
3497 case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG);
3498 case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG);
3499 case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG);
3500 case ISD::ATOMIC_FENCE: return LowerATOMIC_FENCE(Op, DAG);
3501 case ISD::GlobalAddress: return LowerGLOBALADDRESS(Op, DAG);
3502 case ISD::BlockAddress: return LowerBlockAddress(Op, DAG);
3503 case ISD::GLOBAL_OFFSET_TABLE: return LowerGLOBAL_OFFSET_TABLE(Op, DAG);
3504 case ISD::VACOPY: return LowerVACOPY(Op, DAG);
3505 case ISD::VASTART: return LowerVASTART(Op, DAG);
3506 case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG);
3507 case ISD::SETCC: return LowerSETCC(Op, DAG);
3508 case ISD::VSELECT: return LowerVSELECT(Op, DAG);
3509 case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG);
3510 case ISD::INTRINSIC_VOID: return LowerINTRINSIC_VOID(Op, DAG);
3511 // INTRINSIC_W_CHAIN is marked Custom only so HLX memory intrinsics can be
3512 // lowered (handled above via LowerHlxOperation). Any other W_CHAIN
3513 // intrinsic is left unchanged for normal instruction selection.
3514 case ISD::INTRINSIC_W_CHAIN:
3515 return Op;
3516 case ISD::PREFETCH:
3517 return LowerPREFETCH(Op, DAG);
3518 case ISD::FMAXIMUM:
3519 case ISD::FMINIMUM:
3520 return LowerFMINFMAX(Op, DAG);
3521 break;
3522 }
3523
3524 return SDValue();
3525}
3526
3527void
3528HexagonTargetLowering::LowerOperationWrapper(SDNode *N,
3529 SmallVectorImpl<SDValue> &Results,
3530 SelectionDAG &DAG) const {
3531 if (isHvxOperation(N, DAG)) {
3532 LowerHvxOperationWrapper(N, Results, DAG);
3533 if (!Results.empty())
3534 return;
3535 }
3536
3537 SDValue Op(N, 0);
3538 unsigned Opc = N->getOpcode();
3539
3540 switch (Opc) {
3541 case HexagonISD::SSAT:
3542 case HexagonISD::USAT:
3543 Results.push_back(Elt: opJoin(Ops: SplitVectorOp(Op, DAG), dl: SDLoc(Op), DAG));
3544 break;
3545 case ISD::STORE:
3546 // We are only custom-lowering stores to verify the alignment of the
3547 // address if it is a compile-time constant. Since a store can be
3548 // modified during type-legalization (the value being stored may need
3549 // legalization), return empty Results here to indicate that we don't
3550 // really make any changes in the custom lowering.
3551 return;
3552 default:
3553 TargetLowering::LowerOperationWrapper(N, Results, DAG);
3554 break;
3555 }
3556}
3557
3558void
3559HexagonTargetLowering::ReplaceNodeResults(SDNode *N,
3560 SmallVectorImpl<SDValue> &Results,
3561 SelectionDAG &DAG) const {
3562 if (isHvxOperation(N, DAG)) {
3563 ReplaceHvxNodeResults(N, Results, DAG);
3564 if (!Results.empty())
3565 return;
3566 }
3567
3568 const SDLoc &dl(N);
3569 switch (N->getOpcode()) {
3570 case ISD::SRL:
3571 case ISD::SRA:
3572 case ISD::SHL:
3573 return;
3574 case ISD::BITCAST:
3575 // Handle a bitcast from v8i1 to i8.
3576 if (N->getValueType(ResNo: 0) == MVT::i8) {
3577 if (N->getOperand(Num: 0).getValueType() == MVT::v8i1) {
3578 SDValue P = getInstr(MachineOpc: Hexagon::C2_tfrpr, dl, Ty: MVT::i32,
3579 Ops: N->getOperand(Num: 0), DAG);
3580 SDValue T = DAG.getAnyExtOrTrunc(Op: P, DL: dl, VT: MVT::i8);
3581 Results.push_back(Elt: T);
3582 }
3583 }
3584 break;
3585 }
3586}
3587
3588SDValue
3589HexagonTargetLowering::PerformDAGCombine(SDNode *N,
3590 DAGCombinerInfo &DCI) const {
3591 SDValue Op(N, 0);
3592 const SDLoc &dl(Op);
3593 unsigned Opc = Op.getOpcode();
3594
3595 // Combining transformations applicable for arbitrary vector sizes.
3596 if (DCI.isBeforeLegalizeOps()) {
3597 switch (Opc) {
3598 case ISD::VECREDUCE_ADD:
3599 if (SDValue V = splitVecReduceAdd(N, DAG&: DCI.DAG))
3600 return V;
3601 if (SDValue V = expandVecReduceAdd(N, DAG&: DCI.DAG))
3602 return V;
3603 return SDValue();
3604 case ISD::PARTIAL_REDUCE_SMLA:
3605 case ISD::PARTIAL_REDUCE_UMLA:
3606 case ISD::PARTIAL_REDUCE_SUMLA:
3607 if (SDValue V = splitExtendingPartialReduceMLA(N, DAG&: DCI.DAG))
3608 return V;
3609 return SDValue();
3610 }
3611 } else {
3612 switch (Opc) {
3613 case ISD::VSELECT: {
3614 // (vselect (xor x, ptrue), v0, v1) -> (vselect x, v1, v0)
3615 SDValue Cond = Op.getOperand(i: 0);
3616 if (Cond->getOpcode() == ISD::XOR) {
3617 SDValue C0 = Cond.getOperand(i: 0), C1 = Cond.getOperand(i: 1);
3618 if (C1->getOpcode() == HexagonISD::PTRUE) {
3619 SDValue VSel = DCI.DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: ty(Op), N1: C0,
3620 N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 1));
3621 return VSel;
3622 }
3623 }
3624 return SDValue();
3625 }
3626 }
3627 }
3628
3629 if (isHvxOperation(N, DAG&: DCI.DAG)) {
3630 if (SDValue V = PerformHvxDAGCombine(N, DCI))
3631 return V;
3632 return SDValue();
3633 }
3634
3635 if (Opc == ISD::TRUNCATE) {
3636 SDValue Op0 = Op.getOperand(i: 0);
3637 // fold (truncate (build pair x, y)) -> (truncate x) or x
3638 if (Op0.getOpcode() == ISD::BUILD_PAIR) {
3639 EVT TruncTy = Op.getValueType();
3640 SDValue Elem0 = Op0.getOperand(i: 0);
3641 // if we match the low element of the pair, just return it.
3642 if (Elem0.getValueType() == TruncTy)
3643 return Elem0;
3644 // otherwise, if the low part is still too large, apply the truncate.
3645 if (Elem0.getValueType().bitsGT(VT: TruncTy))
3646 return DCI.DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: TruncTy, Operand: Elem0);
3647 }
3648 }
3649
3650 if (DCI.isBeforeLegalizeOps())
3651 return SDValue();
3652
3653 switch (Opc) {
3654 case HexagonISD::P2D: {
3655 SDValue P = Op.getOperand(i: 0);
3656 switch (P.getOpcode()) {
3657 case HexagonISD::PTRUE:
3658 return DCI.DAG.getAllOnesConstant(DL: dl, VT: ty(Op));
3659 case HexagonISD::PFALSE:
3660 return getZero(dl, Ty: ty(Op), DAG&: DCI.DAG);
3661 default:
3662 break;
3663 }
3664 break;
3665 }
3666 case ISD::TRUNCATE: {
3667 SDValue Op0 = Op.getOperand(i: 0);
3668 // fold (truncate (build pair x, y)) -> (truncate x) or x
3669 if (Op0.getOpcode() == ISD::BUILD_PAIR) {
3670 MVT TruncTy = ty(Op);
3671 SDValue Elem0 = Op0.getOperand(i: 0);
3672 // if we match the low element of the pair, just return it.
3673 if (ty(Op: Elem0) == TruncTy)
3674 return Elem0;
3675 // otherwise, if the low part is still too large, apply the truncate.
3676 if (ty(Op: Elem0).bitsGT(VT: TruncTy))
3677 return DCI.DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: TruncTy, Operand: Elem0);
3678 }
3679 break;
3680 }
3681 case ISD::OR: {
3682 // fold (or (shl xx, s), (zext y)) -> (COMBINE (shl xx, s-32), y)
3683 // if s >= 32
3684 auto fold0 = [&, this](SDValue Op) {
3685 if (ty(Op) != MVT::i64)
3686 return SDValue();
3687 SDValue Shl = Op.getOperand(i: 0);
3688 SDValue Zxt = Op.getOperand(i: 1);
3689 if (Shl.getOpcode() != ISD::SHL)
3690 std::swap(a&: Shl, b&: Zxt);
3691
3692 if (Shl.getOpcode() != ISD::SHL || Zxt.getOpcode() != ISD::ZERO_EXTEND)
3693 return SDValue();
3694
3695 SDValue Z = Zxt.getOperand(i: 0);
3696 auto *Amt = dyn_cast<ConstantSDNode>(Val: Shl.getOperand(i: 1));
3697 if (Amt && Amt->getZExtValue() >= 32 && ty(Op: Z).getSizeInBits() <= 32) {
3698 unsigned A = Amt->getZExtValue();
3699 SDValue S = Shl.getOperand(i: 0);
3700 SDValue T0 = DCI.DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: ty(Op: S), N1: S,
3701 N2: DCI.DAG.getConstant(Val: A - 32, DL: dl, VT: MVT::i32));
3702 SDValue T1 = DCI.DAG.getZExtOrTrunc(Op: T0, DL: dl, VT: MVT::i32);
3703 SDValue T2 = DCI.DAG.getZExtOrTrunc(Op: Z, DL: dl, VT: MVT::i32);
3704 return DCI.DAG.getNode(Opcode: HexagonISD::COMBINE, DL: dl, VT: MVT::i64, Ops: {T1, T2});
3705 }
3706 return SDValue();
3707 };
3708
3709 if (SDValue R = fold0(Op))
3710 return R;
3711 break;
3712 }
3713 }
3714
3715 return SDValue();
3716}
3717
3718/// Returns relocation base for the given PIC jumptable.
3719SDValue
3720HexagonTargetLowering::getPICJumpTableRelocBase(SDValue Table,
3721 SelectionDAG &DAG) const {
3722 int Idx = cast<JumpTableSDNode>(Val&: Table)->getIndex();
3723 EVT VT = Table.getValueType();
3724 SDValue T = DAG.getTargetJumpTable(JTI: Idx, VT, TargetFlags: HexagonII::MO_PCREL);
3725 return DAG.getNode(Opcode: HexagonISD::AT_PCREL, DL: SDLoc(Table), VT, Operand: T);
3726}
3727
3728//===----------------------------------------------------------------------===//
3729// Inline Assembly Support
3730//===----------------------------------------------------------------------===//
3731
3732TargetLowering::ConstraintType
3733HexagonTargetLowering::getConstraintType(StringRef Constraint) const {
3734 if (Constraint.size() == 1) {
3735 switch (Constraint[0]) {
3736 case 'q':
3737 case 'v':
3738 if (Subtarget.useHVXOps())
3739 return C_RegisterClass;
3740 break;
3741 case 'a':
3742 return C_RegisterClass;
3743 default:
3744 break;
3745 }
3746 }
3747 return TargetLowering::getConstraintType(Constraint);
3748}
3749
3750std::pair<unsigned, const TargetRegisterClass*>
3751HexagonTargetLowering::getRegForInlineAsmConstraint(
3752 const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const {
3753
3754 if (Constraint.size() == 1) {
3755 switch (Constraint[0]) {
3756 case 'r': // R0-R31
3757 switch (VT.SimpleTy) {
3758 default:
3759 return {0u, nullptr};
3760 case MVT::i1:
3761 case MVT::i8:
3762 case MVT::i16:
3763 case MVT::i32:
3764 case MVT::f32:
3765 return {0u, &Hexagon::IntRegsRegClass};
3766 case MVT::i64:
3767 case MVT::f64:
3768 return {0u, &Hexagon::DoubleRegsRegClass};
3769 }
3770 break;
3771 case 'a': // M0-M1
3772 if (VT != MVT::i32)
3773 return {0u, nullptr};
3774 return {0u, &Hexagon::ModRegsRegClass};
3775 case 'q': // q0-q3
3776 switch (VT.getSizeInBits()) {
3777 default:
3778 return {0u, nullptr};
3779 case 64:
3780 case 128:
3781 return {0u, &Hexagon::HvxQRRegClass};
3782 }
3783 break;
3784 case 'v': // V0-V31
3785 switch (VT.getSizeInBits()) {
3786 default:
3787 return {0u, nullptr};
3788 case 512:
3789 return {0u, &Hexagon::HvxVRRegClass};
3790 case 1024:
3791 if (Subtarget.hasV60Ops() && Subtarget.useHVX128BOps())
3792 return {0u, &Hexagon::HvxVRRegClass};
3793 return {0u, &Hexagon::HvxWRRegClass};
3794 case 2048:
3795 return {0u, &Hexagon::HvxWRRegClass};
3796 }
3797 break;
3798 default:
3799 return {0u, nullptr};
3800 }
3801 }
3802
3803 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
3804}
3805
3806/// Returns how to materialize the specified FP immediate.
3807TargetLowering::FPImmAction
3808HexagonTargetLowering::getFPImmAction(const APFloat &Imm, EVT VT,
3809 bool ForCodeSize) const {
3810 return FPImmAction::Legal;
3811}
3812
3813/// Returns true if it is beneficial to convert a load of a constant
3814/// to just the constant itself.
3815bool HexagonTargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
3816 Type *Ty) const {
3817 if (!ConstantLoadsToImm)
3818 return false;
3819
3820 assert(Ty->isIntegerTy());
3821 unsigned BitSize = Ty->getPrimitiveSizeInBits();
3822 return (BitSize > 0 && BitSize <= 64);
3823}
3824
3825/// isLegalAddressingMode - Return true if the addressing mode represented by
3826/// AM is legal for this target, for a load/store of the specified type.
3827bool HexagonTargetLowering::isLegalAddressingMode(const DataLayout &DL,
3828 const AddrMode &AM, Type *Ty,
3829 unsigned AS, Instruction *I) const {
3830 if (Ty->isSized()) {
3831 // When LSR detects uses of the same base address to access different
3832 // types (e.g. unions), it will assume a conservative type for these
3833 // uses:
3834 // LSR Use: Kind=Address of void in addrspace(4294967295), ...
3835 // The type Ty passed here would then be "void". Skip the alignment
3836 // checks, but do not return false right away, since that confuses
3837 // LSR into crashing.
3838 Align A = DL.getABITypeAlign(Ty);
3839 // The base offset must be a multiple of the alignment.
3840 if (!isAligned(Lhs: A, SizeInBytes: AM.BaseOffs))
3841 return false;
3842 // The shifted offset must fit in 11 bits.
3843 if (!isInt<11>(x: AM.BaseOffs >> Log2(A)))
3844 return false;
3845 }
3846
3847 // No global is ever allowed as a base.
3848 if (AM.BaseGV)
3849 return false;
3850
3851 int Scale = AM.Scale;
3852 if (Scale < 0)
3853 Scale = -Scale;
3854 switch (Scale) {
3855 case 0: // No scale reg, "r+i", "r", or just "i".
3856 break;
3857 default: // No scaled addressing mode.
3858 return false;
3859 }
3860 return true;
3861}
3862
3863/// Return true if folding a constant offset with the given GlobalAddress is
3864/// legal. It is frequently not legal in PIC relocation models.
3865bool HexagonTargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA)
3866 const {
3867 return HTM.getRelocationModel() == Reloc::Static;
3868}
3869
3870/// isLegalICmpImmediate - Return true if the specified immediate is legal
3871/// icmp immediate, that is the target has icmp instructions which can compare
3872/// a register against the immediate without having to materialize the
3873/// immediate into a register.
3874bool HexagonTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
3875 return Imm >= -512 && Imm <= 511;
3876}
3877
3878/// IsEligibleForTailCallOptimization - Check whether the call is eligible
3879/// for tail call optimization. Targets which want to do tail call
3880/// optimization should implement this function.
3881bool HexagonTargetLowering::IsEligibleForTailCallOptimization(
3882 SDValue Callee,
3883 CallingConv::ID CalleeCC,
3884 bool IsVarArg,
3885 bool IsCalleeStructRet,
3886 bool IsCallerStructRet,
3887 const SmallVectorImpl<ISD::OutputArg> &Outs,
3888 const SmallVectorImpl<SDValue> &OutVals,
3889 const SmallVectorImpl<ISD::InputArg> &Ins,
3890 SelectionDAG& DAG) const {
3891 const Function &CallerF = DAG.getMachineFunction().getFunction();
3892 CallingConv::ID CallerCC = CallerF.getCallingConv();
3893 bool CCMatch = CallerCC == CalleeCC;
3894
3895 // ***************************************************************************
3896 // Look for obvious safe cases to perform tail call optimization that do not
3897 // require ABI changes.
3898 // ***************************************************************************
3899
3900 // If this is a tail call via a function pointer, then don't do it!
3901 if (!isa<GlobalAddressSDNode>(Val: Callee) &&
3902 !isa<ExternalSymbolSDNode>(Val: Callee)) {
3903 return false;
3904 }
3905
3906 // Do not optimize if the calling conventions do not match and the conventions
3907 // used are not C or Fast.
3908 if (!CCMatch) {
3909 bool R = (CallerCC == CallingConv::C || CallerCC == CallingConv::Fast);
3910 bool E = (CalleeCC == CallingConv::C || CalleeCC == CallingConv::Fast);
3911 // If R & E, then ok.
3912 if (!R || !E)
3913 return false;
3914 }
3915
3916 // Do not tail call optimize vararg calls.
3917 if (IsVarArg)
3918 return false;
3919
3920 // Also avoid tail call optimization if either caller or callee uses struct
3921 // return semantics.
3922 if (IsCalleeStructRet || IsCallerStructRet)
3923 return false;
3924
3925 // In addition to the cases above, we also disable Tail Call Optimization if
3926 // the calling convention code that at least one outgoing argument needs to
3927 // go on the stack. We cannot check that here because at this point that
3928 // information is not available.
3929 return true;
3930}
3931
3932/// Returns the target specific optimal type for load and store operations as
3933/// a result of memset, memcpy, and memmove lowering.
3934///
3935/// If DstAlign is zero that means it's safe to destination alignment can
3936/// satisfy any constraint. Similarly if SrcAlign is zero it means there isn't
3937/// a need to check it against alignment requirement, probably because the
3938/// source does not need to be loaded. If 'IsMemset' is true, that means it's
3939/// expanding a memset. If 'ZeroMemset' is true, that means it's a memset of
3940/// zero. 'MemcpyStrSrc' indicates whether the memcpy source is constant so it
3941/// does not need to be loaded. It returns EVT::Other if the type should be
3942/// determined using generic target-independent logic.
3943EVT HexagonTargetLowering::getOptimalMemOpType(
3944 LLVMContext &Context, const MemOp &Op,
3945 const AttributeList &FuncAttributes) const {
3946 if (Op.size() >= 8 && Op.isAligned(AlignCheck: Align(8)))
3947 return MVT::i64;
3948 if (Op.size() >= 4 && Op.isAligned(AlignCheck: Align(4)))
3949 return MVT::i32;
3950 if (Op.size() >= 2 && Op.isAligned(AlignCheck: Align(2)))
3951 return MVT::i16;
3952 return MVT::Other;
3953}
3954
3955// The helpers below are versions of llvm::getShuffleReduction and
3956// llvm::getOrderedReduction, adapted to use during DAG passes and simplified as
3957// follows:
3958// - ICmp and FCmp are not handled;
3959// - in every step in getShuffleReduction, the input is split into halves (not
3960// pairwise).
3961
3962static SDValue getOrderedReduction(SDValue Vec, unsigned Op,
3963 SelectionDAG &DAG) {
3964 assert(Op != Instruction::ICmp && Op != Instruction::FCmp);
3965
3966 EVT VT = Vec.getValueType();
3967 EVT EltT = VT.getVectorElementType();
3968 unsigned VF = VT.getVectorNumElements();
3969 assert(VF > 0 &&
3970 "Reduction emission only supported for non-zero length vectors!");
3971
3972 SDLoc DL(Vec);
3973 SDValue Result = DAG.getExtractVectorElt(DL, VT: EltT, Vec, Idx: 0);
3974 for (unsigned ExtractIdx = 1; ExtractIdx < VF; ++ExtractIdx) {
3975 SDValue Ext = DAG.getExtractVectorElt(DL, VT: EltT, Vec, Idx: ExtractIdx);
3976 Result = DAG.getNode(Opcode: Op, DL, VT: EltT, Ops: {Result, Ext});
3977 }
3978
3979 return Result;
3980}
3981
3982static SDValue getShuffleReduction(SDValue Vec, unsigned Op,
3983 SelectionDAG &DAG) {
3984 assert(Op != Instruction::ICmp && Op != Instruction::FCmp);
3985
3986 EVT VT = Vec.getValueType();
3987 unsigned VF = VT.getVectorNumElements();
3988 if (VF == 0)
3989 llvm_unreachable("Vector must be non-zero length");
3990 // VF is a power of 2 so we can emit the reduction using log2(VF) shuffles
3991 // and vector ops, reducing the set of values being computed by half each
3992 // round.
3993 assert(isPowerOf2_32(VF) &&
3994 "Reduction emission only supported for pow2 vectors!");
3995
3996 SDLoc DL(Vec);
3997 // TODO: Is it correct to create double-vector shuffle and fill 3/4 of it with
3998 // undefs?
3999 SmallVector<int, 32> ShuffleMask(VF);
4000 for (unsigned i = VF; i > 1; i >>= 1) {
4001 // Move the upper half of the vector to the lower half.
4002 for (unsigned j = 0; j != i / 2; ++j)
4003 ShuffleMask[j] = i / 2 + j;
4004 // Fill the rest of the mask with undef.
4005 std::fill(first: &ShuffleMask[i / 2], last: ShuffleMask.end(), value: -1);
4006
4007 SDValue Shuf =
4008 DAG.getVectorShuffle(VT, dl: DL, N1: Vec, N2: DAG.getUNDEF(VT), Mask: ShuffleMask);
4009
4010 Vec = DAG.getNode(Opcode: Op, DL, VT, Ops: {Vec, Shuf});
4011 }
4012 // The result is in the first element of the vector.
4013 return DAG.getExtractVectorElt(DL, VT: VT.getVectorElementType(), Vec, Idx: 0);
4014}
4015
4016SDValue HexagonTargetLowering::expandVecReduceAdd(SDNode *N,
4017 SelectionDAG &DAG) const {
4018 // Since we disabled automatic reduction expansion, generate log2 ladder code
4019 // if the vector is of a power-of-two length.
4020 SDValue Input = N->getOperand(Num: 0);
4021 if (isPowerOf2_32(Value: Input.getValueType().getVectorNumElements()))
4022 return getShuffleReduction(Vec: Input, Op: ISD::ADD, DAG);
4023 // Otherwise, reduction will be scalarized.
4024 return getOrderedReduction(Vec: Input, Op: ISD::ADD, DAG);
4025}
4026
4027bool HexagonTargetLowering::allowsMemoryAccess(
4028 LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace,
4029 Align Alignment, MachineMemOperand::Flags Flags, unsigned *Fast) const {
4030 if (!VT.isSimple())
4031 return false;
4032 MVT SVT = VT.getSimpleVT();
4033 if (Subtarget.isHLXVectorType(VecTy: SVT))
4034 return allowsHlxMemoryAccess(VecTy: SVT, Flags, Fast);
4035 if (Subtarget.isHVXVectorType(VecTy: SVT, IncludeBool: true))
4036 return allowsHvxMemoryAccess(VecTy: SVT, Flags, Fast);
4037 return TargetLoweringBase::allowsMemoryAccess(
4038 Context, DL, VT, AddrSpace, Alignment, Flags, Fast);
4039}
4040
4041bool HexagonTargetLowering::allowsMisalignedMemoryAccesses(
4042 EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags,
4043 unsigned *Fast) const {
4044 if (!VT.isSimple())
4045 return false;
4046 MVT SVT = VT.getSimpleVT();
4047 if (Subtarget.isHVXVectorType(VecTy: SVT, IncludeBool: true))
4048 return allowsHvxMisalignedMemoryAccesses(VecTy: SVT, Flags, Fast);
4049 if (Fast)
4050 *Fast = 0;
4051 return false;
4052}
4053
4054std::pair<const TargetRegisterClass*, uint8_t>
4055HexagonTargetLowering::findRepresentativeClass(const TargetRegisterInfo *TRI,
4056 MVT VT) const {
4057 if (Subtarget.isHVXVectorType(VecTy: VT, IncludeBool: true)) {
4058 unsigned BitWidth = VT.getSizeInBits();
4059 unsigned VecWidth = Subtarget.getVectorLength() * 8;
4060
4061 if (VT.getVectorElementType() == MVT::i1)
4062 return std::make_pair(x: &Hexagon::HvxQRRegClass, y: 1);
4063 if (BitWidth == VecWidth)
4064 return std::make_pair(x: &Hexagon::HvxVRRegClass, y: 1);
4065 assert(BitWidth == 2 * VecWidth);
4066 return std::make_pair(x: &Hexagon::HvxWRRegClass, y: 1);
4067 }
4068
4069 return TargetLowering::findRepresentativeClass(TRI, VT);
4070}
4071
4072bool HexagonTargetLowering::shouldReduceLoadWidth(
4073 SDNode *Load, ISD::LoadExtType ExtTy, EVT NewVT,
4074 std::optional<unsigned> ByteOffset) const {
4075 // TODO: This may be worth removing. Check regression tests for diffs.
4076 if (!TargetLoweringBase::shouldReduceLoadWidth(Load, ExtTy, NewVT,
4077 ByteOffset))
4078 return false;
4079
4080 auto *L = cast<LoadSDNode>(Val: Load);
4081 std::pair<SDValue, int> BO = getBaseAndOffset(Addr: L->getBasePtr());
4082 // Small-data object, do not shrink.
4083 if (BO.first.getOpcode() == HexagonISD::CONST32_GP)
4084 return false;
4085 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Val&: BO.first)) {
4086 auto &HTM = static_cast<const HexagonTargetMachine &>(getTargetMachine());
4087 const auto *GO = dyn_cast_or_null<const GlobalObject>(Val: GA->getGlobal());
4088 return !GO || !HTM.getObjFileLowering()->isGlobalInSmallSection(GO, TM: HTM);
4089 }
4090 return true;
4091}
4092
4093void HexagonTargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI,
4094 SDNode *Node) const {
4095 AdjustHvxInstrPostInstrSelection(MI, Node);
4096}
4097
4098Value *HexagonTargetLowering::emitLoadLinked(IRBuilderBase &Builder,
4099 Type *ValueTy, Value *Addr,
4100 AtomicOrdering Ord) const {
4101 unsigned SZ = ValueTy->getPrimitiveSizeInBits();
4102 assert((SZ == 32 || SZ == 64) && "Only 32/64-bit atomic loads supported");
4103 Intrinsic::ID IntID = (SZ == 32) ? Intrinsic::hexagon_L2_loadw_locked
4104 : Intrinsic::hexagon_L4_loadd_locked;
4105
4106 Value *Call =
4107 Builder.CreateIntrinsic(ID: IntID, Args: Addr, /*FMFSource=*/nullptr, Name: "larx");
4108
4109 return Builder.CreateBitCast(V: Call, DestTy: ValueTy);
4110}
4111
4112/// Perform a store-conditional operation to Addr. Return the status of the
4113/// store. This should be 0 if the store succeeded, non-zero otherwise.
4114Value *HexagonTargetLowering::emitStoreConditional(IRBuilderBase &Builder,
4115 Value *Val, Value *Addr,
4116 AtomicOrdering Ord) const {
4117 BasicBlock *BB = Builder.GetInsertBlock();
4118 Module *M = BB->getParent()->getParent();
4119 Type *Ty = Val->getType();
4120 unsigned SZ = Ty->getPrimitiveSizeInBits();
4121
4122 Type *CastTy = Builder.getIntNTy(N: SZ);
4123 assert((SZ == 32 || SZ == 64) && "Only 32/64-bit atomic stores supported");
4124 Intrinsic::ID IntID = (SZ == 32) ? Intrinsic::hexagon_S2_storew_locked
4125 : Intrinsic::hexagon_S4_stored_locked;
4126
4127 Val = Builder.CreateBitCast(V: Val, DestTy: CastTy);
4128
4129 Value *Call = Builder.CreateIntrinsic(ID: IntID, Args: {Addr, Val},
4130 /*FMFSource=*/nullptr, Name: "stcx");
4131 Value *Cmp = Builder.CreateICmpEQ(LHS: Call, RHS: Builder.getInt32(C: 0), Name: "");
4132 Value *Ext = Builder.CreateZExt(V: Cmp, DestTy: Type::getInt32Ty(C&: M->getContext()));
4133 return Ext;
4134}
4135
4136TargetLowering::AtomicExpansionKind
4137HexagonTargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const {
4138 // Do not expand loads and stores that don't exceed 64 bits.
4139 return LI->getType()->getPrimitiveSizeInBits() > 64
4140 ? AtomicExpansionKind::LLOnly
4141 : AtomicExpansionKind::None;
4142}
4143
4144TargetLowering::AtomicExpansionKind
4145HexagonTargetLowering::shouldExpandAtomicStoreInIR(StoreInst *SI) const {
4146 // Do not expand loads and stores that don't exceed 64 bits.
4147 return SI->getValueOperand()->getType()->getPrimitiveSizeInBits() > 64
4148 ? AtomicExpansionKind::Expand
4149 : AtomicExpansionKind::None;
4150}
4151
4152TargetLowering::AtomicExpansionKind
4153HexagonTargetLowering::shouldExpandAtomicCmpXchgInIR(
4154 const AtomicCmpXchgInst *AI) const {
4155 return AtomicExpansionKind::LLSC;
4156}
4157
4158MachineBasicBlock *HexagonTargetLowering::EmitInstrWithCustomInserter(
4159 MachineInstr &MI, MachineBasicBlock *BB) const {
4160 switch (MI.getOpcode()) {
4161 case TargetOpcode::PATCHABLE_EVENT_CALL:
4162 case TargetOpcode::PATCHABLE_TYPED_EVENT_CALL:
4163 // These are lowered in the AsmPrinter.
4164 return BB;
4165 default:
4166 llvm_unreachable("Unexpected instruction with custom inserter");
4167 }
4168}
4169
4170MachineInstr *
4171HexagonTargetLowering::EmitKCFICheck(MachineBasicBlock &MBB,
4172 MachineBasicBlock::instr_iterator &MBBI,
4173 const TargetInstrInfo *TII) const {
4174 assert(MBBI->isCall() && MBBI->getCFIType() &&
4175 "Invalid call instruction for a KCFI check");
4176
4177 switch (MBBI->getOpcode()) {
4178 case Hexagon::J2_callr:
4179 case Hexagon::PS_callr_nr:
4180 break;
4181 default:
4182 llvm_unreachable("Unexpected CFI call opcode");
4183 }
4184
4185 MachineOperand &Target = MBBI->getOperand(i: 0);
4186 assert(Target.isReg() && "Invalid target operand for an indirect call");
4187 Target.setIsRenamable(false);
4188
4189 return BuildMI(BB&: MBB, I: MBBI, MIMD: MBBI->getDebugLoc(), MCID: TII->get(Opcode: Hexagon::KCFI_CHECK))
4190 .addReg(RegNo: Target.getReg())
4191 .addImm(Val: MBBI->getCFIType())
4192 .getInstr();
4193}
4194
4195bool HexagonTargetLowering::isMaskAndCmp0FoldingBeneficial(
4196 const Instruction &AndI) const {
4197 // Only sink 'and' mask to cmp use block if it is masking a single bit since
4198 // this will fold the and/cmp/br into a single tstbit instruction.
4199 ConstantInt *Mask = dyn_cast<ConstantInt>(Val: AndI.getOperand(i: 1));
4200 if (!Mask)
4201 return false;
4202 return Mask->getValue().isPowerOf2();
4203}
4204
4205// Check if the result of the node is only used as a return value, as
4206// otherwise we can't perform a tail-call.
4207bool HexagonTargetLowering::isUsedByReturnOnly(SDNode *N,
4208 SDValue &Chain) const {
4209 if (N->getNumValues() != 1)
4210 return false;
4211 if (!N->hasNUsesOfValue(NUses: 1, Value: 0))
4212 return false;
4213
4214 SDNode *Copy = *N->user_begin();
4215
4216 if (Copy->getOpcode() == ISD::BITCAST) {
4217 return isUsedByReturnOnly(N: Copy, Chain);
4218 }
4219
4220 if (Copy->getOpcode() != ISD::CopyToReg) {
4221 return false;
4222 }
4223
4224 // If the ISD::CopyToReg has a glue operand, we conservatively assume it
4225 // isn't safe to perform a tail call.
4226 if (Copy->getOperand(Num: Copy->getNumOperands() - 1).getValueType() == MVT::Glue)
4227 return false;
4228
4229 // The copy must be used by a HexagonISD::RET_GLUE, and nothing else.
4230 bool HasRet = false;
4231 for (SDNode *Node : Copy->users()) {
4232 if (Node->getOpcode() != HexagonISD::RET_GLUE)
4233 return false;
4234 HasRet = true;
4235 }
4236 if (!HasRet)
4237 return false;
4238
4239 Chain = Copy->getOperand(Num: 0);
4240 return true;
4241}
4242
4243bool HexagonTargetLowering::hasInlineStackProbe(
4244 const MachineFunction &MF) const {
4245 if (MF.getFunction().hasFnAttribute(Kind: "probe-stack"))
4246 return MF.getFunction().getFnAttribute(Kind: "probe-stack").getValueAsString() ==
4247 "inline-asm";
4248 return false;
4249}
4250
4251unsigned HexagonTargetLowering::getStackProbeSize(const MachineFunction &MF,
4252 Align StackAlign) const {
4253 const Function &Fn = MF.getFunction();
4254 unsigned StackProbeSize =
4255 Fn.getFnAttributeAsParsedInteger(Kind: "stack-probe-size", Default: 4096);
4256 // Round down to the stack alignment.
4257 StackProbeSize = alignDown(Value: StackProbeSize, Align: StackAlign.value());
4258 return StackProbeSize ? StackProbeSize : StackAlign.value();
4259}
4260