1//===-- VEISelLowering.cpp - VE DAG Lowering Implementation ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the interfaces that VE uses to lower LLVM code into a
10// selection DAG.
11//
12//===----------------------------------------------------------------------===//
13
14#include "VEISelLowering.h"
15#include "MCTargetDesc/VEMCAsmInfo.h"
16#include "VECustomDAG.h"
17#include "VEInstrBuilder.h"
18#include "VEMachineFunctionInfo.h"
19#include "VERegisterInfo.h"
20#include "VESelectionDAGInfo.h"
21#include "VETargetMachine.h"
22#include "llvm/ADT/StringSwitch.h"
23#include "llvm/CodeGen/CallingConvLower.h"
24#include "llvm/CodeGen/MachineFrameInfo.h"
25#include "llvm/CodeGen/MachineFunction.h"
26#include "llvm/CodeGen/MachineInstrBuilder.h"
27#include "llvm/CodeGen/MachineJumpTableInfo.h"
28#include "llvm/CodeGen/MachineModuleInfo.h"
29#include "llvm/CodeGen/MachineRegisterInfo.h"
30#include "llvm/CodeGen/SelectionDAG.h"
31#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
32#include "llvm/IR/DerivedTypes.h"
33#include "llvm/IR/Function.h"
34#include "llvm/IR/IRBuilder.h"
35#include "llvm/IR/Module.h"
36#include "llvm/Support/ErrorHandling.h"
37using namespace llvm;
38
39#define DEBUG_TYPE "ve-lower"
40
41//===----------------------------------------------------------------------===//
42// Calling Convention Implementation
43//===----------------------------------------------------------------------===//
44
45#define GET_CALLING_CONV_IMPL
46#include "VEGenCallingConv.inc"
47
48CCAssignFn *getReturnCC(CallingConv::ID CallConv) {
49 switch (CallConv) {
50 default:
51 return RetCC_VE_C;
52 case CallingConv::Fast:
53 return RetCC_VE_Fast;
54 }
55}
56
57CCAssignFn *getParamCC(CallingConv::ID CallConv, bool IsVarArg) {
58 if (IsVarArg)
59 return CC_VE2;
60 switch (CallConv) {
61 default:
62 return CC_VE_C;
63 case CallingConv::Fast:
64 return CC_VE_Fast;
65 }
66}
67
68bool VETargetLowering::CanLowerReturn(
69 CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg,
70 const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context,
71 const Type *RetTy) const {
72 CCAssignFn *RetCC = getReturnCC(CallConv);
73 SmallVector<CCValAssign, 16> RVLocs;
74 CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
75 return CCInfo.CheckReturn(Outs, Fn: RetCC);
76}
77
78static const MVT AllVectorVTs[] = {MVT::v256i32, MVT::v512i32, MVT::v256i64,
79 MVT::v256f32, MVT::v512f32, MVT::v256f64};
80
81static const MVT AllMaskVTs[] = {MVT::v256i1, MVT::v512i1};
82
83static const MVT AllPackedVTs[] = {MVT::v512i32, MVT::v512f32};
84
85void VETargetLowering::initRegisterClasses() {
86 // Set up the register classes.
87 addRegisterClass(VT: MVT::i32, RC: &VE::I32RegClass);
88 addRegisterClass(VT: MVT::i64, RC: &VE::I64RegClass);
89 addRegisterClass(VT: MVT::f32, RC: &VE::F32RegClass);
90 addRegisterClass(VT: MVT::f64, RC: &VE::I64RegClass);
91 addRegisterClass(VT: MVT::f128, RC: &VE::F128RegClass);
92
93 if (Subtarget->enableVPU()) {
94 for (MVT VecVT : AllVectorVTs)
95 addRegisterClass(VT: VecVT, RC: &VE::V64RegClass);
96 addRegisterClass(VT: MVT::v256i1, RC: &VE::VMRegClass);
97 addRegisterClass(VT: MVT::v512i1, RC: &VE::VM512RegClass);
98 }
99}
100
101void VETargetLowering::initSPUActions() {
102 const auto &TM = getTargetMachine();
103 /// Load & Store {
104
105 // VE doesn't have i1 sign extending load.
106 for (MVT VT : MVT::integer_valuetypes()) {
107 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: VT, MemVT: MVT::i1, Action: Promote);
108 setLoadExtAction(ExtType: ISD::ZEXTLOAD, ValVT: VT, MemVT: MVT::i1, Action: Promote);
109 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: MVT::i1, Action: Promote);
110 setTruncStoreAction(ValVT: VT, MemVT: MVT::i1, Action: Expand);
111 }
112
113 // VE doesn't have floating point extload/truncstore, so expand them.
114 for (MVT FPVT : MVT::fp_valuetypes()) {
115 for (MVT OtherFPVT : MVT::fp_valuetypes()) {
116 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: FPVT, MemVT: OtherFPVT, Action: Expand);
117 setTruncStoreAction(ValVT: FPVT, MemVT: OtherFPVT, Action: Expand);
118 }
119 }
120
121 // VE doesn't have fp128 load/store, so expand them in custom lower.
122 setOperationAction(Op: ISD::LOAD, VT: MVT::f128, Action: Custom);
123 setOperationAction(Op: ISD::STORE, VT: MVT::f128, Action: Custom);
124
125 /// } Load & Store
126
127 // Custom legalize address nodes into LO/HI parts.
128 MVT PtrVT = MVT::getIntegerVT(BitWidth: TM.getPointerSizeInBits(AS: 0));
129 setOperationAction(Op: ISD::BlockAddress, VT: PtrVT, Action: Custom);
130 setOperationAction(Op: ISD::GlobalAddress, VT: PtrVT, Action: Custom);
131 setOperationAction(Op: ISD::GlobalTLSAddress, VT: PtrVT, Action: Custom);
132 setOperationAction(Op: ISD::ConstantPool, VT: PtrVT, Action: Custom);
133 setOperationAction(Op: ISD::JumpTable, VT: PtrVT, Action: Custom);
134
135 /// VAARG handling {
136 setOperationAction(Op: ISD::VASTART, VT: MVT::Other, Action: Custom);
137 // VAARG needs to be lowered to access with 8 bytes alignment.
138 setOperationAction(Op: ISD::VAARG, VT: MVT::Other, Action: Custom);
139 // Use the default implementation.
140 setOperationAction(Op: ISD::VACOPY, VT: MVT::Other, Action: Expand);
141 setOperationAction(Op: ISD::VAEND, VT: MVT::Other, Action: Expand);
142 /// } VAARG handling
143
144 /// Stack {
145 setOperationAction(Op: ISD::DYNAMIC_STACKALLOC, VT: MVT::i32, Action: Custom);
146 setOperationAction(Op: ISD::DYNAMIC_STACKALLOC, VT: MVT::i64, Action: Custom);
147
148 // Use the default implementation.
149 setOperationAction(Op: ISD::STACKSAVE, VT: MVT::Other, Action: Expand);
150 setOperationAction(Op: ISD::STACKRESTORE, VT: MVT::Other, Action: Expand);
151 /// } Stack
152
153 /// Branch {
154
155 // VE doesn't have BRCOND
156 setOperationAction(Op: ISD::BRCOND, VT: MVT::Other, Action: Expand);
157
158 // BR_JT is not implemented yet.
159 setOperationAction(Op: ISD::BR_JT, VT: MVT::Other, Action: Expand);
160
161 /// } Branch
162
163 /// Int Ops {
164 for (MVT IntVT : {MVT::i32, MVT::i64}) {
165 // VE has no REM or DIVREM operations.
166 setOperationAction(Op: ISD::UREM, VT: IntVT, Action: Expand);
167 setOperationAction(Op: ISD::SREM, VT: IntVT, Action: Expand);
168 setOperationAction(Op: ISD::SDIVREM, VT: IntVT, Action: Expand);
169 setOperationAction(Op: ISD::UDIVREM, VT: IntVT, Action: Expand);
170
171 // VE has no SHL_PARTS/SRA_PARTS/SRL_PARTS operations.
172 setOperationAction(Op: ISD::SHL_PARTS, VT: IntVT, Action: Expand);
173 setOperationAction(Op: ISD::SRA_PARTS, VT: IntVT, Action: Expand);
174 setOperationAction(Op: ISD::SRL_PARTS, VT: IntVT, Action: Expand);
175
176 // VE has no MULHU/S or U/SMUL_LOHI operations.
177 // TODO: Use MPD instruction to implement SMUL_LOHI for i32 type.
178 setOperationAction(Op: ISD::MULHU, VT: IntVT, Action: Expand);
179 setOperationAction(Op: ISD::MULHS, VT: IntVT, Action: Expand);
180 setOperationAction(Op: ISD::UMUL_LOHI, VT: IntVT, Action: Expand);
181 setOperationAction(Op: ISD::SMUL_LOHI, VT: IntVT, Action: Expand);
182
183 // VE has no CTTZ, ROTL, ROTR operations.
184 setOperationAction(Op: ISD::CTTZ, VT: IntVT, Action: Expand);
185 setOperationAction(Op: ISD::ROTL, VT: IntVT, Action: Expand);
186 setOperationAction(Op: ISD::ROTR, VT: IntVT, Action: Expand);
187
188 // VE has 64 bits instruction which works as i64 BSWAP operation. This
189 // instruction works fine as i32 BSWAP operation with an additional
190 // parameter. Use isel patterns to lower BSWAP.
191 setOperationAction(Op: ISD::BSWAP, VT: IntVT, Action: Legal);
192
193 // VE has only 64 bits instructions which work as i64 BITREVERSE/CTLZ/CTPOP
194 // operations. Use isel patterns for i64, promote for i32.
195 LegalizeAction Act = (IntVT == MVT::i32) ? Promote : Legal;
196 setOperationAction(Op: ISD::BITREVERSE, VT: IntVT, Action: Act);
197 setOperationAction(Op: ISD::CTLZ, VT: IntVT, Action: Act);
198 setOperationAction(Op: ISD::CTLZ_ZERO_POISON, VT: IntVT, Action: Act);
199 setOperationAction(Op: ISD::CTPOP, VT: IntVT, Action: Act);
200
201 // VE has only 64 bits instructions which work as i64 AND/OR/XOR operations.
202 // Use isel patterns for i64, promote for i32.
203 setOperationAction(Op: ISD::AND, VT: IntVT, Action: Act);
204 setOperationAction(Op: ISD::OR, VT: IntVT, Action: Act);
205 setOperationAction(Op: ISD::XOR, VT: IntVT, Action: Act);
206
207 // Legal smax and smin
208 setOperationAction(Op: ISD::SMAX, VT: IntVT, Action: Legal);
209 setOperationAction(Op: ISD::SMIN, VT: IntVT, Action: Legal);
210 }
211 /// } Int Ops
212
213 /// Conversion {
214 // VE doesn't have instructions for fp<->uint, so expand them by llvm
215 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i32, Action: Promote); // use i64
216 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::i32, Action: Promote); // use i64
217 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i64, Action: Expand);
218 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::i64, Action: Expand);
219
220 // fp16 not supported
221 for (MVT FPVT : MVT::fp_valuetypes()) {
222 setOperationAction(Op: ISD::FP16_TO_FP, VT: FPVT, Action: Expand);
223 setOperationAction(Op: ISD::FP_TO_FP16, VT: FPVT, Action: Expand);
224 }
225 /// } Conversion
226
227 /// Floating-point Ops {
228 /// Note: Floating-point operations are fneg, fadd, fsub, fmul, fdiv, frem,
229 /// and fcmp.
230
231 // VE doesn't have following floating point operations.
232 for (MVT VT : MVT::fp_valuetypes()) {
233 setOperationAction(Op: ISD::FNEG, VT, Action: Expand);
234 setOperationAction(Op: ISD::FREM, VT, Action: LibCall);
235 }
236
237 // VE doesn't have fdiv of f128.
238 setOperationAction(Op: ISD::FDIV, VT: MVT::f128, Action: Expand);
239
240 for (MVT FPVT : {MVT::f32, MVT::f64}) {
241 // f32 and f64 uses ConstantFP. f128 uses ConstantPool.
242 setOperationAction(Op: ISD::ConstantFP, VT: FPVT, Action: Legal);
243 }
244 /// } Floating-point Ops
245
246 /// Floating-point math functions {
247
248 // VE doesn't have following floating point math functions.
249 for (MVT VT : MVT::fp_valuetypes()) {
250 setOperationAction(Op: ISD::FABS, VT, Action: Expand);
251 setOperationAction(Op: ISD::FCOPYSIGN, VT, Action: Expand);
252 setOperationAction(Op: ISD::FCOS, VT, Action: Expand);
253 setOperationAction(Op: ISD::FMA, VT, Action: Expand);
254 setOperationAction(Op: ISD::FPOW, VT, Action: Expand);
255 setOperationAction(Op: ISD::FSIN, VT, Action: Expand);
256 setOperationAction(Op: ISD::FSQRT, VT, Action: Expand);
257 }
258
259 // VE has single and double FMINNUM and FMAXNUM
260 for (MVT VT : {MVT::f32, MVT::f64}) {
261 setOperationAction(Ops: {ISD::FMAXNUM, ISD::FMINNUM}, VT, Action: Legal);
262 }
263
264 /// } Floating-point math functions
265
266 /// Atomic instructions {
267
268 setMaxAtomicSizeInBitsSupported(64);
269 setMinCmpXchgSizeInBits(32);
270 setSupportsUnalignedAtomics(false);
271
272 // Use custom inserter for ATOMIC_FENCE.
273 setOperationAction(Op: ISD::ATOMIC_FENCE, VT: MVT::Other, Action: Custom);
274
275 // Other atomic instructions.
276 for (MVT VT : MVT::integer_valuetypes()) {
277 // Support i8/i16 atomic swap.
278 setOperationAction(Op: ISD::ATOMIC_SWAP, VT, Action: Custom);
279
280 // FIXME: Support "atmam" instructions.
281 setOperationAction(Op: ISD::ATOMIC_LOAD_ADD, VT, Action: Expand);
282 setOperationAction(Op: ISD::ATOMIC_LOAD_SUB, VT, Action: Expand);
283 setOperationAction(Op: ISD::ATOMIC_LOAD_AND, VT, Action: Expand);
284 setOperationAction(Op: ISD::ATOMIC_LOAD_OR, VT, Action: Expand);
285
286 // VE doesn't have follwing instructions.
287 setOperationAction(Op: ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, VT, Action: Expand);
288 setOperationAction(Op: ISD::ATOMIC_LOAD_CLR, VT, Action: Expand);
289 setOperationAction(Op: ISD::ATOMIC_LOAD_XOR, VT, Action: Expand);
290 setOperationAction(Op: ISD::ATOMIC_LOAD_NAND, VT, Action: Expand);
291 setOperationAction(Op: ISD::ATOMIC_LOAD_MIN, VT, Action: Expand);
292 setOperationAction(Op: ISD::ATOMIC_LOAD_MAX, VT, Action: Expand);
293 setOperationAction(Op: ISD::ATOMIC_LOAD_UMIN, VT, Action: Expand);
294 setOperationAction(Op: ISD::ATOMIC_LOAD_UMAX, VT, Action: Expand);
295 }
296
297 /// } Atomic instructions
298
299 /// SJLJ instructions {
300 setOperationAction(Op: ISD::EH_SJLJ_LONGJMP, VT: MVT::Other, Action: Custom);
301 setOperationAction(Op: ISD::EH_SJLJ_SETJMP, VT: MVT::i32, Action: Custom);
302 setOperationAction(Op: ISD::EH_SJLJ_SETUP_DISPATCH, VT: MVT::Other, Action: Custom);
303 /// } SJLJ instructions
304
305 // Intrinsic instructions
306 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::Other, Action: Custom);
307}
308
309void VETargetLowering::initVPUActions() {
310 for (MVT LegalMaskVT : AllMaskVTs)
311 setOperationAction(Op: ISD::BUILD_VECTOR, VT: LegalMaskVT, Action: Custom);
312
313 for (unsigned Opc : {ISD::AND, ISD::OR, ISD::XOR})
314 setOperationAction(Op: Opc, VT: MVT::v512i1, Action: Custom);
315
316 for (MVT LegalVecVT : AllVectorVTs) {
317 setOperationAction(Op: ISD::BUILD_VECTOR, VT: LegalVecVT, Action: Custom);
318 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: LegalVecVT, Action: Legal);
319 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: LegalVecVT, Action: Legal);
320 // Translate all vector instructions with legal element types to VVP_*
321 // nodes.
322 // TODO We will custom-widen into VVP_* nodes in the future. While we are
323 // buildling the infrastructure for this, we only do this for legal vector
324 // VTs.
325#define HANDLE_VP_TO_VVP(VP_OPC, VVP_NAME) \
326 setOperationAction(ISD::VP_OPC, LegalVecVT, Custom);
327#define ADD_VVP_OP(VVP_NAME, ISD_NAME) \
328 setOperationAction(ISD::ISD_NAME, LegalVecVT, Custom);
329 setOperationAction(Op: ISD::EXPERIMENTAL_VP_STRIDED_LOAD, VT: LegalVecVT, Action: Custom);
330 setOperationAction(Op: ISD::EXPERIMENTAL_VP_STRIDED_STORE, VT: LegalVecVT, Action: Custom);
331#include "VVPNodes.def"
332 }
333
334 for (MVT LegalPackedVT : AllPackedVTs) {
335 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: LegalPackedVT, Action: Custom);
336 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: LegalPackedVT, Action: Custom);
337 }
338
339 // vNt32, vNt64 ops (legal element types)
340 for (MVT VT : MVT::vector_valuetypes()) {
341 MVT ElemVT = VT.getVectorElementType();
342 unsigned ElemBits = ElemVT.getScalarSizeInBits();
343 if (ElemBits != 32 && ElemBits != 64)
344 continue;
345
346 for (unsigned MemOpc : {ISD::MLOAD, ISD::MSTORE, ISD::LOAD, ISD::STORE})
347 setOperationAction(Op: MemOpc, VT, Action: Custom);
348
349 const ISD::NodeType IntReductionOCs[] = {
350 ISD::VECREDUCE_ADD, ISD::VECREDUCE_MUL, ISD::VECREDUCE_AND,
351 ISD::VECREDUCE_OR, ISD::VECREDUCE_XOR, ISD::VECREDUCE_SMIN,
352 ISD::VECREDUCE_SMAX, ISD::VECREDUCE_UMIN, ISD::VECREDUCE_UMAX};
353
354 for (unsigned IntRedOpc : IntReductionOCs)
355 setOperationAction(Op: IntRedOpc, VT, Action: Custom);
356 }
357
358 // v256i1 and v512i1 ops
359 for (MVT MaskVT : AllMaskVTs) {
360 // Custom lower mask ops
361 setOperationAction(Op: ISD::STORE, VT: MaskVT, Action: Custom);
362 setOperationAction(Op: ISD::LOAD, VT: MaskVT, Action: Custom);
363 }
364}
365
366SDValue
367VETargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
368 bool IsVarArg,
369 const SmallVectorImpl<ISD::OutputArg> &Outs,
370 const SmallVectorImpl<SDValue> &OutVals,
371 const SDLoc &DL, SelectionDAG &DAG) const {
372 // CCValAssign - represent the assignment of the return value to locations.
373 SmallVector<CCValAssign, 16> RVLocs;
374
375 // CCState - Info about the registers and stack slot.
376 CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
377 *DAG.getContext());
378
379 // Analyze return values.
380 CCInfo.AnalyzeReturn(Outs, Fn: getReturnCC(CallConv));
381
382 SDValue Glue;
383 SmallVector<SDValue, 4> RetOps(1, Chain);
384
385 // Copy the result values into the output registers.
386 for (unsigned i = 0; i != RVLocs.size(); ++i) {
387 CCValAssign &VA = RVLocs[i];
388 assert(VA.isRegLoc() && "Can only return in registers!");
389 assert(!VA.needsCustom() && "Unexpected custom lowering");
390 SDValue OutVal = OutVals[i];
391
392 // Integer return values must be sign or zero extended by the callee.
393 switch (VA.getLocInfo()) {
394 case CCValAssign::Full:
395 break;
396 case CCValAssign::SExt:
397 OutVal = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: VA.getLocVT(), Operand: OutVal);
398 break;
399 case CCValAssign::ZExt:
400 OutVal = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: VA.getLocVT(), Operand: OutVal);
401 break;
402 case CCValAssign::AExt:
403 OutVal = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: VA.getLocVT(), Operand: OutVal);
404 break;
405 case CCValAssign::BCvt: {
406 // Convert a float return value to i64 with padding.
407 // 63 31 0
408 // +------+------+
409 // | float| 0 |
410 // +------+------+
411 assert(VA.getLocVT() == MVT::i64);
412 assert(VA.getValVT() == MVT::f32);
413 SDValue Undef = SDValue(
414 DAG.getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl: DL, VT: MVT::i64), 0);
415 SDValue Sub_f32 = DAG.getTargetConstant(Val: VE::sub_f32, DL, VT: MVT::i32);
416 OutVal = SDValue(DAG.getMachineNode(Opcode: TargetOpcode::INSERT_SUBREG, dl: DL,
417 VT: MVT::i64, Op1: Undef, Op2: OutVal, Op3: Sub_f32),
418 0);
419 break;
420 }
421 default:
422 llvm_unreachable("Unknown loc info!");
423 }
424
425 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: VA.getLocReg(), N: OutVal, Glue);
426
427 // Guarantee that all emitted copies are stuck together with flags.
428 Glue = Chain.getValue(R: 1);
429 RetOps.push_back(Elt: DAG.getRegister(Reg: VA.getLocReg(), VT: VA.getLocVT()));
430 }
431
432 RetOps[0] = Chain; // Update chain.
433
434 // Add the glue if we have it.
435 if (Glue.getNode())
436 RetOps.push_back(Elt: Glue);
437
438 return DAG.getNode(Opcode: VEISD::RET_GLUE, DL, VT: MVT::Other, Ops: RetOps);
439}
440
441SDValue VETargetLowering::LowerFormalArguments(
442 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
443 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
444 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
445 MachineFunction &MF = DAG.getMachineFunction();
446
447 // Get the base offset of the incoming arguments stack space.
448 unsigned ArgsBaseOffset = Subtarget->getRsaSize();
449 // Get the size of the preserved arguments area
450 unsigned ArgsPreserved = 64;
451
452 // Analyze arguments according to CC_VE.
453 SmallVector<CCValAssign, 16> ArgLocs;
454 CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), ArgLocs,
455 *DAG.getContext());
456 // Allocate the preserved area first.
457 CCInfo.AllocateStack(Size: ArgsPreserved, Alignment: Align(8));
458 // We already allocated the preserved area, so the stack offset computed
459 // by CC_VE would be correct now.
460 CCInfo.AnalyzeFormalArguments(Ins, Fn: getParamCC(CallConv, IsVarArg: false));
461
462 for (const CCValAssign &VA : ArgLocs) {
463 assert(!VA.needsCustom() && "Unexpected custom lowering");
464 if (VA.isRegLoc()) {
465 // This argument is passed in a register.
466 // All integer register arguments are promoted by the caller to i64.
467
468 // Create a virtual register for the promoted live-in value.
469 Register VReg =
470 MF.addLiveIn(PReg: VA.getLocReg(), RC: getRegClassFor(VT: VA.getLocVT()));
471 SDValue Arg = DAG.getCopyFromReg(Chain, dl: DL, Reg: VReg, VT: VA.getLocVT());
472
473 // The caller promoted the argument, so insert an Assert?ext SDNode so we
474 // won't promote the value again in this function.
475 switch (VA.getLocInfo()) {
476 case CCValAssign::SExt:
477 Arg = DAG.getNode(Opcode: ISD::AssertSext, DL, VT: VA.getLocVT(), N1: Arg,
478 N2: DAG.getValueType(VA.getValVT()));
479 break;
480 case CCValAssign::ZExt:
481 Arg = DAG.getNode(Opcode: ISD::AssertZext, DL, VT: VA.getLocVT(), N1: Arg,
482 N2: DAG.getValueType(VA.getValVT()));
483 break;
484 case CCValAssign::BCvt: {
485 // Extract a float argument from i64 with padding.
486 // 63 31 0
487 // +------+------+
488 // | float| 0 |
489 // +------+------+
490 assert(VA.getLocVT() == MVT::i64);
491 assert(VA.getValVT() == MVT::f32);
492 SDValue Sub_f32 = DAG.getTargetConstant(Val: VE::sub_f32, DL, VT: MVT::i32);
493 Arg = SDValue(DAG.getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG, dl: DL,
494 VT: MVT::f32, Op1: Arg, Op2: Sub_f32),
495 0);
496 break;
497 }
498 default:
499 break;
500 }
501
502 // Truncate the register down to the argument type.
503 if (VA.isExtInLoc())
504 Arg = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: VA.getValVT(), Operand: Arg);
505
506 InVals.push_back(Elt: Arg);
507 continue;
508 }
509
510 // The registers are exhausted. This argument was passed on the stack.
511 assert(VA.isMemLoc());
512 // The CC_VE_Full/Half functions compute stack offsets relative to the
513 // beginning of the arguments area at %fp + the size of reserved area.
514 unsigned Offset = VA.getLocMemOffset() + ArgsBaseOffset;
515 unsigned ValSize = VA.getValVT().getSizeInBits() / 8;
516
517 // Adjust offset for a float argument by adding 4 since the argument is
518 // stored in 8 bytes buffer with offset like below. LLVM generates
519 // 4 bytes load instruction, so need to adjust offset here. This
520 // adjustment is required in only LowerFormalArguments. In LowerCall,
521 // a float argument is converted to i64 first, and stored as 8 bytes
522 // data, which is required by ABI, so no need for adjustment.
523 // 0 4
524 // +------+------+
525 // | empty| float|
526 // +------+------+
527 if (VA.getValVT() == MVT::f32)
528 Offset += 4;
529
530 int FI = MF.getFrameInfo().CreateFixedObject(Size: ValSize, SPOffset: Offset, IsImmutable: true);
531 InVals.push_back(
532 Elt: DAG.getLoad(VT: VA.getValVT(), dl: DL, Chain,
533 Ptr: DAG.getFrameIndex(FI, VT: getPointerTy(DL: MF.getDataLayout())),
534 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI)));
535 }
536
537 if (!IsVarArg)
538 return Chain;
539
540 // This function takes variable arguments, some of which may have been passed
541 // in registers %s0-%s8.
542 //
543 // The va_start intrinsic needs to know the offset to the first variable
544 // argument.
545 // TODO: need to calculate offset correctly once we support f128.
546 unsigned ArgOffset = ArgLocs.size() * 8;
547 VEMachineFunctionInfo *FuncInfo = MF.getInfo<VEMachineFunctionInfo>();
548 // Skip the reserved area at the top of stack.
549 FuncInfo->setVarArgsFrameOffset(ArgOffset + ArgsBaseOffset);
550
551 return Chain;
552}
553
554// FIXME? Maybe this could be a TableGen attribute on some registers and
555// this table could be generated automatically from RegInfo.
556Register VETargetLowering::getRegisterByName(const char *RegName, LLT VT,
557 const MachineFunction &MF) const {
558 Register Reg = StringSwitch<Register>(RegName)
559 .Case(S: "sp", Value: VE::SX11) // Stack pointer
560 .Case(S: "fp", Value: VE::SX9) // Frame pointer
561 .Case(S: "sl", Value: VE::SX8) // Stack limit
562 .Case(S: "lr", Value: VE::SX10) // Link register
563 .Case(S: "tp", Value: VE::SX14) // Thread pointer
564 .Case(S: "outer", Value: VE::SX12) // Outer regiser
565 .Case(S: "info", Value: VE::SX17) // Info area register
566 .Case(S: "got", Value: VE::SX15) // Global offset table register
567 .Case(S: "plt", Value: VE::SX16) // Procedure linkage table register
568 .Default(Value: Register());
569 return Reg;
570}
571
572//===----------------------------------------------------------------------===//
573// TargetLowering Implementation
574//===----------------------------------------------------------------------===//
575
576SDValue VETargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
577 SmallVectorImpl<SDValue> &InVals) const {
578 SelectionDAG &DAG = CLI.DAG;
579 SDLoc DL = CLI.DL;
580 SDValue Chain = CLI.Chain;
581 auto PtrVT = getPointerTy(DL: DAG.getDataLayout());
582
583 // VE target does not yet support tail call optimization.
584 CLI.IsTailCall = false;
585
586 // Get the base offset of the outgoing arguments stack space.
587 unsigned ArgsBaseOffset = Subtarget->getRsaSize();
588 // Get the size of the preserved arguments area
589 unsigned ArgsPreserved = 8 * 8u;
590
591 // Analyze operands of the call, assigning locations to each operand.
592 SmallVector<CCValAssign, 16> ArgLocs;
593 CCState CCInfo(CLI.CallConv, CLI.IsVarArg, DAG.getMachineFunction(), ArgLocs,
594 *DAG.getContext());
595 // Allocate the preserved area first.
596 CCInfo.AllocateStack(Size: ArgsPreserved, Alignment: Align(8));
597 // We already allocated the preserved area, so the stack offset computed
598 // by CC_VE would be correct now.
599 CCInfo.AnalyzeCallOperands(Outs: CLI.Outs, Fn: getParamCC(CallConv: CLI.CallConv, IsVarArg: false));
600
601 // VE requires to use both register and stack for varargs or no-prototyped
602 // functions.
603 bool UseBoth = CLI.IsVarArg;
604
605 // Analyze operands again if it is required to store BOTH.
606 SmallVector<CCValAssign, 16> ArgLocs2;
607 CCState CCInfo2(CLI.CallConv, CLI.IsVarArg, DAG.getMachineFunction(),
608 ArgLocs2, *DAG.getContext());
609 if (UseBoth)
610 CCInfo2.AnalyzeCallOperands(Outs: CLI.Outs, Fn: getParamCC(CallConv: CLI.CallConv, IsVarArg: true));
611
612 // Get the size of the outgoing arguments stack space requirement.
613 unsigned ArgsSize = CCInfo.getStackSize();
614
615 // Keep stack frames 16-byte aligned.
616 ArgsSize = alignTo(Value: ArgsSize, Align: 16);
617
618 // Adjust the stack pointer to make room for the arguments.
619 // FIXME: Use hasReservedCallFrame to avoid %sp adjustments around all calls
620 // with more than 6 arguments.
621 Chain = DAG.getCALLSEQ_START(Chain, InSize: ArgsSize, OutSize: 0, DL);
622
623 // Collect the set of registers to pass to the function and their values.
624 // This will be emitted as a sequence of CopyToReg nodes glued to the call
625 // instruction.
626 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
627
628 // Collect chains from all the memory opeations that copy arguments to the
629 // stack. They must follow the stack pointer adjustment above and precede the
630 // call instruction itself.
631 SmallVector<SDValue, 8> MemOpChains;
632
633 // VE needs to get address of callee function in a register
634 // So, prepare to copy it to SX12 here.
635
636 // If the callee is a GlobalAddress node (quite common, every direct call is)
637 // turn it into a TargetGlobalAddress node so that legalize doesn't hack it.
638 // Likewise ExternalSymbol -> TargetExternalSymbol.
639 SDValue Callee = CLI.Callee;
640
641 bool IsPICCall = isPositionIndependent();
642
643 // PC-relative references to external symbols should go through $stub.
644 // If so, we need to prepare GlobalBaseReg first.
645 const TargetMachine &TM = DAG.getTarget();
646 const GlobalValue *GV = nullptr;
647 auto *CalleeG = dyn_cast<GlobalAddressSDNode>(Val&: Callee);
648 if (CalleeG)
649 GV = CalleeG->getGlobal();
650 bool Local = TM.shouldAssumeDSOLocal(GV);
651 bool UsePlt = !Local;
652 MachineFunction &MF = DAG.getMachineFunction();
653
654 // Turn GlobalAddress/ExternalSymbol node into a value node
655 // containing the address of them here.
656 if (CalleeG) {
657 if (IsPICCall) {
658 if (UsePlt)
659 Subtarget->getInstrInfo()->getGlobalBaseReg(MF: &MF);
660 Callee = DAG.getTargetGlobalAddress(GV, DL, VT: PtrVT, offset: 0, TargetFlags: 0);
661 Callee = DAG.getNode(Opcode: VEISD::GETFUNPLT, DL, VT: PtrVT, Operand: Callee);
662 } else {
663 Callee = makeHiLoPair(Op: Callee, HiTF: VE::S_HI32, LoTF: VE::S_LO32, DAG);
664 }
665 } else if (ExternalSymbolSDNode *E = dyn_cast<ExternalSymbolSDNode>(Val&: Callee)) {
666 if (IsPICCall) {
667 if (UsePlt)
668 Subtarget->getInstrInfo()->getGlobalBaseReg(MF: &MF);
669 Callee = DAG.getTargetExternalSymbol(Sym: E->getSymbol(), VT: PtrVT, TargetFlags: 0);
670 Callee = DAG.getNode(Opcode: VEISD::GETFUNPLT, DL, VT: PtrVT, Operand: Callee);
671 } else {
672 Callee = makeHiLoPair(Op: Callee, HiTF: VE::S_HI32, LoTF: VE::S_LO32, DAG);
673 }
674 }
675
676 RegsToPass.push_back(Elt: std::make_pair(x: VE::SX12, y&: Callee));
677
678 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
679 CCValAssign &VA = ArgLocs[i];
680 SDValue Arg = CLI.OutVals[i];
681
682 // Promote the value if needed.
683 switch (VA.getLocInfo()) {
684 default:
685 llvm_unreachable("Unknown location info!");
686 case CCValAssign::Full:
687 break;
688 case CCValAssign::SExt:
689 Arg = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: VA.getLocVT(), Operand: Arg);
690 break;
691 case CCValAssign::ZExt:
692 Arg = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: VA.getLocVT(), Operand: Arg);
693 break;
694 case CCValAssign::AExt:
695 Arg = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: VA.getLocVT(), Operand: Arg);
696 break;
697 case CCValAssign::BCvt: {
698 // Convert a float argument to i64 with padding.
699 // 63 31 0
700 // +------+------+
701 // | float| 0 |
702 // +------+------+
703 assert(VA.getLocVT() == MVT::i64);
704 assert(VA.getValVT() == MVT::f32);
705 SDValue Undef = SDValue(
706 DAG.getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl: DL, VT: MVT::i64), 0);
707 SDValue Sub_f32 = DAG.getTargetConstant(Val: VE::sub_f32, DL, VT: MVT::i32);
708 Arg = SDValue(DAG.getMachineNode(Opcode: TargetOpcode::INSERT_SUBREG, dl: DL,
709 VT: MVT::i64, Op1: Undef, Op2: Arg, Op3: Sub_f32),
710 0);
711 break;
712 }
713 }
714
715 if (VA.isRegLoc()) {
716 RegsToPass.push_back(Elt: std::make_pair(x: VA.getLocReg(), y&: Arg));
717 if (!UseBoth)
718 continue;
719 VA = ArgLocs2[i];
720 }
721
722 assert(VA.isMemLoc());
723
724 // Create a store off the stack pointer for this argument.
725 SDValue StackPtr = DAG.getRegister(Reg: VE::SX11, VT: PtrVT);
726 // The argument area starts at %fp/%sp + the size of reserved area.
727 SDValue PtrOff =
728 DAG.getIntPtrConstant(Val: VA.getLocMemOffset() + ArgsBaseOffset, DL);
729 PtrOff = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: StackPtr, N2: PtrOff);
730 MemOpChains.push_back(
731 Elt: DAG.getStore(Chain, dl: DL, Val: Arg, Ptr: PtrOff, PtrInfo: MachinePointerInfo()));
732 }
733
734 // Emit all stores, make sure they occur before the call.
735 if (!MemOpChains.empty())
736 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: MemOpChains);
737
738 // Build a sequence of CopyToReg nodes glued together with token chain and
739 // glue operands which copy the outgoing args into registers. The InGlue is
740 // necessary since all emitted instructions must be stuck together in order
741 // to pass the live physical registers.
742 SDValue InGlue;
743 for (const auto &[Reg, N] : RegsToPass) {
744 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg, N, Glue: InGlue);
745 InGlue = Chain.getValue(R: 1);
746 }
747
748 // Build the operands for the call instruction itself.
749 SmallVector<SDValue, 8> Ops;
750 Ops.push_back(Elt: Chain);
751 for (const auto &[Reg, N] : RegsToPass)
752 Ops.push_back(Elt: DAG.getRegister(Reg, VT: N.getValueType()));
753
754 // Add a register mask operand representing the call-preserved registers.
755 const VERegisterInfo *TRI = Subtarget->getRegisterInfo();
756 const uint32_t *Mask =
757 TRI->getCallPreservedMask(MF: DAG.getMachineFunction(), CC: CLI.CallConv);
758 assert(Mask && "Missing call preserved mask for calling convention");
759 Ops.push_back(Elt: DAG.getRegisterMask(RegMask: Mask));
760
761 // Make sure the CopyToReg nodes are glued to the call instruction which
762 // consumes the registers.
763 if (InGlue.getNode())
764 Ops.push_back(Elt: InGlue);
765
766 // Now the call itself.
767 SDVTList NodeTys = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
768 Chain = DAG.getNode(Opcode: VEISD::CALL, DL, VTList: NodeTys, Ops);
769 InGlue = Chain.getValue(R: 1);
770
771 // Revert the stack pointer immediately after the call.
772 Chain = DAG.getCALLSEQ_END(Chain, Size1: ArgsSize, Size2: 0, Glue: InGlue, DL);
773 InGlue = Chain.getValue(R: 1);
774
775 // Now extract the return values. This is more or less the same as
776 // LowerFormalArguments.
777
778 // Assign locations to each value returned by this call.
779 SmallVector<CCValAssign, 16> RVLocs;
780 CCState RVInfo(CLI.CallConv, CLI.IsVarArg, DAG.getMachineFunction(), RVLocs,
781 *DAG.getContext());
782
783 // Set inreg flag manually for codegen generated library calls that
784 // return float.
785 if (CLI.Ins.size() == 1 && CLI.Ins[0].VT == MVT::f32 && !CLI.CB)
786 CLI.Ins[0].Flags.setInReg();
787
788 RVInfo.AnalyzeCallResult(Ins: CLI.Ins, Fn: getReturnCC(CallConv: CLI.CallConv));
789
790 // Copy all of the result registers out of their specified physreg.
791 for (unsigned i = 0; i != RVLocs.size(); ++i) {
792 CCValAssign &VA = RVLocs[i];
793 assert(!VA.needsCustom() && "Unexpected custom lowering");
794 Register Reg = VA.getLocReg();
795
796 // When returning 'inreg {i32, i32 }', two consecutive i32 arguments can
797 // reside in the same register in the high and low bits. Reuse the
798 // CopyFromReg previous node to avoid duplicate copies.
799 SDValue RV;
800 if (RegisterSDNode *SrcReg = dyn_cast<RegisterSDNode>(Val: Chain.getOperand(i: 1)))
801 if (SrcReg->getReg() == Reg && Chain->getOpcode() == ISD::CopyFromReg)
802 RV = Chain.getValue(R: 0);
803
804 // But usually we'll create a new CopyFromReg for a different register.
805 if (!RV.getNode()) {
806 RV = DAG.getCopyFromReg(Chain, dl: DL, Reg, VT: RVLocs[i].getLocVT(), Glue: InGlue);
807 Chain = RV.getValue(R: 1);
808 InGlue = Chain.getValue(R: 2);
809 }
810
811 // The callee promoted the return value, so insert an Assert?ext SDNode so
812 // we won't promote the value again in this function.
813 switch (VA.getLocInfo()) {
814 case CCValAssign::SExt:
815 RV = DAG.getNode(Opcode: ISD::AssertSext, DL, VT: VA.getLocVT(), N1: RV,
816 N2: DAG.getValueType(VA.getValVT()));
817 break;
818 case CCValAssign::ZExt:
819 RV = DAG.getNode(Opcode: ISD::AssertZext, DL, VT: VA.getLocVT(), N1: RV,
820 N2: DAG.getValueType(VA.getValVT()));
821 break;
822 case CCValAssign::BCvt: {
823 // Extract a float return value from i64 with padding.
824 // 63 31 0
825 // +------+------+
826 // | float| 0 |
827 // +------+------+
828 assert(VA.getLocVT() == MVT::i64);
829 assert(VA.getValVT() == MVT::f32);
830 SDValue Sub_f32 = DAG.getTargetConstant(Val: VE::sub_f32, DL, VT: MVT::i32);
831 RV = SDValue(DAG.getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG, dl: DL,
832 VT: MVT::f32, Op1: RV, Op2: Sub_f32),
833 0);
834 break;
835 }
836 default:
837 break;
838 }
839
840 // Truncate the register down to the return value type.
841 if (VA.isExtInLoc())
842 RV = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: VA.getValVT(), Operand: RV);
843
844 InVals.push_back(Elt: RV);
845 }
846
847 return Chain;
848}
849
850bool VETargetLowering::isOffsetFoldingLegal(
851 const GlobalAddressSDNode *GA) const {
852 // VE uses 64 bit addressing, so we need multiple instructions to generate
853 // an address. Folding address with offset increases the number of
854 // instructions, so that we disable it here. Offsets will be folded in
855 // the DAG combine later if it worth to do so.
856 return false;
857}
858
859/// isFPImmLegal - Returns true if the target can instruction select the
860/// specified FP immediate natively. If false, the legalizer will
861/// materialize the FP immediate as a load from a constant pool.
862bool VETargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
863 bool ForCodeSize) const {
864 return VT == MVT::f32 || VT == MVT::f64;
865}
866
867/// Determine if the target supports unaligned memory accesses.
868///
869/// This function returns true if the target allows unaligned memory accesses
870/// of the specified type in the given address space. If true, it also returns
871/// whether the unaligned memory access is "fast" in the last argument by
872/// reference. This is used, for example, in situations where an array
873/// copy/move/set is converted to a sequence of store operations. Its use
874/// helps to ensure that such replacements don't generate code that causes an
875/// alignment error (trap) on the target machine.
876bool VETargetLowering::allowsMisalignedMemoryAccesses(EVT VT,
877 unsigned AddrSpace,
878 Align A,
879 MachineMemOperand::Flags,
880 unsigned *Fast) const {
881 if (Fast) {
882 // It's fast anytime on VE
883 *Fast = 1;
884 }
885 return true;
886}
887
888VETargetLowering::VETargetLowering(const TargetMachine &TM,
889 const VESubtarget &STI)
890 : TargetLowering(TM, STI), Subtarget(&STI) {
891 // Instructions which use registers as conditionals examine all the
892 // bits (as does the pseudo SELECT_CC expansion). I don't think it
893 // matters much whether it's ZeroOrOneBooleanContent, or
894 // ZeroOrNegativeOneBooleanContent, so, arbitrarily choose the
895 // former.
896 setBooleanContents(ZeroOrOneBooleanContent);
897 setBooleanVectorContents(ZeroOrOneBooleanContent);
898
899 initRegisterClasses();
900 initSPUActions();
901 initVPUActions();
902
903 setStackPointerRegisterToSaveRestore(VE::SX11);
904
905 // We have target-specific dag combine patterns for the following nodes:
906 setTargetDAGCombine(ISD::TRUNCATE);
907 setTargetDAGCombine(ISD::SELECT);
908 setTargetDAGCombine(ISD::SELECT_CC);
909
910 // Set function alignment to 16 bytes
911 setMinFunctionAlignment(Align(16));
912
913 // VE stores all argument by 8 bytes alignment
914 setMinStackArgumentAlignment(Align(8));
915
916 computeRegisterProperties(TRI: Subtarget->getRegisterInfo());
917}
918
919EVT VETargetLowering::getSetCCResultType(const DataLayout &,
920 LLVMContext &Context, EVT VT) const {
921 if (VT.isVector())
922 return VT.changeVectorElementType(Context, EltVT: MVT::i1);
923 return MVT::i32;
924}
925
926// Convert to a target node and set target flags.
927SDValue VETargetLowering::withTargetFlags(SDValue Op, unsigned TF,
928 SelectionDAG &DAG) const {
929 if (const GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Val&: Op))
930 return DAG.getTargetGlobalAddress(GV: GA->getGlobal(), DL: SDLoc(GA),
931 VT: GA->getValueType(ResNo: 0), offset: GA->getOffset(), TargetFlags: TF);
932
933 if (const BlockAddressSDNode *BA = dyn_cast<BlockAddressSDNode>(Val&: Op))
934 return DAG.getTargetBlockAddress(BA: BA->getBlockAddress(), VT: Op.getValueType(),
935 Offset: 0, TargetFlags: TF);
936
937 if (const ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(Val&: Op))
938 return DAG.getTargetConstantPool(C: CP->getConstVal(), VT: CP->getValueType(ResNo: 0),
939 Align: CP->getAlign(), Offset: CP->getOffset(), TargetFlags: TF);
940
941 if (const ExternalSymbolSDNode *ES = dyn_cast<ExternalSymbolSDNode>(Val&: Op))
942 return DAG.getTargetExternalSymbol(Sym: ES->getSymbol(), VT: ES->getValueType(ResNo: 0),
943 TargetFlags: TF);
944
945 if (const JumpTableSDNode *JT = dyn_cast<JumpTableSDNode>(Val&: Op))
946 return DAG.getTargetJumpTable(JTI: JT->getIndex(), VT: JT->getValueType(ResNo: 0), TargetFlags: TF);
947
948 llvm_unreachable("Unhandled address SDNode");
949}
950
951// Split Op into high and low parts according to HiTF and LoTF.
952// Return an ADD node combining the parts.
953SDValue VETargetLowering::makeHiLoPair(SDValue Op, unsigned HiTF, unsigned LoTF,
954 SelectionDAG &DAG) const {
955 SDLoc DL(Op);
956 EVT VT = Op.getValueType();
957 SDValue Hi = DAG.getNode(Opcode: VEISD::Hi, DL, VT, Operand: withTargetFlags(Op, TF: HiTF, DAG));
958 SDValue Lo = DAG.getNode(Opcode: VEISD::Lo, DL, VT, Operand: withTargetFlags(Op, TF: LoTF, DAG));
959 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Hi, N2: Lo);
960}
961
962// Build SDNodes for producing an address from a GlobalAddress, ConstantPool,
963// or ExternalSymbol SDNode.
964SDValue VETargetLowering::makeAddress(SDValue Op, SelectionDAG &DAG) const {
965 SDLoc DL(Op);
966 EVT PtrVT = Op.getValueType();
967
968 // Handle PIC mode first. VE needs a got load for every variable!
969 if (isPositionIndependent()) {
970 auto GlobalN = dyn_cast<GlobalAddressSDNode>(Val&: Op);
971
972 if (isa<ConstantPoolSDNode>(Val: Op) || isa<JumpTableSDNode>(Val: Op) ||
973 (GlobalN && GlobalN->getGlobal()->hasLocalLinkage())) {
974 // Create following instructions for local linkage PIC code.
975 // lea %reg, label@gotoff_lo
976 // and %reg, %reg, (32)0
977 // lea.sl %reg, label@gotoff_hi(%reg, %got)
978 SDValue HiLo =
979 makeHiLoPair(Op, HiTF: VE::S_GOTOFF_HI32, LoTF: VE::S_GOTOFF_LO32, DAG);
980 SDValue GlobalBase = DAG.getNode(Opcode: VEISD::GLOBAL_BASE_REG, DL, VT: PtrVT);
981 return DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: GlobalBase, N2: HiLo);
982 }
983 // Create following instructions for not local linkage PIC code.
984 // lea %reg, label@got_lo
985 // and %reg, %reg, (32)0
986 // lea.sl %reg, label@got_hi(%reg)
987 // ld %reg, (%reg, %got)
988 SDValue HiLo = makeHiLoPair(Op, HiTF: VE::S_GOT_HI32, LoTF: VE::S_GOT_LO32, DAG);
989 SDValue GlobalBase = DAG.getNode(Opcode: VEISD::GLOBAL_BASE_REG, DL, VT: PtrVT);
990 SDValue AbsAddr = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: GlobalBase, N2: HiLo);
991 return DAG.getLoad(VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: AbsAddr,
992 PtrInfo: MachinePointerInfo::getGOT(MF&: DAG.getMachineFunction()));
993 }
994
995 // This is one of the absolute code models.
996 switch (getTargetMachine().getCodeModel()) {
997 default:
998 llvm_unreachable("Unsupported absolute code model");
999 case CodeModel::Small:
1000 case CodeModel::Medium:
1001 case CodeModel::Large:
1002 // abs64.
1003 return makeHiLoPair(Op, HiTF: VE::S_HI32, LoTF: VE::S_LO32, DAG);
1004 }
1005}
1006
1007/// Custom Lower {
1008
1009// The mappings for emitLeading/TrailingFence for VE is designed by following
1010// http://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html
1011Instruction *VETargetLowering::emitLeadingFence(IRBuilderBase &Builder,
1012 Instruction *Inst,
1013 AtomicOrdering Ord) const {
1014 switch (Ord) {
1015 case AtomicOrdering::NotAtomic:
1016 case AtomicOrdering::Unordered:
1017 llvm_unreachable("Invalid fence: unordered/non-atomic");
1018 case AtomicOrdering::Monotonic:
1019 case AtomicOrdering::Acquire:
1020 return nullptr; // Nothing to do
1021 case AtomicOrdering::Release:
1022 case AtomicOrdering::AcquireRelease:
1023 return Builder.CreateFence(Ordering: AtomicOrdering::Release);
1024 case AtomicOrdering::SequentiallyConsistent:
1025 if (!Inst->hasAtomicStore())
1026 return nullptr; // Nothing to do
1027 return Builder.CreateFence(Ordering: AtomicOrdering::SequentiallyConsistent);
1028 }
1029 llvm_unreachable("Unknown fence ordering in emitLeadingFence");
1030}
1031
1032Instruction *VETargetLowering::emitTrailingFence(IRBuilderBase &Builder,
1033 Instruction *Inst,
1034 AtomicOrdering Ord) const {
1035 switch (Ord) {
1036 case AtomicOrdering::NotAtomic:
1037 case AtomicOrdering::Unordered:
1038 llvm_unreachable("Invalid fence: unordered/not-atomic");
1039 case AtomicOrdering::Monotonic:
1040 case AtomicOrdering::Release:
1041 return nullptr; // Nothing to do
1042 case AtomicOrdering::Acquire:
1043 case AtomicOrdering::AcquireRelease:
1044 return Builder.CreateFence(Ordering: AtomicOrdering::Acquire);
1045 case AtomicOrdering::SequentiallyConsistent:
1046 return Builder.CreateFence(Ordering: AtomicOrdering::SequentiallyConsistent);
1047 }
1048 llvm_unreachable("Unknown fence ordering in emitTrailingFence");
1049}
1050
1051SDValue VETargetLowering::lowerATOMIC_FENCE(SDValue Op,
1052 SelectionDAG &DAG) const {
1053 SDLoc DL(Op);
1054 AtomicOrdering FenceOrdering =
1055 static_cast<AtomicOrdering>(Op.getConstantOperandVal(i: 1));
1056 SyncScope::ID FenceSSID =
1057 static_cast<SyncScope::ID>(Op.getConstantOperandVal(i: 2));
1058
1059 // VE uses Release consistency, so need a fence instruction if it is a
1060 // cross-thread fence.
1061 if (FenceSSID == SyncScope::System) {
1062 switch (FenceOrdering) {
1063 case AtomicOrdering::NotAtomic:
1064 case AtomicOrdering::Unordered:
1065 case AtomicOrdering::Monotonic:
1066 // No need to generate fencem instruction here.
1067 break;
1068 case AtomicOrdering::Acquire:
1069 // Generate "fencem 2" as acquire fence.
1070 return SDValue(DAG.getMachineNode(Opcode: VE::FENCEM, dl: DL, VT: MVT::Other,
1071 Op1: DAG.getTargetConstant(Val: 2, DL, VT: MVT::i32),
1072 Op2: Op.getOperand(i: 0)),
1073 0);
1074 case AtomicOrdering::Release:
1075 // Generate "fencem 1" as release fence.
1076 return SDValue(DAG.getMachineNode(Opcode: VE::FENCEM, dl: DL, VT: MVT::Other,
1077 Op1: DAG.getTargetConstant(Val: 1, DL, VT: MVT::i32),
1078 Op2: Op.getOperand(i: 0)),
1079 0);
1080 case AtomicOrdering::AcquireRelease:
1081 case AtomicOrdering::SequentiallyConsistent:
1082 // Generate "fencem 3" as acq_rel and seq_cst fence.
1083 // FIXME: "fencem 3" doesn't wait for PCIe deveices accesses,
1084 // so seq_cst may require more instruction for them.
1085 return SDValue(DAG.getMachineNode(Opcode: VE::FENCEM, dl: DL, VT: MVT::Other,
1086 Op1: DAG.getTargetConstant(Val: 3, DL, VT: MVT::i32),
1087 Op2: Op.getOperand(i: 0)),
1088 0);
1089 }
1090 }
1091
1092 // MEMBARRIER is a compiler barrier; it codegens to a no-op.
1093 return DAG.getNode(Opcode: ISD::MEMBARRIER, DL, VT: MVT::Other, Operand: Op.getOperand(i: 0));
1094}
1095
1096TargetLowering::AtomicExpansionKind
1097VETargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
1098 // We have TS1AM implementation for i8/i16/i32/i64, so use it.
1099 if (AI->getOperation() == AtomicRMWInst::Xchg) {
1100 return AtomicExpansionKind::None;
1101 }
1102 // FIXME: Support "ATMAM" instruction for LOAD_ADD/SUB/AND/OR.
1103
1104 // Otherwise, expand it using compare and exchange instruction to not call
1105 // __sync_fetch_and_* functions.
1106 return AtomicExpansionKind::CmpXChg;
1107}
1108
1109static SDValue prepareTS1AM(SDValue Op, SelectionDAG &DAG, SDValue &Flag,
1110 SDValue &Bits) {
1111 SDLoc DL(Op);
1112 AtomicSDNode *N = cast<AtomicSDNode>(Val&: Op);
1113 SDValue Ptr = N->getOperand(Num: 1);
1114 SDValue Val = N->getOperand(Num: 2);
1115 EVT PtrVT = Ptr.getValueType();
1116 bool Byte = N->getMemoryVT() == MVT::i8;
1117 // Remainder = AND Ptr, 3
1118 // Flag = 1 << Remainder ; If Byte is true (1 byte swap flag)
1119 // Flag = 3 << Remainder ; If Byte is false (2 bytes swap flag)
1120 // Bits = Remainder << 3
1121 // NewVal = Val << Bits
1122 SDValue Const3 = DAG.getConstant(Val: 3, DL, VT: PtrVT);
1123 SDValue Remainder = DAG.getNode(Opcode: ISD::AND, DL, VT: PtrVT, Ops: {Ptr, Const3});
1124 SDValue Mask = Byte ? DAG.getConstant(Val: 1, DL, VT: MVT::i32)
1125 : DAG.getConstant(Val: 3, DL, VT: MVT::i32);
1126 Flag = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i32, Ops: {Mask, Remainder});
1127 Bits = DAG.getNode(Opcode: ISD::SHL, DL, VT: PtrVT, Ops: {Remainder, Const3});
1128 return DAG.getNode(Opcode: ISD::SHL, DL, VT: Val.getValueType(), Ops: {Val, Bits});
1129}
1130
1131static SDValue finalizeTS1AM(SDValue Op, SelectionDAG &DAG, SDValue Data,
1132 SDValue Bits) {
1133 SDLoc DL(Op);
1134 EVT VT = Data.getValueType();
1135 bool Byte = cast<AtomicSDNode>(Val&: Op)->getMemoryVT() == MVT::i8;
1136 // NewData = Data >> Bits
1137 // Result = NewData & 0xff ; If Byte is true (1 byte)
1138 // Result = NewData & 0xffff ; If Byte is false (2 bytes)
1139
1140 SDValue NewData = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Data, N2: Bits);
1141 return DAG.getNode(Opcode: ISD::AND, DL, VT,
1142 Ops: {NewData, DAG.getConstant(Val: Byte ? 0xff : 0xffff, DL, VT)});
1143}
1144
1145SDValue VETargetLowering::lowerATOMIC_SWAP(SDValue Op,
1146 SelectionDAG &DAG) const {
1147 SDLoc DL(Op);
1148 AtomicSDNode *N = cast<AtomicSDNode>(Val&: Op);
1149
1150 if (N->getMemoryVT() == MVT::i8) {
1151 // For i8, use "ts1am"
1152 // Input:
1153 // ATOMIC_SWAP Ptr, Val, Order
1154 //
1155 // Output:
1156 // Remainder = AND Ptr, 3
1157 // Flag = 1 << Remainder ; 1 byte swap flag for TS1AM inst.
1158 // Bits = Remainder << 3
1159 // NewVal = Val << Bits
1160 //
1161 // Aligned = AND Ptr, -4
1162 // Data = TS1AM Aligned, Flag, NewVal
1163 //
1164 // NewData = Data >> Bits
1165 // Result = NewData & 0xff ; 1 byte result
1166 SDValue Flag;
1167 SDValue Bits;
1168 SDValue NewVal = prepareTS1AM(Op, DAG, Flag, Bits);
1169
1170 SDValue Ptr = N->getOperand(Num: 1);
1171 SDValue Aligned =
1172 DAG.getNode(Opcode: ISD::AND, DL, VT: Ptr.getValueType(),
1173 Ops: {Ptr, DAG.getSignedConstant(Val: -4, DL, VT: MVT::i64)});
1174 SDValue TS1AM =
1175 DAG.getMemIntrinsicNode(Opcode: VEISD::TS1AM, dl: DL,
1176 VTList: DAG.getVTList(VT1: Op.getNode()->getValueType(ResNo: 0),
1177 VT2: Op.getNode()->getValueType(ResNo: 1)),
1178 Ops: {N->getChain(), Aligned, Flag, NewVal},
1179 MemVT: N->getMemoryVT(), MMO: N->getMemOperand());
1180
1181 SDValue Result = finalizeTS1AM(Op, DAG, Data: TS1AM, Bits);
1182 SDValue Chain = TS1AM.getValue(R: 1);
1183 return DAG.getMergeValues(Ops: {Result, Chain}, dl: DL);
1184 }
1185 if (N->getMemoryVT() == MVT::i16) {
1186 // For i16, use "ts1am"
1187 SDValue Flag;
1188 SDValue Bits;
1189 SDValue NewVal = prepareTS1AM(Op, DAG, Flag, Bits);
1190
1191 SDValue Ptr = N->getOperand(Num: 1);
1192 SDValue Aligned =
1193 DAG.getNode(Opcode: ISD::AND, DL, VT: Ptr.getValueType(),
1194 Ops: {Ptr, DAG.getSignedConstant(Val: -4, DL, VT: MVT::i64)});
1195 SDValue TS1AM =
1196 DAG.getMemIntrinsicNode(Opcode: VEISD::TS1AM, dl: DL,
1197 VTList: DAG.getVTList(VT1: Op.getNode()->getValueType(ResNo: 0),
1198 VT2: Op.getNode()->getValueType(ResNo: 1)),
1199 Ops: {N->getChain(), Aligned, Flag, NewVal},
1200 MemVT: N->getMemoryVT(), MMO: N->getMemOperand());
1201
1202 SDValue Result = finalizeTS1AM(Op, DAG, Data: TS1AM, Bits);
1203 SDValue Chain = TS1AM.getValue(R: 1);
1204 return DAG.getMergeValues(Ops: {Result, Chain}, dl: DL);
1205 }
1206 // Otherwise, let llvm legalize it.
1207 return Op;
1208}
1209
1210SDValue VETargetLowering::lowerGlobalAddress(SDValue Op,
1211 SelectionDAG &DAG) const {
1212 return makeAddress(Op, DAG);
1213}
1214
1215SDValue VETargetLowering::lowerBlockAddress(SDValue Op,
1216 SelectionDAG &DAG) const {
1217 return makeAddress(Op, DAG);
1218}
1219
1220SDValue VETargetLowering::lowerConstantPool(SDValue Op,
1221 SelectionDAG &DAG) const {
1222 return makeAddress(Op, DAG);
1223}
1224
1225SDValue
1226VETargetLowering::lowerToTLSGeneralDynamicModel(SDValue Op,
1227 SelectionDAG &DAG) const {
1228 SDLoc DL(Op);
1229
1230 // Generate the following code:
1231 // t1: ch,glue = callseq_start t0, 0, 0
1232 // t2: i64,ch,glue = VEISD::GETTLSADDR t1, label, t1:1
1233 // t3: ch,glue = callseq_end t2, 0, 0, t2:2
1234 // t4: i64,ch,glue = CopyFromReg t3, Register:i64 $sx0, t3:1
1235 SDValue Label = withTargetFlags(Op, TF: 0, DAG);
1236 EVT PtrVT = Op.getValueType();
1237
1238 // Lowering the machine isd will make sure everything is in the right
1239 // location.
1240 SDValue Chain = DAG.getEntryNode();
1241 SDVTList NodeTys = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
1242 const uint32_t *Mask = Subtarget->getRegisterInfo()->getCallPreservedMask(
1243 MF: DAG.getMachineFunction(), CC: CallingConv::C);
1244 Chain = DAG.getCALLSEQ_START(Chain, InSize: 64, OutSize: 0, DL);
1245 SDValue Args[] = {Chain, Label, DAG.getRegisterMask(RegMask: Mask), Chain.getValue(R: 1)};
1246 Chain = DAG.getNode(Opcode: VEISD::GETTLSADDR, DL, VTList: NodeTys, Ops: Args);
1247 Chain = DAG.getCALLSEQ_END(Chain, Size1: 64, Size2: 0, Glue: Chain.getValue(R: 1), DL);
1248 Chain = DAG.getCopyFromReg(Chain, dl: DL, Reg: VE::SX0, VT: PtrVT, Glue: Chain.getValue(R: 1));
1249
1250 // GETTLSADDR will be codegen'ed as call. Inform MFI that function has calls.
1251 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
1252 MFI.setHasCalls(true);
1253
1254 // Also generate code to prepare a GOT register if it is PIC.
1255 if (isPositionIndependent()) {
1256 MachineFunction &MF = DAG.getMachineFunction();
1257 Subtarget->getInstrInfo()->getGlobalBaseReg(MF: &MF);
1258 }
1259
1260 return Chain;
1261}
1262
1263SDValue VETargetLowering::lowerGlobalTLSAddress(SDValue Op,
1264 SelectionDAG &DAG) const {
1265 // The current implementation of nld (2.26) doesn't allow local exec model
1266 // code described in VE-tls_v1.1.pdf (*1) as its input. Instead, we always
1267 // generate the general dynamic model code sequence.
1268 //
1269 // *1: https://www.nec.com/en/global/prod/hpc/aurora/document/VE-tls_v1.1.pdf
1270 return lowerToTLSGeneralDynamicModel(Op, DAG);
1271}
1272
1273SDValue VETargetLowering::lowerJumpTable(SDValue Op, SelectionDAG &DAG) const {
1274 return makeAddress(Op, DAG);
1275}
1276
1277// Lower a f128 load into two f64 loads.
1278static SDValue lowerLoadF128(SDValue Op, SelectionDAG &DAG) {
1279 SDLoc DL(Op);
1280 LoadSDNode *LdNode = dyn_cast<LoadSDNode>(Val: Op.getNode());
1281 assert(LdNode && LdNode->getOffset().isUndef() && "Unexpected node type");
1282 Align Alignment = LdNode->getAlign();
1283 if (Alignment > 8)
1284 Alignment = Align(8);
1285
1286 SDValue Lo64 =
1287 DAG.getLoad(VT: MVT::f64, dl: DL, Chain: LdNode->getChain(), Ptr: LdNode->getBasePtr(),
1288 PtrInfo: LdNode->getPointerInfo(), Alignment,
1289 MMOFlags: LdNode->isVolatile() ? MachineMemOperand::MOVolatile
1290 : MachineMemOperand::MONone);
1291 EVT AddrVT = LdNode->getBasePtr().getValueType();
1292 SDValue HiPtr = DAG.getNode(Opcode: ISD::ADD, DL, VT: AddrVT, N1: LdNode->getBasePtr(),
1293 N2: DAG.getConstant(Val: 8, DL, VT: AddrVT));
1294 SDValue Hi64 =
1295 DAG.getLoad(VT: MVT::f64, dl: DL, Chain: LdNode->getChain(), Ptr: HiPtr,
1296 PtrInfo: LdNode->getPointerInfo(), Alignment,
1297 MMOFlags: LdNode->isVolatile() ? MachineMemOperand::MOVolatile
1298 : MachineMemOperand::MONone);
1299
1300 SDValue SubRegEven = DAG.getTargetConstant(Val: VE::sub_even, DL, VT: MVT::i32);
1301 SDValue SubRegOdd = DAG.getTargetConstant(Val: VE::sub_odd, DL, VT: MVT::i32);
1302
1303 // VE stores Hi64 to 8(addr) and Lo64 to 0(addr)
1304 SDNode *InFP128 =
1305 DAG.getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl: DL, VT: MVT::f128);
1306 InFP128 = DAG.getMachineNode(Opcode: TargetOpcode::INSERT_SUBREG, dl: DL, VT: MVT::f128,
1307 Op1: SDValue(InFP128, 0), Op2: Hi64, Op3: SubRegEven);
1308 InFP128 = DAG.getMachineNode(Opcode: TargetOpcode::INSERT_SUBREG, dl: DL, VT: MVT::f128,
1309 Op1: SDValue(InFP128, 0), Op2: Lo64, Op3: SubRegOdd);
1310 SDValue OutChains[2] = {SDValue(Lo64.getNode(), 1),
1311 SDValue(Hi64.getNode(), 1)};
1312 SDValue OutChain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: OutChains);
1313 SDValue Ops[2] = {SDValue(InFP128, 0), OutChain};
1314 return DAG.getMergeValues(Ops, dl: DL);
1315}
1316
1317// Lower a vXi1 load into following instructions
1318// LDrii %1, (,%addr)
1319// LVMxir %vm, 0, %1
1320// LDrii %2, 8(,%addr)
1321// LVMxir %vm, 0, %2
1322// ...
1323static SDValue lowerLoadI1(SDValue Op, SelectionDAG &DAG) {
1324 SDLoc DL(Op);
1325 LoadSDNode *LdNode = dyn_cast<LoadSDNode>(Val: Op.getNode());
1326 assert(LdNode && LdNode->getOffset().isUndef() && "Unexpected node type");
1327
1328 SDValue BasePtr = LdNode->getBasePtr();
1329 Align Alignment = LdNode->getAlign();
1330 if (Alignment > 8)
1331 Alignment = Align(8);
1332
1333 EVT AddrVT = BasePtr.getValueType();
1334 EVT MemVT = LdNode->getMemoryVT();
1335 if (MemVT == MVT::v256i1 || MemVT == MVT::v4i64) {
1336 SDValue OutChains[4];
1337 SDNode *VM = DAG.getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl: DL, VT: MemVT);
1338 for (int i = 0; i < 4; ++i) {
1339 // Generate load dag and prepare chains.
1340 SDValue Addr = DAG.getNode(Opcode: ISD::ADD, DL, VT: AddrVT, N1: BasePtr,
1341 N2: DAG.getConstant(Val: 8 * i, DL, VT: AddrVT));
1342 SDValue Val =
1343 DAG.getLoad(VT: MVT::i64, dl: DL, Chain: LdNode->getChain(), Ptr: Addr,
1344 PtrInfo: LdNode->getPointerInfo(), Alignment,
1345 MMOFlags: LdNode->isVolatile() ? MachineMemOperand::MOVolatile
1346 : MachineMemOperand::MONone);
1347 OutChains[i] = SDValue(Val.getNode(), 1);
1348
1349 VM = DAG.getMachineNode(Opcode: VE::LVMir_m, dl: DL, VT: MVT::i64,
1350 Op1: DAG.getTargetConstant(Val: i, DL, VT: MVT::i64), Op2: Val,
1351 Op3: SDValue(VM, 0));
1352 }
1353 SDValue OutChain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: OutChains);
1354 SDValue Ops[2] = {SDValue(VM, 0), OutChain};
1355 return DAG.getMergeValues(Ops, dl: DL);
1356 } else if (MemVT == MVT::v512i1 || MemVT == MVT::v8i64) {
1357 SDValue OutChains[8];
1358 SDNode *VM = DAG.getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl: DL, VT: MemVT);
1359 for (int i = 0; i < 8; ++i) {
1360 // Generate load dag and prepare chains.
1361 SDValue Addr = DAG.getNode(Opcode: ISD::ADD, DL, VT: AddrVT, N1: BasePtr,
1362 N2: DAG.getConstant(Val: 8 * i, DL, VT: AddrVT));
1363 SDValue Val =
1364 DAG.getLoad(VT: MVT::i64, dl: DL, Chain: LdNode->getChain(), Ptr: Addr,
1365 PtrInfo: LdNode->getPointerInfo(), Alignment,
1366 MMOFlags: LdNode->isVolatile() ? MachineMemOperand::MOVolatile
1367 : MachineMemOperand::MONone);
1368 OutChains[i] = SDValue(Val.getNode(), 1);
1369
1370 VM = DAG.getMachineNode(Opcode: VE::LVMyir_y, dl: DL, VT: MVT::i64,
1371 Op1: DAG.getTargetConstant(Val: i, DL, VT: MVT::i64), Op2: Val,
1372 Op3: SDValue(VM, 0));
1373 }
1374 SDValue OutChain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: OutChains);
1375 SDValue Ops[2] = {SDValue(VM, 0), OutChain};
1376 return DAG.getMergeValues(Ops, dl: DL);
1377 } else {
1378 // Otherwise, ask llvm to expand it.
1379 return SDValue();
1380 }
1381}
1382
1383SDValue VETargetLowering::lowerLOAD(SDValue Op, SelectionDAG &DAG) const {
1384 LoadSDNode *LdNode = cast<LoadSDNode>(Val: Op.getNode());
1385 EVT MemVT = LdNode->getMemoryVT();
1386
1387 // If VPU is enabled, always expand non-mask vector loads to VVP
1388 if (Subtarget->enableVPU() && MemVT.isVector() && !isMaskType(SomeVT: MemVT))
1389 return lowerToVVP(Op, DAG);
1390
1391 SDValue BasePtr = LdNode->getBasePtr();
1392 if (isa<FrameIndexSDNode>(Val: BasePtr.getNode())) {
1393 // Do not expand store instruction with frame index here because of
1394 // dependency problems. We expand it later in eliminateFrameIndex().
1395 return Op;
1396 }
1397
1398 if (MemVT == MVT::f128)
1399 return lowerLoadF128(Op, DAG);
1400 if (isMaskType(SomeVT: MemVT))
1401 return lowerLoadI1(Op, DAG);
1402
1403 return Op;
1404}
1405
1406// Lower a f128 store into two f64 stores.
1407static SDValue lowerStoreF128(SDValue Op, SelectionDAG &DAG) {
1408 SDLoc DL(Op);
1409 StoreSDNode *StNode = dyn_cast<StoreSDNode>(Val: Op.getNode());
1410 assert(StNode && StNode->getOffset().isUndef() && "Unexpected node type");
1411
1412 SDValue SubRegEven = DAG.getTargetConstant(Val: VE::sub_even, DL, VT: MVT::i32);
1413 SDValue SubRegOdd = DAG.getTargetConstant(Val: VE::sub_odd, DL, VT: MVT::i32);
1414
1415 SDNode *Hi64 = DAG.getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG, dl: DL, VT: MVT::i64,
1416 Op1: StNode->getValue(), Op2: SubRegEven);
1417 SDNode *Lo64 = DAG.getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG, dl: DL, VT: MVT::i64,
1418 Op1: StNode->getValue(), Op2: SubRegOdd);
1419
1420 Align Alignment = StNode->getAlign();
1421 if (Alignment > 8)
1422 Alignment = Align(8);
1423
1424 // VE stores Hi64 to 8(addr) and Lo64 to 0(addr)
1425 SDValue OutChains[2];
1426 OutChains[0] =
1427 DAG.getStore(Chain: StNode->getChain(), dl: DL, Val: SDValue(Lo64, 0),
1428 Ptr: StNode->getBasePtr(), PtrInfo: MachinePointerInfo(), Alignment,
1429 MMOFlags: StNode->isVolatile() ? MachineMemOperand::MOVolatile
1430 : MachineMemOperand::MONone);
1431 EVT AddrVT = StNode->getBasePtr().getValueType();
1432 SDValue HiPtr = DAG.getNode(Opcode: ISD::ADD, DL, VT: AddrVT, N1: StNode->getBasePtr(),
1433 N2: DAG.getConstant(Val: 8, DL, VT: AddrVT));
1434 OutChains[1] =
1435 DAG.getStore(Chain: StNode->getChain(), dl: DL, Val: SDValue(Hi64, 0), Ptr: HiPtr,
1436 PtrInfo: MachinePointerInfo(), Alignment,
1437 MMOFlags: StNode->isVolatile() ? MachineMemOperand::MOVolatile
1438 : MachineMemOperand::MONone);
1439 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: OutChains);
1440}
1441
1442// Lower a vXi1 store into following instructions
1443// SVMi %1, %vm, 0
1444// STrii %1, (,%addr)
1445// SVMi %2, %vm, 1
1446// STrii %2, 8(,%addr)
1447// ...
1448static SDValue lowerStoreI1(SDValue Op, SelectionDAG &DAG) {
1449 SDLoc DL(Op);
1450 StoreSDNode *StNode = dyn_cast<StoreSDNode>(Val: Op.getNode());
1451 assert(StNode && StNode->getOffset().isUndef() && "Unexpected node type");
1452
1453 SDValue BasePtr = StNode->getBasePtr();
1454 Align Alignment = StNode->getAlign();
1455 if (Alignment > 8)
1456 Alignment = Align(8);
1457 EVT AddrVT = BasePtr.getValueType();
1458 EVT MemVT = StNode->getMemoryVT();
1459 if (MemVT == MVT::v256i1 || MemVT == MVT::v4i64) {
1460 SDValue OutChains[4];
1461 for (int i = 0; i < 4; ++i) {
1462 SDNode *V =
1463 DAG.getMachineNode(Opcode: VE::SVMmi, dl: DL, VT: MVT::i64, Op1: StNode->getValue(),
1464 Op2: DAG.getTargetConstant(Val: i, DL, VT: MVT::i64));
1465 SDValue Addr = DAG.getNode(Opcode: ISD::ADD, DL, VT: AddrVT, N1: BasePtr,
1466 N2: DAG.getConstant(Val: 8 * i, DL, VT: AddrVT));
1467 OutChains[i] =
1468 DAG.getStore(Chain: StNode->getChain(), dl: DL, Val: SDValue(V, 0), Ptr: Addr,
1469 PtrInfo: MachinePointerInfo(), Alignment,
1470 MMOFlags: StNode->isVolatile() ? MachineMemOperand::MOVolatile
1471 : MachineMemOperand::MONone);
1472 }
1473 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: OutChains);
1474 } else if (MemVT == MVT::v512i1 || MemVT == MVT::v8i64) {
1475 SDValue OutChains[8];
1476 for (int i = 0; i < 8; ++i) {
1477 SDNode *V =
1478 DAG.getMachineNode(Opcode: VE::SVMyi, dl: DL, VT: MVT::i64, Op1: StNode->getValue(),
1479 Op2: DAG.getTargetConstant(Val: i, DL, VT: MVT::i64));
1480 SDValue Addr = DAG.getNode(Opcode: ISD::ADD, DL, VT: AddrVT, N1: BasePtr,
1481 N2: DAG.getConstant(Val: 8 * i, DL, VT: AddrVT));
1482 OutChains[i] =
1483 DAG.getStore(Chain: StNode->getChain(), dl: DL, Val: SDValue(V, 0), Ptr: Addr,
1484 PtrInfo: MachinePointerInfo(), Alignment,
1485 MMOFlags: StNode->isVolatile() ? MachineMemOperand::MOVolatile
1486 : MachineMemOperand::MONone);
1487 }
1488 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: OutChains);
1489 } else {
1490 // Otherwise, ask llvm to expand it.
1491 return SDValue();
1492 }
1493}
1494
1495SDValue VETargetLowering::lowerSTORE(SDValue Op, SelectionDAG &DAG) const {
1496 StoreSDNode *StNode = cast<StoreSDNode>(Val: Op.getNode());
1497 assert(StNode && StNode->getOffset().isUndef() && "Unexpected node type");
1498 EVT MemVT = StNode->getMemoryVT();
1499
1500 // If VPU is enabled, always expand non-mask vector stores to VVP
1501 if (Subtarget->enableVPU() && MemVT.isVector() && !isMaskType(SomeVT: MemVT))
1502 return lowerToVVP(Op, DAG);
1503
1504 SDValue BasePtr = StNode->getBasePtr();
1505 if (isa<FrameIndexSDNode>(Val: BasePtr.getNode())) {
1506 // Do not expand store instruction with frame index here because of
1507 // dependency problems. We expand it later in eliminateFrameIndex().
1508 return Op;
1509 }
1510
1511 if (MemVT == MVT::f128)
1512 return lowerStoreF128(Op, DAG);
1513 if (isMaskType(SomeVT: MemVT))
1514 return lowerStoreI1(Op, DAG);
1515
1516 // Otherwise, ask llvm to expand it.
1517 return SDValue();
1518}
1519
1520SDValue VETargetLowering::lowerVASTART(SDValue Op, SelectionDAG &DAG) const {
1521 MachineFunction &MF = DAG.getMachineFunction();
1522 VEMachineFunctionInfo *FuncInfo = MF.getInfo<VEMachineFunctionInfo>();
1523 auto PtrVT = getPointerTy(DL: DAG.getDataLayout());
1524
1525 // Need frame address to find the address of VarArgsFrameIndex.
1526 MF.getFrameInfo().setFrameAddressIsTaken(true);
1527
1528 // vastart just stores the address of the VarArgsFrameIndex slot into the
1529 // memory location argument.
1530 SDLoc DL(Op);
1531 SDValue Offset =
1532 DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: DAG.getRegister(Reg: VE::SX9, VT: PtrVT),
1533 N2: DAG.getIntPtrConstant(Val: FuncInfo->getVarArgsFrameOffset(), DL));
1534 const Value *SV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 2))->getValue();
1535 return DAG.getStore(Chain: Op.getOperand(i: 0), dl: DL, Val: Offset, Ptr: Op.getOperand(i: 1),
1536 PtrInfo: MachinePointerInfo(SV));
1537}
1538
1539SDValue VETargetLowering::lowerVAARG(SDValue Op, SelectionDAG &DAG) const {
1540 SDNode *Node = Op.getNode();
1541 EVT VT = Node->getValueType(ResNo: 0);
1542 SDValue InChain = Node->getOperand(Num: 0);
1543 SDValue VAListPtr = Node->getOperand(Num: 1);
1544 EVT PtrVT = VAListPtr.getValueType();
1545 const Value *SV = cast<SrcValueSDNode>(Val: Node->getOperand(Num: 2))->getValue();
1546 SDLoc DL(Node);
1547 SDValue VAList =
1548 DAG.getLoad(VT: PtrVT, dl: DL, Chain: InChain, Ptr: VAListPtr, PtrInfo: MachinePointerInfo(SV));
1549 SDValue Chain = VAList.getValue(R: 1);
1550 SDValue NextPtr;
1551
1552 if (VT == MVT::f128) {
1553 // VE f128 values must be stored with 16 bytes alignment. We don't
1554 // know the actual alignment of VAList, so we take alignment of it
1555 // dynamically.
1556 int Align = 16;
1557 VAList = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: VAList,
1558 N2: DAG.getConstant(Val: Align - 1, DL, VT: PtrVT));
1559 VAList = DAG.getNode(Opcode: ISD::AND, DL, VT: PtrVT, N1: VAList,
1560 N2: DAG.getSignedConstant(Val: -Align, DL, VT: PtrVT));
1561 // Increment the pointer, VAList, by 16 to the next vaarg.
1562 NextPtr =
1563 DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: VAList, N2: DAG.getIntPtrConstant(Val: 16, DL));
1564 } else if (VT == MVT::f32) {
1565 // float --> need special handling like below.
1566 // 0 4
1567 // +------+------+
1568 // | empty| float|
1569 // +------+------+
1570 // Increment the pointer, VAList, by 8 to the next vaarg.
1571 NextPtr =
1572 DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: VAList, N2: DAG.getIntPtrConstant(Val: 8, DL));
1573 // Then, adjust VAList.
1574 unsigned InternalOffset = 4;
1575 VAList = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: VAList,
1576 N2: DAG.getConstant(Val: InternalOffset, DL, VT: PtrVT));
1577 } else {
1578 // Increment the pointer, VAList, by 8 to the next vaarg.
1579 NextPtr =
1580 DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: VAList, N2: DAG.getIntPtrConstant(Val: 8, DL));
1581 }
1582
1583 // Store the incremented VAList to the legalized pointer.
1584 InChain = DAG.getStore(Chain, dl: DL, Val: NextPtr, Ptr: VAListPtr, PtrInfo: MachinePointerInfo(SV));
1585
1586 // Load the actual argument out of the pointer VAList.
1587 // We can't count on greater alignment than the word size.
1588 return DAG.getLoad(
1589 VT, dl: DL, Chain: InChain, Ptr: VAList, PtrInfo: MachinePointerInfo(),
1590 Alignment: Align(std::min(a: PtrVT.getSizeInBits(), b: VT.getSizeInBits()) / 8));
1591}
1592
1593SDValue VETargetLowering::lowerDYNAMIC_STACKALLOC(SDValue Op,
1594 SelectionDAG &DAG) const {
1595 // Generate following code.
1596 // (void)__llvm_grow_stack(size);
1597 // ret = GETSTACKTOP; // pseudo instruction
1598 SDLoc DL(Op);
1599
1600 // Get the inputs.
1601 SDNode *Node = Op.getNode();
1602 SDValue Chain = Op.getOperand(i: 0);
1603 SDValue Size = Op.getOperand(i: 1);
1604 MaybeAlign Alignment(Op.getConstantOperandVal(i: 2));
1605 EVT VT = Node->getValueType(ResNo: 0);
1606
1607 // Chain the dynamic stack allocation so that it doesn't modify the stack
1608 // pointer when other instructions are using the stack.
1609 Chain = DAG.getCALLSEQ_START(Chain, InSize: 0, OutSize: 0, DL);
1610
1611 const TargetFrameLowering &TFI = *Subtarget->getFrameLowering();
1612 Align StackAlign = TFI.getStackAlign();
1613 bool NeedsAlign = Alignment.valueOrOne() > StackAlign;
1614
1615 // Prepare arguments
1616 TargetLowering::ArgListTy Args;
1617 Args.emplace_back(args&: Size, args: Size.getValueType().getTypeForEVT(Context&: *DAG.getContext()));
1618 if (NeedsAlign) {
1619 SDValue Align = DAG.getConstant(Val: ~(Alignment->value() - 1ULL), DL, VT);
1620 Args.emplace_back(args&: Align,
1621 args: Align.getValueType().getTypeForEVT(Context&: *DAG.getContext()));
1622 }
1623 Type *RetTy = Type::getVoidTy(C&: *DAG.getContext());
1624
1625 EVT PtrVT = Op.getValueType();
1626 SDValue Callee;
1627 if (NeedsAlign) {
1628 Callee = DAG.getTargetExternalSymbol(Sym: "__ve_grow_stack_align", VT: PtrVT, TargetFlags: 0);
1629 } else {
1630 Callee = DAG.getTargetExternalSymbol(Sym: "__ve_grow_stack", VT: PtrVT, TargetFlags: 0);
1631 }
1632
1633 TargetLowering::CallLoweringInfo CLI(DAG);
1634 CLI.setDebugLoc(DL)
1635 .setChain(Chain)
1636 .setCallee(CC: CallingConv::PreserveAll, ResultType: RetTy, Target: Callee, ArgsList: std::move(Args))
1637 .setDiscardResult(true);
1638 std::pair<SDValue, SDValue> pair = LowerCallTo(CLI);
1639 Chain = pair.second;
1640 SDValue Result = DAG.getNode(Opcode: VEISD::GETSTACKTOP, DL, VT, Operand: Chain);
1641 if (NeedsAlign) {
1642 Result = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Result,
1643 N2: DAG.getConstant(Val: (Alignment->value() - 1ULL), DL, VT));
1644 Result = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Result,
1645 N2: DAG.getConstant(Val: ~(Alignment->value() - 1ULL), DL, VT));
1646 }
1647 // Chain = Result.getValue(1);
1648 Chain = DAG.getCALLSEQ_END(Chain, Size1: 0, Size2: 0, Glue: SDValue(), DL);
1649
1650 SDValue Ops[2] = {Result, Chain};
1651 return DAG.getMergeValues(Ops, dl: DL);
1652}
1653
1654SDValue VETargetLowering::lowerEH_SJLJ_LONGJMP(SDValue Op,
1655 SelectionDAG &DAG) const {
1656 SDLoc DL(Op);
1657 return DAG.getNode(Opcode: VEISD::EH_SJLJ_LONGJMP, DL, VT: MVT::Other, N1: Op.getOperand(i: 0),
1658 N2: Op.getOperand(i: 1));
1659}
1660
1661SDValue VETargetLowering::lowerEH_SJLJ_SETJMP(SDValue Op,
1662 SelectionDAG &DAG) const {
1663 SDLoc DL(Op);
1664 return DAG.getNode(Opcode: VEISD::EH_SJLJ_SETJMP, DL,
1665 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other), N1: Op.getOperand(i: 0),
1666 N2: Op.getOperand(i: 1));
1667}
1668
1669SDValue VETargetLowering::lowerEH_SJLJ_SETUP_DISPATCH(SDValue Op,
1670 SelectionDAG &DAG) const {
1671 SDLoc DL(Op);
1672 return DAG.getNode(Opcode: VEISD::EH_SJLJ_SETUP_DISPATCH, DL, VT: MVT::Other,
1673 Operand: Op.getOperand(i: 0));
1674}
1675
1676static SDValue lowerFRAMEADDR(SDValue Op, SelectionDAG &DAG,
1677 const VETargetLowering &TLI,
1678 const VESubtarget *Subtarget) {
1679 SDLoc DL(Op);
1680 MachineFunction &MF = DAG.getMachineFunction();
1681 EVT PtrVT = TLI.getPointerTy(DL: MF.getDataLayout());
1682
1683 MachineFrameInfo &MFI = MF.getFrameInfo();
1684 MFI.setFrameAddressIsTaken(true);
1685
1686 unsigned Depth = Op.getConstantOperandVal(i: 0);
1687 const VERegisterInfo *RegInfo = Subtarget->getRegisterInfo();
1688 Register FrameReg = RegInfo->getFrameRegister(MF);
1689 SDValue FrameAddr =
1690 DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: DL, Reg: FrameReg, VT: PtrVT);
1691 while (Depth--)
1692 FrameAddr = DAG.getLoad(VT: Op.getValueType(), dl: DL, Chain: DAG.getEntryNode(),
1693 Ptr: FrameAddr, PtrInfo: MachinePointerInfo());
1694 return FrameAddr;
1695}
1696
1697static SDValue lowerRETURNADDR(SDValue Op, SelectionDAG &DAG,
1698 const VETargetLowering &TLI,
1699 const VESubtarget *Subtarget) {
1700 MachineFunction &MF = DAG.getMachineFunction();
1701 MachineFrameInfo &MFI = MF.getFrameInfo();
1702 MFI.setReturnAddressIsTaken(true);
1703
1704 SDValue FrameAddr = lowerFRAMEADDR(Op, DAG, TLI, Subtarget);
1705
1706 SDLoc DL(Op);
1707 EVT VT = Op.getValueType();
1708 SDValue Offset = DAG.getConstant(Val: 8, DL, VT);
1709 return DAG.getLoad(VT, dl: DL, Chain: DAG.getEntryNode(),
1710 Ptr: DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: FrameAddr, N2: Offset),
1711 PtrInfo: MachinePointerInfo());
1712}
1713
1714SDValue VETargetLowering::lowerINTRINSIC_WO_CHAIN(SDValue Op,
1715 SelectionDAG &DAG) const {
1716 SDLoc DL(Op);
1717 unsigned IntNo = Op.getConstantOperandVal(i: 0);
1718 switch (IntNo) {
1719 default: // Don't custom lower most intrinsics.
1720 return SDValue();
1721 case Intrinsic::eh_sjlj_lsda: {
1722 MachineFunction &MF = DAG.getMachineFunction();
1723 MVT VT = Op.getSimpleValueType();
1724 const VETargetMachine *TM =
1725 static_cast<const VETargetMachine *>(&DAG.getTarget());
1726
1727 // Create GCC_except_tableXX string. The real symbol for that will be
1728 // generated in EHStreamer::emitExceptionTable() later. So, we just
1729 // borrow it's name here.
1730 TM->getStrList()->push_back(x: std::string(
1731 (Twine("GCC_except_table") + Twine(MF.getFunctionNumber())).str()));
1732 SDValue Addr =
1733 DAG.getTargetExternalSymbol(Sym: TM->getStrList()->back().c_str(), VT, TargetFlags: 0);
1734 if (isPositionIndependent()) {
1735 Addr = makeHiLoPair(Op: Addr, HiTF: VE::S_GOTOFF_HI32, LoTF: VE::S_GOTOFF_LO32, DAG);
1736 SDValue GlobalBase = DAG.getNode(Opcode: VEISD::GLOBAL_BASE_REG, DL, VT);
1737 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: GlobalBase, N2: Addr);
1738 }
1739 return makeHiLoPair(Op: Addr, HiTF: VE::S_HI32, LoTF: VE::S_LO32, DAG);
1740 }
1741 }
1742}
1743
1744static bool getUniqueInsertion(SDNode *N, unsigned &UniqueIdx) {
1745 if (!isa<BuildVectorSDNode>(Val: N))
1746 return false;
1747 const auto *BVN = cast<BuildVectorSDNode>(Val: N);
1748
1749 // Find first non-undef insertion.
1750 unsigned Idx;
1751 for (Idx = 0; Idx < BVN->getNumOperands(); ++Idx) {
1752 auto ElemV = BVN->getOperand(Num: Idx);
1753 if (!ElemV->isUndef())
1754 break;
1755 }
1756 // Catch the (hypothetical) all-undef case.
1757 if (Idx == BVN->getNumOperands())
1758 return false;
1759 // Remember insertion.
1760 UniqueIdx = Idx++;
1761 // Verify that all other insertions are undef.
1762 for (; Idx < BVN->getNumOperands(); ++Idx) {
1763 auto ElemV = BVN->getOperand(Num: Idx);
1764 if (!ElemV->isUndef())
1765 return false;
1766 }
1767 return true;
1768}
1769
1770static SDValue getSplatValue(SDNode *N) {
1771 if (auto *BuildVec = dyn_cast<BuildVectorSDNode>(Val: N)) {
1772 return BuildVec->getSplatValue();
1773 }
1774 return SDValue();
1775}
1776
1777SDValue VETargetLowering::lowerBUILD_VECTOR(SDValue Op,
1778 SelectionDAG &DAG) const {
1779 VECustomDAG CDAG(DAG, Op);
1780 MVT ResultVT = Op.getSimpleValueType();
1781
1782 // If there is just one element, expand to INSERT_VECTOR_ELT.
1783 unsigned UniqueIdx;
1784 if (getUniqueInsertion(N: Op.getNode(), UniqueIdx)) {
1785 SDValue AccuV = CDAG.getUNDEF(VT: Op.getValueType());
1786 auto ElemV = Op->getOperand(Num: UniqueIdx);
1787 SDValue IdxV = CDAG.getConstant(Val: UniqueIdx, VT: MVT::i64);
1788 return CDAG.getNode(OC: ISD::INSERT_VECTOR_ELT, ResVT: ResultVT, OpV: {AccuV, ElemV, IdxV});
1789 }
1790
1791 // Else emit a broadcast.
1792 if (SDValue ScalarV = getSplatValue(N: Op.getNode())) {
1793 unsigned NumEls = ResultVT.getVectorNumElements();
1794 auto AVL = CDAG.getConstant(Val: NumEls, VT: MVT::i32);
1795 return CDAG.getBroadcast(ResultVT, Scalar: ScalarV, AVL);
1796 }
1797
1798 // Expand
1799 return SDValue();
1800}
1801
1802TargetLowering::LegalizeAction
1803VETargetLowering::getCustomOperationAction(SDNode &Op) const {
1804 // Custom legalization on VVP_* and VEC_* opcodes is required to pack-legalize
1805 // these operations (transform nodes such that their AVL parameter refers to
1806 // packs of 64bit, instead of number of elements.
1807
1808 // Packing opcodes are created with a pack-legal AVL (LEGALAVL). No need to
1809 // re-visit them.
1810 if (isPackingSupportOpcode(Opc: Op.getOpcode()))
1811 return Legal;
1812
1813 // Custom lower to legalize AVL for packed mode.
1814 if (isVVPOrVEC(Op.getOpcode()))
1815 return Custom;
1816 return Legal;
1817}
1818
1819SDValue VETargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
1820 LLVM_DEBUG(dbgs() << "::LowerOperation "; Op.dump(&DAG));
1821 unsigned Opcode = Op.getOpcode();
1822
1823 /// Scalar isel.
1824 switch (Opcode) {
1825 case ISD::ATOMIC_FENCE:
1826 return lowerATOMIC_FENCE(Op, DAG);
1827 case ISD::ATOMIC_SWAP:
1828 return lowerATOMIC_SWAP(Op, DAG);
1829 case ISD::BlockAddress:
1830 return lowerBlockAddress(Op, DAG);
1831 case ISD::ConstantPool:
1832 return lowerConstantPool(Op, DAG);
1833 case ISD::DYNAMIC_STACKALLOC:
1834 return lowerDYNAMIC_STACKALLOC(Op, DAG);
1835 case ISD::EH_SJLJ_LONGJMP:
1836 return lowerEH_SJLJ_LONGJMP(Op, DAG);
1837 case ISD::EH_SJLJ_SETJMP:
1838 return lowerEH_SJLJ_SETJMP(Op, DAG);
1839 case ISD::EH_SJLJ_SETUP_DISPATCH:
1840 return lowerEH_SJLJ_SETUP_DISPATCH(Op, DAG);
1841 case ISD::FRAMEADDR:
1842 return lowerFRAMEADDR(Op, DAG, TLI: *this, Subtarget);
1843 case ISD::GlobalAddress:
1844 return lowerGlobalAddress(Op, DAG);
1845 case ISD::GlobalTLSAddress:
1846 return lowerGlobalTLSAddress(Op, DAG);
1847 case ISD::INTRINSIC_WO_CHAIN:
1848 return lowerINTRINSIC_WO_CHAIN(Op, DAG);
1849 case ISD::JumpTable:
1850 return lowerJumpTable(Op, DAG);
1851 case ISD::LOAD:
1852 return lowerLOAD(Op, DAG);
1853 case ISD::RETURNADDR:
1854 return lowerRETURNADDR(Op, DAG, TLI: *this, Subtarget);
1855 case ISD::BUILD_VECTOR:
1856 return lowerBUILD_VECTOR(Op, DAG);
1857 case ISD::STORE:
1858 return lowerSTORE(Op, DAG);
1859 case ISD::VASTART:
1860 return lowerVASTART(Op, DAG);
1861 case ISD::VAARG:
1862 return lowerVAARG(Op, DAG);
1863
1864 case ISD::INSERT_VECTOR_ELT:
1865 return lowerINSERT_VECTOR_ELT(Op, DAG);
1866 case ISD::EXTRACT_VECTOR_ELT:
1867 return lowerEXTRACT_VECTOR_ELT(Op, DAG);
1868 }
1869
1870 /// Vector isel.
1871 if (ISD::isVPOpcode(Opcode))
1872 return lowerToVVP(Op, DAG);
1873
1874 switch (Opcode) {
1875 default:
1876 llvm_unreachable("Should not custom lower this!");
1877
1878 // Legalize the AVL of this internal node.
1879 case VEISD::VEC_BROADCAST:
1880#define ADD_VVP_OP(VVP_NAME, ...) case VEISD::VVP_NAME:
1881#include "VVPNodes.def"
1882 // AVL already legalized.
1883 if (getAnnotatedNodeAVL(Op).second)
1884 return Op;
1885 return legalizeInternalVectorOp(Op, DAG);
1886
1887 // Translate into a VEC_*/VVP_* layer operation.
1888 case ISD::MLOAD:
1889 case ISD::MSTORE:
1890#define ADD_VVP_OP(VVP_NAME, ISD_NAME) case ISD::ISD_NAME:
1891#include "VVPNodes.def"
1892 if (isMaskArithmetic(Op) && isPackedVectorType(SomeVT: Op.getValueType()))
1893 return splitMaskArithmetic(Op, DAG);
1894 return lowerToVVP(Op, DAG);
1895 }
1896}
1897/// } Custom Lower
1898
1899void VETargetLowering::ReplaceNodeResults(SDNode *N,
1900 SmallVectorImpl<SDValue> &Results,
1901 SelectionDAG &DAG) const {
1902 switch (N->getOpcode()) {
1903 case ISD::ATOMIC_SWAP:
1904 // Let LLVM expand atomic swap instruction through LowerOperation.
1905 return;
1906 default:
1907 LLVM_DEBUG(N->dumpr(&DAG));
1908 llvm_unreachable("Do not know how to custom type legalize this operation!");
1909 }
1910}
1911
1912/// JumpTable for VE.
1913///
1914/// VE cannot generate relocatable symbol in jump table. VE cannot
1915/// generate expressions using symbols in both text segment and data
1916/// segment like below.
1917/// .4byte .LBB0_2-.LJTI0_0
1918/// So, we generate offset from the top of function like below as
1919/// a custom label.
1920/// .4byte .LBB0_2-<function name>
1921
1922unsigned VETargetLowering::getJumpTableEncoding() const {
1923 // Use custom label for PIC.
1924 if (isPositionIndependent())
1925 return MachineJumpTableInfo::EK_Custom32;
1926
1927 // Otherwise, use the normal jump table encoding heuristics.
1928 return TargetLowering::getJumpTableEncoding();
1929}
1930
1931const MCExpr *VETargetLowering::LowerCustomJumpTableEntry(
1932 const MachineJumpTableInfo *MJTI, const MachineBasicBlock *MBB,
1933 unsigned Uid, MCContext &Ctx) const {
1934 assert(isPositionIndependent());
1935
1936 // Generate custom label for PIC like below.
1937 // .4bytes .LBB0_2-<function name>
1938 const auto *Value = MCSymbolRefExpr::create(Symbol: MBB->getSymbol(), Ctx);
1939 MCSymbol *Sym = Ctx.getOrCreateSymbol(Name: MBB->getParent()->getName().data());
1940 const auto *Base = MCSymbolRefExpr::create(Symbol: Sym, Ctx);
1941 return MCBinaryExpr::createSub(LHS: Value, RHS: Base, Ctx);
1942}
1943
1944SDValue VETargetLowering::getPICJumpTableRelocBase(SDValue Table,
1945 SelectionDAG &DAG) const {
1946 assert(isPositionIndependent());
1947 SDLoc DL(Table);
1948 Function *Function = &DAG.getMachineFunction().getFunction();
1949 assert(Function != nullptr);
1950 auto PtrTy = getPointerTy(DL: DAG.getDataLayout(), AS: Function->getAddressSpace());
1951
1952 // In the jump table, we have following values in PIC mode.
1953 // .4bytes .LBB0_2-<function name>
1954 // We need to add this value and the address of this function to generate
1955 // .LBB0_2 label correctly under PIC mode. So, we want to generate following
1956 // instructions:
1957 // lea %reg, fun@gotoff_lo
1958 // and %reg, %reg, (32)0
1959 // lea.sl %reg, fun@gotoff_hi(%reg, %got)
1960 // In order to do so, we need to genarate correctly marked DAG node using
1961 // makeHiLoPair.
1962 SDValue Op = DAG.getGlobalAddress(GV: Function, DL, VT: PtrTy);
1963 SDValue HiLo = makeHiLoPair(Op, HiTF: VE::S_GOTOFF_HI32, LoTF: VE::S_GOTOFF_LO32, DAG);
1964 SDValue GlobalBase = DAG.getNode(Opcode: VEISD::GLOBAL_BASE_REG, DL, VT: PtrTy);
1965 return DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrTy, N1: GlobalBase, N2: HiLo);
1966}
1967
1968Register VETargetLowering::prepareMBB(MachineBasicBlock &MBB,
1969 MachineBasicBlock::iterator I,
1970 MachineBasicBlock *TargetBB,
1971 const DebugLoc &DL) const {
1972 MachineFunction *MF = MBB.getParent();
1973 MachineRegisterInfo &MRI = MF->getRegInfo();
1974 const VEInstrInfo *TII = Subtarget->getInstrInfo();
1975
1976 const TargetRegisterClass *RC = &VE::I64RegClass;
1977 Register Tmp1 = MRI.createVirtualRegister(RegClass: RC);
1978 Register Tmp2 = MRI.createVirtualRegister(RegClass: RC);
1979 Register Result = MRI.createVirtualRegister(RegClass: RC);
1980
1981 if (isPositionIndependent()) {
1982 // Create following instructions for local linkage PIC code.
1983 // lea %Tmp1, TargetBB@gotoff_lo
1984 // and %Tmp2, %Tmp1, (32)0
1985 // lea.sl %Result, TargetBB@gotoff_hi(%Tmp2, %s15) ; %s15 is GOT
1986 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: VE::LEAzii), DestReg: Tmp1)
1987 .addImm(Val: 0)
1988 .addImm(Val: 0)
1989 .addMBB(MBB: TargetBB, TargetFlags: VE::S_GOTOFF_LO32);
1990 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: VE::ANDrm), DestReg: Tmp2)
1991 .addReg(RegNo: Tmp1, Flags: getKillRegState(B: true))
1992 .addImm(Val: M0(Val: 32));
1993 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: VE::LEASLrri), DestReg: Result)
1994 .addReg(RegNo: VE::SX15)
1995 .addReg(RegNo: Tmp2, Flags: getKillRegState(B: true))
1996 .addMBB(MBB: TargetBB, TargetFlags: VE::S_GOTOFF_HI32);
1997 } else {
1998 // Create following instructions for non-PIC code.
1999 // lea %Tmp1, TargetBB@lo
2000 // and %Tmp2, %Tmp1, (32)0
2001 // lea.sl %Result, TargetBB@hi(%Tmp2)
2002 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: VE::LEAzii), DestReg: Tmp1)
2003 .addImm(Val: 0)
2004 .addImm(Val: 0)
2005 .addMBB(MBB: TargetBB, TargetFlags: VE::S_LO32);
2006 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: VE::ANDrm), DestReg: Tmp2)
2007 .addReg(RegNo: Tmp1, Flags: getKillRegState(B: true))
2008 .addImm(Val: M0(Val: 32));
2009 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: VE::LEASLrii), DestReg: Result)
2010 .addReg(RegNo: Tmp2, Flags: getKillRegState(B: true))
2011 .addImm(Val: 0)
2012 .addMBB(MBB: TargetBB, TargetFlags: VE::S_HI32);
2013 }
2014 return Result;
2015}
2016
2017Register VETargetLowering::prepareSymbol(MachineBasicBlock &MBB,
2018 MachineBasicBlock::iterator I,
2019 StringRef Symbol, const DebugLoc &DL,
2020 bool IsLocal = false,
2021 bool IsCall = false) const {
2022 MachineFunction *MF = MBB.getParent();
2023 MachineRegisterInfo &MRI = MF->getRegInfo();
2024 const VEInstrInfo *TII = Subtarget->getInstrInfo();
2025
2026 const TargetRegisterClass *RC = &VE::I64RegClass;
2027 Register Result = MRI.createVirtualRegister(RegClass: RC);
2028
2029 if (isPositionIndependent()) {
2030 if (IsCall && !IsLocal) {
2031 // Create following instructions for non-local linkage PIC code function
2032 // calls. These instructions uses IC and magic number -24, so we expand
2033 // them in VEAsmPrinter.cpp from GETFUNPLT pseudo instruction.
2034 // lea %Reg, Symbol@plt_lo(-24)
2035 // and %Reg, %Reg, (32)0
2036 // sic %s16
2037 // lea.sl %Result, Symbol@plt_hi(%Reg, %s16) ; %s16 is PLT
2038 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: VE::GETFUNPLT), DestReg: Result)
2039 .addExternalSymbol(FnName: "abort");
2040 } else if (IsLocal) {
2041 Register Tmp1 = MRI.createVirtualRegister(RegClass: RC);
2042 Register Tmp2 = MRI.createVirtualRegister(RegClass: RC);
2043 // Create following instructions for local linkage PIC code.
2044 // lea %Tmp1, Symbol@gotoff_lo
2045 // and %Tmp2, %Tmp1, (32)0
2046 // lea.sl %Result, Symbol@gotoff_hi(%Tmp2, %s15) ; %s15 is GOT
2047 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: VE::LEAzii), DestReg: Tmp1)
2048 .addImm(Val: 0)
2049 .addImm(Val: 0)
2050 .addExternalSymbol(FnName: Symbol.data(), TargetFlags: VE::S_GOTOFF_LO32);
2051 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: VE::ANDrm), DestReg: Tmp2)
2052 .addReg(RegNo: Tmp1, Flags: getKillRegState(B: true))
2053 .addImm(Val: M0(Val: 32));
2054 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: VE::LEASLrri), DestReg: Result)
2055 .addReg(RegNo: VE::SX15)
2056 .addReg(RegNo: Tmp2, Flags: getKillRegState(B: true))
2057 .addExternalSymbol(FnName: Symbol.data(), TargetFlags: VE::S_GOTOFF_HI32);
2058 } else {
2059 Register Tmp1 = MRI.createVirtualRegister(RegClass: RC);
2060 Register Tmp2 = MRI.createVirtualRegister(RegClass: RC);
2061 // Create following instructions for not local linkage PIC code.
2062 // lea %Tmp1, Symbol@got_lo
2063 // and %Tmp2, %Tmp1, (32)0
2064 // lea.sl %Tmp3, Symbol@gotoff_hi(%Tmp2, %s15) ; %s15 is GOT
2065 // ld %Result, 0(%Tmp3)
2066 Register Tmp3 = MRI.createVirtualRegister(RegClass: RC);
2067 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: VE::LEAzii), DestReg: Tmp1)
2068 .addImm(Val: 0)
2069 .addImm(Val: 0)
2070 .addExternalSymbol(FnName: Symbol.data(), TargetFlags: VE::S_GOT_LO32);
2071 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: VE::ANDrm), DestReg: Tmp2)
2072 .addReg(RegNo: Tmp1, Flags: getKillRegState(B: true))
2073 .addImm(Val: M0(Val: 32));
2074 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: VE::LEASLrri), DestReg: Tmp3)
2075 .addReg(RegNo: VE::SX15)
2076 .addReg(RegNo: Tmp2, Flags: getKillRegState(B: true))
2077 .addExternalSymbol(FnName: Symbol.data(), TargetFlags: VE::S_GOT_HI32);
2078 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: VE::LDrii), DestReg: Result)
2079 .addReg(RegNo: Tmp3, Flags: getKillRegState(B: true))
2080 .addImm(Val: 0)
2081 .addImm(Val: 0);
2082 }
2083 } else {
2084 Register Tmp1 = MRI.createVirtualRegister(RegClass: RC);
2085 Register Tmp2 = MRI.createVirtualRegister(RegClass: RC);
2086 // Create following instructions for non-PIC code.
2087 // lea %Tmp1, Symbol@lo
2088 // and %Tmp2, %Tmp1, (32)0
2089 // lea.sl %Result, Symbol@hi(%Tmp2)
2090 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: VE::LEAzii), DestReg: Tmp1)
2091 .addImm(Val: 0)
2092 .addImm(Val: 0)
2093 .addExternalSymbol(FnName: Symbol.data(), TargetFlags: VE::S_LO32);
2094 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: VE::ANDrm), DestReg: Tmp2)
2095 .addReg(RegNo: Tmp1, Flags: getKillRegState(B: true))
2096 .addImm(Val: M0(Val: 32));
2097 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: VE::LEASLrii), DestReg: Result)
2098 .addReg(RegNo: Tmp2, Flags: getKillRegState(B: true))
2099 .addImm(Val: 0)
2100 .addExternalSymbol(FnName: Symbol.data(), TargetFlags: VE::S_HI32);
2101 }
2102 return Result;
2103}
2104
2105void VETargetLowering::setupEntryBlockForSjLj(MachineInstr &MI,
2106 MachineBasicBlock *MBB,
2107 MachineBasicBlock *DispatchBB,
2108 int FI, int Offset) const {
2109 DebugLoc DL = MI.getDebugLoc();
2110 const VEInstrInfo *TII = Subtarget->getInstrInfo();
2111
2112 Register LabelReg =
2113 prepareMBB(MBB&: *MBB, I: MachineBasicBlock::iterator(MI), TargetBB: DispatchBB, DL);
2114
2115 // Store an address of DispatchBB to a given jmpbuf[1] where has next IC
2116 // referenced by longjmp (throw) later.
2117 MachineInstrBuilder MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: VE::STrii));
2118 addFrameReference(MIB, FI, Offset); // jmpbuf[1]
2119 MIB.addReg(RegNo: LabelReg, Flags: getKillRegState(B: true));
2120}
2121
2122MachineBasicBlock *
2123VETargetLowering::emitEHSjLjSetJmp(MachineInstr &MI,
2124 MachineBasicBlock *MBB) const {
2125 DebugLoc DL = MI.getDebugLoc();
2126 MachineFunction *MF = MBB->getParent();
2127 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
2128 const TargetRegisterInfo *TRI = Subtarget->getRegisterInfo();
2129 MachineRegisterInfo &MRI = MF->getRegInfo();
2130
2131 const BasicBlock *BB = MBB->getBasicBlock();
2132 MachineFunction::iterator I = ++MBB->getIterator();
2133
2134 // Memory Reference.
2135 SmallVector<MachineMemOperand *, 2> MMOs(MI.memoperands());
2136 Register BufReg = MI.getOperand(i: 1).getReg();
2137
2138 Register DstReg;
2139
2140 DstReg = MI.getOperand(i: 0).getReg();
2141 const TargetRegisterClass *RC = MRI.getRegClass(Reg: DstReg);
2142 assert(TRI->isTypeLegalForClass(*RC, MVT::i32) && "Invalid destination!");
2143 (void)TRI;
2144 Register MainDestReg = MRI.createVirtualRegister(RegClass: RC);
2145 Register RestoreDestReg = MRI.createVirtualRegister(RegClass: RC);
2146
2147 // For `v = call @llvm.eh.sjlj.setjmp(buf)`, we generate following
2148 // instructions. SP/FP must be saved in jmpbuf before `llvm.eh.sjlj.setjmp`.
2149 //
2150 // ThisMBB:
2151 // buf[3] = %s17 iff %s17 is used as BP
2152 // buf[1] = RestoreMBB as IC after longjmp
2153 // # SjLjSetup RestoreMBB
2154 //
2155 // MainMBB:
2156 // v_main = 0
2157 //
2158 // SinkMBB:
2159 // v = phi(v_main, MainMBB, v_restore, RestoreMBB)
2160 // ...
2161 //
2162 // RestoreMBB:
2163 // %s17 = buf[3] = iff %s17 is used as BP
2164 // v_restore = 1
2165 // goto SinkMBB
2166
2167 MachineBasicBlock *ThisMBB = MBB;
2168 MachineBasicBlock *MainMBB = MF->CreateMachineBasicBlock(BB);
2169 MachineBasicBlock *SinkMBB = MF->CreateMachineBasicBlock(BB);
2170 MachineBasicBlock *RestoreMBB = MF->CreateMachineBasicBlock(BB);
2171 MF->insert(MBBI: I, MBB: MainMBB);
2172 MF->insert(MBBI: I, MBB: SinkMBB);
2173 MF->push_back(MBB: RestoreMBB);
2174 RestoreMBB->setMachineBlockAddressTaken();
2175
2176 // Transfer the remainder of BB and its successor edges to SinkMBB.
2177 SinkMBB->splice(Where: SinkMBB->begin(), Other: MBB,
2178 From: std::next(x: MachineBasicBlock::iterator(MI)), To: MBB->end());
2179 SinkMBB->transferSuccessorsAndUpdatePHIs(FromMBB: MBB);
2180
2181 // ThisMBB:
2182 Register LabelReg =
2183 prepareMBB(MBB&: *MBB, I: MachineBasicBlock::iterator(MI), TargetBB: RestoreMBB, DL);
2184
2185 // Store BP in buf[3] iff this function is using BP.
2186 const VEFrameLowering *TFI = Subtarget->getFrameLowering();
2187 if (TFI->hasBP(MF: *MF)) {
2188 MachineInstrBuilder MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: VE::STrii));
2189 MIB.addReg(RegNo: BufReg);
2190 MIB.addImm(Val: 0);
2191 MIB.addImm(Val: 24);
2192 MIB.addReg(RegNo: VE::SX17);
2193 MIB.setMemRefs(MMOs);
2194 }
2195
2196 // Store IP in buf[1].
2197 MachineInstrBuilder MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: VE::STrii));
2198 MIB.add(MO: MI.getOperand(i: 1)); // we can preserve the kill flags here.
2199 MIB.addImm(Val: 0);
2200 MIB.addImm(Val: 8);
2201 MIB.addReg(RegNo: LabelReg, Flags: getKillRegState(B: true));
2202 MIB.setMemRefs(MMOs);
2203
2204 // SP/FP are already stored in jmpbuf before `llvm.eh.sjlj.setjmp`.
2205
2206 // Insert setup.
2207 MIB =
2208 BuildMI(BB&: *ThisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: VE::EH_SjLj_Setup)).addMBB(MBB: RestoreMBB);
2209
2210 const VERegisterInfo *RegInfo = Subtarget->getRegisterInfo();
2211 MIB.addRegMask(Mask: RegInfo->getNoPreservedMask());
2212 ThisMBB->addSuccessor(Succ: MainMBB);
2213 ThisMBB->addSuccessor(Succ: RestoreMBB);
2214
2215 // MainMBB:
2216 BuildMI(BB: MainMBB, MIMD: DL, MCID: TII->get(Opcode: VE::LEAzii), DestReg: MainDestReg)
2217 .addImm(Val: 0)
2218 .addImm(Val: 0)
2219 .addImm(Val: 0);
2220 MainMBB->addSuccessor(Succ: SinkMBB);
2221
2222 // SinkMBB:
2223 BuildMI(BB&: *SinkMBB, I: SinkMBB->begin(), MIMD: DL, MCID: TII->get(Opcode: VE::PHI), DestReg: DstReg)
2224 .addReg(RegNo: MainDestReg)
2225 .addMBB(MBB: MainMBB)
2226 .addReg(RegNo: RestoreDestReg)
2227 .addMBB(MBB: RestoreMBB);
2228
2229 // RestoreMBB:
2230 // Restore BP from buf[3] iff this function is using BP. The address of
2231 // buf is in SX10.
2232 // FIXME: Better to not use SX10 here
2233 if (TFI->hasBP(MF: *MF)) {
2234 MachineInstrBuilder MIB =
2235 BuildMI(BB: RestoreMBB, MIMD: DL, MCID: TII->get(Opcode: VE::LDrii), DestReg: VE::SX17);
2236 MIB.addReg(RegNo: VE::SX10);
2237 MIB.addImm(Val: 0);
2238 MIB.addImm(Val: 24);
2239 MIB.setMemRefs(MMOs);
2240 }
2241 BuildMI(BB: RestoreMBB, MIMD: DL, MCID: TII->get(Opcode: VE::LEAzii), DestReg: RestoreDestReg)
2242 .addImm(Val: 0)
2243 .addImm(Val: 0)
2244 .addImm(Val: 1);
2245 BuildMI(BB: RestoreMBB, MIMD: DL, MCID: TII->get(Opcode: VE::BRCFLa_t)).addMBB(MBB: SinkMBB);
2246 RestoreMBB->addSuccessor(Succ: SinkMBB);
2247
2248 MI.eraseFromParent();
2249 return SinkMBB;
2250}
2251
2252MachineBasicBlock *
2253VETargetLowering::emitEHSjLjLongJmp(MachineInstr &MI,
2254 MachineBasicBlock *MBB) const {
2255 DebugLoc DL = MI.getDebugLoc();
2256 MachineFunction *MF = MBB->getParent();
2257 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
2258 MachineRegisterInfo &MRI = MF->getRegInfo();
2259
2260 // Memory Reference.
2261 SmallVector<MachineMemOperand *, 2> MMOs(MI.memoperands());
2262 Register BufReg = MI.getOperand(i: 0).getReg();
2263
2264 Register Tmp = MRI.createVirtualRegister(RegClass: &VE::I64RegClass);
2265 // Since FP is only updated here but NOT referenced, it's treated as GPR.
2266 Register FP = VE::SX9;
2267 Register SP = VE::SX11;
2268
2269 MachineInstrBuilder MIB;
2270
2271 MachineBasicBlock *ThisMBB = MBB;
2272
2273 // For `call @llvm.eh.sjlj.longjmp(buf)`, we generate following instructions.
2274 //
2275 // ThisMBB:
2276 // %fp = load buf[0]
2277 // %jmp = load buf[1]
2278 // %s10 = buf ; Store an address of buf to SX10 for RestoreMBB
2279 // %sp = load buf[2] ; generated by llvm.eh.sjlj.setjmp.
2280 // jmp %jmp
2281
2282 // Reload FP.
2283 MIB = BuildMI(BB&: *ThisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: VE::LDrii), DestReg: FP);
2284 MIB.addReg(RegNo: BufReg);
2285 MIB.addImm(Val: 0);
2286 MIB.addImm(Val: 0);
2287 MIB.setMemRefs(MMOs);
2288
2289 // Reload IP.
2290 MIB = BuildMI(BB&: *ThisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: VE::LDrii), DestReg: Tmp);
2291 MIB.addReg(RegNo: BufReg);
2292 MIB.addImm(Val: 0);
2293 MIB.addImm(Val: 8);
2294 MIB.setMemRefs(MMOs);
2295
2296 // Copy BufReg to SX10 for later use in setjmp.
2297 // FIXME: Better to not use SX10 here
2298 BuildMI(BB&: *ThisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: VE::ORri), DestReg: VE::SX10)
2299 .addReg(RegNo: BufReg)
2300 .addImm(Val: 0);
2301
2302 // Reload SP.
2303 MIB = BuildMI(BB&: *ThisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: VE::LDrii), DestReg: SP);
2304 MIB.add(MO: MI.getOperand(i: 0)); // we can preserve the kill flags here.
2305 MIB.addImm(Val: 0);
2306 MIB.addImm(Val: 16);
2307 MIB.setMemRefs(MMOs);
2308
2309 // Jump.
2310 BuildMI(BB&: *ThisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: VE::BCFLari_t))
2311 .addReg(RegNo: Tmp, Flags: getKillRegState(B: true))
2312 .addImm(Val: 0);
2313
2314 MI.eraseFromParent();
2315 return ThisMBB;
2316}
2317
2318MachineBasicBlock *
2319VETargetLowering::emitSjLjDispatchBlock(MachineInstr &MI,
2320 MachineBasicBlock *BB) const {
2321 DebugLoc DL = MI.getDebugLoc();
2322 MachineFunction *MF = BB->getParent();
2323 MachineFrameInfo &MFI = MF->getFrameInfo();
2324 MachineRegisterInfo &MRI = MF->getRegInfo();
2325 const VEInstrInfo *TII = Subtarget->getInstrInfo();
2326 int FI = MFI.getFunctionContextIndex();
2327
2328 // Get a mapping of the call site numbers to all of the landing pads they're
2329 // associated with.
2330 DenseMap<unsigned, SmallVector<MachineBasicBlock *, 2>> CallSiteNumToLPad;
2331 unsigned MaxCSNum = 0;
2332 for (auto &MBB : *MF) {
2333 if (!MBB.isEHPad())
2334 continue;
2335
2336 MCSymbol *Sym = nullptr;
2337 for (const auto &MI : MBB) {
2338 if (MI.isDebugInstr())
2339 continue;
2340
2341 assert(MI.isEHLabel() && "expected EH_LABEL");
2342 Sym = MI.getOperand(i: 0).getMCSymbol();
2343 break;
2344 }
2345
2346 if (!MF->hasCallSiteLandingPad(Sym))
2347 continue;
2348
2349 for (unsigned CSI : MF->getCallSiteLandingPad(Sym)) {
2350 CallSiteNumToLPad[CSI].push_back(Elt: &MBB);
2351 MaxCSNum = std::max(a: MaxCSNum, b: CSI);
2352 }
2353 }
2354
2355 // Get an ordered list of the machine basic blocks for the jump table.
2356 std::vector<MachineBasicBlock *> LPadList;
2357 SmallPtrSet<MachineBasicBlock *, 32> InvokeBBs;
2358 LPadList.reserve(n: CallSiteNumToLPad.size());
2359
2360 for (unsigned CSI = 1; CSI <= MaxCSNum; ++CSI) {
2361 for (auto &LP : CallSiteNumToLPad[CSI]) {
2362 LPadList.push_back(x: LP);
2363 InvokeBBs.insert_range(R: LP->predecessors());
2364 }
2365 }
2366
2367 assert(!LPadList.empty() &&
2368 "No landing pad destinations for the dispatch jump table!");
2369
2370 // The %fn_context is allocated like below (from --print-after=sjljehprepare):
2371 // %fn_context = alloca { i8*, i64, [4 x i64], i8*, i8*, [5 x i8*] }
2372 //
2373 // This `[5 x i8*]` is jmpbuf, so jmpbuf[1] is FI+72.
2374 // First `i64` is callsite, so callsite is FI+8.
2375 static const int OffsetIC = 72;
2376 static const int OffsetCS = 8;
2377
2378 // Create the MBBs for the dispatch code like following:
2379 //
2380 // ThisMBB:
2381 // Prepare DispatchBB address and store it to buf[1].
2382 // ...
2383 //
2384 // DispatchBB:
2385 // %s15 = GETGOT iff isPositionIndependent
2386 // %callsite = load callsite
2387 // brgt.l.t #size of callsites, %callsite, DispContBB
2388 //
2389 // TrapBB:
2390 // Call abort.
2391 //
2392 // DispContBB:
2393 // %breg = address of jump table
2394 // %pc = load and calculate next pc from %breg and %callsite
2395 // jmp %pc
2396
2397 // Shove the dispatch's address into the return slot in the function context.
2398 MachineBasicBlock *DispatchBB = MF->CreateMachineBasicBlock();
2399 DispatchBB->setIsEHPad(true);
2400
2401 // Trap BB will causes trap like `assert(0)`.
2402 MachineBasicBlock *TrapBB = MF->CreateMachineBasicBlock();
2403 DispatchBB->addSuccessor(Succ: TrapBB);
2404
2405 MachineBasicBlock *DispContBB = MF->CreateMachineBasicBlock();
2406 DispatchBB->addSuccessor(Succ: DispContBB);
2407
2408 // Insert MBBs.
2409 MF->push_back(MBB: DispatchBB);
2410 MF->push_back(MBB: DispContBB);
2411 MF->push_back(MBB: TrapBB);
2412
2413 // Insert code to call abort in the TrapBB.
2414 Register Abort = prepareSymbol(MBB&: *TrapBB, I: TrapBB->end(), Symbol: "abort", DL,
2415 /* Local */ IsLocal: false, /* Call */ IsCall: true);
2416 BuildMI(BB: TrapBB, MIMD: DL, MCID: TII->get(Opcode: VE::BSICrii), DestReg: VE::SX10)
2417 .addReg(RegNo: Abort, Flags: getKillRegState(B: true))
2418 .addImm(Val: 0)
2419 .addImm(Val: 0);
2420
2421 // Insert code into the entry block that creates and registers the function
2422 // context.
2423 setupEntryBlockForSjLj(MI, MBB: BB, DispatchBB, FI, Offset: OffsetIC);
2424
2425 // Create the jump table and associated information
2426 unsigned JTE = getJumpTableEncoding();
2427 MachineJumpTableInfo *JTI = MF->getOrCreateJumpTableInfo(JTEntryKind: JTE);
2428 unsigned MJTI = JTI->createJumpTableIndex(DestBBs: LPadList);
2429
2430 const VERegisterInfo &RI = TII->getRegisterInfo();
2431 // Add a register mask with no preserved registers. This results in all
2432 // registers being marked as clobbered.
2433 BuildMI(BB: DispatchBB, MIMD: DL, MCID: TII->get(Opcode: VE::NOP))
2434 .addRegMask(Mask: RI.getNoPreservedMask());
2435
2436 if (isPositionIndependent()) {
2437 // Force to generate GETGOT, since current implementation doesn't store GOT
2438 // register.
2439 BuildMI(BB: DispatchBB, MIMD: DL, MCID: TII->get(Opcode: VE::GETGOT), DestReg: VE::SX15);
2440 }
2441
2442 // IReg is used as an index in a memory operand and therefore can't be SP
2443 const TargetRegisterClass *RC = &VE::I64RegClass;
2444 Register IReg = MRI.createVirtualRegister(RegClass: RC);
2445 addFrameReference(MIB: BuildMI(BB: DispatchBB, MIMD: DL, MCID: TII->get(Opcode: VE::LDLZXrii), DestReg: IReg), FI,
2446 Offset: OffsetCS);
2447 if (LPadList.size() < 64) {
2448 BuildMI(BB: DispatchBB, MIMD: DL, MCID: TII->get(Opcode: VE::BRCFLir_t))
2449 .addImm(Val: VECC::CC_ILE)
2450 .addImm(Val: LPadList.size())
2451 .addReg(RegNo: IReg)
2452 .addMBB(MBB: TrapBB);
2453 } else {
2454 assert(LPadList.size() <= 0x7FFFFFFF && "Too large Landing Pad!");
2455 Register TmpReg = MRI.createVirtualRegister(RegClass: RC);
2456 BuildMI(BB: DispatchBB, MIMD: DL, MCID: TII->get(Opcode: VE::LEAzii), DestReg: TmpReg)
2457 .addImm(Val: 0)
2458 .addImm(Val: 0)
2459 .addImm(Val: LPadList.size());
2460 BuildMI(BB: DispatchBB, MIMD: DL, MCID: TII->get(Opcode: VE::BRCFLrr_t))
2461 .addImm(Val: VECC::CC_ILE)
2462 .addReg(RegNo: TmpReg, Flags: getKillRegState(B: true))
2463 .addReg(RegNo: IReg)
2464 .addMBB(MBB: TrapBB);
2465 }
2466
2467 Register BReg = MRI.createVirtualRegister(RegClass: RC);
2468 Register Tmp1 = MRI.createVirtualRegister(RegClass: RC);
2469 Register Tmp2 = MRI.createVirtualRegister(RegClass: RC);
2470
2471 if (isPositionIndependent()) {
2472 // Create following instructions for local linkage PIC code.
2473 // lea %Tmp1, .LJTI0_0@gotoff_lo
2474 // and %Tmp2, %Tmp1, (32)0
2475 // lea.sl %BReg, .LJTI0_0@gotoff_hi(%Tmp2, %s15) ; %s15 is GOT
2476 BuildMI(BB: DispContBB, MIMD: DL, MCID: TII->get(Opcode: VE::LEAzii), DestReg: Tmp1)
2477 .addImm(Val: 0)
2478 .addImm(Val: 0)
2479 .addJumpTableIndex(Idx: MJTI, TargetFlags: VE::S_GOTOFF_LO32);
2480 BuildMI(BB: DispContBB, MIMD: DL, MCID: TII->get(Opcode: VE::ANDrm), DestReg: Tmp2)
2481 .addReg(RegNo: Tmp1, Flags: getKillRegState(B: true))
2482 .addImm(Val: M0(Val: 32));
2483 BuildMI(BB: DispContBB, MIMD: DL, MCID: TII->get(Opcode: VE::LEASLrri), DestReg: BReg)
2484 .addReg(RegNo: VE::SX15)
2485 .addReg(RegNo: Tmp2, Flags: getKillRegState(B: true))
2486 .addJumpTableIndex(Idx: MJTI, TargetFlags: VE::S_GOTOFF_HI32);
2487 } else {
2488 // Create following instructions for non-PIC code.
2489 // lea %Tmp1, .LJTI0_0@lo
2490 // and %Tmp2, %Tmp1, (32)0
2491 // lea.sl %BReg, .LJTI0_0@hi(%Tmp2)
2492 BuildMI(BB: DispContBB, MIMD: DL, MCID: TII->get(Opcode: VE::LEAzii), DestReg: Tmp1)
2493 .addImm(Val: 0)
2494 .addImm(Val: 0)
2495 .addJumpTableIndex(Idx: MJTI, TargetFlags: VE::S_LO32);
2496 BuildMI(BB: DispContBB, MIMD: DL, MCID: TII->get(Opcode: VE::ANDrm), DestReg: Tmp2)
2497 .addReg(RegNo: Tmp1, Flags: getKillRegState(B: true))
2498 .addImm(Val: M0(Val: 32));
2499 BuildMI(BB: DispContBB, MIMD: DL, MCID: TII->get(Opcode: VE::LEASLrii), DestReg: BReg)
2500 .addReg(RegNo: Tmp2, Flags: getKillRegState(B: true))
2501 .addImm(Val: 0)
2502 .addJumpTableIndex(Idx: MJTI, TargetFlags: VE::S_HI32);
2503 }
2504
2505 switch (JTE) {
2506 case MachineJumpTableInfo::EK_BlockAddress: {
2507 // Generate simple block address code for no-PIC model.
2508 // sll %Tmp1, %IReg, 3
2509 // lds %TReg, 0(%Tmp1, %BReg)
2510 // bcfla %TReg
2511
2512 Register TReg = MRI.createVirtualRegister(RegClass: RC);
2513 Register Tmp1 = MRI.createVirtualRegister(RegClass: RC);
2514
2515 BuildMI(BB: DispContBB, MIMD: DL, MCID: TII->get(Opcode: VE::SLLri), DestReg: Tmp1)
2516 .addReg(RegNo: IReg, Flags: getKillRegState(B: true))
2517 .addImm(Val: 3);
2518 BuildMI(BB: DispContBB, MIMD: DL, MCID: TII->get(Opcode: VE::LDrri), DestReg: TReg)
2519 .addReg(RegNo: BReg, Flags: getKillRegState(B: true))
2520 .addReg(RegNo: Tmp1, Flags: getKillRegState(B: true))
2521 .addImm(Val: 0);
2522 BuildMI(BB: DispContBB, MIMD: DL, MCID: TII->get(Opcode: VE::BCFLari_t))
2523 .addReg(RegNo: TReg, Flags: getKillRegState(B: true))
2524 .addImm(Val: 0);
2525 break;
2526 }
2527 case MachineJumpTableInfo::EK_Custom32: {
2528 // Generate block address code using differences from the function pointer
2529 // for PIC model.
2530 // sll %Tmp1, %IReg, 2
2531 // ldl.zx %OReg, 0(%Tmp1, %BReg)
2532 // Prepare function address in BReg2.
2533 // adds.l %TReg, %BReg2, %OReg
2534 // bcfla %TReg
2535
2536 assert(isPositionIndependent());
2537 Register OReg = MRI.createVirtualRegister(RegClass: RC);
2538 Register TReg = MRI.createVirtualRegister(RegClass: RC);
2539 Register Tmp1 = MRI.createVirtualRegister(RegClass: RC);
2540
2541 BuildMI(BB: DispContBB, MIMD: DL, MCID: TII->get(Opcode: VE::SLLri), DestReg: Tmp1)
2542 .addReg(RegNo: IReg, Flags: getKillRegState(B: true))
2543 .addImm(Val: 2);
2544 BuildMI(BB: DispContBB, MIMD: DL, MCID: TII->get(Opcode: VE::LDLZXrri), DestReg: OReg)
2545 .addReg(RegNo: BReg, Flags: getKillRegState(B: true))
2546 .addReg(RegNo: Tmp1, Flags: getKillRegState(B: true))
2547 .addImm(Val: 0);
2548 Register BReg2 =
2549 prepareSymbol(MBB&: *DispContBB, I: DispContBB->end(),
2550 Symbol: DispContBB->getParent()->getName(), DL, /* Local */ IsLocal: true);
2551 BuildMI(BB: DispContBB, MIMD: DL, MCID: TII->get(Opcode: VE::ADDSLrr), DestReg: TReg)
2552 .addReg(RegNo: OReg, Flags: getKillRegState(B: true))
2553 .addReg(RegNo: BReg2, Flags: getKillRegState(B: true));
2554 BuildMI(BB: DispContBB, MIMD: DL, MCID: TII->get(Opcode: VE::BCFLari_t))
2555 .addReg(RegNo: TReg, Flags: getKillRegState(B: true))
2556 .addImm(Val: 0);
2557 break;
2558 }
2559 default:
2560 llvm_unreachable("Unexpected jump table encoding");
2561 }
2562
2563 // Add the jump table entries as successors to the MBB.
2564 SmallPtrSet<MachineBasicBlock *, 8> SeenMBBs;
2565 for (auto &LP : LPadList)
2566 if (SeenMBBs.insert(Ptr: LP).second)
2567 DispContBB->addSuccessor(Succ: LP);
2568
2569 // N.B. the order the invoke BBs are processed in doesn't matter here.
2570 SmallVector<MachineBasicBlock *, 64> MBBLPads;
2571 const MCPhysReg *SavedRegs = MF->getRegInfo().getCalleeSavedRegs();
2572 for (MachineBasicBlock *MBB : InvokeBBs) {
2573 // Remove the landing pad successor from the invoke block and replace it
2574 // with the new dispatch block.
2575 // Keep a copy of Successors since it's modified inside the loop.
2576 SmallVector<MachineBasicBlock *, 8> Successors(MBB->succ_rbegin(),
2577 MBB->succ_rend());
2578 // FIXME: Avoid quadratic complexity.
2579 for (auto *MBBS : Successors) {
2580 if (MBBS->isEHPad()) {
2581 MBB->removeSuccessor(Succ: MBBS);
2582 MBBLPads.push_back(Elt: MBBS);
2583 }
2584 }
2585
2586 MBB->addSuccessor(Succ: DispatchBB);
2587
2588 // Find the invoke call and mark all of the callee-saved registers as
2589 // 'implicit defined' so that they're spilled. This prevents code from
2590 // moving instructions to before the EH block, where they will never be
2591 // executed.
2592 for (auto &II : reverse(C&: *MBB)) {
2593 if (!II.isCall())
2594 continue;
2595
2596 DenseSet<Register> DefRegs;
2597 for (auto &MOp : II.operands())
2598 if (MOp.isReg())
2599 DefRegs.insert(V: MOp.getReg());
2600
2601 MachineInstrBuilder MIB(*MF, &II);
2602 for (unsigned RI = 0; SavedRegs[RI]; ++RI) {
2603 Register Reg = SavedRegs[RI];
2604 if (!DefRegs.contains(V: Reg))
2605 MIB.addReg(RegNo: Reg, Flags: RegState::ImplicitDefine | RegState::Dead);
2606 }
2607
2608 break;
2609 }
2610 }
2611
2612 // Mark all former landing pads as non-landing pads. The dispatch is the only
2613 // landing pad now.
2614 for (auto &LP : MBBLPads)
2615 LP->setIsEHPad(false);
2616
2617 // The instruction is gone now.
2618 MI.eraseFromParent();
2619 return BB;
2620}
2621
2622MachineBasicBlock *
2623VETargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
2624 MachineBasicBlock *BB) const {
2625 switch (MI.getOpcode()) {
2626 default:
2627 llvm_unreachable("Unknown Custom Instruction!");
2628 case VE::EH_SjLj_LongJmp:
2629 return emitEHSjLjLongJmp(MI, MBB: BB);
2630 case VE::EH_SjLj_SetJmp:
2631 return emitEHSjLjSetJmp(MI, MBB: BB);
2632 case VE::EH_SjLj_Setup_Dispatch:
2633 return emitSjLjDispatchBlock(MI, BB);
2634 }
2635}
2636
2637static bool isSimm7(SDValue V) {
2638 EVT VT = V.getValueType();
2639 if (VT.isVector())
2640 return false;
2641
2642 if (VT.isInteger()) {
2643 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: V))
2644 return isInt<7>(x: C->getSExtValue());
2645 } else if (VT.isFloatingPoint()) {
2646 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val&: V)) {
2647 if (VT == MVT::f32 || VT == MVT::f64) {
2648 const APInt &Imm = C->getValueAPF().bitcastToAPInt();
2649 uint64_t Val = Imm.getSExtValue();
2650 if (Imm.getBitWidth() == 32)
2651 Val <<= 32; // Immediate value of float place at higher bits on VE.
2652 return isInt<7>(x: Val);
2653 }
2654 }
2655 }
2656 return false;
2657}
2658
2659static bool isMImm(SDValue V) {
2660 EVT VT = V.getValueType();
2661 if (VT.isVector())
2662 return false;
2663
2664 if (VT.isInteger()) {
2665 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: V))
2666 return isMImmVal(Val: getImmVal(N: C));
2667 } else if (VT.isFloatingPoint()) {
2668 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val&: V)) {
2669 if (VT == MVT::f32) {
2670 // Float value places at higher bits, so ignore lower 32 bits.
2671 return isMImm32Val(Val: getFpImmVal(N: C) >> 32);
2672 } else if (VT == MVT::f64) {
2673 return isMImmVal(Val: getFpImmVal(N: C));
2674 }
2675 }
2676 }
2677 return false;
2678}
2679
2680static unsigned decideComp(EVT SrcVT, ISD::CondCode CC) {
2681 if (SrcVT.isFloatingPoint()) {
2682 if (SrcVT == MVT::f128)
2683 return VEISD::CMPQ;
2684 return VEISD::CMPF;
2685 }
2686 return isSignedIntSetCC(Code: CC) ? VEISD::CMPI : VEISD::CMPU;
2687}
2688
2689static EVT decideCompType(EVT SrcVT) {
2690 if (SrcVT == MVT::f128)
2691 return MVT::f64;
2692 return SrcVT;
2693}
2694
2695static bool safeWithoutCompWithNull(EVT SrcVT, ISD::CondCode CC,
2696 bool WithCMov) {
2697 if (SrcVT.isFloatingPoint()) {
2698 // For the case of floating point setcc, only unordered comparison
2699 // or general comparison with -enable-no-nans-fp-math option reach
2700 // here, so it is safe even if values are NaN. Only f128 doesn't
2701 // safe since VE uses f64 result of f128 comparison.
2702 return SrcVT != MVT::f128;
2703 }
2704 if (isIntEqualitySetCC(Code: CC)) {
2705 // For the case of equal or not equal, it is safe without comparison with 0.
2706 return true;
2707 }
2708 if (WithCMov) {
2709 // For the case of integer setcc with cmov, all signed comparison with 0
2710 // are safe.
2711 return isSignedIntSetCC(Code: CC);
2712 }
2713 // For the case of integer setcc, only signed 64 bits comparison is safe.
2714 // For unsigned, "CMPU 0x80000000, 0" has to be greater than 0, but it becomes
2715 // less than 0 witout CMPU. For 32 bits, other half of 32 bits are
2716 // uncoditional, so it is not safe too without CMPI..
2717 return isSignedIntSetCC(Code: CC) && SrcVT == MVT::i64;
2718}
2719
2720static SDValue generateComparison(EVT VT, SDValue LHS, SDValue RHS,
2721 ISD::CondCode CC, bool WithCMov,
2722 const SDLoc &DL, SelectionDAG &DAG) {
2723 // Compare values. If RHS is 0 and it is safe to calculate without
2724 // comparison, we don't generate an instruction for comparison.
2725 EVT CompVT = decideCompType(SrcVT: VT);
2726 if (CompVT == VT && safeWithoutCompWithNull(SrcVT: VT, CC, WithCMov) &&
2727 (isNullConstant(V: RHS) || isNullFPConstant(V: RHS))) {
2728 return LHS;
2729 }
2730 return DAG.getNode(Opcode: decideComp(SrcVT: VT, CC), DL, VT: CompVT, N1: LHS, N2: RHS);
2731}
2732
2733SDValue VETargetLowering::combineSelect(SDNode *N,
2734 DAGCombinerInfo &DCI) const {
2735 assert(N->getOpcode() == ISD::SELECT &&
2736 "Should be called with a SELECT node");
2737 ISD::CondCode CC = ISD::CondCode::SETNE;
2738 SDValue Cond = N->getOperand(Num: 0);
2739 SDValue True = N->getOperand(Num: 1);
2740 SDValue False = N->getOperand(Num: 2);
2741
2742 // We handle only scalar SELECT.
2743 EVT VT = N->getValueType(ResNo: 0);
2744 if (VT.isVector())
2745 return SDValue();
2746
2747 // Peform combineSelect after leagalize DAG.
2748 if (!DCI.isAfterLegalizeDAG())
2749 return SDValue();
2750
2751 EVT VT0 = Cond.getValueType();
2752 if (isMImm(V: True)) {
2753 // VE's condition move can handle MImm in True clause, so nothing to do.
2754 } else if (isMImm(V: False)) {
2755 // VE's condition move can handle MImm in True clause, so swap True and
2756 // False clauses if False has MImm value. And, update condition code.
2757 std::swap(a&: True, b&: False);
2758 CC = getSetCCInverse(Operation: CC, Type: VT0);
2759 }
2760
2761 SDLoc DL(N);
2762 SelectionDAG &DAG = DCI.DAG;
2763 VECC::CondCode VECCVal;
2764 if (VT0.isFloatingPoint()) {
2765 VECCVal = fpCondCode2Fcc(CC);
2766 } else {
2767 VECCVal = intCondCode2Icc(CC);
2768 }
2769 SDValue Ops[] = {Cond, True, False,
2770 DAG.getConstant(Val: VECCVal, DL, VT: MVT::i32)};
2771 return DAG.getNode(Opcode: VEISD::CMOV, DL, VT, Ops);
2772}
2773
2774SDValue VETargetLowering::combineSelectCC(SDNode *N,
2775 DAGCombinerInfo &DCI) const {
2776 assert(N->getOpcode() == ISD::SELECT_CC &&
2777 "Should be called with a SELECT_CC node");
2778 ISD::CondCode CC = cast<CondCodeSDNode>(Val: N->getOperand(Num: 4))->get();
2779 SDValue LHS = N->getOperand(Num: 0);
2780 SDValue RHS = N->getOperand(Num: 1);
2781 SDValue True = N->getOperand(Num: 2);
2782 SDValue False = N->getOperand(Num: 3);
2783
2784 // We handle only scalar SELECT_CC.
2785 EVT VT = N->getValueType(ResNo: 0);
2786 if (VT.isVector())
2787 return SDValue();
2788
2789 // Peform combineSelectCC after leagalize DAG.
2790 if (!DCI.isAfterLegalizeDAG())
2791 return SDValue();
2792
2793 // We handle only i32/i64/f32/f64/f128 comparisons.
2794 EVT LHSVT = LHS.getValueType();
2795 assert(LHSVT == RHS.getValueType());
2796 switch (LHSVT.getSimpleVT().SimpleTy) {
2797 case MVT::i32:
2798 case MVT::i64:
2799 case MVT::f32:
2800 case MVT::f64:
2801 case MVT::f128:
2802 break;
2803 default:
2804 // Return SDValue to let llvm handle other types.
2805 return SDValue();
2806 }
2807
2808 if (isMImm(V: RHS)) {
2809 // VE's comparison can handle MImm in RHS, so nothing to do.
2810 } else if (isSimm7(V: RHS)) {
2811 // VE's comparison can handle Simm7 in LHS, so swap LHS and RHS, and
2812 // update condition code.
2813 std::swap(a&: LHS, b&: RHS);
2814 CC = getSetCCSwappedOperands(Operation: CC);
2815 }
2816 if (isMImm(V: True)) {
2817 // VE's condition move can handle MImm in True clause, so nothing to do.
2818 } else if (isMImm(V: False)) {
2819 // VE's condition move can handle MImm in True clause, so swap True and
2820 // False clauses if False has MImm value. And, update condition code.
2821 std::swap(a&: True, b&: False);
2822 CC = getSetCCInverse(Operation: CC, Type: LHSVT);
2823 }
2824
2825 SDLoc DL(N);
2826 SelectionDAG &DAG = DCI.DAG;
2827
2828 bool WithCMov = true;
2829 SDValue CompNode = generateComparison(VT: LHSVT, LHS, RHS, CC, WithCMov, DL, DAG);
2830
2831 VECC::CondCode VECCVal;
2832 if (LHSVT.isFloatingPoint()) {
2833 VECCVal = fpCondCode2Fcc(CC);
2834 } else {
2835 VECCVal = intCondCode2Icc(CC);
2836 }
2837 SDValue Ops[] = {CompNode, True, False,
2838 DAG.getConstant(Val: VECCVal, DL, VT: MVT::i32)};
2839 return DAG.getNode(Opcode: VEISD::CMOV, DL, VT, Ops);
2840}
2841
2842static bool isI32InsnAllUses(const SDNode *User, const SDNode *N);
2843static bool isI32Insn(const SDNode *User, const SDNode *N) {
2844 switch (User->getOpcode()) {
2845 default:
2846 return false;
2847 case ISD::ADD:
2848 case ISD::SUB:
2849 case ISD::MUL:
2850 case ISD::SDIV:
2851 case ISD::UDIV:
2852 case ISD::SETCC:
2853 case ISD::SMIN:
2854 case ISD::SMAX:
2855 case ISD::SHL:
2856 case ISD::SRA:
2857 case ISD::BSWAP:
2858 case ISD::SINT_TO_FP:
2859 case ISD::UINT_TO_FP:
2860 case ISD::BR_CC:
2861 case ISD::BITCAST:
2862 case ISD::ATOMIC_CMP_SWAP:
2863 case ISD::ATOMIC_SWAP:
2864 case VEISD::CMPU:
2865 case VEISD::CMPI:
2866 return true;
2867 case ISD::SRL:
2868 if (N->getOperand(Num: 0).getOpcode() != ISD::SRL)
2869 return true;
2870 // (srl (trunc (srl ...))) may be optimized by combining srl, so
2871 // doesn't optimize trunc now.
2872 return false;
2873 case ISD::SELECT_CC:
2874 if (User->getOperand(Num: 2).getNode() != N &&
2875 User->getOperand(Num: 3).getNode() != N)
2876 return true;
2877 return isI32InsnAllUses(User, N);
2878 case VEISD::CMOV:
2879 // CMOV in (cmov (trunc ...), true, false, int-comparison) is safe.
2880 // However, trunc in true or false clauses is not safe.
2881 if (User->getOperand(Num: 1).getNode() != N &&
2882 User->getOperand(Num: 2).getNode() != N &&
2883 isa<ConstantSDNode>(Val: User->getOperand(Num: 3))) {
2884 VECC::CondCode VECCVal =
2885 static_cast<VECC::CondCode>(User->getConstantOperandVal(Num: 3));
2886 return isIntVECondCode(CC: VECCVal);
2887 }
2888 [[fallthrough]];
2889 case ISD::AND:
2890 case ISD::OR:
2891 case ISD::XOR:
2892 case ISD::SELECT:
2893 case ISD::CopyToReg:
2894 // Check all use of selections, bit operations, and copies. If all of them
2895 // are safe, optimize truncate to extract_subreg.
2896 return isI32InsnAllUses(User, N);
2897 }
2898}
2899
2900static bool isI32InsnAllUses(const SDNode *User, const SDNode *N) {
2901 // Check all use of User node. If all of them are safe, optimize
2902 // truncate to extract_subreg.
2903 for (const SDNode *U : User->users()) {
2904 switch (U->getOpcode()) {
2905 default:
2906 // If the use is an instruction which treats the source operand as i32,
2907 // it is safe to avoid truncate here.
2908 if (isI32Insn(User: U, N))
2909 continue;
2910 break;
2911 case ISD::ANY_EXTEND:
2912 case ISD::SIGN_EXTEND:
2913 case ISD::ZERO_EXTEND: {
2914 // Special optimizations to the combination of ext and trunc.
2915 // (ext ... (select ... (trunc ...))) is safe to avoid truncate here
2916 // since this truncate instruction clears higher 32 bits which is filled
2917 // by one of ext instructions later.
2918 assert(N->getValueType(0) == MVT::i32 &&
2919 "find truncate to not i32 integer");
2920 if (User->getOpcode() == ISD::SELECT_CC ||
2921 User->getOpcode() == ISD::SELECT || User->getOpcode() == VEISD::CMOV)
2922 continue;
2923 break;
2924 }
2925 }
2926 return false;
2927 }
2928 return true;
2929}
2930
2931// Optimize TRUNCATE in DAG combining. Optimizing it in CUSTOM lower is
2932// sometime too early. Optimizing it in DAG pattern matching in VEInstrInfo.td
2933// is sometime too late. So, doing it at here.
2934SDValue VETargetLowering::combineTRUNCATE(SDNode *N,
2935 DAGCombinerInfo &DCI) const {
2936 assert(N->getOpcode() == ISD::TRUNCATE &&
2937 "Should be called with a TRUNCATE node");
2938
2939 SelectionDAG &DAG = DCI.DAG;
2940 SDLoc DL(N);
2941 EVT VT = N->getValueType(ResNo: 0);
2942
2943 // We prefer to do this when all types are legal.
2944 if (!DCI.isAfterLegalizeDAG())
2945 return SDValue();
2946
2947 // Skip combine TRUNCATE atm if the operand of TRUNCATE might be a constant.
2948 if (N->getOperand(Num: 0)->getOpcode() == ISD::SELECT_CC &&
2949 isa<ConstantSDNode>(Val: N->getOperand(Num: 0)->getOperand(Num: 0)) &&
2950 isa<ConstantSDNode>(Val: N->getOperand(Num: 0)->getOperand(Num: 1)))
2951 return SDValue();
2952
2953 // Check all use of this TRUNCATE.
2954 for (const SDNode *User : N->users()) {
2955 // Make sure that we're not going to replace TRUNCATE for non i32
2956 // instructions.
2957 //
2958 // FIXME: Although we could sometimes handle this, and it does occur in
2959 // practice that one of the condition inputs to the select is also one of
2960 // the outputs, we currently can't deal with this.
2961 if (isI32Insn(User, N))
2962 continue;
2963
2964 return SDValue();
2965 }
2966
2967 SDValue SubI32 = DAG.getTargetConstant(Val: VE::sub_i32, DL, VT: MVT::i32);
2968 return SDValue(DAG.getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG, dl: DL, VT,
2969 Op1: N->getOperand(Num: 0), Op2: SubI32),
2970 0);
2971}
2972
2973SDValue VETargetLowering::PerformDAGCombine(SDNode *N,
2974 DAGCombinerInfo &DCI) const {
2975 switch (N->getOpcode()) {
2976 default:
2977 break;
2978 case ISD::SELECT:
2979 return combineSelect(N, DCI);
2980 case ISD::SELECT_CC:
2981 return combineSelectCC(N, DCI);
2982 case ISD::TRUNCATE:
2983 return combineTRUNCATE(N, DCI);
2984 }
2985
2986 return SDValue();
2987}
2988
2989//===----------------------------------------------------------------------===//
2990// VE Inline Assembly Support
2991//===----------------------------------------------------------------------===//
2992
2993VETargetLowering::ConstraintType
2994VETargetLowering::getConstraintType(StringRef Constraint) const {
2995 if (Constraint.size() == 1) {
2996 switch (Constraint[0]) {
2997 default:
2998 break;
2999 case 'v': // vector registers
3000 return C_RegisterClass;
3001 }
3002 }
3003 return TargetLowering::getConstraintType(Constraint);
3004}
3005
3006std::pair<unsigned, const TargetRegisterClass *>
3007VETargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
3008 StringRef Constraint,
3009 MVT VT) const {
3010 const TargetRegisterClass *RC = nullptr;
3011 if (Constraint.size() == 1) {
3012 switch (Constraint[0]) {
3013 default:
3014 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
3015 case 'r':
3016 RC = &VE::I64RegClass;
3017 break;
3018 case 'v':
3019 RC = &VE::V64RegClass;
3020 break;
3021 }
3022 return std::make_pair(x: 0U, y&: RC);
3023 }
3024
3025 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
3026}
3027
3028//===----------------------------------------------------------------------===//
3029// VE Target Optimization Support
3030//===----------------------------------------------------------------------===//
3031
3032unsigned VETargetLowering::getMinimumJumpTableEntries() const {
3033 // Specify 8 for PIC model to relieve the impact of PIC load instructions.
3034 if (isJumpTableRelative())
3035 return 8;
3036
3037 return TargetLowering::getMinimumJumpTableEntries();
3038}
3039
3040bool VETargetLowering::hasAndNot(SDValue Y) const {
3041 EVT VT = Y.getValueType();
3042
3043 // VE doesn't have vector and not instruction.
3044 if (VT.isVector())
3045 return false;
3046
3047 // VE allows different immediate values for X and Y where ~X & Y.
3048 // Only simm7 works for X, and only mimm works for Y on VE. However, this
3049 // function is used to check whether an immediate value is OK for and-not
3050 // instruction as both X and Y. Generating additional instruction to
3051 // retrieve an immediate value is no good since the purpose of this
3052 // function is to convert a series of 3 instructions to another series of
3053 // 3 instructions with better parallelism. Therefore, we return false
3054 // for all immediate values now.
3055 // FIXME: Change hasAndNot function to have two operands to make it work
3056 // correctly with Aurora VE.
3057 if (isa<ConstantSDNode>(Val: Y))
3058 return false;
3059
3060 // It's ok for generic registers.
3061 return true;
3062}
3063
3064SDValue VETargetLowering::lowerEXTRACT_VECTOR_ELT(SDValue Op,
3065 SelectionDAG &DAG) const {
3066 assert(Op.getOpcode() == ISD::EXTRACT_VECTOR_ELT && "Unknown opcode!");
3067 MVT VT = Op.getOperand(i: 0).getSimpleValueType();
3068
3069 // Special treatment for packed V64 types.
3070 assert(VT == MVT::v512i32 || VT == MVT::v512f32);
3071 (void)VT;
3072 // Example of codes:
3073 // %packed_v = extractelt %vr, %idx / 2
3074 // %v = %packed_v >> (%idx % 2 * 32)
3075 // %res = %v & 0xffffffff
3076
3077 SDValue Vec = Op.getOperand(i: 0);
3078 SDValue Idx = Op.getOperand(i: 1);
3079 SDLoc DL(Op);
3080 SDValue Result = Op;
3081 if (false /* Idx->isConstant() */) {
3082 // TODO: optimized implementation using constant values
3083 } else {
3084 SDValue Const1 = DAG.getConstant(Val: 1, DL, VT: MVT::i64);
3085 SDValue HalfIdx = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i64, Ops: {Idx, Const1});
3086 SDValue PackedElt =
3087 SDValue(DAG.getMachineNode(Opcode: VE::LVSvr, dl: DL, VT: MVT::i64, Ops: {Vec, HalfIdx}), 0);
3088 SDValue AndIdx = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i64, Ops: {Idx, Const1});
3089 SDValue Shift = DAG.getNode(Opcode: ISD::XOR, DL, VT: MVT::i64, Ops: {AndIdx, Const1});
3090 SDValue Const5 = DAG.getConstant(Val: 5, DL, VT: MVT::i64);
3091 Shift = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i64, Ops: {Shift, Const5});
3092 PackedElt = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i64, Ops: {PackedElt, Shift});
3093 SDValue Mask = DAG.getConstant(Val: 0xFFFFFFFFL, DL, VT: MVT::i64);
3094 PackedElt = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i64, Ops: {PackedElt, Mask});
3095 SDValue SubI32 = DAG.getTargetConstant(Val: VE::sub_i32, DL, VT: MVT::i32);
3096 Result = SDValue(DAG.getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG, dl: DL,
3097 VT: MVT::i32, Op1: PackedElt, Op2: SubI32),
3098 0);
3099
3100 if (Op.getSimpleValueType() == MVT::f32) {
3101 Result = DAG.getBitcast(VT: MVT::f32, V: Result);
3102 } else {
3103 assert(Op.getSimpleValueType() == MVT::i32);
3104 }
3105 }
3106 return Result;
3107}
3108
3109SDValue VETargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op,
3110 SelectionDAG &DAG) const {
3111 assert(Op.getOpcode() == ISD::INSERT_VECTOR_ELT && "Unknown opcode!");
3112 MVT VT = Op.getOperand(i: 0).getSimpleValueType();
3113
3114 // Special treatment for packed V64 types.
3115 assert(VT == MVT::v512i32 || VT == MVT::v512f32);
3116 (void)VT;
3117 // The v512i32 and v512f32 starts from upper bits (0..31). This "upper
3118 // bits" required `val << 32` from C implementation's point of view.
3119 //
3120 // Example of codes:
3121 // %packed_elt = extractelt %vr, (%idx >> 1)
3122 // %shift = ((%idx & 1) ^ 1) << 5
3123 // %packed_elt &= 0xffffffff00000000 >> shift
3124 // %packed_elt |= (zext %val) << shift
3125 // %vr = insertelt %vr, %packed_elt, (%idx >> 1)
3126
3127 SDLoc DL(Op);
3128 SDValue Vec = Op.getOperand(i: 0);
3129 SDValue Val = Op.getOperand(i: 1);
3130 SDValue Idx = Op.getOperand(i: 2);
3131 if (Idx.getSimpleValueType() == MVT::i32)
3132 Idx = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i64, Operand: Idx);
3133 if (Val.getSimpleValueType() == MVT::f32)
3134 Val = DAG.getBitcast(VT: MVT::i32, V: Val);
3135 assert(Val.getSimpleValueType() == MVT::i32);
3136 Val = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i64, Operand: Val);
3137
3138 SDValue Result = Op;
3139 if (false /* Idx->isConstant()*/) {
3140 // TODO: optimized implementation using constant values
3141 } else {
3142 SDValue Const1 = DAG.getConstant(Val: 1, DL, VT: MVT::i64);
3143 SDValue HalfIdx = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i64, Ops: {Idx, Const1});
3144 SDValue PackedElt =
3145 SDValue(DAG.getMachineNode(Opcode: VE::LVSvr, dl: DL, VT: MVT::i64, Ops: {Vec, HalfIdx}), 0);
3146 SDValue AndIdx = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i64, Ops: {Idx, Const1});
3147 SDValue Shift = DAG.getNode(Opcode: ISD::XOR, DL, VT: MVT::i64, Ops: {AndIdx, Const1});
3148 SDValue Const5 = DAG.getConstant(Val: 5, DL, VT: MVT::i64);
3149 Shift = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i64, Ops: {Shift, Const5});
3150 SDValue Mask = DAG.getConstant(Val: 0xFFFFFFFF00000000L, DL, VT: MVT::i64);
3151 Mask = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i64, Ops: {Mask, Shift});
3152 PackedElt = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i64, Ops: {PackedElt, Mask});
3153 Val = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i64, Ops: {Val, Shift});
3154 PackedElt = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i64, Ops: {PackedElt, Val});
3155 Result =
3156 SDValue(DAG.getMachineNode(Opcode: VE::LSVrr_v, dl: DL, VT: Vec.getSimpleValueType(),
3157 Ops: {HalfIdx, PackedElt, Vec}),
3158 0);
3159 }
3160 return Result;
3161}
3162