1//===-- PPCISelLowering.cpp - PPC DAG Lowering Implementation -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the PPCISelLowering class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "PPCISelLowering.h"
14#include "MCTargetDesc/PPCMCTargetDesc.h"
15#include "MCTargetDesc/PPCPredicates.h"
16#include "PPC.h"
17#include "PPCCallingConv.h"
18#include "PPCFrameLowering.h"
19#include "PPCInstrInfo.h"
20#include "PPCMachineFunctionInfo.h"
21#include "PPCPerfectShuffle.h"
22#include "PPCRegisterInfo.h"
23#include "PPCSelectionDAGInfo.h"
24#include "PPCSubtarget.h"
25#include "PPCTargetMachine.h"
26#include "llvm/ADT/APFloat.h"
27#include "llvm/ADT/APInt.h"
28#include "llvm/ADT/APSInt.h"
29#include "llvm/ADT/ArrayRef.h"
30#include "llvm/ADT/DenseMap.h"
31#include "llvm/ADT/STLExtras.h"
32#include "llvm/ADT/SmallPtrSet.h"
33#include "llvm/ADT/SmallVector.h"
34#include "llvm/ADT/Statistic.h"
35#include "llvm/ADT/StringRef.h"
36#include "llvm/CodeGen/CallingConvLower.h"
37#include "llvm/CodeGen/ISDOpcodes.h"
38#include "llvm/CodeGen/LivePhysRegs.h"
39#include "llvm/CodeGen/MachineBasicBlock.h"
40#include "llvm/CodeGen/MachineFrameInfo.h"
41#include "llvm/CodeGen/MachineFunction.h"
42#include "llvm/CodeGen/MachineInstr.h"
43#include "llvm/CodeGen/MachineInstrBuilder.h"
44#include "llvm/CodeGen/MachineJumpTableInfo.h"
45#include "llvm/CodeGen/MachineLoopInfo.h"
46#include "llvm/CodeGen/MachineMemOperand.h"
47#include "llvm/CodeGen/MachineModuleInfo.h"
48#include "llvm/CodeGen/MachineOperand.h"
49#include "llvm/CodeGen/MachineRegisterInfo.h"
50#include "llvm/CodeGen/SelectionDAG.h"
51#include "llvm/CodeGen/SelectionDAGNodes.h"
52#include "llvm/CodeGen/TargetInstrInfo.h"
53#include "llvm/CodeGen/TargetLowering.h"
54#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
55#include "llvm/CodeGen/TargetRegisterInfo.h"
56#include "llvm/CodeGen/ValueTypes.h"
57#include "llvm/CodeGenTypes/MachineValueType.h"
58#include "llvm/IR/CallingConv.h"
59#include "llvm/IR/Constant.h"
60#include "llvm/IR/Constants.h"
61#include "llvm/IR/DataLayout.h"
62#include "llvm/IR/DebugLoc.h"
63#include "llvm/IR/DerivedTypes.h"
64#include "llvm/IR/Function.h"
65#include "llvm/IR/GlobalValue.h"
66#include "llvm/IR/IRBuilder.h"
67#include "llvm/IR/Instructions.h"
68#include "llvm/IR/Intrinsics.h"
69#include "llvm/IR/IntrinsicsPowerPC.h"
70#include "llvm/IR/Module.h"
71#include "llvm/IR/Type.h"
72#include "llvm/IR/Use.h"
73#include "llvm/IR/Value.h"
74#include "llvm/MC/MCContext.h"
75#include "llvm/MC/MCExpr.h"
76#include "llvm/MC/MCSectionXCOFF.h"
77#include "llvm/MC/MCSymbolXCOFF.h"
78#include "llvm/Support/AtomicOrdering.h"
79#include "llvm/Support/BranchProbability.h"
80#include "llvm/Support/Casting.h"
81#include "llvm/Support/CodeGen.h"
82#include "llvm/Support/CommandLine.h"
83#include "llvm/Support/Compiler.h"
84#include "llvm/Support/Debug.h"
85#include "llvm/Support/ErrorHandling.h"
86#include "llvm/Support/Format.h"
87#include "llvm/Support/KnownBits.h"
88#include "llvm/Support/MathExtras.h"
89#include "llvm/Support/raw_ostream.h"
90#include "llvm/Target/TargetMachine.h"
91#include "llvm/Target/TargetOptions.h"
92#include <algorithm>
93#include <cassert>
94#include <cstdint>
95#include <iterator>
96#include <list>
97#include <optional>
98#include <utility>
99#include <vector>
100
101using namespace llvm;
102
103#define DEBUG_TYPE "ppc-lowering"
104
105static cl::opt<bool> DisableP10StoreForward(
106 "disable-p10-store-forward",
107 cl::desc("disable P10 store forward-friendly conversion"), cl::Hidden,
108 cl::init(Val: false));
109
110static cl::opt<bool> DisablePPCPreinc("disable-ppc-preinc",
111cl::desc("disable preincrement load/store generation on PPC"), cl::Hidden);
112
113static cl::opt<bool> DisableILPPref("disable-ppc-ilp-pref",
114cl::desc("disable setting the node scheduling preference to ILP on PPC"), cl::Hidden);
115
116static cl::opt<bool> DisablePPCUnaligned("disable-ppc-unaligned",
117cl::desc("disable unaligned load/store generation on PPC"), cl::Hidden);
118
119static cl::opt<bool> DisableSCO("disable-ppc-sco",
120cl::desc("disable sibling call optimization on ppc"), cl::Hidden);
121
122static cl::opt<bool> DisableInnermostLoopAlign32("disable-ppc-innermost-loop-align32",
123cl::desc("don't always align innermost loop to 32 bytes on ppc"), cl::Hidden);
124
125static cl::opt<bool> UseAbsoluteJumpTables("ppc-use-absolute-jumptables",
126cl::desc("use absolute jump tables on ppc"), cl::Hidden);
127
128static cl::opt<bool>
129 DisablePerfectShuffle("ppc-disable-perfect-shuffle",
130 cl::desc("disable vector permute decomposition"),
131 cl::init(Val: true), cl::Hidden);
132
133cl::opt<bool> DisableAutoPairedVecSt(
134 "disable-auto-paired-vec-st",
135 cl::desc("disable automatically generated 32byte paired vector stores"),
136 cl::init(Val: true), cl::Hidden);
137
138static cl::opt<unsigned> PPCMinimumJumpTableEntries(
139 "ppc-min-jump-table-entries", cl::init(Val: 64), cl::Hidden,
140 cl::desc("Set minimum number of entries to use a jump table on PPC"));
141
142static cl::opt<unsigned> PPCMinimumBitTestCmps(
143 "ppc-min-bit-test-cmps", cl::init(Val: 3), cl::Hidden,
144 cl::desc("Set minimum of largest number of comparisons to use bit test for "
145 "switch on PPC."));
146
147static cl::opt<unsigned> PPCGatherAllAliasesMaxDepth(
148 "ppc-gather-alias-max-depth", cl::init(Val: 18), cl::Hidden,
149 cl::desc("max depth when checking alias info in GatherAllAliases()"));
150
151static cl::opt<unsigned> PPCAIXTLSModelOptUseIEForLDLimit(
152 "ppc-aix-shared-lib-tls-model-opt-limit", cl::init(Val: 1), cl::Hidden,
153 cl::desc("Set inclusive limit count of TLS local-dynamic access(es) in a "
154 "function to use initial-exec"));
155
156STATISTIC(NumTailCalls, "Number of tail calls");
157STATISTIC(NumSiblingCalls, "Number of sibling calls");
158STATISTIC(ShufflesHandledWithVPERM,
159 "Number of shuffles lowered to a VPERM or XXPERM");
160STATISTIC(NumDynamicAllocaProbed, "Number of dynamic stack allocation probed");
161
162static bool isNByteElemShuffleMask(ShuffleVectorSDNode *, unsigned, int);
163
164static SDValue widenVec(SelectionDAG &DAG, SDValue Vec, const SDLoc &dl);
165
166static void signExtendOperandIfUnknown(MachineInstr &MI, MachineBasicBlock *BB,
167 unsigned OpIdx, bool IsByte,
168 const PPCInstrInfo *TII);
169
170// A faster local-[exec|dynamic] TLS access sequence (enabled with the
171// -maix-small-local-[exec|dynamic]-tls option) can be produced for TLS
172// variables; consistent with the IBM XL compiler, we apply a max size of
173// slightly under 32KB.
174constexpr uint64_t AIXSmallTlsPolicySizeLimit = 32751;
175
176// FIXME: Remove this once the bug has been fixed!
177extern cl::opt<bool> ANDIGlueBug;
178
179PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM,
180 const PPCSubtarget &STI)
181 : TargetLowering(TM, STI), Subtarget(STI) {
182 // Initialize map that relates the PPC addressing modes to the computed flags
183 // of a load/store instruction. The map is used to determine the optimal
184 // addressing mode when selecting load and stores.
185 initializeAddrModeMap();
186 // On PPC32/64, arguments smaller than 4/8 bytes are extended, so all
187 // arguments are at least 4/8 bytes aligned.
188 bool isPPC64 = Subtarget.isPPC64();
189 setMinStackArgumentAlignment(isPPC64 ? Align(8) : Align(4));
190 const MVT RegVT = Subtarget.getScalarIntVT();
191
192 // Set up the register classes.
193 addRegisterClass(VT: MVT::i32, RC: &PPC::GPRCRegClass);
194 if (!useSoftFloat()) {
195 if (hasSPE()) {
196 addRegisterClass(VT: MVT::f32, RC: &PPC::GPRCRegClass);
197 // EFPU2 APU only supports f32
198 if (!Subtarget.hasEFPU2())
199 addRegisterClass(VT: MVT::f64, RC: &PPC::SPERCRegClass);
200 } else {
201 addRegisterClass(VT: MVT::f32, RC: &PPC::F4RCRegClass);
202 addRegisterClass(VT: MVT::f64, RC: &PPC::F8RCRegClass);
203 }
204 }
205
206 setOperationAction(Op: ISD::UADDO, VT: RegVT, Action: Custom);
207 setOperationAction(Op: ISD::USUBO, VT: RegVT, Action: Custom);
208
209 // PowerPC uses addo_carry,subo_carry to propagate carry.
210 setOperationAction(Op: ISD::UADDO_CARRY, VT: RegVT, Action: Custom);
211 setOperationAction(Op: ISD::USUBO_CARRY, VT: RegVT, Action: Custom);
212
213 // On P10, the default lowering generates better code using the
214 // setbc instruction.
215 if (!Subtarget.hasP10Vector()) {
216 setOperationAction(Op: ISD::SSUBO, VT: MVT::i32, Action: Custom);
217 setOperationAction(Op: ISD::SADDO, VT: MVT::i32, Action: Custom);
218 if (isPPC64) {
219 setOperationAction(Op: ISD::SSUBO, VT: MVT::i64, Action: Custom);
220 setOperationAction(Op: ISD::SADDO, VT: MVT::i64, Action: Custom);
221 }
222 }
223
224 // Match BITREVERSE to customized fast code sequence in the td file.
225 setOperationAction(Op: ISD::BITREVERSE, VT: MVT::i32, Action: Legal);
226 setOperationAction(Op: ISD::BITREVERSE, VT: MVT::i64, Action: Legal);
227
228 // Sub-word ATOMIC_CMP_SWAP need to ensure that the input is zero-extended.
229 setOperationAction(Op: ISD::ATOMIC_CMP_SWAP, VT: MVT::i32, Action: Custom);
230
231 // Custom lower inline assembly to check for special registers.
232 setOperationAction(Op: ISD::INLINEASM, VT: MVT::Other, Action: Custom);
233 setOperationAction(Op: ISD::INLINEASM_BR, VT: MVT::Other, Action: Custom);
234
235 // PowerPC has an i16 but no i8 (or i1) SEXTLOAD.
236 for (MVT VT : MVT::integer_valuetypes()) {
237 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: VT, MemVT: MVT::i1, Action: Promote);
238 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: VT, MemVT: MVT::i8, Action: Expand);
239 }
240
241 setTruncStoreAction(ValVT: MVT::f128, MemVT: MVT::f16, Action: Expand);
242 setOperationAction(Op: ISD::FP_TO_FP16, VT: MVT::f128, Action: Expand);
243
244 if (Subtarget.isISA3_0()) {
245 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f128, MemVT: MVT::f16, Action: Legal);
246 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::f16, Action: Legal);
247 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f32, MemVT: MVT::f16, Action: Legal);
248 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f16, Action: Legal);
249 setTruncStoreAction(ValVT: MVT::f32, MemVT: MVT::f16, Action: Legal);
250 } else {
251 // No extending loads from f16 or HW conversions back and forth.
252 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f128, MemVT: MVT::f16, Action: Expand);
253 setOperationAction(Op: ISD::FP16_TO_FP, VT: MVT::f128, Action: Expand);
254 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::f16, Action: Expand);
255 setOperationAction(Op: ISD::FP16_TO_FP, VT: MVT::f64, Action: Expand);
256 setOperationAction(Op: ISD::FP_TO_FP16, VT: MVT::f64, Action: Expand);
257 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f32, MemVT: MVT::f16, Action: Expand);
258 setOperationAction(Op: ISD::FP16_TO_FP, VT: MVT::f32, Action: Expand);
259 setOperationAction(Op: ISD::FP_TO_FP16, VT: MVT::f32, Action: Expand);
260 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f16, Action: Expand);
261 setTruncStoreAction(ValVT: MVT::f32, MemVT: MVT::f16, Action: Expand);
262 }
263
264 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f32, Action: Expand);
265
266 // PowerPC has pre-inc load and store's.
267 setIndexedLoadAction(IdxModes: ISD::PRE_INC, VT: MVT::i1, Action: Legal);
268 setIndexedLoadAction(IdxModes: ISD::PRE_INC, VT: MVT::i8, Action: Legal);
269 setIndexedLoadAction(IdxModes: ISD::PRE_INC, VT: MVT::i16, Action: Legal);
270 setIndexedLoadAction(IdxModes: ISD::PRE_INC, VT: MVT::i32, Action: Legal);
271 setIndexedLoadAction(IdxModes: ISD::PRE_INC, VT: MVT::i64, Action: Legal);
272 setIndexedStoreAction(IdxModes: ISD::PRE_INC, VT: MVT::i1, Action: Legal);
273 setIndexedStoreAction(IdxModes: ISD::PRE_INC, VT: MVT::i8, Action: Legal);
274 setIndexedStoreAction(IdxModes: ISD::PRE_INC, VT: MVT::i16, Action: Legal);
275 setIndexedStoreAction(IdxModes: ISD::PRE_INC, VT: MVT::i32, Action: Legal);
276 setIndexedStoreAction(IdxModes: ISD::PRE_INC, VT: MVT::i64, Action: Legal);
277 if (!Subtarget.hasSPE()) {
278 setIndexedLoadAction(IdxModes: ISD::PRE_INC, VT: MVT::f32, Action: Legal);
279 setIndexedLoadAction(IdxModes: ISD::PRE_INC, VT: MVT::f64, Action: Legal);
280 setIndexedStoreAction(IdxModes: ISD::PRE_INC, VT: MVT::f32, Action: Legal);
281 setIndexedStoreAction(IdxModes: ISD::PRE_INC, VT: MVT::f64, Action: Legal);
282 }
283
284 if (Subtarget.useCRBits()) {
285 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i1, Action: Expand);
286
287 if (isPPC64 || Subtarget.hasFPCVT()) {
288 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::i1, Action: Promote);
289 AddPromotedToType(Opc: ISD::STRICT_SINT_TO_FP, OrigVT: MVT::i1, DestVT: RegVT);
290 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::i1, Action: Promote);
291 AddPromotedToType(Opc: ISD::STRICT_UINT_TO_FP, OrigVT: MVT::i1, DestVT: RegVT);
292
293 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i1, Action: Promote);
294 AddPromotedToType(Opc: ISD::SINT_TO_FP, OrigVT: MVT::i1, DestVT: RegVT);
295 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::i1, Action: Promote);
296 AddPromotedToType(Opc: ISD::UINT_TO_FP, OrigVT: MVT::i1, DestVT: RegVT);
297
298 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::i1, Action: Promote);
299 AddPromotedToType(Opc: ISD::STRICT_FP_TO_SINT, OrigVT: MVT::i1, DestVT: RegVT);
300 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::i1, Action: Promote);
301 AddPromotedToType(Opc: ISD::STRICT_FP_TO_UINT, OrigVT: MVT::i1, DestVT: RegVT);
302
303 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::i1, Action: Promote);
304 AddPromotedToType(Opc: ISD::FP_TO_SINT, OrigVT: MVT::i1, DestVT: RegVT);
305 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i1, Action: Promote);
306 AddPromotedToType(Opc: ISD::FP_TO_UINT, OrigVT: MVT::i1, DestVT: RegVT);
307 } else {
308 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::i1, Action: Custom);
309 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::i1, Action: Custom);
310 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i1, Action: Custom);
311 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::i1, Action: Custom);
312 }
313
314 // PowerPC does not support direct load/store of condition registers.
315 setOperationAction(Op: ISD::LOAD, VT: MVT::i1, Action: Custom);
316 setOperationAction(Op: ISD::STORE, VT: MVT::i1, Action: Custom);
317
318 // FIXME: Remove this once the ANDI glue bug is fixed:
319 if (ANDIGlueBug)
320 setOperationAction(Op: ISD::TRUNCATE, VT: MVT::i1, Action: Custom);
321
322 for (MVT VT : MVT::integer_valuetypes()) {
323 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: VT, MemVT: MVT::i1, Action: Promote);
324 setLoadExtAction(ExtType: ISD::ZEXTLOAD, ValVT: VT, MemVT: MVT::i1, Action: Promote);
325 setTruncStoreAction(ValVT: VT, MemVT: MVT::i1, Action: Expand);
326 }
327
328 addRegisterClass(VT: MVT::i1, RC: &PPC::CRBITRCRegClass);
329 }
330
331 // Expand ppcf128 to i32 by hand for the benefit of llvm-gcc bootstrap on
332 // PPC (the libcall is not available).
333 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::ppcf128, Action: Custom);
334 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::ppcf128, Action: Custom);
335 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::ppcf128, Action: Custom);
336 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::ppcf128, Action: Custom);
337
338 // We do not currently implement these libm ops for PowerPC.
339 setOperationAction(Op: ISD::FFLOOR, VT: MVT::ppcf128, Action: Expand);
340 setOperationAction(Op: ISD::FCEIL, VT: MVT::ppcf128, Action: Expand);
341 setOperationAction(Op: ISD::FTRUNC, VT: MVT::ppcf128, Action: Expand);
342 setOperationAction(Op: ISD::FRINT, VT: MVT::ppcf128, Action: Expand);
343 setOperationAction(Op: ISD::FNEARBYINT, VT: MVT::ppcf128, Action: Expand);
344 setOperationAction(Op: ISD::FREM, VT: MVT::ppcf128, Action: LibCall);
345
346 // PowerPC has no SREM/UREM instructions unless we are on P9
347 // On P9 we may use a hardware instruction to compute the remainder.
348 // When the result of both the remainder and the division is required it is
349 // more efficient to compute the remainder from the result of the division
350 // rather than use the remainder instruction. The instructions are legalized
351 // directly because the DivRemPairsPass performs the transformation at the IR
352 // level.
353 if (Subtarget.isISA3_0()) {
354 setOperationAction(Op: ISD::SREM, VT: MVT::i32, Action: Legal);
355 setOperationAction(Op: ISD::UREM, VT: MVT::i32, Action: Legal);
356 setOperationAction(Op: ISD::SREM, VT: MVT::i64, Action: Legal);
357 setOperationAction(Op: ISD::UREM, VT: MVT::i64, Action: Legal);
358 } else {
359 setOperationAction(Op: ISD::SREM, VT: MVT::i32, Action: Expand);
360 setOperationAction(Op: ISD::UREM, VT: MVT::i32, Action: Expand);
361 setOperationAction(Op: ISD::SREM, VT: MVT::i64, Action: Expand);
362 setOperationAction(Op: ISD::UREM, VT: MVT::i64, Action: Expand);
363 }
364
365 // Don't use SMUL_LOHI/UMUL_LOHI or SDIVREM/UDIVREM to lower SREM/UREM.
366 setOperationAction(Op: ISD::UMUL_LOHI, VT: MVT::i32, Action: Expand);
367 setOperationAction(Op: ISD::SMUL_LOHI, VT: MVT::i32, Action: Expand);
368 setOperationAction(Op: ISD::UMUL_LOHI, VT: MVT::i64, Action: Expand);
369 setOperationAction(Op: ISD::SMUL_LOHI, VT: MVT::i64, Action: Expand);
370 setOperationAction(Op: ISD::UDIVREM, VT: MVT::i32, Action: Expand);
371 setOperationAction(Op: ISD::SDIVREM, VT: MVT::i32, Action: Expand);
372 setOperationAction(Op: ISD::UDIVREM, VT: MVT::i64, Action: Expand);
373 setOperationAction(Op: ISD::SDIVREM, VT: MVT::i64, Action: Expand);
374
375 // Handle constrained floating-point operations of scalar.
376 // TODO: Handle SPE specific operation.
377 setOperationAction(Op: ISD::STRICT_FADD, VT: MVT::f32, Action: Legal);
378 setOperationAction(Op: ISD::STRICT_FSUB, VT: MVT::f32, Action: Legal);
379 setOperationAction(Op: ISD::STRICT_FMUL, VT: MVT::f32, Action: Legal);
380 setOperationAction(Op: ISD::STRICT_FDIV, VT: MVT::f32, Action: Legal);
381 setOperationAction(Op: ISD::STRICT_FP_ROUND, VT: MVT::f32, Action: Legal);
382
383 setOperationAction(Op: ISD::STRICT_FADD, VT: MVT::f64, Action: Legal);
384 setOperationAction(Op: ISD::STRICT_FSUB, VT: MVT::f64, Action: Legal);
385 setOperationAction(Op: ISD::STRICT_FMUL, VT: MVT::f64, Action: Legal);
386 setOperationAction(Op: ISD::STRICT_FDIV, VT: MVT::f64, Action: Legal);
387
388 if (!Subtarget.hasSPE()) {
389 setOperationAction(Op: ISD::STRICT_FMA, VT: MVT::f32, Action: Legal);
390 setOperationAction(Op: ISD::STRICT_FMA, VT: MVT::f64, Action: Legal);
391 }
392
393 if (Subtarget.hasVSX()) {
394 setOperationAction(Op: ISD::STRICT_FRINT, VT: MVT::f32, Action: Legal);
395 setOperationAction(Op: ISD::STRICT_FRINT, VT: MVT::f64, Action: Legal);
396 }
397
398 if (Subtarget.hasFSQRT()) {
399 setOperationAction(Op: ISD::STRICT_FSQRT, VT: MVT::f32, Action: Legal);
400 setOperationAction(Op: ISD::STRICT_FSQRT, VT: MVT::f64, Action: Legal);
401 }
402
403 if (Subtarget.hasFPRND()) {
404 setOperationAction(Op: ISD::STRICT_FFLOOR, VT: MVT::f32, Action: Legal);
405 setOperationAction(Op: ISD::STRICT_FCEIL, VT: MVT::f32, Action: Legal);
406 setOperationAction(Op: ISD::STRICT_FTRUNC, VT: MVT::f32, Action: Legal);
407 setOperationAction(Op: ISD::STRICT_FROUND, VT: MVT::f32, Action: Legal);
408
409 setOperationAction(Op: ISD::STRICT_FFLOOR, VT: MVT::f64, Action: Legal);
410 setOperationAction(Op: ISD::STRICT_FCEIL, VT: MVT::f64, Action: Legal);
411 setOperationAction(Op: ISD::STRICT_FTRUNC, VT: MVT::f64, Action: Legal);
412 setOperationAction(Op: ISD::STRICT_FROUND, VT: MVT::f64, Action: Legal);
413 }
414
415 // We don't support sin/cos/sqrt/fmod/pow
416 setOperationAction(Op: ISD::FSIN , VT: MVT::f64, Action: Expand);
417 setOperationAction(Op: ISD::FCOS , VT: MVT::f64, Action: Expand);
418 setOperationAction(Op: ISD::FSINCOS, VT: MVT::f64, Action: Expand);
419 setOperationAction(Op: ISD::FREM, VT: MVT::f64, Action: LibCall);
420 setOperationAction(Op: ISD::FPOW , VT: MVT::f64, Action: Expand);
421 setOperationAction(Op: ISD::FSIN , VT: MVT::f32, Action: Expand);
422 setOperationAction(Op: ISD::FCOS , VT: MVT::f32, Action: Expand);
423 setOperationAction(Op: ISD::FSINCOS, VT: MVT::f32, Action: Expand);
424 setOperationAction(Op: ISD::FREM, VT: MVT::f32, Action: LibCall);
425 setOperationAction(Op: ISD::FPOW , VT: MVT::f32, Action: Expand);
426
427 // MASS transformation for LLVM intrinsics with replicating fast-math flag
428 // to be consistent to PPCGenScalarMASSEntries pass
429 if (TM.getOptLevel() == CodeGenOptLevel::Aggressive) {
430 setOperationAction(Op: ISD::FSIN , VT: MVT::f64, Action: Custom);
431 setOperationAction(Op: ISD::FCOS , VT: MVT::f64, Action: Custom);
432 setOperationAction(Op: ISD::FPOW , VT: MVT::f64, Action: Custom);
433 setOperationAction(Op: ISD::FLOG, VT: MVT::f64, Action: Custom);
434 setOperationAction(Op: ISD::FLOG10, VT: MVT::f64, Action: Custom);
435 setOperationAction(Op: ISD::FEXP, VT: MVT::f64, Action: Custom);
436 setOperationAction(Op: ISD::FSIN , VT: MVT::f32, Action: Custom);
437 setOperationAction(Op: ISD::FCOS , VT: MVT::f32, Action: Custom);
438 setOperationAction(Op: ISD::FPOW , VT: MVT::f32, Action: Custom);
439 setOperationAction(Op: ISD::FLOG, VT: MVT::f32, Action: Custom);
440 setOperationAction(Op: ISD::FLOG10, VT: MVT::f32, Action: Custom);
441 setOperationAction(Op: ISD::FEXP, VT: MVT::f32, Action: Custom);
442 }
443
444 if (Subtarget.hasSPE()) {
445 setOperationAction(Op: ISD::FMA , VT: MVT::f64, Action: Expand);
446 setOperationAction(Op: ISD::FMA , VT: MVT::f32, Action: Expand);
447 } else {
448 setOperationAction(Op: ISD::FMA , VT: MVT::f64, Action: Legal);
449 setOperationAction(Op: ISD::FMA , VT: MVT::f32, Action: Legal);
450 setOperationAction(Op: ISD::GET_ROUNDING, VT: MVT::i32, Action: Custom);
451 setOperationAction(Op: ISD::SET_ROUNDING, VT: MVT::Other, Action: Custom);
452 }
453
454 if (Subtarget.hasSPE())
455 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::f32, Action: Expand);
456
457 // If we're enabling GP optimizations, use hardware square root
458 if (!Subtarget.hasFSQRT() && !(Subtarget.hasFRSQRTE() && Subtarget.hasFRE()))
459 setOperationAction(Op: ISD::FSQRT, VT: MVT::f64, Action: Expand);
460
461 if (!Subtarget.hasFSQRT() &&
462 !(Subtarget.hasFRSQRTES() && Subtarget.hasFRES()))
463 setOperationAction(Op: ISD::FSQRT, VT: MVT::f32, Action: Expand);
464
465 if (Subtarget.hasFCPSGN()) {
466 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::f64, Action: Legal);
467 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::f32, Action: Legal);
468 } else {
469 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::f64, Action: Expand);
470 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::f32, Action: Expand);
471 }
472
473 if (Subtarget.hasFPRND()) {
474 setOperationAction(Op: ISD::FFLOOR, VT: MVT::f64, Action: Legal);
475 setOperationAction(Op: ISD::FCEIL, VT: MVT::f64, Action: Legal);
476 setOperationAction(Op: ISD::FTRUNC, VT: MVT::f64, Action: Legal);
477 setOperationAction(Op: ISD::FROUND, VT: MVT::f64, Action: Legal);
478
479 setOperationAction(Op: ISD::FFLOOR, VT: MVT::f32, Action: Legal);
480 setOperationAction(Op: ISD::FCEIL, VT: MVT::f32, Action: Legal);
481 setOperationAction(Op: ISD::FTRUNC, VT: MVT::f32, Action: Legal);
482 setOperationAction(Op: ISD::FROUND, VT: MVT::f32, Action: Legal);
483 }
484
485 // Prior to P10, PowerPC does not have BSWAP, but we can use vector BSWAP
486 // instruction xxbrd to speed up scalar BSWAP64.
487 if (Subtarget.isISA3_1()) {
488 setOperationAction(Op: ISD::BSWAP, VT: MVT::i32, Action: Legal);
489 setOperationAction(Op: ISD::BSWAP, VT: MVT::i64, Action: Legal);
490 } else {
491 setOperationAction(Op: ISD::BSWAP, VT: MVT::i32, Action: Expand);
492 setOperationAction(Op: ISD::BSWAP, VT: MVT::i64,
493 Action: ((Subtarget.hasP8Vector()) && isPPC64) ? Custom
494 : Expand);
495 }
496
497 // CTPOP or CTTZ were introduced in P8/P9 respectively
498 if (Subtarget.isISA3_0()) {
499 setOperationAction(Op: ISD::CTTZ , VT: MVT::i32 , Action: Legal);
500 setOperationAction(Op: ISD::CTTZ , VT: MVT::i64 , Action: Legal);
501 } else {
502 setOperationAction(Op: ISD::CTTZ , VT: MVT::i32 , Action: Expand);
503 setOperationAction(Op: ISD::CTTZ , VT: MVT::i64 , Action: Expand);
504 }
505
506 if (Subtarget.hasPOPCNTD() == PPCSubtarget::POPCNTD_Fast) {
507 setOperationAction(Op: ISD::CTPOP, VT: MVT::i32 , Action: Legal);
508 setOperationAction(Op: ISD::CTPOP, VT: MVT::i64 , Action: Legal);
509 } else {
510 setOperationAction(Op: ISD::CTPOP, VT: MVT::i32 , Action: Expand);
511 setOperationAction(Op: ISD::CTPOP, VT: MVT::i64 , Action: Expand);
512 }
513
514 // PowerPC does not have ROTR
515 setOperationAction(Op: ISD::ROTR, VT: MVT::i32 , Action: Expand);
516 setOperationAction(Op: ISD::ROTR, VT: MVT::i64 , Action: Expand);
517
518 if (!Subtarget.useCRBits()) {
519 // PowerPC does not have Select
520 setOperationAction(Op: ISD::SELECT, VT: MVT::i32, Action: Expand);
521 setOperationAction(Op: ISD::SELECT, VT: MVT::i64, Action: Expand);
522 setOperationAction(Op: ISD::SELECT, VT: MVT::f32, Action: Expand);
523 setOperationAction(Op: ISD::SELECT, VT: MVT::f64, Action: Expand);
524 }
525
526 // PowerPC wants to turn select_cc of FP into fsel when possible.
527 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::f32, Action: Custom);
528 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::f64, Action: Custom);
529
530 // PowerPC wants to optimize integer setcc a bit
531 if (!Subtarget.useCRBits())
532 setOperationAction(Op: ISD::SETCC, VT: MVT::i32, Action: Custom);
533
534 if (Subtarget.hasFPU()) {
535 setOperationAction(Op: ISD::STRICT_FSETCC, VT: MVT::f32, Action: Legal);
536 setOperationAction(Op: ISD::STRICT_FSETCC, VT: MVT::f64, Action: Legal);
537 setOperationAction(Op: ISD::STRICT_FSETCC, VT: MVT::f128, Action: Legal);
538
539 setOperationAction(Op: ISD::STRICT_FSETCCS, VT: MVT::f32, Action: Legal);
540 setOperationAction(Op: ISD::STRICT_FSETCCS, VT: MVT::f64, Action: Legal);
541 setOperationAction(Op: ISD::STRICT_FSETCCS, VT: MVT::f128, Action: Legal);
542 }
543
544 // PowerPC does not have BRCOND which requires SetCC
545 if (!Subtarget.useCRBits())
546 setOperationAction(Op: ISD::BRCOND, VT: MVT::Other, Action: Expand);
547
548 setOperationAction(Op: ISD::BR_JT, VT: MVT::Other, Action: Expand);
549
550 if (Subtarget.hasSPE()) {
551 // SPE has built-in conversions
552 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::i32, Action: Legal);
553 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::i32, Action: Legal);
554 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::i32, Action: Legal);
555 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::i32, Action: Legal);
556 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i32, Action: Legal);
557 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::i32, Action: Legal);
558
559 // SPE supports signaling compare of f32/f64.
560 // But it doesn't comply IEEE-754 rules for comparing
561 // special values like NaNs, Infs.
562 setOperationAction(Op: ISD::SETCC, VT: MVT::f32, Action: Custom);
563 setOperationAction(Op: ISD::SETCC, VT: MVT::f64, Action: Custom);
564 setOperationAction(Op: ISD::STRICT_FSETCCS, VT: MVT::f32, Action: Custom);
565 setOperationAction(Op: ISD::STRICT_FSETCCS, VT: MVT::f64, Action: Custom);
566 setOperationAction(Op: ISD::STRICT_FSETCC, VT: MVT::f32, Action: Custom);
567 setOperationAction(Op: ISD::STRICT_FSETCC, VT: MVT::f64, Action: Custom);
568 setOperationAction(Op: ISD::BR_CC, VT: MVT::f32, Action: Custom);
569 setOperationAction(Op: ISD::BR_CC, VT: MVT::f64, Action: Custom);
570 } else {
571 // PowerPC turns FP_TO_SINT into FCTIWZ and some load/stores.
572 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::i32, Action: Custom);
573 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::i32, Action: Custom);
574
575 // PowerPC does not have [U|S]INT_TO_FP
576 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::i32, Action: Expand);
577 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::i32, Action: Expand);
578 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i32, Action: Expand);
579 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::i32, Action: Expand);
580 }
581
582 if (Subtarget.hasDirectMove() && isPPC64) {
583 setOperationAction(Op: ISD::BITCAST, VT: MVT::f32, Action: Legal);
584 setOperationAction(Op: ISD::BITCAST, VT: MVT::i32, Action: Legal);
585 setOperationAction(Op: ISD::BITCAST, VT: MVT::i64, Action: Legal);
586 setOperationAction(Op: ISD::BITCAST, VT: MVT::f64, Action: Legal);
587
588 setOperationAction(Op: ISD::STRICT_LRINT, VT: MVT::f64, Action: Custom);
589 setOperationAction(Op: ISD::STRICT_LRINT, VT: MVT::f32, Action: Custom);
590 setOperationAction(Op: ISD::STRICT_LLRINT, VT: MVT::f64, Action: Custom);
591 setOperationAction(Op: ISD::STRICT_LLRINT, VT: MVT::f32, Action: Custom);
592 setOperationAction(Op: ISD::STRICT_LROUND, VT: MVT::f64, Action: Custom);
593 setOperationAction(Op: ISD::STRICT_LROUND, VT: MVT::f32, Action: Custom);
594 setOperationAction(Op: ISD::STRICT_LLROUND, VT: MVT::f64, Action: Custom);
595 setOperationAction(Op: ISD::STRICT_LLROUND, VT: MVT::f32, Action: Custom);
596 } else {
597 setOperationAction(Op: ISD::BITCAST, VT: MVT::f32, Action: Expand);
598 setOperationAction(Op: ISD::BITCAST, VT: MVT::i32, Action: Expand);
599 setOperationAction(Op: ISD::BITCAST, VT: MVT::i64, Action: Expand);
600 setOperationAction(Op: ISD::BITCAST, VT: MVT::f64, Action: Expand);
601 }
602
603 // We cannot sextinreg(i1). Expand to shifts.
604 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i1, Action: Expand);
605
606 // Custom handling for PowerPC ucmp instruction
607 setOperationAction(Op: ISD::UCMP, VT: MVT::i32, Action: Custom);
608 setOperationAction(Op: ISD::UCMP, VT: MVT::i64, Action: isPPC64 ? Custom : Expand);
609 setOperationAction(Op: ISD::ABDU, VT: MVT::i32, Action: Custom);
610 setOperationAction(Op: ISD::ABDU, VT: MVT::i64, Action: isPPC64 ? Custom : Expand);
611
612 // NOTE: EH_SJLJ_SETJMP/_LONGJMP supported here is NOT intended to support
613 // SjLj exception handling but a light-weight setjmp/longjmp replacement to
614 // support continuation, user-level threading, and etc.. As a result, no
615 // other SjLj exception interfaces are implemented and please don't build
616 // your own exception handling based on them.
617 // LLVM/Clang supports zero-cost DWARF exception handling.
618 setOperationAction(Op: ISD::EH_SJLJ_SETJMP, VT: MVT::i32, Action: Custom);
619 setOperationAction(Op: ISD::EH_SJLJ_LONGJMP, VT: MVT::Other, Action: Custom);
620
621 // We want to legalize GlobalAddress and ConstantPool nodes into the
622 // appropriate instructions to materialize the address.
623 setOperationAction(Op: ISD::GlobalAddress, VT: MVT::i32, Action: Custom);
624 setOperationAction(Op: ISD::GlobalTLSAddress, VT: MVT::i32, Action: Custom);
625 setOperationAction(Op: ISD::BlockAddress, VT: MVT::i32, Action: Custom);
626 setOperationAction(Op: ISD::ConstantPool, VT: MVT::i32, Action: Custom);
627 setOperationAction(Op: ISD::JumpTable, VT: MVT::i32, Action: Custom);
628 setOperationAction(Op: ISD::GlobalAddress, VT: MVT::i64, Action: Custom);
629 setOperationAction(Op: ISD::GlobalTLSAddress, VT: MVT::i64, Action: Custom);
630 setOperationAction(Op: ISD::BlockAddress, VT: MVT::i64, Action: Custom);
631 setOperationAction(Op: ISD::ConstantPool, VT: MVT::i64, Action: Custom);
632 setOperationAction(Op: ISD::JumpTable, VT: MVT::i64, Action: Custom);
633
634 // TRAP is legal.
635 setOperationAction(Op: ISD::TRAP, VT: MVT::Other, Action: Legal);
636
637 // TRAMPOLINE is custom lowered.
638 setOperationAction(Op: ISD::INIT_TRAMPOLINE, VT: MVT::Other, Action: Custom);
639 setOperationAction(Op: ISD::ADJUST_TRAMPOLINE, VT: MVT::Other, Action: Custom);
640
641 // VASTART needs to be custom lowered to use the VarArgsFrameIndex
642 setOperationAction(Op: ISD::VASTART , VT: MVT::Other, Action: Custom);
643
644 if (Subtarget.is64BitELFABI()) {
645 // VAARG always uses double-word chunks, so promote anything smaller.
646 setOperationAction(Op: ISD::VAARG, VT: MVT::i1, Action: Promote);
647 AddPromotedToType(Opc: ISD::VAARG, OrigVT: MVT::i1, DestVT: MVT::i64);
648 setOperationAction(Op: ISD::VAARG, VT: MVT::i8, Action: Promote);
649 AddPromotedToType(Opc: ISD::VAARG, OrigVT: MVT::i8, DestVT: MVT::i64);
650 setOperationAction(Op: ISD::VAARG, VT: MVT::i16, Action: Promote);
651 AddPromotedToType(Opc: ISD::VAARG, OrigVT: MVT::i16, DestVT: MVT::i64);
652 setOperationAction(Op: ISD::VAARG, VT: MVT::i32, Action: Promote);
653 AddPromotedToType(Opc: ISD::VAARG, OrigVT: MVT::i32, DestVT: MVT::i64);
654 setOperationAction(Op: ISD::VAARG, VT: MVT::Other, Action: Expand);
655 } else if (Subtarget.is32BitELFABI()) {
656 // VAARG is custom lowered with the 32-bit SVR4 ABI.
657 setOperationAction(Op: ISD::VAARG, VT: MVT::Other, Action: Custom);
658 setOperationAction(Op: ISD::VAARG, VT: MVT::i64, Action: Custom);
659 } else
660 setOperationAction(Op: ISD::VAARG, VT: MVT::Other, Action: Expand);
661
662 // VACOPY is custom lowered with the 32-bit SVR4 ABI.
663 if (Subtarget.is32BitELFABI())
664 setOperationAction(Op: ISD::VACOPY , VT: MVT::Other, Action: Custom);
665 else
666 setOperationAction(Op: ISD::VACOPY , VT: MVT::Other, Action: Expand);
667
668 // Use the default implementation.
669 setOperationAction(Op: ISD::VAEND , VT: MVT::Other, Action: Expand);
670 setOperationAction(Op: ISD::STACKSAVE , VT: MVT::Other, Action: Expand);
671 setOperationAction(Op: ISD::STACKRESTORE , VT: MVT::Other, Action: Custom);
672 setOperationAction(Op: ISD::DYNAMIC_STACKALLOC, VT: MVT::i32 , Action: Custom);
673 setOperationAction(Op: ISD::DYNAMIC_STACKALLOC, VT: MVT::i64 , Action: Custom);
674 setOperationAction(Op: ISD::GET_DYNAMIC_AREA_OFFSET, VT: MVT::i32, Action: Custom);
675 setOperationAction(Op: ISD::GET_DYNAMIC_AREA_OFFSET, VT: MVT::i64, Action: Custom);
676 setOperationAction(Op: ISD::EH_DWARF_CFA, VT: MVT::i32, Action: Custom);
677 setOperationAction(Op: ISD::EH_DWARF_CFA, VT: MVT::i64, Action: Custom);
678
679 if (Subtarget.isISA3_0() && isPPC64) {
680 setOperationAction(Op: ISD::VP_STORE, VT: MVT::v16i1, Action: Custom);
681 setOperationAction(Op: ISD::VP_STORE, VT: MVT::v8i1, Action: Custom);
682 setOperationAction(Op: ISD::VP_STORE, VT: MVT::v4i1, Action: Custom);
683 setOperationAction(Op: ISD::VP_STORE, VT: MVT::v2i1, Action: Custom);
684 setOperationAction(Op: ISD::VP_LOAD, VT: MVT::v16i1, Action: Custom);
685 setOperationAction(Op: ISD::VP_LOAD, VT: MVT::v8i1, Action: Custom);
686 setOperationAction(Op: ISD::VP_LOAD, VT: MVT::v4i1, Action: Custom);
687 setOperationAction(Op: ISD::VP_LOAD, VT: MVT::v2i1, Action: Custom);
688 }
689
690 // We want to custom lower some of our intrinsics.
691 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::Other, Action: Custom);
692 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::f64, Action: Custom);
693 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::ppcf128, Action: Custom);
694 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::v4f32, Action: Custom);
695 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::v2f64, Action: Custom);
696
697 // To handle counter-based loop conditions.
698 setOperationAction(Op: ISD::INTRINSIC_W_CHAIN, VT: MVT::i1, Action: Custom);
699 setOperationAction(Op: ISD::INTRINSIC_W_CHAIN, VT: MVT::Other, Action: Custom);
700
701 setOperationAction(Op: ISD::INTRINSIC_VOID, VT: MVT::i8, Action: Custom);
702 setOperationAction(Op: ISD::INTRINSIC_VOID, VT: MVT::i16, Action: Custom);
703 setOperationAction(Op: ISD::INTRINSIC_VOID, VT: MVT::i32, Action: Custom);
704 setOperationAction(Op: ISD::INTRINSIC_VOID, VT: MVT::Other, Action: Custom);
705
706 // Comparisons that require checking two conditions.
707 if (Subtarget.hasSPE()) {
708 setCondCodeAction(CCs: ISD::SETO, VT: MVT::f32, Action: Expand);
709 setCondCodeAction(CCs: ISD::SETO, VT: MVT::f64, Action: Expand);
710 setCondCodeAction(CCs: ISD::SETUO, VT: MVT::f32, Action: Expand);
711 setCondCodeAction(CCs: ISD::SETUO, VT: MVT::f64, Action: Expand);
712 }
713 setCondCodeAction(CCs: ISD::SETULT, VT: MVT::f32, Action: Expand);
714 setCondCodeAction(CCs: ISD::SETULT, VT: MVT::f64, Action: Expand);
715 setCondCodeAction(CCs: ISD::SETUGT, VT: MVT::f32, Action: Expand);
716 setCondCodeAction(CCs: ISD::SETUGT, VT: MVT::f64, Action: Expand);
717 setCondCodeAction(CCs: ISD::SETUEQ, VT: MVT::f32, Action: Expand);
718 setCondCodeAction(CCs: ISD::SETUEQ, VT: MVT::f64, Action: Expand);
719 setCondCodeAction(CCs: ISD::SETOGE, VT: MVT::f32, Action: Expand);
720 setCondCodeAction(CCs: ISD::SETOGE, VT: MVT::f64, Action: Expand);
721 setCondCodeAction(CCs: ISD::SETOLE, VT: MVT::f32, Action: Expand);
722 setCondCodeAction(CCs: ISD::SETOLE, VT: MVT::f64, Action: Expand);
723 setCondCodeAction(CCs: ISD::SETONE, VT: MVT::f32, Action: Expand);
724 setCondCodeAction(CCs: ISD::SETONE, VT: MVT::f64, Action: Expand);
725
726 setOperationAction(Op: ISD::STRICT_FP_EXTEND, VT: MVT::f32, Action: Legal);
727 setOperationAction(Op: ISD::STRICT_FP_EXTEND, VT: MVT::f64, Action: Legal);
728
729 if (Subtarget.has64BitSupport()) {
730 // They also have instructions for converting between i64 and fp.
731 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::i64, Action: Custom);
732 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::i64, Action: Expand);
733 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::i64, Action: Custom);
734 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::i64, Action: Expand);
735 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::i64, Action: Custom);
736 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i64, Action: Expand);
737 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i64, Action: Custom);
738 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::i64, Action: Expand);
739 // This is just the low 32 bits of a (signed) fp->i64 conversion.
740 // We cannot do this with Promote because i64 is not a legal type.
741 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::i32, Action: Custom);
742 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i32, Action: Custom);
743
744 if (Subtarget.hasLFIWAX() || isPPC64) {
745 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i32, Action: Custom);
746 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::i32, Action: Custom);
747 }
748 } else {
749 // PowerPC does not have FP_TO_UINT on 32-bit implementations.
750 if (Subtarget.hasSPE()) {
751 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::i32, Action: Legal);
752 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i32, Action: Legal);
753 } else {
754 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::i32, Action: Expand);
755 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i32, Action: Expand);
756 }
757 }
758
759 // With the instructions enabled under FPCVT, we can do everything.
760 if (Subtarget.hasFPCVT()) {
761 if (Subtarget.has64BitSupport()) {
762 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::i64, Action: Custom);
763 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::i64, Action: Custom);
764 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::i64, Action: Custom);
765 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::i64, Action: Custom);
766 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::i64, Action: Custom);
767 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i64, Action: Custom);
768 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i64, Action: Custom);
769 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::i64, Action: Custom);
770 }
771
772 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::i32, Action: Custom);
773 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::i32, Action: Custom);
774 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::i32, Action: Custom);
775 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::i32, Action: Custom);
776 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::i32, Action: Custom);
777 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i32, Action: Custom);
778 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i32, Action: Custom);
779 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::i32, Action: Custom);
780 }
781
782 if (Subtarget.use64BitRegs()) {
783 // 64-bit PowerPC implementations can support i64 types directly
784 addRegisterClass(VT: MVT::i64, RC: &PPC::G8RCRegClass);
785 // BUILD_PAIR can't be handled natively, and should be expanded to shl/or
786 setOperationAction(Op: ISD::BUILD_PAIR, VT: MVT::i64, Action: Expand);
787 // 64-bit PowerPC wants to expand i128 shifts itself.
788 setOperationAction(Op: ISD::SHL_PARTS, VT: MVT::i64, Action: Custom);
789 setOperationAction(Op: ISD::SRA_PARTS, VT: MVT::i64, Action: Custom);
790 setOperationAction(Op: ISD::SRL_PARTS, VT: MVT::i64, Action: Custom);
791 } else {
792 // 32-bit PowerPC wants to expand i64 shifts itself.
793 setOperationAction(Op: ISD::SHL_PARTS, VT: MVT::i32, Action: Custom);
794 setOperationAction(Op: ISD::SRA_PARTS, VT: MVT::i32, Action: Custom);
795 setOperationAction(Op: ISD::SRL_PARTS, VT: MVT::i32, Action: Custom);
796 }
797
798 // PowerPC has better expansions for funnel shifts than the generic
799 // TargetLowering::expandFunnelShift.
800 if (Subtarget.has64BitSupport()) {
801 setOperationAction(Op: ISD::FSHL, VT: MVT::i64, Action: Custom);
802 setOperationAction(Op: ISD::FSHR, VT: MVT::i64, Action: Custom);
803 }
804 setOperationAction(Op: ISD::FSHL, VT: MVT::i32, Action: Custom);
805 setOperationAction(Op: ISD::FSHR, VT: MVT::i32, Action: Custom);
806
807 if (Subtarget.hasVSX()) {
808 setOperationAction(Op: ISD::FMAXNUM_IEEE, VT: MVT::f64, Action: Legal);
809 setOperationAction(Op: ISD::FMAXNUM_IEEE, VT: MVT::f32, Action: Legal);
810 setOperationAction(Op: ISD::FMINNUM_IEEE, VT: MVT::f64, Action: Legal);
811 setOperationAction(Op: ISD::FMINNUM_IEEE, VT: MVT::f32, Action: Legal);
812 setOperationAction(Op: ISD::FMAXNUM, VT: MVT::f64, Action: Legal);
813 setOperationAction(Op: ISD::FMAXNUM, VT: MVT::f32, Action: Legal);
814 setOperationAction(Op: ISD::FMINNUM, VT: MVT::f64, Action: Legal);
815 setOperationAction(Op: ISD::FMINNUM, VT: MVT::f32, Action: Legal);
816 setOperationAction(Op: ISD::FCANONICALIZE, VT: MVT::f64, Action: Legal);
817 setOperationAction(Op: ISD::FCANONICALIZE, VT: MVT::f32, Action: Legal);
818 }
819
820 if (Subtarget.hasAltivec()) {
821 for (MVT VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32 }) {
822 setOperationAction(Op: ISD::AVGCEILS, VT, Action: Legal);
823 setOperationAction(Op: ISD::AVGCEILU, VT, Action: Legal);
824 setOperationAction(Op: ISD::SADDSAT, VT, Action: Legal);
825 setOperationAction(Op: ISD::SSUBSAT, VT, Action: Legal);
826 setOperationAction(Op: ISD::UADDSAT, VT, Action: Legal);
827 setOperationAction(Op: ISD::USUBSAT, VT, Action: Legal);
828 }
829 // First set operation action for all vector types to expand. Then we
830 // will selectively turn on ones that can be effectively codegen'd.
831 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
832 // add/sub are legal for all supported vector VT's.
833 setOperationAction(Op: ISD::ADD, VT, Action: Legal);
834 setOperationAction(Op: ISD::SUB, VT, Action: Legal);
835
836 // For v2i64, these are only valid with P8Vector. This is corrected after
837 // the loop.
838 if (VT.getSizeInBits() <= 128 && VT.getScalarSizeInBits() <= 64) {
839 setOperationAction(Op: ISD::SMAX, VT, Action: Legal);
840 setOperationAction(Op: ISD::SMIN, VT, Action: Legal);
841 setOperationAction(Op: ISD::UMAX, VT, Action: Legal);
842 setOperationAction(Op: ISD::UMIN, VT, Action: Legal);
843 }
844 else {
845 setOperationAction(Op: ISD::SMAX, VT, Action: Expand);
846 setOperationAction(Op: ISD::SMIN, VT, Action: Expand);
847 setOperationAction(Op: ISD::UMAX, VT, Action: Expand);
848 setOperationAction(Op: ISD::UMIN, VT, Action: Expand);
849 }
850
851 if (Subtarget.hasVSX()) {
852 setOperationAction(Op: ISD::FMAXNUM_IEEE, VT, Action: Legal);
853 setOperationAction(Op: ISD::FMINNUM_IEEE, VT, Action: Legal);
854 setOperationAction(Op: ISD::FMAXNUM, VT, Action: Legal);
855 setOperationAction(Op: ISD::FMINNUM, VT, Action: Legal);
856 setOperationAction(Op: ISD::FCANONICALIZE, VT, Action: Legal);
857 }
858
859 // Vector instructions introduced in P8
860 if (Subtarget.hasP8Altivec() && (VT.SimpleTy != MVT::v1i128)) {
861 setOperationAction(Op: ISD::CTPOP, VT, Action: Legal);
862 setOperationAction(Op: ISD::CTLZ, VT, Action: Legal);
863 }
864 else {
865 setOperationAction(Op: ISD::CTPOP, VT, Action: Expand);
866 setOperationAction(Op: ISD::CTLZ, VT, Action: Expand);
867 }
868
869 // Vector instructions introduced in P9
870 if (Subtarget.hasP9Altivec() && (VT.SimpleTy != MVT::v1i128))
871 setOperationAction(Op: ISD::CTTZ, VT, Action: Legal);
872 else
873 setOperationAction(Op: ISD::CTTZ, VT, Action: Expand);
874
875 // We promote all shuffles to v16i8.
876 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT, Action: Promote);
877 AddPromotedToType (Opc: ISD::VECTOR_SHUFFLE, OrigVT: VT, DestVT: MVT::v16i8);
878
879 // We promote all non-typed operations to v4i32.
880 setOperationAction(Op: ISD::AND , VT, Action: Promote);
881 AddPromotedToType (Opc: ISD::AND , OrigVT: VT, DestVT: MVT::v4i32);
882 setOperationAction(Op: ISD::OR , VT, Action: Promote);
883 AddPromotedToType (Opc: ISD::OR , OrigVT: VT, DestVT: MVT::v4i32);
884 setOperationAction(Op: ISD::XOR , VT, Action: Promote);
885 AddPromotedToType (Opc: ISD::XOR , OrigVT: VT, DestVT: MVT::v4i32);
886 setOperationAction(Op: ISD::LOAD , VT, Action: Promote);
887 AddPromotedToType (Opc: ISD::LOAD , OrigVT: VT, DestVT: MVT::v4i32);
888 setOperationAction(Op: ISD::SELECT, VT, Action: Promote);
889 AddPromotedToType (Opc: ISD::SELECT, OrigVT: VT, DestVT: MVT::v4i32);
890 setOperationAction(Op: ISD::VSELECT, VT, Action: Legal);
891 setOperationAction(Op: ISD::SELECT_CC, VT, Action: Promote);
892 AddPromotedToType (Opc: ISD::SELECT_CC, OrigVT: VT, DestVT: MVT::v4i32);
893 setOperationAction(Op: ISD::STORE, VT, Action: Promote);
894 AddPromotedToType (Opc: ISD::STORE, OrigVT: VT, DestVT: MVT::v4i32);
895
896 // No other operations are legal.
897 setOperationAction(Op: ISD::MUL , VT, Action: Expand);
898 setOperationAction(Op: ISD::SDIV, VT, Action: Expand);
899 setOperationAction(Op: ISD::SREM, VT, Action: Expand);
900 setOperationAction(Op: ISD::UDIV, VT, Action: Expand);
901 setOperationAction(Op: ISD::UREM, VT, Action: Expand);
902 setOperationAction(Op: ISD::FDIV, VT, Action: Expand);
903 setOperationAction(Op: ISD::FREM, VT, Action: Expand);
904 setOperationAction(Op: ISD::FNEG, VT, Action: Expand);
905 setOperationAction(Op: ISD::FSQRT, VT, Action: Expand);
906 setOperationAction(Op: ISD::FLOG, VT, Action: Expand);
907 setOperationAction(Op: ISD::FLOG10, VT, Action: Expand);
908 setOperationAction(Op: ISD::FLOG2, VT, Action: Expand);
909 setOperationAction(Op: ISD::FEXP, VT, Action: Expand);
910 setOperationAction(Op: ISD::FEXP2, VT, Action: Expand);
911 setOperationAction(Op: ISD::FSIN, VT, Action: Expand);
912 setOperationAction(Op: ISD::FCOS, VT, Action: Expand);
913 setOperationAction(Op: ISD::FABS, VT, Action: Expand);
914 setOperationAction(Op: ISD::FFLOOR, VT, Action: Expand);
915 setOperationAction(Op: ISD::FCEIL, VT, Action: Expand);
916 setOperationAction(Op: ISD::FTRUNC, VT, Action: Expand);
917 setOperationAction(Op: ISD::FRINT, VT, Action: Expand);
918 setOperationAction(Op: ISD::FLDEXP, VT, Action: Expand);
919 setOperationAction(Op: ISD::FNEARBYINT, VT, Action: Expand);
920 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT, Action: Expand);
921 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT, Action: Expand);
922 setOperationAction(Op: ISD::BUILD_VECTOR, VT, Action: Expand);
923 setOperationAction(Op: ISD::MULHU, VT, Action: Expand);
924 setOperationAction(Op: ISD::MULHS, VT, Action: Expand);
925 setOperationAction(Op: ISD::UMUL_LOHI, VT, Action: Expand);
926 setOperationAction(Op: ISD::SMUL_LOHI, VT, Action: Expand);
927 setOperationAction(Op: ISD::UDIVREM, VT, Action: Expand);
928 setOperationAction(Op: ISD::SDIVREM, VT, Action: Expand);
929 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT, Action: Expand);
930 setOperationAction(Op: ISD::FPOW, VT, Action: Expand);
931 setOperationAction(Op: ISD::BSWAP, VT, Action: Expand);
932 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT, Action: Expand);
933 setOperationAction(Op: ISD::ROTL, VT, Action: Expand);
934 setOperationAction(Op: ISD::ROTR, VT, Action: Expand);
935
936 for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) {
937 setTruncStoreAction(ValVT: VT, MemVT: InnerVT, Action: Expand);
938 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: VT, MemVT: InnerVT, Action: Expand);
939 setLoadExtAction(ExtType: ISD::ZEXTLOAD, ValVT: VT, MemVT: InnerVT, Action: Expand);
940 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: InnerVT, Action: Expand);
941 }
942 }
943 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::v4i32, Action: Expand);
944 if (!Subtarget.hasP8Vector()) {
945 setOperationAction(Op: ISD::SMAX, VT: MVT::v2i64, Action: Expand);
946 setOperationAction(Op: ISD::SMIN, VT: MVT::v2i64, Action: Expand);
947 setOperationAction(Op: ISD::UMAX, VT: MVT::v2i64, Action: Expand);
948 setOperationAction(Op: ISD::UMIN, VT: MVT::v2i64, Action: Expand);
949 }
950
951 // We can custom expand all VECTOR_SHUFFLEs to VPERM, others we can handle
952 // with merges, splats, etc.
953 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT: MVT::v16i8, Action: Custom);
954
955 // Vector truncates to sub-word integer that fit in an Altivec/VSX register
956 // are cheap, so handle them before they get expanded to scalar.
957 setOperationAction(Op: ISD::TRUNCATE, VT: MVT::v8i8, Action: Custom);
958 setOperationAction(Op: ISD::TRUNCATE, VT: MVT::v4i8, Action: Custom);
959 setOperationAction(Op: ISD::TRUNCATE, VT: MVT::v2i8, Action: Custom);
960 setOperationAction(Op: ISD::TRUNCATE, VT: MVT::v4i16, Action: Custom);
961 setOperationAction(Op: ISD::TRUNCATE, VT: MVT::v2i16, Action: Custom);
962
963 setOperationAction(Op: ISD::AND , VT: MVT::v4i32, Action: Legal);
964 setOperationAction(Op: ISD::OR , VT: MVT::v4i32, Action: Legal);
965 setOperationAction(Op: ISD::XOR , VT: MVT::v4i32, Action: Legal);
966 setOperationAction(Op: ISD::LOAD , VT: MVT::v4i32, Action: Legal);
967 setOperationAction(Op: ISD::SELECT, VT: MVT::v4i32,
968 Action: Subtarget.useCRBits() ? Legal : Expand);
969 setOperationAction(Op: ISD::STORE , VT: MVT::v4i32, Action: Legal);
970 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::v4i32, Action: Legal);
971 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::v4i32, Action: Legal);
972 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::v4i32, Action: Legal);
973 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::v4i32, Action: Legal);
974 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::v4i32, Action: Legal);
975 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::v4i32, Action: Legal);
976 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v4i32, Action: Legal);
977 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v4i32, Action: Legal);
978 setOperationAction(Op: ISD::FFLOOR, VT: MVT::v4f32, Action: Legal);
979 setOperationAction(Op: ISD::FCEIL, VT: MVT::v4f32, Action: Legal);
980 setOperationAction(Op: ISD::FTRUNC, VT: MVT::v4f32, Action: Legal);
981 setOperationAction(Op: ISD::FNEARBYINT, VT: MVT::v4f32, Action: Legal);
982
983 // Custom lowering ROTL v1i128 to VECTOR_SHUFFLE v16i8.
984 setOperationAction(Op: ISD::ROTL, VT: MVT::v1i128, Action: Custom);
985 // With hasAltivec set, we can lower ISD::ROTL to vrl(b|h|w).
986 if (Subtarget.hasAltivec())
987 for (auto VT : {MVT::v4i32, MVT::v8i16, MVT::v16i8})
988 setOperationAction(Op: ISD::ROTL, VT, Action: Legal);
989 // With hasP8Altivec set, we can lower ISD::ROTL to vrld.
990 if (Subtarget.hasP8Altivec())
991 setOperationAction(Op: ISD::ROTL, VT: MVT::v2i64, Action: Legal);
992
993 addRegisterClass(VT: MVT::v4f32, RC: &PPC::VRRCRegClass);
994 addRegisterClass(VT: MVT::v4i32, RC: &PPC::VRRCRegClass);
995 addRegisterClass(VT: MVT::v8i16, RC: &PPC::VRRCRegClass);
996 addRegisterClass(VT: MVT::v16i8, RC: &PPC::VRRCRegClass);
997
998 setOperationAction(Op: ISD::MUL, VT: MVT::v4f32, Action: Legal);
999 setOperationAction(Op: ISD::FMA, VT: MVT::v4f32, Action: Legal);
1000
1001 if (Subtarget.hasVSX()) {
1002 setOperationAction(Op: ISD::FDIV, VT: MVT::v4f32, Action: Legal);
1003 setOperationAction(Op: ISD::FSQRT, VT: MVT::v4f32, Action: Legal);
1004 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v2f64, Action: Custom);
1005 }
1006
1007 if (Subtarget.hasP8Altivec())
1008 setOperationAction(Op: ISD::MUL, VT: MVT::v4i32, Action: Legal);
1009 else
1010 setOperationAction(Op: ISD::MUL, VT: MVT::v4i32, Action: Custom);
1011
1012 if (Subtarget.isISA3_1()) {
1013 setOperationAction(Op: ISD::MUL, VT: MVT::v2i64, Action: Legal);
1014 setOperationAction(Op: ISD::MULHS, VT: MVT::v2i64, Action: Legal);
1015 setOperationAction(Op: ISD::MULHU, VT: MVT::v2i64, Action: Legal);
1016 setOperationAction(Op: ISD::MULHS, VT: MVT::v4i32, Action: Legal);
1017 setOperationAction(Op: ISD::MULHU, VT: MVT::v4i32, Action: Legal);
1018 setOperationAction(Op: ISD::UDIV, VT: MVT::v2i64, Action: Legal);
1019 setOperationAction(Op: ISD::SDIV, VT: MVT::v2i64, Action: Legal);
1020 setOperationAction(Op: ISD::UDIV, VT: MVT::v4i32, Action: Legal);
1021 setOperationAction(Op: ISD::SDIV, VT: MVT::v4i32, Action: Legal);
1022 setOperationAction(Op: ISD::UREM, VT: MVT::v2i64, Action: Legal);
1023 setOperationAction(Op: ISD::SREM, VT: MVT::v2i64, Action: Legal);
1024 setOperationAction(Op: ISD::UREM, VT: MVT::v4i32, Action: Legal);
1025 setOperationAction(Op: ISD::SREM, VT: MVT::v4i32, Action: Legal);
1026 setOperationAction(Op: ISD::UREM, VT: MVT::v1i128, Action: Legal);
1027 setOperationAction(Op: ISD::SREM, VT: MVT::v1i128, Action: Legal);
1028 setOperationAction(Op: ISD::UDIV, VT: MVT::v1i128, Action: Legal);
1029 setOperationAction(Op: ISD::SDIV, VT: MVT::v1i128, Action: Legal);
1030 setOperationAction(Op: ISD::ROTL, VT: MVT::v1i128, Action: Legal);
1031 }
1032
1033 setOperationAction(Op: ISD::MUL, VT: MVT::v8i16, Action: Legal);
1034 setOperationAction(Op: ISD::MUL, VT: MVT::v16i8, Action: Custom);
1035
1036 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v4f32, Action: Custom);
1037 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v4i32, Action: Custom);
1038 // LE is P8+/64-bit so direct moves are supported and these operations
1039 // are legal. The custom transformation requires 64-bit since we need a
1040 // pair of stores that will cover a 128-bit load for P10.
1041 if (!DisableP10StoreForward && isPPC64 && !Subtarget.isLittleEndian()) {
1042 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v2i64, Action: Custom);
1043 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v8i16, Action: Custom);
1044 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v16i8, Action: Custom);
1045 }
1046
1047 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::v16i8, Action: Custom);
1048 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::v8i16, Action: Custom);
1049 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::v4i32, Action: Custom);
1050 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::v4f32, Action: Custom);
1051
1052 // Altivec does not contain unordered floating-point compare instructions
1053 setCondCodeAction(CCs: ISD::SETUO, VT: MVT::v4f32, Action: Expand);
1054 setCondCodeAction(CCs: ISD::SETUEQ, VT: MVT::v4f32, Action: Expand);
1055 setCondCodeAction(CCs: ISD::SETO, VT: MVT::v4f32, Action: Expand);
1056 setCondCodeAction(CCs: ISD::SETONE, VT: MVT::v4f32, Action: Expand);
1057
1058 if (Subtarget.hasVSX()) {
1059 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v2f64, Action: Legal);
1060 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: MVT::v2f64, Action: Legal);
1061 if (Subtarget.hasP8Vector()) {
1062 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v4f32, Action: Legal);
1063 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: MVT::v4f32, Action: Legal);
1064 }
1065 if (Subtarget.hasDirectMove() && isPPC64) {
1066 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v16i8, Action: Legal);
1067 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v8i16, Action: Legal);
1068 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v4i32, Action: Legal);
1069 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v2i64, Action: Legal);
1070 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: MVT::v16i8, Action: Legal);
1071 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: MVT::v8i16, Action: Legal);
1072 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: MVT::v4i32, Action: Legal);
1073 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: MVT::v2i64, Action: Legal);
1074 }
1075 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: MVT::v2f64, Action: Legal);
1076
1077 // The nearbyint variants are not allowed to raise the inexact exception
1078 // so we can only code-gen them with fpexcept.ignore.
1079 setOperationAction(Op: ISD::STRICT_FNEARBYINT, VT: MVT::f64, Action: Custom);
1080 setOperationAction(Op: ISD::STRICT_FNEARBYINT, VT: MVT::f32, Action: Custom);
1081 setOperationAction(Op: ISD::STRICT_FNEARBYINT, VT: MVT::v2f64, Action: Custom);
1082 setOperationAction(Op: ISD::STRICT_FNEARBYINT, VT: MVT::v4f32, Action: Custom);
1083
1084 setOperationAction(Op: ISD::FFLOOR, VT: MVT::v2f64, Action: Legal);
1085 setOperationAction(Op: ISD::FCEIL, VT: MVT::v2f64, Action: Legal);
1086 setOperationAction(Op: ISD::FTRUNC, VT: MVT::v2f64, Action: Legal);
1087 setOperationAction(Op: ISD::FRINT, VT: MVT::v2f64, Action: Legal);
1088 setOperationAction(Op: ISD::FROUND, VT: MVT::v2f64, Action: Legal);
1089 setOperationAction(Op: ISD::FROUND, VT: MVT::f64, Action: Legal);
1090 setOperationAction(Op: ISD::FRINT, VT: MVT::f64, Action: Legal);
1091
1092 setOperationAction(Op: ISD::FRINT, VT: MVT::v4f32, Action: Legal);
1093 setOperationAction(Op: ISD::FROUND, VT: MVT::v4f32, Action: Legal);
1094 setOperationAction(Op: ISD::FROUND, VT: MVT::f32, Action: Legal);
1095 setOperationAction(Op: ISD::FRINT, VT: MVT::f32, Action: Legal);
1096
1097 setOperationAction(Op: ISD::MUL, VT: MVT::v2f64, Action: Legal);
1098 setOperationAction(Op: ISD::FMA, VT: MVT::v2f64, Action: Legal);
1099
1100 setOperationAction(Op: ISD::FDIV, VT: MVT::v2f64, Action: Legal);
1101 setOperationAction(Op: ISD::FSQRT, VT: MVT::v2f64, Action: Legal);
1102
1103 // Share the Altivec comparison restrictions.
1104 setCondCodeAction(CCs: ISD::SETUO, VT: MVT::v2f64, Action: Expand);
1105 setCondCodeAction(CCs: ISD::SETUEQ, VT: MVT::v2f64, Action: Expand);
1106 setCondCodeAction(CCs: ISD::SETO, VT: MVT::v2f64, Action: Expand);
1107 setCondCodeAction(CCs: ISD::SETONE, VT: MVT::v2f64, Action: Expand);
1108
1109 setOperationAction(Op: ISD::LOAD, VT: MVT::v2f64, Action: Legal);
1110 setOperationAction(Op: ISD::STORE, VT: MVT::v2f64, Action: Legal);
1111
1112 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT: MVT::v2f64, Action: Custom);
1113
1114 if (Subtarget.hasP8Vector())
1115 addRegisterClass(VT: MVT::f32, RC: &PPC::VSSRCRegClass);
1116
1117 addRegisterClass(VT: MVT::f64, RC: &PPC::VSFRCRegClass);
1118
1119 addRegisterClass(VT: MVT::v4i32, RC: &PPC::VSRCRegClass);
1120 addRegisterClass(VT: MVT::v4f32, RC: &PPC::VSRCRegClass);
1121 addRegisterClass(VT: MVT::v2f64, RC: &PPC::VSRCRegClass);
1122
1123 if (Subtarget.hasP8Altivec()) {
1124 setOperationAction(Op: ISD::SHL, VT: MVT::v2i64, Action: Legal);
1125 setOperationAction(Op: ISD::SRA, VT: MVT::v2i64, Action: Legal);
1126 setOperationAction(Op: ISD::SRL, VT: MVT::v2i64, Action: Legal);
1127
1128 // 128 bit shifts can be accomplished via 3 instructions for SHL and
1129 // SRL, but not for SRA because of the instructions available:
1130 // VS{RL} and VS{RL}O. However due to direct move costs, it's not worth
1131 // doing
1132 setOperationAction(Op: ISD::SHL, VT: MVT::v1i128, Action: Expand);
1133 setOperationAction(Op: ISD::SRL, VT: MVT::v1i128, Action: Expand);
1134 setOperationAction(Op: ISD::SRA, VT: MVT::v1i128, Action: Expand);
1135
1136 setOperationAction(Op: ISD::SETCC, VT: MVT::v2i64, Action: Legal);
1137 }
1138 else {
1139 setOperationAction(Op: ISD::SHL, VT: MVT::v2i64, Action: Expand);
1140 setOperationAction(Op: ISD::SRA, VT: MVT::v2i64, Action: Expand);
1141 setOperationAction(Op: ISD::SRL, VT: MVT::v2i64, Action: Expand);
1142
1143 setOperationAction(Op: ISD::SETCC, VT: MVT::v2i64, Action: Custom);
1144
1145 // VSX v2i64 only supports non-arithmetic operations.
1146 setOperationAction(Op: ISD::ADD, VT: MVT::v2i64, Action: Expand);
1147 setOperationAction(Op: ISD::SUB, VT: MVT::v2i64, Action: Expand);
1148 }
1149
1150 if (Subtarget.isISA3_1())
1151 setOperationAction(Op: ISD::SETCC, VT: MVT::v1i128, Action: Legal);
1152 else
1153 setOperationAction(Op: ISD::SETCC, VT: MVT::v1i128, Action: Expand);
1154
1155 setOperationAction(Op: ISD::LOAD, VT: MVT::v2i64, Action: Promote);
1156 AddPromotedToType (Opc: ISD::LOAD, OrigVT: MVT::v2i64, DestVT: MVT::v2f64);
1157 setOperationAction(Op: ISD::STORE, VT: MVT::v2i64, Action: Promote);
1158 AddPromotedToType (Opc: ISD::STORE, OrigVT: MVT::v2i64, DestVT: MVT::v2f64);
1159
1160 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT: MVT::v2i64, Action: Custom);
1161
1162 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::v2i64, Action: Legal);
1163 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::v2i64, Action: Legal);
1164 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::v2i64, Action: Legal);
1165 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::v2i64, Action: Legal);
1166 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v2i64, Action: Legal);
1167 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v2i64, Action: Legal);
1168 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::v2i64, Action: Legal);
1169 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::v2i64, Action: Legal);
1170
1171 // Custom handling for partial vectors of integers converted to
1172 // floating point. We already have optimal handling for v2i32 through
1173 // the DAG combine, so those aren't necessary.
1174 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::v2i8, Action: Custom);
1175 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::v4i8, Action: Custom);
1176 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::v2i16, Action: Custom);
1177 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::v4i16, Action: Custom);
1178 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::v2i8, Action: Custom);
1179 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::v4i8, Action: Custom);
1180 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::v2i16, Action: Custom);
1181 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::v4i16, Action: Custom);
1182 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v2i8, Action: Custom);
1183 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v4i8, Action: Custom);
1184 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v2i16, Action: Custom);
1185 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v4i16, Action: Custom);
1186 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v2i8, Action: Custom);
1187 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v4i8, Action: Custom);
1188 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v2i16, Action: Custom);
1189 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v4i16, Action: Custom);
1190
1191 setOperationAction(Op: ISD::FNEG, VT: MVT::v4f32, Action: Legal);
1192 setOperationAction(Op: ISD::FNEG, VT: MVT::v2f64, Action: Legal);
1193 setOperationAction(Op: ISD::FABS, VT: MVT::v4f32, Action: Legal);
1194 setOperationAction(Op: ISD::FABS, VT: MVT::v2f64, Action: Legal);
1195 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::v4f32, Action: Legal);
1196 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::v2f64, Action: Legal);
1197
1198 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::v2i64, Action: Custom);
1199 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::v2f64, Action: Custom);
1200
1201 // Handle constrained floating-point operations of vector.
1202 // The predictor is `hasVSX` because altivec instruction has
1203 // no exception but VSX vector instruction has.
1204 setOperationAction(Op: ISD::STRICT_FADD, VT: MVT::v4f32, Action: Legal);
1205 setOperationAction(Op: ISD::STRICT_FSUB, VT: MVT::v4f32, Action: Legal);
1206 setOperationAction(Op: ISD::STRICT_FMUL, VT: MVT::v4f32, Action: Legal);
1207 setOperationAction(Op: ISD::STRICT_FDIV, VT: MVT::v4f32, Action: Legal);
1208 setOperationAction(Op: ISD::STRICT_FMA, VT: MVT::v4f32, Action: Legal);
1209 setOperationAction(Op: ISD::STRICT_FSQRT, VT: MVT::v4f32, Action: Legal);
1210 setOperationAction(Op: ISD::STRICT_FMAXNUM, VT: MVT::v4f32, Action: Legal);
1211 setOperationAction(Op: ISD::STRICT_FMINNUM, VT: MVT::v4f32, Action: Legal);
1212 setOperationAction(Op: ISD::STRICT_FRINT, VT: MVT::v4f32, Action: Legal);
1213 setOperationAction(Op: ISD::STRICT_FFLOOR, VT: MVT::v4f32, Action: Legal);
1214 setOperationAction(Op: ISD::STRICT_FCEIL, VT: MVT::v4f32, Action: Legal);
1215 setOperationAction(Op: ISD::STRICT_FTRUNC, VT: MVT::v4f32, Action: Legal);
1216 setOperationAction(Op: ISD::STRICT_FROUND, VT: MVT::v4f32, Action: Legal);
1217
1218 setOperationAction(Op: ISD::STRICT_FADD, VT: MVT::v2f64, Action: Legal);
1219 setOperationAction(Op: ISD::STRICT_FSUB, VT: MVT::v2f64, Action: Legal);
1220 setOperationAction(Op: ISD::STRICT_FMUL, VT: MVT::v2f64, Action: Legal);
1221 setOperationAction(Op: ISD::STRICT_FDIV, VT: MVT::v2f64, Action: Legal);
1222 setOperationAction(Op: ISD::STRICT_FMA, VT: MVT::v2f64, Action: Legal);
1223 setOperationAction(Op: ISD::STRICT_FSQRT, VT: MVT::v2f64, Action: Legal);
1224 setOperationAction(Op: ISD::STRICT_FMAXNUM, VT: MVT::v2f64, Action: Legal);
1225 setOperationAction(Op: ISD::STRICT_FMINNUM, VT: MVT::v2f64, Action: Legal);
1226 setOperationAction(Op: ISD::STRICT_FRINT, VT: MVT::v2f64, Action: Legal);
1227 setOperationAction(Op: ISD::STRICT_FFLOOR, VT: MVT::v2f64, Action: Legal);
1228 setOperationAction(Op: ISD::STRICT_FCEIL, VT: MVT::v2f64, Action: Legal);
1229 setOperationAction(Op: ISD::STRICT_FTRUNC, VT: MVT::v2f64, Action: Legal);
1230 setOperationAction(Op: ISD::STRICT_FROUND, VT: MVT::v2f64, Action: Legal);
1231
1232 addRegisterClass(VT: MVT::v2i64, RC: &PPC::VSRCRegClass);
1233 addRegisterClass(VT: MVT::f128, RC: &PPC::VRRCRegClass);
1234
1235 for (MVT FPT : MVT::fp_valuetypes())
1236 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f128, MemVT: FPT, Action: Expand);
1237
1238 // Expand the SELECT to SELECT_CC
1239 setOperationAction(Op: ISD::SELECT, VT: MVT::f128, Action: Expand);
1240
1241 setTruncStoreAction(ValVT: MVT::f128, MemVT: MVT::f64, Action: Expand);
1242 setTruncStoreAction(ValVT: MVT::f128, MemVT: MVT::f32, Action: Expand);
1243
1244 // No implementation for these ops for PowerPC.
1245 setOperationAction(Op: ISD::FSINCOS, VT: MVT::f128, Action: Expand);
1246 setOperationAction(Op: ISD::FSIN, VT: MVT::f128, Action: Expand);
1247 setOperationAction(Op: ISD::FCOS, VT: MVT::f128, Action: Expand);
1248 setOperationAction(Op: ISD::FPOW, VT: MVT::f128, Action: Expand);
1249 setOperationAction(Op: ISD::FPOWI, VT: MVT::f128, Action: Expand);
1250 setOperationAction(Op: ISD::FREM, VT: MVT::f128, Action: LibCall);
1251 }
1252
1253 if (Subtarget.hasP8Altivec()) {
1254 addRegisterClass(VT: MVT::v2i64, RC: &PPC::VRRCRegClass);
1255 addRegisterClass(VT: MVT::v1i128, RC: &PPC::VRRCRegClass);
1256 }
1257
1258 if (Subtarget.hasP9Vector()) {
1259 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v4i32, Action: Custom);
1260 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v4f32, Action: Custom);
1261
1262 // Test data class instructions store results in CR bits.
1263 if (Subtarget.useCRBits()) {
1264 setOperationAction(Op: ISD::IS_FPCLASS, VT: MVT::f32, Action: Custom);
1265 setOperationAction(Op: ISD::IS_FPCLASS, VT: MVT::f64, Action: Custom);
1266 setOperationAction(Op: ISD::IS_FPCLASS, VT: MVT::f128, Action: Custom);
1267 setOperationAction(Op: ISD::IS_FPCLASS, VT: MVT::ppcf128, Action: Custom);
1268 }
1269
1270 // 128 bit shifts can be accomplished via 3 instructions for SHL and
1271 // SRL, but not for SRA because of the instructions available:
1272 // VS{RL} and VS{RL}O.
1273 setOperationAction(Op: ISD::SHL, VT: MVT::v1i128, Action: Legal);
1274 setOperationAction(Op: ISD::SRL, VT: MVT::v1i128, Action: Legal);
1275 setOperationAction(Op: ISD::SRA, VT: MVT::v1i128, Action: Expand);
1276
1277 setOperationAction(Op: ISD::FADD, VT: MVT::f128, Action: Legal);
1278 setOperationAction(Op: ISD::FSUB, VT: MVT::f128, Action: Legal);
1279 setOperationAction(Op: ISD::FDIV, VT: MVT::f128, Action: Legal);
1280 setOperationAction(Op: ISD::FMUL, VT: MVT::f128, Action: Legal);
1281 setOperationAction(Op: ISD::FP_EXTEND, VT: MVT::f128, Action: Legal);
1282
1283 setOperationAction(Op: ISD::FMA, VT: MVT::f128, Action: Legal);
1284 setCondCodeAction(CCs: ISD::SETULT, VT: MVT::f128, Action: Expand);
1285 setCondCodeAction(CCs: ISD::SETUGT, VT: MVT::f128, Action: Expand);
1286 setCondCodeAction(CCs: ISD::SETUEQ, VT: MVT::f128, Action: Expand);
1287 setCondCodeAction(CCs: ISD::SETOGE, VT: MVT::f128, Action: Expand);
1288 setCondCodeAction(CCs: ISD::SETOLE, VT: MVT::f128, Action: Expand);
1289 setCondCodeAction(CCs: ISD::SETONE, VT: MVT::f128, Action: Expand);
1290
1291 setOperationAction(Op: ISD::FTRUNC, VT: MVT::f128, Action: Legal);
1292 setOperationAction(Op: ISD::FRINT, VT: MVT::f128, Action: Legal);
1293 setOperationAction(Op: ISD::FFLOOR, VT: MVT::f128, Action: Legal);
1294 setOperationAction(Op: ISD::FCEIL, VT: MVT::f128, Action: Legal);
1295 setOperationAction(Op: ISD::FNEARBYINT, VT: MVT::f128, Action: Legal);
1296 setOperationAction(Op: ISD::FROUND, VT: MVT::f128, Action: Legal);
1297
1298 setOperationAction(Op: ISD::FP_ROUND, VT: MVT::f64, Action: Legal);
1299 setOperationAction(Op: ISD::FP_ROUND, VT: MVT::f32, Action: Legal);
1300 setOperationAction(Op: ISD::BITCAST, VT: MVT::i128, Action: Custom);
1301
1302 // Handle constrained floating-point operations of fp128
1303 setOperationAction(Op: ISD::STRICT_FADD, VT: MVT::f128, Action: Legal);
1304 setOperationAction(Op: ISD::STRICT_FSUB, VT: MVT::f128, Action: Legal);
1305 setOperationAction(Op: ISD::STRICT_FMUL, VT: MVT::f128, Action: Legal);
1306 setOperationAction(Op: ISD::STRICT_FDIV, VT: MVT::f128, Action: Legal);
1307 setOperationAction(Op: ISD::STRICT_FMA, VT: MVT::f128, Action: Legal);
1308 setOperationAction(Op: ISD::STRICT_FSQRT, VT: MVT::f128, Action: Legal);
1309 setOperationAction(Op: ISD::STRICT_FP_EXTEND, VT: MVT::f128, Action: Legal);
1310 setOperationAction(Op: ISD::STRICT_FP_ROUND, VT: MVT::f64, Action: Legal);
1311 setOperationAction(Op: ISD::STRICT_FP_ROUND, VT: MVT::f32, Action: Legal);
1312 setOperationAction(Op: ISD::STRICT_FRINT, VT: MVT::f128, Action: Legal);
1313 setOperationAction(Op: ISD::STRICT_FNEARBYINT, VT: MVT::f128, Action: Legal);
1314 setOperationAction(Op: ISD::STRICT_FFLOOR, VT: MVT::f128, Action: Legal);
1315 setOperationAction(Op: ISD::STRICT_FCEIL, VT: MVT::f128, Action: Legal);
1316 setOperationAction(Op: ISD::STRICT_FTRUNC, VT: MVT::f128, Action: Legal);
1317 setOperationAction(Op: ISD::STRICT_FROUND, VT: MVT::f128, Action: Legal);
1318 setOperationAction(Op: ISD::FP_EXTEND, VT: MVT::v2f32, Action: Custom);
1319 setOperationAction(Op: ISD::BSWAP, VT: MVT::v8i16, Action: Legal);
1320 setOperationAction(Op: ISD::BSWAP, VT: MVT::v4i32, Action: Legal);
1321 setOperationAction(Op: ISD::BSWAP, VT: MVT::v2i64, Action: Legal);
1322 setOperationAction(Op: ISD::BSWAP, VT: MVT::v1i128, Action: Legal);
1323 } else if (Subtarget.hasVSX()) {
1324 setOperationAction(Op: ISD::LOAD, VT: MVT::f128, Action: Promote);
1325 setOperationAction(Op: ISD::STORE, VT: MVT::f128, Action: Promote);
1326
1327 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::f128, DestVT: MVT::v4i32);
1328 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::f128, DestVT: MVT::v4i32);
1329
1330 // Set FADD/FSUB as libcall to avoid the legalizer to expand the
1331 // fp_to_uint and int_to_fp.
1332 setOperationAction(Op: ISD::FADD, VT: MVT::f128, Action: LibCall);
1333 setOperationAction(Op: ISD::FSUB, VT: MVT::f128, Action: LibCall);
1334
1335 setOperationAction(Op: ISD::FMUL, VT: MVT::f128, Action: Expand);
1336 setOperationAction(Op: ISD::FDIV, VT: MVT::f128, Action: Expand);
1337 setOperationAction(Op: ISD::FNEG, VT: MVT::f128, Action: Expand);
1338 setOperationAction(Op: ISD::FABS, VT: MVT::f128, Action: Expand);
1339 setOperationAction(Op: ISD::FSQRT, VT: MVT::f128, Action: Expand);
1340 setOperationAction(Op: ISD::FMA, VT: MVT::f128, Action: Expand);
1341 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::f128, Action: Expand);
1342
1343 // Expand the fp_extend if the target type is fp128.
1344 setOperationAction(Op: ISD::FP_EXTEND, VT: MVT::f128, Action: Expand);
1345 setOperationAction(Op: ISD::STRICT_FP_EXTEND, VT: MVT::f128, Action: Expand);
1346
1347 // Expand the fp_round if the source type is fp128.
1348 for (MVT VT : {MVT::f32, MVT::f64}) {
1349 setOperationAction(Op: ISD::FP_ROUND, VT, Action: Custom);
1350 setOperationAction(Op: ISD::STRICT_FP_ROUND, VT, Action: Custom);
1351 }
1352
1353 setOperationAction(Op: ISD::SETCC, VT: MVT::f128, Action: Custom);
1354 setOperationAction(Op: ISD::STRICT_FSETCC, VT: MVT::f128, Action: Custom);
1355 setOperationAction(Op: ISD::STRICT_FSETCCS, VT: MVT::f128, Action: Custom);
1356 setOperationAction(Op: ISD::BR_CC, VT: MVT::f128, Action: Expand);
1357
1358 // Lower following f128 select_cc pattern:
1359 // select_cc x, y, tv, fv, cc -> select_cc (setcc x, y, cc), 0, tv, fv, NE
1360 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::f128, Action: Custom);
1361
1362 // We need to handle f128 SELECT_CC with integer result type.
1363 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::i32, Action: Custom);
1364 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::i64, Action: isPPC64 ? Custom : Expand);
1365 }
1366
1367 if (Subtarget.hasP9Altivec()) {
1368 if (Subtarget.isISA3_1()) {
1369 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v2i64, Action: Legal);
1370 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v8i16, Action: Legal);
1371 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v16i8, Action: Legal);
1372 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v4i32, Action: Legal);
1373 } else {
1374 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v8i16, Action: Custom);
1375 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v16i8, Action: Custom);
1376 }
1377 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::v4i8, Action: Legal);
1378 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::v4i16, Action: Legal);
1379 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::v4i32, Action: Legal);
1380 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::v2i8, Action: Legal);
1381 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::v2i16, Action: Legal);
1382 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::v2i32, Action: Legal);
1383 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::v2i64, Action: Legal);
1384
1385 setOperationAction(Op: ISD::ABDU, VT: MVT::v16i8, Action: Legal);
1386 setOperationAction(Op: ISD::ABDU, VT: MVT::v8i16, Action: Legal);
1387 setOperationAction(Op: ISD::ABDU, VT: MVT::v4i32, Action: Legal);
1388 setOperationAction(Op: ISD::ABDS, VT: MVT::v4i32, Action: Legal);
1389 }
1390
1391 if (Subtarget.hasP10Vector()) {
1392 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::f128, Action: Custom);
1393 }
1394
1395 setOperationAction(Op: ISD::PARTIAL_REDUCE_UMLA, VT: MVT::v16i32, Action: Custom);
1396 setPartialReduceMLAAction(Opc: ISD::PARTIAL_REDUCE_UMLA, AccVT: MVT::v4i32, InputVT: MVT::v8i16,
1397 Action: Legal);
1398 setPartialReduceMLAAction(Opc: ISD::PARTIAL_REDUCE_SMLA, AccVT: MVT::v4i32, InputVT: MVT::v8i16,
1399 Action: Legal);
1400 setPartialReduceMLAAction(Opc: ISD::PARTIAL_REDUCE_UMLA, AccVT: MVT::v4i32, InputVT: MVT::v16i8,
1401 Action: Legal);
1402 setPartialReduceMLAAction(Opc: ISD::PARTIAL_REDUCE_SUMLA, AccVT: MVT::v4i32, InputVT: MVT::v16i8,
1403 Action: Legal);
1404 }
1405
1406 if (Subtarget.pairedVectorMemops()) {
1407 addRegisterClass(VT: MVT::v256i1, RC: &PPC::VSRpRCRegClass);
1408 setOperationAction(Op: ISD::LOAD, VT: MVT::v256i1, Action: Custom);
1409 setOperationAction(Op: ISD::STORE, VT: MVT::v256i1, Action: Custom);
1410 }
1411 if (Subtarget.hasMMA()) {
1412 if (Subtarget.isISAFuture()) {
1413 addRegisterClass(VT: MVT::v512i1, RC: &PPC::WACCRCRegClass);
1414 addRegisterClass(VT: MVT::v1024i1, RC: &PPC::DMRRCRegClass);
1415 addRegisterClass(VT: MVT::v2048i1, RC: &PPC::DMRpRCRegClass);
1416 setOperationAction(Op: ISD::LOAD, VT: MVT::v1024i1, Action: Custom);
1417 setOperationAction(Op: ISD::STORE, VT: MVT::v1024i1, Action: Custom);
1418 setOperationAction(Op: ISD::LOAD, VT: MVT::v2048i1, Action: Custom);
1419 setOperationAction(Op: ISD::STORE, VT: MVT::v2048i1, Action: Custom);
1420 } else {
1421 addRegisterClass(VT: MVT::v512i1, RC: &PPC::UACCRCRegClass);
1422 }
1423 setOperationAction(Op: ISD::LOAD, VT: MVT::v512i1, Action: Custom);
1424 setOperationAction(Op: ISD::STORE, VT: MVT::v512i1, Action: Custom);
1425 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::v512i1, Action: Custom);
1426 }
1427
1428 if (Subtarget.has64BitSupport())
1429 setOperationAction(Op: ISD::PREFETCH, VT: MVT::Other, Action: Legal);
1430
1431 if (Subtarget.isISA3_1())
1432 setOperationAction(Op: ISD::SRA, VT: MVT::v1i128, Action: Legal);
1433
1434 setOperationAction(Op: ISD::READCYCLECOUNTER, VT: MVT::i64, Action: isPPC64 ? Legal : Custom);
1435
1436 if (!isPPC64) {
1437 setOperationAction(Op: ISD::ATOMIC_LOAD, VT: MVT::i64, Action: Expand);
1438 setOperationAction(Op: ISD::ATOMIC_STORE, VT: MVT::i64, Action: Expand);
1439 }
1440
1441 if (shouldInlineQuadwordAtomics()) {
1442 setOperationAction(Op: ISD::ATOMIC_LOAD, VT: MVT::i128, Action: Custom);
1443 setOperationAction(Op: ISD::ATOMIC_STORE, VT: MVT::i128, Action: Custom);
1444 setOperationAction(Op: ISD::INTRINSIC_VOID, VT: MVT::i128, Action: Custom);
1445 }
1446
1447 setBooleanContents(ZeroOrOneBooleanContent);
1448
1449 if (Subtarget.hasAltivec()) {
1450 // Altivec instructions set fields to all zeros or all ones.
1451 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
1452 }
1453
1454 if (shouldInlineQuadwordAtomics())
1455 setMaxAtomicSizeInBitsSupported(128);
1456 else if (isPPC64)
1457 setMaxAtomicSizeInBitsSupported(64);
1458 else
1459 setMaxAtomicSizeInBitsSupported(32);
1460
1461 setStackPointerRegisterToSaveRestore(isPPC64 ? PPC::X1 : PPC::R1);
1462
1463 // We have target-specific dag combine patterns for the following nodes:
1464 setTargetDAGCombine({ISD::AND, ISD::ADD, ISD::XOR, ISD::SHL, ISD::SRA,
1465 ISD::SRL, ISD::MUL, ISD::FMA, ISD::SINT_TO_FP,
1466 ISD::BUILD_VECTOR});
1467 if (Subtarget.hasFPCVT())
1468 setTargetDAGCombine(ISD::UINT_TO_FP);
1469 setTargetDAGCombine({ISD::LOAD, ISD::STORE, ISD::BR_CC});
1470 if (Subtarget.useCRBits())
1471 setTargetDAGCombine(ISD::BRCOND);
1472 setTargetDAGCombine({ISD::BSWAP, ISD::INTRINSIC_WO_CHAIN,
1473 ISD::INTRINSIC_W_CHAIN, ISD::INTRINSIC_VOID});
1474
1475 setTargetDAGCombine({ISD::SIGN_EXTEND, ISD::ZERO_EXTEND, ISD::ANY_EXTEND});
1476
1477 setTargetDAGCombine({ISD::TRUNCATE, ISD::VECTOR_SHUFFLE});
1478
1479 if (Subtarget.useCRBits()) {
1480 setTargetDAGCombine({ISD::SETCC, ISD::SELECT_CC});
1481 }
1482
1483 if (Subtarget.hasP8Vector())
1484 setTargetDAGCombine(ISD::BITCAST);
1485
1486 // With 32 condition bits, we don't need to sink (and duplicate) compares
1487 // aggressively in CodeGenPrep.
1488 if (Subtarget.useCRBits()) {
1489 setJumpIsExpensive();
1490 }
1491
1492 // TODO: The default entry number is set to 64. This stops most jump table
1493 // generation on PPC. But it is good for current PPC HWs because the indirect
1494 // branch instruction mtctr to the jump table may lead to bad branch predict.
1495 // Re-evaluate this value on future HWs that can do better with mtctr.
1496 setMinimumJumpTableEntries(PPCMinimumJumpTableEntries);
1497
1498 // The default minimum of largest number in a BitTest cluster is 3.
1499 setMinimumBitTestCmps(PPCMinimumBitTestCmps);
1500
1501 setMinFunctionAlignment(Align(4));
1502 setMinCmpXchgSizeInBits(Subtarget.hasPartwordAtomics() ? 8 : 32);
1503
1504 auto CPUDirective = Subtarget.getCPUDirective();
1505 switch (CPUDirective) {
1506 default: break;
1507 case PPC::DIR_970:
1508 case PPC::DIR_A2:
1509 case PPC::DIR_E500:
1510 case PPC::DIR_E500mc:
1511 case PPC::DIR_E5500:
1512 case PPC::DIR_PWR4:
1513 case PPC::DIR_PWR5:
1514 case PPC::DIR_PWR5X:
1515 case PPC::DIR_PWR6:
1516 case PPC::DIR_PWR6X:
1517 case PPC::DIR_PWR7:
1518 case PPC::DIR_PWR8:
1519 case PPC::DIR_PWR9:
1520 case PPC::DIR_PWR10:
1521 case PPC::DIR_PWR11:
1522 case PPC::DIR_PWR_FUTURE:
1523 setPrefLoopAlignment(Align(16));
1524 setPrefFunctionAlignment(Align(16));
1525 break;
1526 }
1527
1528 if (Subtarget.enableMachineScheduler())
1529 setSchedulingPreference(Sched::Source);
1530 else
1531 setSchedulingPreference(Sched::Hybrid);
1532
1533 computeRegisterProperties(TRI: STI.getRegisterInfo());
1534
1535 // The Freescale cores do better with aggressive inlining of memcpy and
1536 // friends. GCC uses same threshold of 128 bytes (= 32 word stores).
1537 if (CPUDirective == PPC::DIR_E500mc || CPUDirective == PPC::DIR_E5500) {
1538 MaxStoresPerMemset = 32;
1539 MaxStoresPerMemsetOptSize = 16;
1540 MaxStoresPerMemcpy = 32;
1541 MaxStoresPerMemcpyOptSize = 8;
1542 MaxStoresPerMemmove = 32;
1543 MaxStoresPerMemmoveOptSize = 8;
1544 } else if (CPUDirective == PPC::DIR_A2) {
1545 // The A2 also benefits from (very) aggressive inlining of memcpy and
1546 // friends. The overhead of a the function call, even when warm, can be
1547 // over one hundred cycles.
1548 MaxStoresPerMemset = 128;
1549 MaxStoresPerMemcpy = 128;
1550 MaxStoresPerMemmove = 128;
1551 MaxLoadsPerMemcmp = 128;
1552 } else {
1553 MaxLoadsPerMemcmp = 8;
1554 MaxLoadsPerMemcmpOptSize = 4;
1555 }
1556
1557 // Enable generation of STXVP instructions by default for mcpu=future.
1558 if (CPUDirective == PPC::DIR_PWR_FUTURE &&
1559 DisableAutoPairedVecSt.getNumOccurrences() == 0)
1560 DisableAutoPairedVecSt = false;
1561
1562 IsStrictFPEnabled = true;
1563
1564 // Let the subtarget (CPU) decide if a predictable select is more expensive
1565 // than the corresponding branch. This information is used in CGP to decide
1566 // when to convert selects into branches.
1567 PredictableSelectIsExpensive = Subtarget.isPredictableSelectIsExpensive();
1568
1569 GatherAllAliasesMaxDepth = PPCGatherAllAliasesMaxDepth;
1570}
1571
1572// *********************************** NOTE ************************************
1573// For selecting load and store instructions, the addressing modes are defined
1574// as ComplexPatterns in PPCInstrInfo.td, which are then utilized in the TD
1575// patterns to match the load the store instructions.
1576//
1577// The TD definitions for the addressing modes correspond to their respective
1578// Select<AddrMode>Form() function in PPCISelDAGToDAG.cpp. These functions rely
1579// on SelectOptimalAddrMode(), which calls computeMOFlags() to compute the
1580// address mode flags of a particular node. Afterwards, the computed address
1581// flags are passed into getAddrModeForFlags() in order to retrieve the optimal
1582// addressing mode. SelectOptimalAddrMode() then sets the Base and Displacement
1583// accordingly, based on the preferred addressing mode.
1584//
1585// Within PPCISelLowering.h, there are two enums: MemOpFlags and AddrMode.
1586// MemOpFlags contains all the possible flags that can be used to compute the
1587// optimal addressing mode for load and store instructions.
1588// AddrMode contains all the possible load and store addressing modes available
1589// on Power (such as DForm, DSForm, DQForm, XForm, etc.)
1590//
1591// When adding new load and store instructions, it is possible that new address
1592// flags may need to be added into MemOpFlags, and a new addressing mode will
1593// need to be added to AddrMode. An entry of the new addressing mode (consisting
1594// of the minimal and main distinguishing address flags for the new load/store
1595// instructions) will need to be added into initializeAddrModeMap() below.
1596// Finally, when adding new addressing modes, the getAddrModeForFlags() will
1597// need to be updated to account for selecting the optimal addressing mode.
1598// *****************************************************************************
1599/// Initialize the map that relates the different addressing modes of the load
1600/// and store instructions to a set of flags. This ensures the load/store
1601/// instruction is correctly matched during instruction selection.
1602void PPCTargetLowering::initializeAddrModeMap() {
1603 AddrModesMap[PPC::AM_DForm] = {
1604 // LWZ, STW
1605 PPC::MOF_ZExt | PPC::MOF_RPlusSImm16 | PPC::MOF_WordInt,
1606 PPC::MOF_ZExt | PPC::MOF_RPlusLo | PPC::MOF_WordInt,
1607 PPC::MOF_ZExt | PPC::MOF_NotAddNorCst | PPC::MOF_WordInt,
1608 PPC::MOF_ZExt | PPC::MOF_AddrIsSImm32 | PPC::MOF_WordInt,
1609 // LBZ, LHZ, STB, STH
1610 PPC::MOF_ZExt | PPC::MOF_RPlusSImm16 | PPC::MOF_SubWordInt,
1611 PPC::MOF_ZExt | PPC::MOF_RPlusLo | PPC::MOF_SubWordInt,
1612 PPC::MOF_ZExt | PPC::MOF_NotAddNorCst | PPC::MOF_SubWordInt,
1613 PPC::MOF_ZExt | PPC::MOF_AddrIsSImm32 | PPC::MOF_SubWordInt,
1614 // LHA
1615 PPC::MOF_SExt | PPC::MOF_RPlusSImm16 | PPC::MOF_SubWordInt,
1616 PPC::MOF_SExt | PPC::MOF_RPlusLo | PPC::MOF_SubWordInt,
1617 PPC::MOF_SExt | PPC::MOF_NotAddNorCst | PPC::MOF_SubWordInt,
1618 PPC::MOF_SExt | PPC::MOF_AddrIsSImm32 | PPC::MOF_SubWordInt,
1619 // LFS, LFD, STFS, STFD
1620 PPC::MOF_RPlusSImm16 | PPC::MOF_ScalarFloat | PPC::MOF_SubtargetBeforeP9,
1621 PPC::MOF_RPlusLo | PPC::MOF_ScalarFloat | PPC::MOF_SubtargetBeforeP9,
1622 PPC::MOF_NotAddNorCst | PPC::MOF_ScalarFloat | PPC::MOF_SubtargetBeforeP9,
1623 PPC::MOF_AddrIsSImm32 | PPC::MOF_ScalarFloat | PPC::MOF_SubtargetBeforeP9,
1624 };
1625 AddrModesMap[PPC::AM_DSForm] = {
1626 // LWA
1627 PPC::MOF_SExt | PPC::MOF_RPlusSImm16Mult4 | PPC::MOF_WordInt,
1628 PPC::MOF_SExt | PPC::MOF_NotAddNorCst | PPC::MOF_WordInt,
1629 PPC::MOF_SExt | PPC::MOF_AddrIsSImm32 | PPC::MOF_WordInt,
1630 // LD, STD
1631 PPC::MOF_RPlusSImm16Mult4 | PPC::MOF_DoubleWordInt,
1632 PPC::MOF_NotAddNorCst | PPC::MOF_DoubleWordInt,
1633 PPC::MOF_AddrIsSImm32 | PPC::MOF_DoubleWordInt,
1634 // DFLOADf32, DFLOADf64, DSTOREf32, DSTOREf64
1635 PPC::MOF_RPlusSImm16Mult4 | PPC::MOF_ScalarFloat | PPC::MOF_SubtargetP9,
1636 PPC::MOF_NotAddNorCst | PPC::MOF_ScalarFloat | PPC::MOF_SubtargetP9,
1637 PPC::MOF_AddrIsSImm32 | PPC::MOF_ScalarFloat | PPC::MOF_SubtargetP9,
1638 };
1639 AddrModesMap[PPC::AM_DQForm] = {
1640 // LXV, STXV
1641 PPC::MOF_RPlusSImm16Mult16 | PPC::MOF_Vector | PPC::MOF_SubtargetP9,
1642 PPC::MOF_NotAddNorCst | PPC::MOF_Vector | PPC::MOF_SubtargetP9,
1643 PPC::MOF_AddrIsSImm32 | PPC::MOF_Vector | PPC::MOF_SubtargetP9,
1644 };
1645 AddrModesMap[PPC::AM_PrefixDForm] = {PPC::MOF_RPlusSImm34 |
1646 PPC::MOF_SubtargetP10};
1647 // TODO: Add mapping for quadword load/store.
1648}
1649
1650/// getMaxByValAlign - Helper for getByValTypeAlignment to determine
1651/// the desired ByVal argument alignment.
1652static void getMaxByValAlign(Type *Ty, Align &MaxAlign, Align MaxMaxAlign) {
1653 if (MaxAlign == MaxMaxAlign)
1654 return;
1655 if (VectorType *VTy = dyn_cast<VectorType>(Val: Ty)) {
1656 if (MaxMaxAlign >= 32 &&
1657 VTy->getPrimitiveSizeInBits().getFixedValue() >= 256)
1658 MaxAlign = Align(32);
1659 else if (VTy->getPrimitiveSizeInBits().getFixedValue() >= 128 &&
1660 MaxAlign < 16)
1661 MaxAlign = Align(16);
1662 } else if (ArrayType *ATy = dyn_cast<ArrayType>(Val: Ty)) {
1663 Align EltAlign;
1664 getMaxByValAlign(Ty: ATy->getElementType(), MaxAlign&: EltAlign, MaxMaxAlign);
1665 if (EltAlign > MaxAlign)
1666 MaxAlign = EltAlign;
1667 } else if (StructType *STy = dyn_cast<StructType>(Val: Ty)) {
1668 for (auto *EltTy : STy->elements()) {
1669 Align EltAlign;
1670 getMaxByValAlign(Ty: EltTy, MaxAlign&: EltAlign, MaxMaxAlign);
1671 if (EltAlign > MaxAlign)
1672 MaxAlign = EltAlign;
1673 if (MaxAlign == MaxMaxAlign)
1674 break;
1675 }
1676 }
1677}
1678
1679/// getByValTypeAlignment - Return the desired alignment for ByVal aggregate
1680/// function arguments in the caller parameter area.
1681Align PPCTargetLowering::getByValTypeAlignment(Type *Ty,
1682 const DataLayout &DL) const {
1683 // 16byte and wider vectors are passed on 16byte boundary.
1684 // The rest is 8 on PPC64 and 4 on PPC32 boundary.
1685 Align Alignment = Subtarget.isPPC64() ? Align(8) : Align(4);
1686 if (Subtarget.hasAltivec())
1687 getMaxByValAlign(Ty, MaxAlign&: Alignment, MaxMaxAlign: Align(16));
1688 return Alignment;
1689}
1690
1691bool PPCTargetLowering::useSoftFloat() const {
1692 return Subtarget.useSoftFloat();
1693}
1694
1695bool PPCTargetLowering::hasSPE() const {
1696 return Subtarget.hasSPE();
1697}
1698
1699bool PPCTargetLowering::preferIncOfAddToSubOfNot(EVT VT) const {
1700 return VT.isScalarInteger();
1701}
1702
1703bool PPCTargetLowering::shallExtractConstSplatVectorElementToStore(
1704 Type *VectorTy, unsigned ElemSizeInBits, unsigned &Index) const {
1705 if (!Subtarget.isPPC64() || !Subtarget.hasVSX())
1706 return false;
1707
1708 if (auto *VTy = dyn_cast<VectorType>(Val: VectorTy)) {
1709 if (VTy->getScalarType()->isIntegerTy()) {
1710 // ElemSizeInBits 8/16 can fit in immediate field, not needed here.
1711 if (ElemSizeInBits == 32) {
1712 Index = Subtarget.isLittleEndian() ? 2 : 1;
1713 return true;
1714 }
1715 if (ElemSizeInBits == 64) {
1716 Index = Subtarget.isLittleEndian() ? 1 : 0;
1717 return true;
1718 }
1719 }
1720 }
1721 return false;
1722}
1723
1724EVT PPCTargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &C,
1725 EVT VT) const {
1726 if (!VT.isVector())
1727 return Subtarget.useCRBits() ? MVT::i1 : MVT::i32;
1728
1729 return VT.changeVectorElementTypeToInteger();
1730}
1731
1732bool PPCTargetLowering::enableAggressiveFMAFusion(EVT VT) const {
1733 assert(VT.isFloatingPoint() && "Non-floating-point FMA?");
1734 return true;
1735}
1736
1737//===----------------------------------------------------------------------===//
1738// Node matching predicates, for use by the tblgen matching code.
1739//===----------------------------------------------------------------------===//
1740
1741/// isFloatingPointZero - Return true if this is 0.0 or -0.0.
1742static bool isFloatingPointZero(SDValue Op) {
1743 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Val&: Op))
1744 return CFP->getValueAPF().isZero();
1745 else if (ISD::isEXTLoad(N: Op.getNode()) || ISD::isNON_EXTLoad(N: Op.getNode())) {
1746 // Maybe this has already been legalized into the constant pool?
1747 if (ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(Val: Op.getOperand(i: 1)))
1748 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(Val: CP->getConstVal()))
1749 return CFP->getValueAPF().isZero();
1750 }
1751 return false;
1752}
1753
1754/// isConstantOrUndef - Op is either an undef node or a ConstantSDNode. Return
1755/// true if Op is undef or if it matches the specified value.
1756static bool isConstantOrUndef(int Op, int Val) {
1757 return Op < 0 || Op == Val;
1758}
1759
1760/// isVPKUHUMShuffleMask - Return true if this is the shuffle mask for a
1761/// VPKUHUM instruction.
1762/// The ShuffleKind distinguishes between big-endian operations with
1763/// two different inputs (0), either-endian operations with two identical
1764/// inputs (1), and little-endian operations with two different inputs (2).
1765/// For the latter, the input operands are swapped (see PPCInstrAltivec.td).
1766bool PPC::isVPKUHUMShuffleMask(ShuffleVectorSDNode *N, unsigned ShuffleKind,
1767 SelectionDAG &DAG) {
1768 bool IsLE = DAG.getDataLayout().isLittleEndian();
1769 if (ShuffleKind == 0) {
1770 if (IsLE)
1771 return false;
1772 for (unsigned i = 0; i != 16; ++i)
1773 if (!isConstantOrUndef(Op: N->getMaskElt(Idx: i), Val: i*2+1))
1774 return false;
1775 } else if (ShuffleKind == 2) {
1776 if (!IsLE)
1777 return false;
1778 for (unsigned i = 0; i != 16; ++i)
1779 if (!isConstantOrUndef(Op: N->getMaskElt(Idx: i), Val: i*2))
1780 return false;
1781 } else if (ShuffleKind == 1) {
1782 unsigned j = IsLE ? 0 : 1;
1783 for (unsigned i = 0; i != 8; ++i)
1784 if (!isConstantOrUndef(Op: N->getMaskElt(Idx: i), Val: i*2+j) ||
1785 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+8), Val: i*2+j))
1786 return false;
1787 }
1788 return true;
1789}
1790
1791/// isVPKUWUMShuffleMask - Return true if this is the shuffle mask for a
1792/// VPKUWUM instruction.
1793/// The ShuffleKind distinguishes between big-endian operations with
1794/// two different inputs (0), either-endian operations with two identical
1795/// inputs (1), and little-endian operations with two different inputs (2).
1796/// For the latter, the input operands are swapped (see PPCInstrAltivec.td).
1797bool PPC::isVPKUWUMShuffleMask(ShuffleVectorSDNode *N, unsigned ShuffleKind,
1798 SelectionDAG &DAG) {
1799 bool IsLE = DAG.getDataLayout().isLittleEndian();
1800 if (ShuffleKind == 0) {
1801 if (IsLE)
1802 return false;
1803 for (unsigned i = 0; i != 16; i += 2)
1804 if (!isConstantOrUndef(Op: N->getMaskElt(Idx: i ), Val: i*2+2) ||
1805 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+1), Val: i*2+3))
1806 return false;
1807 } else if (ShuffleKind == 2) {
1808 if (!IsLE)
1809 return false;
1810 for (unsigned i = 0; i != 16; i += 2)
1811 if (!isConstantOrUndef(Op: N->getMaskElt(Idx: i ), Val: i*2) ||
1812 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+1), Val: i*2+1))
1813 return false;
1814 } else if (ShuffleKind == 1) {
1815 unsigned j = IsLE ? 0 : 2;
1816 for (unsigned i = 0; i != 8; i += 2)
1817 if (!isConstantOrUndef(Op: N->getMaskElt(Idx: i ), Val: i*2+j) ||
1818 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+1), Val: i*2+j+1) ||
1819 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+8), Val: i*2+j) ||
1820 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+9), Val: i*2+j+1))
1821 return false;
1822 }
1823 return true;
1824}
1825
1826/// isVPKUDUMShuffleMask - Return true if this is the shuffle mask for a
1827/// VPKUDUM instruction, AND the VPKUDUM instruction exists for the
1828/// current subtarget.
1829///
1830/// The ShuffleKind distinguishes between big-endian operations with
1831/// two different inputs (0), either-endian operations with two identical
1832/// inputs (1), and little-endian operations with two different inputs (2).
1833/// For the latter, the input operands are swapped (see PPCInstrAltivec.td).
1834bool PPC::isVPKUDUMShuffleMask(ShuffleVectorSDNode *N, unsigned ShuffleKind,
1835 SelectionDAG &DAG) {
1836 const PPCSubtarget &Subtarget = DAG.getSubtarget<PPCSubtarget>();
1837 if (!Subtarget.hasP8Vector())
1838 return false;
1839
1840 bool IsLE = DAG.getDataLayout().isLittleEndian();
1841 if (ShuffleKind == 0) {
1842 if (IsLE)
1843 return false;
1844 for (unsigned i = 0; i != 16; i += 4)
1845 if (!isConstantOrUndef(Op: N->getMaskElt(Idx: i ), Val: i*2+4) ||
1846 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+1), Val: i*2+5) ||
1847 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+2), Val: i*2+6) ||
1848 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+3), Val: i*2+7))
1849 return false;
1850 } else if (ShuffleKind == 2) {
1851 if (!IsLE)
1852 return false;
1853 for (unsigned i = 0; i != 16; i += 4)
1854 if (!isConstantOrUndef(Op: N->getMaskElt(Idx: i ), Val: i*2) ||
1855 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+1), Val: i*2+1) ||
1856 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+2), Val: i*2+2) ||
1857 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+3), Val: i*2+3))
1858 return false;
1859 } else if (ShuffleKind == 1) {
1860 unsigned j = IsLE ? 0 : 4;
1861 for (unsigned i = 0; i != 8; i += 4)
1862 if (!isConstantOrUndef(Op: N->getMaskElt(Idx: i ), Val: i*2+j) ||
1863 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+1), Val: i*2+j+1) ||
1864 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+2), Val: i*2+j+2) ||
1865 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+3), Val: i*2+j+3) ||
1866 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+8), Val: i*2+j) ||
1867 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+9), Val: i*2+j+1) ||
1868 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+10), Val: i*2+j+2) ||
1869 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+11), Val: i*2+j+3))
1870 return false;
1871 }
1872 return true;
1873}
1874
1875/// isVMerge - Common function, used to match vmrg* shuffles.
1876///
1877static bool isVMerge(ShuffleVectorSDNode *N, unsigned UnitSize,
1878 unsigned LHSStart, unsigned RHSStart) {
1879 if (N->getValueType(ResNo: 0) != MVT::v16i8)
1880 return false;
1881 assert((UnitSize == 1 || UnitSize == 2 || UnitSize == 4) &&
1882 "Unsupported merge size!");
1883
1884 for (unsigned i = 0; i != 8/UnitSize; ++i) // Step over units
1885 for (unsigned j = 0; j != UnitSize; ++j) { // Step over bytes within unit
1886 if (!isConstantOrUndef(Op: N->getMaskElt(Idx: i*UnitSize*2+j),
1887 Val: LHSStart+j+i*UnitSize) ||
1888 !isConstantOrUndef(Op: N->getMaskElt(Idx: i*UnitSize*2+UnitSize+j),
1889 Val: RHSStart+j+i*UnitSize))
1890 return false;
1891 }
1892 return true;
1893}
1894
1895/// isVMRGLShuffleMask - Return true if this is a shuffle mask suitable for
1896/// a VMRGL* instruction with the specified unit size (1,2 or 4 bytes).
1897/// The ShuffleKind distinguishes between big-endian merges with two
1898/// different inputs (0), either-endian merges with two identical inputs (1),
1899/// and little-endian merges with two different inputs (2). For the latter,
1900/// the input operands are swapped (see PPCInstrAltivec.td).
1901bool PPC::isVMRGLShuffleMask(ShuffleVectorSDNode *N, unsigned UnitSize,
1902 unsigned ShuffleKind, SelectionDAG &DAG) {
1903 if (DAG.getDataLayout().isLittleEndian()) {
1904 if (ShuffleKind == 1) // unary
1905 return isVMerge(N, UnitSize, LHSStart: 0, RHSStart: 0);
1906 else if (ShuffleKind == 2) // swapped
1907 return isVMerge(N, UnitSize, LHSStart: 0, RHSStart: 16);
1908 else
1909 return false;
1910 } else {
1911 if (ShuffleKind == 1) // unary
1912 return isVMerge(N, UnitSize, LHSStart: 8, RHSStart: 8);
1913 else if (ShuffleKind == 0) // normal
1914 return isVMerge(N, UnitSize, LHSStart: 8, RHSStart: 24);
1915 else
1916 return false;
1917 }
1918}
1919
1920/// isVMRGHShuffleMask - Return true if this is a shuffle mask suitable for
1921/// a VMRGH* instruction with the specified unit size (1,2 or 4 bytes).
1922/// The ShuffleKind distinguishes between big-endian merges with two
1923/// different inputs (0), either-endian merges with two identical inputs (1),
1924/// and little-endian merges with two different inputs (2). For the latter,
1925/// the input operands are swapped (see PPCInstrAltivec.td).
1926bool PPC::isVMRGHShuffleMask(ShuffleVectorSDNode *N, unsigned UnitSize,
1927 unsigned ShuffleKind, SelectionDAG &DAG) {
1928 if (DAG.getDataLayout().isLittleEndian()) {
1929 if (ShuffleKind == 1) // unary
1930 return isVMerge(N, UnitSize, LHSStart: 8, RHSStart: 8);
1931 else if (ShuffleKind == 2) // swapped
1932 return isVMerge(N, UnitSize, LHSStart: 8, RHSStart: 24);
1933 else
1934 return false;
1935 } else {
1936 if (ShuffleKind == 1) // unary
1937 return isVMerge(N, UnitSize, LHSStart: 0, RHSStart: 0);
1938 else if (ShuffleKind == 0) // normal
1939 return isVMerge(N, UnitSize, LHSStart: 0, RHSStart: 16);
1940 else
1941 return false;
1942 }
1943}
1944
1945/**
1946 * Common function used to match vmrgew and vmrgow shuffles
1947 *
1948 * The indexOffset determines whether to look for even or odd words in
1949 * the shuffle mask. This is based on the of the endianness of the target
1950 * machine.
1951 * - Little Endian:
1952 * - Use offset of 0 to check for odd elements
1953 * - Use offset of 4 to check for even elements
1954 * - Big Endian:
1955 * - Use offset of 0 to check for even elements
1956 * - Use offset of 4 to check for odd elements
1957 * A detailed description of the vector element ordering for little endian and
1958 * big endian can be found at
1959 * http://www.ibm.com/developerworks/library/l-ibm-xl-c-cpp-compiler/index.html
1960 * Targeting your applications - what little endian and big endian IBM XL C/C++
1961 * compiler differences mean to you
1962 *
1963 * The mask to the shuffle vector instruction specifies the indices of the
1964 * elements from the two input vectors to place in the result. The elements are
1965 * numbered in array-access order, starting with the first vector. These vectors
1966 * are always of type v16i8, thus each vector will contain 16 elements of size
1967 * 8. More info on the shuffle vector can be found in the
1968 * http://llvm.org/docs/LangRef.html#shufflevector-instruction
1969 * Language Reference.
1970 *
1971 * The RHSStartValue indicates whether the same input vectors are used (unary)
1972 * or two different input vectors are used, based on the following:
1973 * - If the instruction uses the same vector for both inputs, the range of the
1974 * indices will be 0 to 15. In this case, the RHSStart value passed should
1975 * be 0.
1976 * - If the instruction has two different vectors then the range of the
1977 * indices will be 0 to 31. In this case, the RHSStart value passed should
1978 * be 16 (indices 0-15 specify elements in the first vector while indices 16
1979 * to 31 specify elements in the second vector).
1980 *
1981 * \param[in] N The shuffle vector SD Node to analyze
1982 * \param[in] IndexOffset Specifies whether to look for even or odd elements
1983 * \param[in] RHSStartValue Specifies the starting index for the righthand input
1984 * vector to the shuffle_vector instruction
1985 * \return true iff this shuffle vector represents an even or odd word merge
1986 */
1987static bool isVMerge(ShuffleVectorSDNode *N, unsigned IndexOffset,
1988 unsigned RHSStartValue) {
1989 if (N->getValueType(ResNo: 0) != MVT::v16i8)
1990 return false;
1991
1992 for (unsigned i = 0; i < 2; ++i)
1993 for (unsigned j = 0; j < 4; ++j)
1994 if (!isConstantOrUndef(Op: N->getMaskElt(Idx: i*4+j),
1995 Val: i*RHSStartValue+j+IndexOffset) ||
1996 !isConstantOrUndef(Op: N->getMaskElt(Idx: i*4+j+8),
1997 Val: i*RHSStartValue+j+IndexOffset+8))
1998 return false;
1999 return true;
2000}
2001
2002/**
2003 * Determine if the specified shuffle mask is suitable for the vmrgew or
2004 * vmrgow instructions.
2005 *
2006 * \param[in] N The shuffle vector SD Node to analyze
2007 * \param[in] CheckEven Check for an even merge (true) or an odd merge (false)
2008 * \param[in] ShuffleKind Identify the type of merge:
2009 * - 0 = big-endian merge with two different inputs;
2010 * - 1 = either-endian merge with two identical inputs;
2011 * - 2 = little-endian merge with two different inputs (inputs are swapped for
2012 * little-endian merges).
2013 * \param[in] DAG The current SelectionDAG
2014 * \return true iff this shuffle mask
2015 */
2016bool PPC::isVMRGEOShuffleMask(ShuffleVectorSDNode *N, bool CheckEven,
2017 unsigned ShuffleKind, SelectionDAG &DAG) {
2018 if (DAG.getDataLayout().isLittleEndian()) {
2019 unsigned indexOffset = CheckEven ? 4 : 0;
2020 if (ShuffleKind == 1) // Unary
2021 return isVMerge(N, IndexOffset: indexOffset, RHSStartValue: 0);
2022 else if (ShuffleKind == 2) // swapped
2023 return isVMerge(N, IndexOffset: indexOffset, RHSStartValue: 16);
2024 else
2025 return false;
2026 }
2027 else {
2028 unsigned indexOffset = CheckEven ? 0 : 4;
2029 if (ShuffleKind == 1) // Unary
2030 return isVMerge(N, IndexOffset: indexOffset, RHSStartValue: 0);
2031 else if (ShuffleKind == 0) // Normal
2032 return isVMerge(N, IndexOffset: indexOffset, RHSStartValue: 16);
2033 else
2034 return false;
2035 }
2036 return false;
2037}
2038
2039/// isVSLDOIShuffleMask - If this is a vsldoi shuffle mask, return the shift
2040/// amount, otherwise return -1.
2041/// The ShuffleKind distinguishes between big-endian operations with two
2042/// different inputs (0), either-endian operations with two identical inputs
2043/// (1), and little-endian operations with two different inputs (2). For the
2044/// latter, the input operands are swapped (see PPCInstrAltivec.td).
2045int PPC::isVSLDOIShuffleMask(SDNode *N, unsigned ShuffleKind,
2046 SelectionDAG &DAG) {
2047 if (N->getValueType(ResNo: 0) != MVT::v16i8)
2048 return -1;
2049
2050 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Val: N);
2051
2052 // Find the first non-undef value in the shuffle mask.
2053 unsigned i;
2054 for (i = 0; i != 16 && SVOp->getMaskElt(Idx: i) < 0; ++i)
2055 /*search*/;
2056
2057 if (i == 16) return -1; // all undef.
2058
2059 // Otherwise, check to see if the rest of the elements are consecutively
2060 // numbered from this value.
2061 unsigned ShiftAmt = SVOp->getMaskElt(Idx: i);
2062 if (ShiftAmt < i) return -1;
2063
2064 ShiftAmt -= i;
2065 bool isLE = DAG.getDataLayout().isLittleEndian();
2066
2067 if ((ShuffleKind == 0 && !isLE) || (ShuffleKind == 2 && isLE)) {
2068 // Check the rest of the elements to see if they are consecutive.
2069 for (++i; i != 16; ++i)
2070 if (!isConstantOrUndef(Op: SVOp->getMaskElt(Idx: i), Val: ShiftAmt+i))
2071 return -1;
2072 } else if (ShuffleKind == 1) {
2073 // Check the rest of the elements to see if they are consecutive.
2074 for (++i; i != 16; ++i)
2075 if (!isConstantOrUndef(Op: SVOp->getMaskElt(Idx: i), Val: (ShiftAmt+i) & 15))
2076 return -1;
2077 } else
2078 return -1;
2079
2080 if (isLE)
2081 ShiftAmt = 16 - ShiftAmt;
2082
2083 return ShiftAmt;
2084}
2085
2086/// isSplatShuffleMask - Return true if the specified VECTOR_SHUFFLE operand
2087/// specifies a splat of a single element that is suitable for input to
2088/// one of the splat operations (VSPLTB/VSPLTH/VSPLTW/XXSPLTW/LXVDSX/etc.).
2089bool PPC::isSplatShuffleMask(ShuffleVectorSDNode *N, unsigned EltSize) {
2090 EVT VT = N->getValueType(ResNo: 0);
2091 if (VT == MVT::v2i64 || VT == MVT::v2f64)
2092 return EltSize == 8 && N->getMaskElt(Idx: 0) == N->getMaskElt(Idx: 1);
2093
2094 assert(VT == MVT::v16i8 && isPowerOf2_32(EltSize) &&
2095 EltSize <= 8 && "Can only handle 1,2,4,8 byte element sizes");
2096
2097 // The consecutive indices need to specify an element, not part of two
2098 // different elements. So abandon ship early if this isn't the case.
2099 if (N->getMaskElt(Idx: 0) % EltSize != 0)
2100 return false;
2101
2102 // This is a splat operation if each element of the permute is the same, and
2103 // if the value doesn't reference the second vector.
2104 unsigned ElementBase = N->getMaskElt(Idx: 0);
2105
2106 // FIXME: Handle UNDEF elements too!
2107 if (ElementBase >= 16)
2108 return false;
2109
2110 // Check that the indices are consecutive, in the case of a multi-byte element
2111 // splatted with a v16i8 mask.
2112 for (unsigned i = 1; i != EltSize; ++i)
2113 if (N->getMaskElt(Idx: i) < 0 || N->getMaskElt(Idx: i) != (int)(i+ElementBase))
2114 return false;
2115
2116 for (unsigned i = EltSize, e = 16; i != e; i += EltSize) {
2117 // An UNDEF element is a sequence of UNDEF bytes.
2118 if (N->getMaskElt(Idx: i) < 0) {
2119 for (unsigned j = 1; j != EltSize; ++j)
2120 if (N->getMaskElt(Idx: i + j) >= 0)
2121 return false;
2122 } else
2123 for (unsigned j = 0; j != EltSize; ++j)
2124 if (N->getMaskElt(Idx: i + j) != N->getMaskElt(Idx: j))
2125 return false;
2126 }
2127 return true;
2128}
2129
2130/// Check that the mask is shuffling N byte elements. Within each N byte
2131/// element of the mask, the indices could be either in increasing or
2132/// decreasing order as long as they are consecutive.
2133/// \param[in] N the shuffle vector SD Node to analyze
2134/// \param[in] Width the element width in bytes, could be 2/4/8/16 (HalfWord/
2135/// Word/DoubleWord/QuadWord).
2136/// \param[in] StepLen the delta indices number among the N byte element, if
2137/// the mask is in increasing/decreasing order then it is 1/-1.
2138/// \return true iff the mask is shuffling N byte elements.
2139static bool isNByteElemShuffleMask(ShuffleVectorSDNode *N, unsigned Width,
2140 int StepLen) {
2141 assert((Width == 2 || Width == 4 || Width == 8 || Width == 16) &&
2142 "Unexpected element width.");
2143 assert((StepLen == 1 || StepLen == -1) && "Unexpected element width.");
2144
2145 unsigned NumOfElem = 16 / Width;
2146 unsigned MaskVal[16]; // Width is never greater than 16
2147 for (unsigned i = 0; i < NumOfElem; ++i) {
2148 MaskVal[0] = N->getMaskElt(Idx: i * Width);
2149 if ((StepLen == 1) && (MaskVal[0] % Width)) {
2150 return false;
2151 } else if ((StepLen == -1) && ((MaskVal[0] + 1) % Width)) {
2152 return false;
2153 }
2154
2155 for (unsigned int j = 1; j < Width; ++j) {
2156 MaskVal[j] = N->getMaskElt(Idx: i * Width + j);
2157 if (MaskVal[j] != MaskVal[j-1] + StepLen) {
2158 return false;
2159 }
2160 }
2161 }
2162
2163 return true;
2164}
2165
2166bool PPC::isXXINSERTWMask(ShuffleVectorSDNode *N, unsigned &ShiftElts,
2167 unsigned &InsertAtByte, bool &Swap, bool IsLE) {
2168 if (!isNByteElemShuffleMask(N, Width: 4, StepLen: 1))
2169 return false;
2170
2171 // Now we look at mask elements 0,4,8,12
2172 unsigned M0 = N->getMaskElt(Idx: 0) / 4;
2173 unsigned M1 = N->getMaskElt(Idx: 4) / 4;
2174 unsigned M2 = N->getMaskElt(Idx: 8) / 4;
2175 unsigned M3 = N->getMaskElt(Idx: 12) / 4;
2176 unsigned LittleEndianShifts[] = { 2, 1, 0, 3 };
2177 unsigned BigEndianShifts[] = { 3, 0, 1, 2 };
2178
2179 // Below, let H and L be arbitrary elements of the shuffle mask
2180 // where H is in the range [4,7] and L is in the range [0,3].
2181 // H, 1, 2, 3 or L, 5, 6, 7
2182 if ((M0 > 3 && M1 == 1 && M2 == 2 && M3 == 3) ||
2183 (M0 < 4 && M1 == 5 && M2 == 6 && M3 == 7)) {
2184 ShiftElts = IsLE ? LittleEndianShifts[M0 & 0x3] : BigEndianShifts[M0 & 0x3];
2185 InsertAtByte = IsLE ? 12 : 0;
2186 Swap = M0 < 4;
2187 return true;
2188 }
2189 // 0, H, 2, 3 or 4, L, 6, 7
2190 if ((M1 > 3 && M0 == 0 && M2 == 2 && M3 == 3) ||
2191 (M1 < 4 && M0 == 4 && M2 == 6 && M3 == 7)) {
2192 ShiftElts = IsLE ? LittleEndianShifts[M1 & 0x3] : BigEndianShifts[M1 & 0x3];
2193 InsertAtByte = IsLE ? 8 : 4;
2194 Swap = M1 < 4;
2195 return true;
2196 }
2197 // 0, 1, H, 3 or 4, 5, L, 7
2198 if ((M2 > 3 && M0 == 0 && M1 == 1 && M3 == 3) ||
2199 (M2 < 4 && M0 == 4 && M1 == 5 && M3 == 7)) {
2200 ShiftElts = IsLE ? LittleEndianShifts[M2 & 0x3] : BigEndianShifts[M2 & 0x3];
2201 InsertAtByte = IsLE ? 4 : 8;
2202 Swap = M2 < 4;
2203 return true;
2204 }
2205 // 0, 1, 2, H or 4, 5, 6, L
2206 if ((M3 > 3 && M0 == 0 && M1 == 1 && M2 == 2) ||
2207 (M3 < 4 && M0 == 4 && M1 == 5 && M2 == 6)) {
2208 ShiftElts = IsLE ? LittleEndianShifts[M3 & 0x3] : BigEndianShifts[M3 & 0x3];
2209 InsertAtByte = IsLE ? 0 : 12;
2210 Swap = M3 < 4;
2211 return true;
2212 }
2213
2214 // If both vector operands for the shuffle are the same vector, the mask will
2215 // contain only elements from the first one and the second one will be undef.
2216 if (N->getOperand(Num: 1).isUndef()) {
2217 ShiftElts = 0;
2218 Swap = true;
2219 unsigned XXINSERTWSrcElem = IsLE ? 2 : 1;
2220 if (M0 == XXINSERTWSrcElem && M1 == 1 && M2 == 2 && M3 == 3) {
2221 InsertAtByte = IsLE ? 12 : 0;
2222 return true;
2223 }
2224 if (M0 == 0 && M1 == XXINSERTWSrcElem && M2 == 2 && M3 == 3) {
2225 InsertAtByte = IsLE ? 8 : 4;
2226 return true;
2227 }
2228 if (M0 == 0 && M1 == 1 && M2 == XXINSERTWSrcElem && M3 == 3) {
2229 InsertAtByte = IsLE ? 4 : 8;
2230 return true;
2231 }
2232 if (M0 == 0 && M1 == 1 && M2 == 2 && M3 == XXINSERTWSrcElem) {
2233 InsertAtByte = IsLE ? 0 : 12;
2234 return true;
2235 }
2236 }
2237
2238 return false;
2239}
2240
2241bool PPC::isXXSLDWIShuffleMask(ShuffleVectorSDNode *N, unsigned &ShiftElts,
2242 bool &Swap, bool IsLE) {
2243 assert(N->getValueType(0) == MVT::v16i8 && "Shuffle vector expects v16i8");
2244 // Ensure each byte index of the word is consecutive.
2245 if (!isNByteElemShuffleMask(N, Width: 4, StepLen: 1))
2246 return false;
2247
2248 // Now we look at mask elements 0,4,8,12, which are the beginning of words.
2249 unsigned M0 = N->getMaskElt(Idx: 0) / 4;
2250 unsigned M1 = N->getMaskElt(Idx: 4) / 4;
2251 unsigned M2 = N->getMaskElt(Idx: 8) / 4;
2252 unsigned M3 = N->getMaskElt(Idx: 12) / 4;
2253
2254 // If both vector operands for the shuffle are the same vector, the mask will
2255 // contain only elements from the first one and the second one will be undef.
2256 if (N->getOperand(Num: 1).isUndef()) {
2257 assert(M0 < 4 && "Indexing into an undef vector?");
2258 if (M1 != (M0 + 1) % 4 || M2 != (M1 + 1) % 4 || M3 != (M2 + 1) % 4)
2259 return false;
2260
2261 ShiftElts = IsLE ? (4 - M0) % 4 : M0;
2262 Swap = false;
2263 return true;
2264 }
2265
2266 // Ensure each word index of the ShuffleVector Mask is consecutive.
2267 if (M1 != (M0 + 1) % 8 || M2 != (M1 + 1) % 8 || M3 != (M2 + 1) % 8)
2268 return false;
2269
2270 if (IsLE) {
2271 if (M0 == 0 || M0 == 7 || M0 == 6 || M0 == 5) {
2272 // Input vectors don't need to be swapped if the leading element
2273 // of the result is one of the 3 left elements of the second vector
2274 // (or if there is no shift to be done at all).
2275 Swap = false;
2276 ShiftElts = (8 - M0) % 8;
2277 } else if (M0 == 4 || M0 == 3 || M0 == 2 || M0 == 1) {
2278 // Input vectors need to be swapped if the leading element
2279 // of the result is one of the 3 left elements of the first vector
2280 // (or if we're shifting by 4 - thereby simply swapping the vectors).
2281 Swap = true;
2282 ShiftElts = (4 - M0) % 4;
2283 }
2284
2285 return true;
2286 } else { // BE
2287 if (M0 == 0 || M0 == 1 || M0 == 2 || M0 == 3) {
2288 // Input vectors don't need to be swapped if the leading element
2289 // of the result is one of the 4 elements of the first vector.
2290 Swap = false;
2291 ShiftElts = M0;
2292 } else if (M0 == 4 || M0 == 5 || M0 == 6 || M0 == 7) {
2293 // Input vectors need to be swapped if the leading element
2294 // of the result is one of the 4 elements of the right vector.
2295 Swap = true;
2296 ShiftElts = M0 - 4;
2297 }
2298
2299 return true;
2300 }
2301}
2302
2303bool static isXXBRShuffleMaskHelper(ShuffleVectorSDNode *N, int Width) {
2304 assert(N->getValueType(0) == MVT::v16i8 && "Shuffle vector expects v16i8");
2305
2306 if (!isNByteElemShuffleMask(N, Width, StepLen: -1))
2307 return false;
2308
2309 for (int i = 0; i < 16; i += Width)
2310 if (N->getMaskElt(Idx: i) != i + Width - 1)
2311 return false;
2312
2313 return true;
2314}
2315
2316bool PPC::isXXBRHShuffleMask(ShuffleVectorSDNode *N) {
2317 return isXXBRShuffleMaskHelper(N, Width: 2);
2318}
2319
2320bool PPC::isXXBRWShuffleMask(ShuffleVectorSDNode *N) {
2321 return isXXBRShuffleMaskHelper(N, Width: 4);
2322}
2323
2324bool PPC::isXXBRDShuffleMask(ShuffleVectorSDNode *N) {
2325 return isXXBRShuffleMaskHelper(N, Width: 8);
2326}
2327
2328bool PPC::isXXBRQShuffleMask(ShuffleVectorSDNode *N) {
2329 return isXXBRShuffleMaskHelper(N, Width: 16);
2330}
2331
2332/// Can node \p N be lowered to an XXPERMDI instruction? If so, set \p Swap
2333/// if the inputs to the instruction should be swapped and set \p DM to the
2334/// value for the immediate.
2335/// Specifically, set \p Swap to true only if \p N can be lowered to XXPERMDI
2336/// AND element 0 of the result comes from the first input (LE) or second input
2337/// (BE). Set \p DM to the calculated result (0-3) only if \p N can be lowered.
2338/// \return true iff the given mask of shuffle node \p N is a XXPERMDI shuffle
2339/// mask.
2340bool PPC::isXXPERMDIShuffleMask(ShuffleVectorSDNode *N, unsigned &DM,
2341 bool &Swap, bool IsLE) {
2342 assert(N->getValueType(0) == MVT::v16i8 && "Shuffle vector expects v16i8");
2343
2344 // Ensure each byte index of the double word is consecutive.
2345 if (!isNByteElemShuffleMask(N, Width: 8, StepLen: 1))
2346 return false;
2347
2348 unsigned M0 = N->getMaskElt(Idx: 0) / 8;
2349 unsigned M1 = N->getMaskElt(Idx: 8) / 8;
2350 assert(((M0 | M1) < 4) && "A mask element out of bounds?");
2351
2352 // If both vector operands for the shuffle are the same vector, the mask will
2353 // contain only elements from the first one and the second one will be undef.
2354 if (N->getOperand(Num: 1).isUndef()) {
2355 if ((M0 | M1) < 2) {
2356 DM = IsLE ? (((~M1) & 1) << 1) + ((~M0) & 1) : (M0 << 1) + (M1 & 1);
2357 Swap = false;
2358 return true;
2359 } else
2360 return false;
2361 }
2362
2363 if (IsLE) {
2364 if (M0 > 1 && M1 < 2) {
2365 Swap = false;
2366 } else if (M0 < 2 && M1 > 1) {
2367 M0 = (M0 + 2) % 4;
2368 M1 = (M1 + 2) % 4;
2369 Swap = true;
2370 } else
2371 return false;
2372
2373 // Note: if control flow comes here that means Swap is already set above
2374 DM = (((~M1) & 1) << 1) + ((~M0) & 1);
2375 return true;
2376 } else { // BE
2377 if (M0 < 2 && M1 > 1) {
2378 Swap = false;
2379 } else if (M0 > 1 && M1 < 2) {
2380 M0 = (M0 + 2) % 4;
2381 M1 = (M1 + 2) % 4;
2382 Swap = true;
2383 } else
2384 return false;
2385
2386 // Note: if control flow comes here that means Swap is already set above
2387 DM = (M0 << 1) + (M1 & 1);
2388 return true;
2389 }
2390}
2391
2392
2393/// getSplatIdxForPPCMnemonics - Return the splat index as a value that is
2394/// appropriate for PPC mnemonics (which have a big endian bias - namely
2395/// elements are counted from the left of the vector register).
2396unsigned PPC::getSplatIdxForPPCMnemonics(SDNode *N, unsigned EltSize,
2397 SelectionDAG &DAG) {
2398 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Val: N);
2399 assert(isSplatShuffleMask(SVOp, EltSize));
2400 EVT VT = SVOp->getValueType(ResNo: 0);
2401
2402 if (VT == MVT::v2i64 || VT == MVT::v2f64)
2403 return DAG.getDataLayout().isLittleEndian() ? 1 - SVOp->getMaskElt(Idx: 0)
2404 : SVOp->getMaskElt(Idx: 0);
2405
2406 if (DAG.getDataLayout().isLittleEndian())
2407 return (16 / EltSize) - 1 - (SVOp->getMaskElt(Idx: 0) / EltSize);
2408 else
2409 return SVOp->getMaskElt(Idx: 0) / EltSize;
2410}
2411
2412/// get_VSPLTI_elt - If this is a build_vector of constants which can be formed
2413/// by using a vspltis[bhw] instruction of the specified element size, return
2414/// the constant being splatted. The ByteSize field indicates the number of
2415/// bytes of each element [124] -> [bhw].
2416SDValue PPC::get_VSPLTI_elt(SDNode *N, unsigned ByteSize, SelectionDAG &DAG) {
2417 SDValue OpVal;
2418
2419 // If ByteSize of the splat is bigger than the element size of the
2420 // build_vector, then we have a case where we are checking for a splat where
2421 // multiple elements of the buildvector are folded together into a single
2422 // logical element of the splat (e.g. "vsplish 1" to splat {0,1}*8).
2423 unsigned EltSize = 16/N->getNumOperands();
2424 if (EltSize < ByteSize) {
2425 unsigned Multiple = ByteSize/EltSize; // Number of BV entries per spltval.
2426 SDValue UniquedVals[4];
2427 assert(Multiple > 1 && Multiple <= 4 && "How can this happen?");
2428
2429 // See if all of the elements in the buildvector agree across.
2430 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
2431 if (N->getOperand(Num: i).isUndef()) continue;
2432 // If the element isn't a constant, bail fully out.
2433 if (!isa<ConstantSDNode>(Val: N->getOperand(Num: i))) return SDValue();
2434
2435 if (!UniquedVals[i&(Multiple-1)].getNode())
2436 UniquedVals[i&(Multiple-1)] = N->getOperand(Num: i);
2437 else if (UniquedVals[i&(Multiple-1)] != N->getOperand(Num: i))
2438 return SDValue(); // no match.
2439 }
2440
2441 // Okay, if we reached this point, UniquedVals[0..Multiple-1] contains
2442 // either constant or undef values that are identical for each chunk. See
2443 // if these chunks can form into a larger vspltis*.
2444
2445 // Check to see if all of the leading entries are either 0 or -1. If
2446 // neither, then this won't fit into the immediate field.
2447 bool LeadingZero = true;
2448 bool LeadingOnes = true;
2449 for (unsigned i = 0; i != Multiple-1; ++i) {
2450 if (!UniquedVals[i].getNode()) continue; // Must have been undefs.
2451
2452 LeadingZero &= isNullConstant(V: UniquedVals[i]);
2453 LeadingOnes &= isAllOnesConstant(V: UniquedVals[i]);
2454 }
2455 // Finally, check the least significant entry.
2456 if (LeadingZero) {
2457 if (!UniquedVals[Multiple-1].getNode())
2458 return DAG.getTargetConstant(Val: 0, DL: SDLoc(N), VT: MVT::i32); // 0,0,0,undef
2459 int Val = UniquedVals[Multiple - 1]->getAsZExtVal();
2460 if (Val < 16) // 0,0,0,4 -> vspltisw(4)
2461 return DAG.getTargetConstant(Val, DL: SDLoc(N), VT: MVT::i32);
2462 }
2463 if (LeadingOnes) {
2464 if (!UniquedVals[Multiple-1].getNode())
2465 return DAG.getTargetConstant(Val: ~0U, DL: SDLoc(N), VT: MVT::i32); // -1,-1,-1,undef
2466 int Val =cast<ConstantSDNode>(Val&: UniquedVals[Multiple-1])->getSExtValue();
2467 if (Val >= -16) // -1,-1,-1,-2 -> vspltisw(-2)
2468 return DAG.getTargetConstant(Val, DL: SDLoc(N), VT: MVT::i32);
2469 }
2470
2471 return SDValue();
2472 }
2473
2474 // Check to see if this buildvec has a single non-undef value in its elements.
2475 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
2476 if (N->getOperand(Num: i).isUndef()) continue;
2477 if (!OpVal.getNode())
2478 OpVal = N->getOperand(Num: i);
2479 else if (OpVal != N->getOperand(Num: i))
2480 return SDValue();
2481 }
2482
2483 if (!OpVal.getNode()) return SDValue(); // All UNDEF: use implicit def.
2484
2485 unsigned ValSizeInBytes = EltSize;
2486 uint64_t Value = 0;
2487 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val&: OpVal)) {
2488 Value = CN->getZExtValue();
2489 } else if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(Val&: OpVal)) {
2490 assert(CN->getValueType(0) == MVT::f32 && "Only one legal FP vector type!");
2491 Value = llvm::bit_cast<uint32_t>(from: CN->getValueAPF().convertToFloat());
2492 }
2493
2494 // If the splat value is larger than the element value, then we can never do
2495 // this splat. The only case that we could fit the replicated bits into our
2496 // immediate field for would be zero, and we prefer to use vxor for it.
2497 if (ValSizeInBytes < ByteSize) return SDValue();
2498
2499 // If the element value is larger than the splat value, check if it consists
2500 // of a repeated bit pattern of size ByteSize.
2501 if (!APInt(ValSizeInBytes * 8, Value).isSplat(SplatSizeInBits: ByteSize * 8))
2502 return SDValue();
2503
2504 // Properly sign extend the value.
2505 int MaskVal = SignExtend32(X: Value, B: ByteSize * 8);
2506
2507 // If this is zero, don't match, zero matches ISD::isBuildVectorAllZeros.
2508 if (MaskVal == 0) return SDValue();
2509
2510 // Finally, if this value fits in a 5 bit sext field, return it
2511 if (SignExtend32<5>(X: MaskVal) == MaskVal)
2512 return DAG.getSignedTargetConstant(Val: MaskVal, DL: SDLoc(N), VT: MVT::i32);
2513 return SDValue();
2514}
2515
2516//===----------------------------------------------------------------------===//
2517// Addressing Mode Selection
2518//===----------------------------------------------------------------------===//
2519
2520/// isIntS16Immediate - This method tests to see if the node is either a 32-bit
2521/// or 64-bit immediate, and if the value can be accurately represented as a
2522/// sign extension from a 16-bit value. If so, this returns true and the
2523/// immediate.
2524bool llvm::isIntS16Immediate(SDNode *N, int16_t &Imm) {
2525 if (!isa<ConstantSDNode>(Val: N))
2526 return false;
2527
2528 Imm = (int16_t)N->getAsZExtVal();
2529 if (N->getValueType(ResNo: 0) == MVT::i32)
2530 return Imm == (int32_t)N->getAsZExtVal();
2531 else
2532 return Imm == (int64_t)N->getAsZExtVal();
2533}
2534bool llvm::isIntS16Immediate(SDValue Op, int16_t &Imm) {
2535 return isIntS16Immediate(N: Op.getNode(), Imm);
2536}
2537
2538/// Used when computing address flags for selecting loads and stores.
2539/// If we have an OR, check if the LHS and RHS are provably disjoint.
2540/// An OR of two provably disjoint values is equivalent to an ADD.
2541/// Most PPC load/store instructions compute the effective address as a sum,
2542/// so doing this conversion is useful.
2543static bool provablyDisjointOr(SelectionDAG &DAG, const SDValue &N) {
2544 if (N.getOpcode() != ISD::OR)
2545 return false;
2546 KnownBits LHSKnown = DAG.computeKnownBits(Op: N.getOperand(i: 0));
2547 if (!LHSKnown.Zero.getBoolValue())
2548 return false;
2549 KnownBits RHSKnown = DAG.computeKnownBits(Op: N.getOperand(i: 1));
2550 return (~(LHSKnown.Zero | RHSKnown.Zero) == 0);
2551}
2552
2553/// SelectAddressEVXRegReg - Given the specified address, check to see if it can
2554/// be represented as an indexed [r+r] operation.
2555bool PPCTargetLowering::SelectAddressEVXRegReg(SDValue N, SDValue &Base,
2556 SDValue &Index,
2557 SelectionDAG &DAG) const {
2558 for (SDNode *U : N->users()) {
2559 if (MemSDNode *Memop = dyn_cast<MemSDNode>(Val: U)) {
2560 if (Memop->getMemoryVT() == MVT::f64) {
2561 Base = N.getOperand(i: 0);
2562 Index = N.getOperand(i: 1);
2563 return true;
2564 }
2565 }
2566 }
2567 return false;
2568}
2569
2570/// isIntS34Immediate - This method tests if value of node given can be
2571/// accurately represented as a sign extension from a 34-bit value. If so,
2572/// this returns true and the immediate.
2573bool llvm::isIntS34Immediate(SDNode *N, int64_t &Imm) {
2574 if (!isa<ConstantSDNode>(Val: N))
2575 return false;
2576
2577 Imm = cast<ConstantSDNode>(Val: N)->getSExtValue();
2578 return isInt<34>(x: Imm);
2579}
2580bool llvm::isIntS34Immediate(SDValue Op, int64_t &Imm) {
2581 return isIntS34Immediate(N: Op.getNode(), Imm);
2582}
2583
2584/// SelectAddressRegReg - Given the specified addressed, check to see if it
2585/// can be represented as an indexed [r+r] operation. Returns false if it
2586/// can be more efficiently represented as [r+imm]. If \p EncodingAlignment is
2587/// non-zero and N can be represented by a base register plus a signed 16-bit
2588/// displacement, make a more precise judgement by checking (displacement % \p
2589/// EncodingAlignment).
2590bool PPCTargetLowering::SelectAddressRegReg(
2591 SDValue N, SDValue &Base, SDValue &Index, SelectionDAG &DAG,
2592 MaybeAlign EncodingAlignment) const {
2593 // If we have a PC Relative target flag don't select as [reg+reg]. It will be
2594 // a [pc+imm].
2595 if (SelectAddressPCRel(N, Base))
2596 return false;
2597
2598 int16_t Imm = 0;
2599 if (N.getOpcode() == ISD::ADD) {
2600 // Is there any SPE load/store (f64), which can't handle 16bit offset?
2601 // SPE load/store can only handle 8-bit offsets.
2602 if (hasSPE() && SelectAddressEVXRegReg(N, Base, Index, DAG))
2603 return true;
2604 if (isIntS16Immediate(Op: N.getOperand(i: 1), Imm) &&
2605 (!EncodingAlignment || isAligned(Lhs: *EncodingAlignment, SizeInBytes: Imm)))
2606 return false; // r+i
2607 if (N.getOperand(i: 1).getOpcode() == PPCISD::Lo)
2608 return false; // r+i
2609
2610 Base = N.getOperand(i: 0);
2611 Index = N.getOperand(i: 1);
2612 return true;
2613 } else if (N.getOpcode() == ISD::OR) {
2614 if (isIntS16Immediate(Op: N.getOperand(i: 1), Imm) &&
2615 (!EncodingAlignment || isAligned(Lhs: *EncodingAlignment, SizeInBytes: Imm)))
2616 return false; // r+i can fold it if we can.
2617
2618 // If this is an or of disjoint bitfields, we can codegen this as an add
2619 // (for better address arithmetic) if the LHS and RHS of the OR are provably
2620 // disjoint.
2621 KnownBits LHSKnown = DAG.computeKnownBits(Op: N.getOperand(i: 0));
2622
2623 if (LHSKnown.Zero.getBoolValue()) {
2624 KnownBits RHSKnown = DAG.computeKnownBits(Op: N.getOperand(i: 1));
2625 // If all of the bits are known zero on the LHS or RHS, the add won't
2626 // carry.
2627 if (~(LHSKnown.Zero | RHSKnown.Zero) == 0) {
2628 Base = N.getOperand(i: 0);
2629 Index = N.getOperand(i: 1);
2630 return true;
2631 }
2632 }
2633 }
2634
2635 return false;
2636}
2637
2638// If we happen to be doing an i64 load or store into a stack slot that has
2639// less than a 4-byte alignment, then the frame-index elimination may need to
2640// use an indexed load or store instruction (because the offset may not be a
2641// multiple of 4). The extra register needed to hold the offset comes from the
2642// register scavenger, and it is possible that the scavenger will need to use
2643// an emergency spill slot. As a result, we need to make sure that a spill slot
2644// is allocated when doing an i64 load/store into a less-than-4-byte-aligned
2645// stack slot.
2646static void fixupFuncForFI(SelectionDAG &DAG, int FrameIdx, EVT VT) {
2647 // FIXME: This does not handle the LWA case.
2648 if (VT != MVT::i64)
2649 return;
2650
2651 // NOTE: We'll exclude negative FIs here, which come from argument
2652 // lowering, because there are no known test cases triggering this problem
2653 // using packed structures (or similar). We can remove this exclusion if
2654 // we find such a test case. The reason why this is so test-case driven is
2655 // because this entire 'fixup' is only to prevent crashes (from the
2656 // register scavenger) on not-really-valid inputs. For example, if we have:
2657 // %a = alloca i1
2658 // %b = bitcast i1* %a to i64*
2659 // store i64* a, i64 b
2660 // then the store should really be marked as 'align 1', but is not. If it
2661 // were marked as 'align 1' then the indexed form would have been
2662 // instruction-selected initially, and the problem this 'fixup' is preventing
2663 // won't happen regardless.
2664 if (FrameIdx < 0)
2665 return;
2666
2667 MachineFunction &MF = DAG.getMachineFunction();
2668 MachineFrameInfo &MFI = MF.getFrameInfo();
2669
2670 if (MFI.getObjectAlign(ObjectIdx: FrameIdx) >= Align(4))
2671 return;
2672
2673 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
2674 FuncInfo->setHasNonRISpills();
2675}
2676
2677/// Returns true if the address N can be represented by a base register plus
2678/// a signed 16-bit displacement [r+imm], and if it is not better
2679/// represented as reg+reg. If \p EncodingAlignment is non-zero, only accept
2680/// displacements that are multiples of that value.
2681bool PPCTargetLowering::SelectAddressRegImm(
2682 SDValue N, SDValue &Disp, SDValue &Base, SelectionDAG &DAG,
2683 MaybeAlign EncodingAlignment) const {
2684 // FIXME dl should come from parent load or store, not from address
2685 SDLoc dl(N);
2686
2687 // If we have a PC Relative target flag don't select as [reg+imm]. It will be
2688 // a [pc+imm].
2689 if (SelectAddressPCRel(N, Base))
2690 return false;
2691
2692 // If this can be more profitably realized as r+r, fail.
2693 if (SelectAddressRegReg(N, Base&: Disp, Index&: Base, DAG, EncodingAlignment))
2694 return false;
2695
2696 if (N.getOpcode() == ISD::ADD) {
2697 int16_t imm = 0;
2698 if (isIntS16Immediate(Op: N.getOperand(i: 1), Imm&: imm) &&
2699 (!EncodingAlignment || isAligned(Lhs: *EncodingAlignment, SizeInBytes: imm))) {
2700 Disp = DAG.getSignedTargetConstant(Val: imm, DL: dl, VT: N.getValueType());
2701 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val: N.getOperand(i: 0))) {
2702 Base = DAG.getTargetFrameIndex(FI: FI->getIndex(), VT: N.getValueType());
2703 fixupFuncForFI(DAG, FrameIdx: FI->getIndex(), VT: N.getValueType());
2704 } else {
2705 Base = N.getOperand(i: 0);
2706 }
2707 return true; // [r+i]
2708 } else if (N.getOperand(i: 1).getOpcode() == PPCISD::Lo) {
2709 // Match LOAD (ADD (X, Lo(G))).
2710 assert(!N.getOperand(1).getConstantOperandVal(1) &&
2711 "Cannot handle constant offsets yet!");
2712 Disp = N.getOperand(i: 1).getOperand(i: 0); // The global address.
2713 assert(Disp.getOpcode() == ISD::TargetGlobalAddress ||
2714 Disp.getOpcode() == ISD::TargetGlobalTLSAddress ||
2715 Disp.getOpcode() == ISD::TargetConstantPool ||
2716 Disp.getOpcode() == ISD::TargetJumpTable);
2717 Base = N.getOperand(i: 0);
2718 return true; // [&g+r]
2719 }
2720 } else if (N.getOpcode() == ISD::OR) {
2721 int16_t imm = 0;
2722 if (isIntS16Immediate(Op: N.getOperand(i: 1), Imm&: imm) &&
2723 (!EncodingAlignment || isAligned(Lhs: *EncodingAlignment, SizeInBytes: imm))) {
2724 // If this is an or of disjoint bitfields, we can codegen this as an add
2725 // (for better address arithmetic) if the LHS and RHS of the OR are
2726 // provably disjoint.
2727 KnownBits LHSKnown = DAG.computeKnownBits(Op: N.getOperand(i: 0));
2728
2729 if ((LHSKnown.Zero.getZExtValue()|~(uint64_t)imm) == ~0ULL) {
2730 // If all of the bits are known zero on the LHS or RHS, the add won't
2731 // carry.
2732 if (FrameIndexSDNode *FI =
2733 dyn_cast<FrameIndexSDNode>(Val: N.getOperand(i: 0))) {
2734 Base = DAG.getTargetFrameIndex(FI: FI->getIndex(), VT: N.getValueType());
2735 fixupFuncForFI(DAG, FrameIdx: FI->getIndex(), VT: N.getValueType());
2736 } else {
2737 Base = N.getOperand(i: 0);
2738 }
2739 Disp = DAG.getTargetConstant(Val: imm, DL: dl, VT: N.getValueType());
2740 return true;
2741 }
2742 }
2743 } else if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val&: N)) {
2744 // Loading from a constant address.
2745
2746 // If this address fits entirely in a 16-bit sext immediate field, codegen
2747 // this as "d, 0"
2748 int16_t Imm;
2749 if (isIntS16Immediate(N: CN, Imm) &&
2750 (!EncodingAlignment || isAligned(Lhs: *EncodingAlignment, SizeInBytes: Imm))) {
2751 Disp = DAG.getTargetConstant(Val: Imm, DL: dl, VT: CN->getValueType(ResNo: 0));
2752 Base = DAG.getRegister(Reg: Subtarget.isPPC64() ? PPC::ZERO8 : PPC::ZERO,
2753 VT: CN->getValueType(ResNo: 0));
2754 return true;
2755 }
2756
2757 // Handle 32-bit sext immediates with LIS + addr mode.
2758 if ((CN->getValueType(ResNo: 0) == MVT::i32 ||
2759 (int64_t)CN->getZExtValue() == (int)CN->getZExtValue()) &&
2760 (!EncodingAlignment ||
2761 isAligned(Lhs: *EncodingAlignment, SizeInBytes: CN->getZExtValue()))) {
2762 int Addr = (int)CN->getZExtValue();
2763
2764 // Otherwise, break this down into an LIS + disp.
2765 Disp = DAG.getTargetConstant(Val: (short)Addr, DL: dl, VT: MVT::i32);
2766
2767 Base = DAG.getTargetConstant(Val: (Addr - (signed short)Addr) >> 16, DL: dl,
2768 VT: MVT::i32);
2769 unsigned Opc = CN->getValueType(ResNo: 0) == MVT::i32 ? PPC::LIS : PPC::LIS8;
2770 Base = SDValue(DAG.getMachineNode(Opcode: Opc, dl, VT: CN->getValueType(ResNo: 0), Op1: Base), 0);
2771 return true;
2772 }
2773 }
2774
2775 Disp = DAG.getTargetConstant(Val: 0, DL: dl, VT: getPointerTy(DL: DAG.getDataLayout()));
2776 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: N)) {
2777 Base = DAG.getTargetFrameIndex(FI: FI->getIndex(), VT: N.getValueType());
2778 fixupFuncForFI(DAG, FrameIdx: FI->getIndex(), VT: N.getValueType());
2779 } else
2780 Base = N;
2781 return true; // [r+0]
2782}
2783
2784/// Similar to the 16-bit case but for instructions that take a 34-bit
2785/// displacement field (prefixed loads/stores).
2786bool PPCTargetLowering::SelectAddressRegImm34(SDValue N, SDValue &Disp,
2787 SDValue &Base,
2788 SelectionDAG &DAG) const {
2789 // Only on 64-bit targets.
2790 if (N.getValueType() != MVT::i64)
2791 return false;
2792
2793 SDLoc dl(N);
2794 int64_t Imm = 0;
2795
2796 if (N.getOpcode() == ISD::ADD) {
2797 if (!isIntS34Immediate(Op: N.getOperand(i: 1), Imm))
2798 return false;
2799 Disp = DAG.getSignedTargetConstant(Val: Imm, DL: dl, VT: N.getValueType());
2800 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val: N.getOperand(i: 0)))
2801 Base = DAG.getTargetFrameIndex(FI: FI->getIndex(), VT: N.getValueType());
2802 else
2803 Base = N.getOperand(i: 0);
2804 return true;
2805 }
2806
2807 if (N.getOpcode() == ISD::OR) {
2808 if (!isIntS34Immediate(Op: N.getOperand(i: 1), Imm))
2809 return false;
2810 // If this is an or of disjoint bitfields, we can codegen this as an add
2811 // (for better address arithmetic) if the LHS and RHS of the OR are
2812 // provably disjoint.
2813 KnownBits LHSKnown = DAG.computeKnownBits(Op: N.getOperand(i: 0));
2814 if ((LHSKnown.Zero.getZExtValue() | ~(uint64_t)Imm) != ~0ULL)
2815 return false;
2816 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val: N.getOperand(i: 0)))
2817 Base = DAG.getTargetFrameIndex(FI: FI->getIndex(), VT: N.getValueType());
2818 else
2819 Base = N.getOperand(i: 0);
2820 Disp = DAG.getSignedTargetConstant(Val: Imm, DL: dl, VT: N.getValueType());
2821 return true;
2822 }
2823
2824 if (isIntS34Immediate(Op: N, Imm)) { // If the address is a 34-bit const.
2825 Disp = DAG.getSignedTargetConstant(Val: Imm, DL: dl, VT: N.getValueType());
2826 Base = DAG.getRegister(Reg: PPC::ZERO8, VT: N.getValueType());
2827 return true;
2828 }
2829
2830 return false;
2831}
2832
2833/// SelectAddressRegRegOnly - Given the specified addressed, force it to be
2834/// represented as an indexed [r+r] operation.
2835bool PPCTargetLowering::SelectAddressRegRegOnly(SDValue N, SDValue &Base,
2836 SDValue &Index,
2837 SelectionDAG &DAG) const {
2838 // Check to see if we can easily represent this as an [r+r] address. This
2839 // will fail if it thinks that the address is more profitably represented as
2840 // reg+imm, e.g. where imm = 0.
2841 if (SelectAddressRegReg(N, Base, Index, DAG))
2842 return true;
2843
2844 // If the address is the result of an add, we will utilize the fact that the
2845 // address calculation includes an implicit add. However, we can reduce
2846 // register pressure if we do not materialize a constant just for use as the
2847 // index register. We only get rid of the add if it is not an add of a
2848 // value and a 16-bit signed constant and both have a single use.
2849 int16_t imm = 0;
2850 if (N.getOpcode() == ISD::ADD &&
2851 (!isIntS16Immediate(Op: N.getOperand(i: 1), Imm&: imm) ||
2852 !N.getOperand(i: 1).hasOneUse() || !N.getOperand(i: 0).hasOneUse())) {
2853 Base = N.getOperand(i: 0);
2854 Index = N.getOperand(i: 1);
2855 return true;
2856 }
2857
2858 // Otherwise, do it the hard way, using R0 as the base register.
2859 Base = DAG.getRegister(Reg: Subtarget.isPPC64() ? PPC::ZERO8 : PPC::ZERO,
2860 VT: N.getValueType());
2861 Index = N;
2862 return true;
2863}
2864
2865template <typename Ty> static bool isValidPCRelNode(SDValue N) {
2866 Ty *PCRelCand = dyn_cast<Ty>(N);
2867 return PCRelCand && (PPCInstrInfo::hasPCRelFlag(TF: PCRelCand->getTargetFlags()));
2868}
2869
2870/// Returns true if this address is a PC Relative address.
2871/// PC Relative addresses are marked with the flag PPCII::MO_PCREL_FLAG
2872/// or if the node opcode is PPCISD::MAT_PCREL_ADDR.
2873bool PPCTargetLowering::SelectAddressPCRel(SDValue N, SDValue &Base) const {
2874 // This is a materialize PC Relative node. Always select this as PC Relative.
2875 Base = N;
2876 if (N.getOpcode() == PPCISD::MAT_PCREL_ADDR)
2877 return true;
2878 if (isValidPCRelNode<ConstantPoolSDNode>(N) ||
2879 isValidPCRelNode<GlobalAddressSDNode>(N) ||
2880 isValidPCRelNode<JumpTableSDNode>(N) ||
2881 isValidPCRelNode<BlockAddressSDNode>(N))
2882 return true;
2883 return false;
2884}
2885
2886/// Returns true if we should use a direct load into vector instruction
2887/// (such as lxsd or lfd), instead of a load into gpr + direct move sequence.
2888static bool usePartialVectorLoads(SDNode *N, const PPCSubtarget& ST) {
2889
2890 // If there are any other uses other than scalar to vector, then we should
2891 // keep it as a scalar load -> direct move pattern to prevent multiple
2892 // loads.
2893 LoadSDNode *LD = dyn_cast<LoadSDNode>(Val: N);
2894 if (!LD)
2895 return false;
2896
2897 EVT MemVT = LD->getMemoryVT();
2898 if (!MemVT.isSimple())
2899 return false;
2900 switch(MemVT.getSimpleVT().SimpleTy) {
2901 case MVT::i64:
2902 break;
2903 case MVT::i32:
2904 if (!ST.hasP8Vector())
2905 return false;
2906 break;
2907 case MVT::i16:
2908 case MVT::i8:
2909 if (!ST.hasP9Vector())
2910 return false;
2911 break;
2912 default:
2913 return false;
2914 }
2915
2916 SDValue LoadedVal(N, 0);
2917 if (!LoadedVal.hasOneUse())
2918 return false;
2919
2920 for (SDUse &Use : LD->uses())
2921 if (Use.getResNo() == 0 &&
2922 Use.getUser()->getOpcode() != ISD::SCALAR_TO_VECTOR &&
2923 Use.getUser()->getOpcode() != PPCISD::SCALAR_TO_VECTOR_PERMUTED)
2924 return false;
2925
2926 return true;
2927}
2928
2929/// getPreIndexedAddressParts - returns true by value, base pointer and
2930/// offset pointer and addressing mode by reference if the node's address
2931/// can be legally represented as pre-indexed load / store address.
2932bool PPCTargetLowering::getPreIndexedAddressParts(SDNode *N, SDValue &Base,
2933 SDValue &Offset,
2934 ISD::MemIndexedMode &AM,
2935 SelectionDAG &DAG) const {
2936 if (DisablePPCPreinc) return false;
2937
2938 bool isLoad = true;
2939 SDValue Ptr;
2940 EVT VT;
2941 Align Alignment;
2942 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Val: N)) {
2943 Ptr = LD->getBasePtr();
2944 VT = LD->getMemoryVT();
2945 Alignment = LD->getAlign();
2946 } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(Val: N)) {
2947 Ptr = ST->getBasePtr();
2948 VT = ST->getMemoryVT();
2949 Alignment = ST->getAlign();
2950 isLoad = false;
2951 } else
2952 return false;
2953
2954 // Do not generate pre-inc forms for specific loads that feed scalar_to_vector
2955 // instructions because we can fold these into a more efficient instruction
2956 // instead, (such as LXSD).
2957 if (isLoad && usePartialVectorLoads(N, ST: Subtarget)) {
2958 return false;
2959 }
2960
2961 // PowerPC doesn't have preinc load/store instructions for vectors
2962 if (VT.isVector())
2963 return false;
2964
2965 if (SelectAddressRegReg(N: Ptr, Base, Index&: Offset, DAG)) {
2966 // Common code will reject creating a pre-inc form if the base pointer
2967 // is a frame index, or if N is a store and the base pointer is either
2968 // the same as or a predecessor of the value being stored. Check for
2969 // those situations here, and try with swapped Base/Offset instead.
2970 bool Swap = false;
2971
2972 if (isa<FrameIndexSDNode>(Val: Base) || isa<RegisterSDNode>(Val: Base))
2973 Swap = true;
2974 else if (!isLoad) {
2975 SDValue Val = cast<StoreSDNode>(Val: N)->getValue();
2976 if (Val == Base || Base.getNode()->isPredecessorOf(N: Val.getNode()))
2977 Swap = true;
2978 }
2979
2980 if (Swap)
2981 std::swap(a&: Base, b&: Offset);
2982
2983 AM = ISD::PRE_INC;
2984 return true;
2985 }
2986
2987 // LDU/STU can only handle immediates that are a multiple of 4.
2988 if (VT != MVT::i64) {
2989 if (!SelectAddressRegImm(N: Ptr, Disp&: Offset, Base, DAG, EncodingAlignment: std::nullopt))
2990 return false;
2991 } else {
2992 // LDU/STU need an address with at least 4-byte alignment.
2993 if (Alignment < Align(4))
2994 return false;
2995
2996 if (!SelectAddressRegImm(N: Ptr, Disp&: Offset, Base, DAG, EncodingAlignment: Align(4)))
2997 return false;
2998 }
2999
3000 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Val: N)) {
3001 // PPC64 doesn't have lwau, but it does have lwaux. Reject preinc load of
3002 // sext i32 to i64 when addr mode is r+i.
3003 if (LD->getValueType(ResNo: 0) == MVT::i64 && LD->getMemoryVT() == MVT::i32 &&
3004 LD->getExtensionType() == ISD::SEXTLOAD &&
3005 isa<ConstantSDNode>(Val: Offset))
3006 return false;
3007 }
3008
3009 AM = ISD::PRE_INC;
3010 return true;
3011}
3012
3013//===----------------------------------------------------------------------===//
3014// LowerOperation implementation
3015//===----------------------------------------------------------------------===//
3016
3017/// Return true if we should reference labels using a PICBase, set the HiOpFlags
3018/// and LoOpFlags to the target MO flags.
3019static void getLabelAccessInfo(bool IsPIC, const PPCSubtarget &Subtarget,
3020 unsigned &HiOpFlags, unsigned &LoOpFlags,
3021 const GlobalValue *GV = nullptr) {
3022 HiOpFlags = PPCII::MO_HA;
3023 LoOpFlags = PPCII::MO_LO;
3024
3025 // Don't use the pic base if not in PIC relocation model.
3026 if (IsPIC) {
3027 HiOpFlags = PPCII::MO_PIC_HA_FLAG;
3028 LoOpFlags = PPCII::MO_PIC_LO_FLAG;
3029 }
3030}
3031
3032static SDValue LowerLabelRef(SDValue HiPart, SDValue LoPart, bool isPIC,
3033 SelectionDAG &DAG) {
3034 SDLoc DL(HiPart);
3035 EVT PtrVT = HiPart.getValueType();
3036 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: PtrVT);
3037
3038 SDValue Hi = DAG.getNode(Opcode: PPCISD::Hi, DL, VT: PtrVT, N1: HiPart, N2: Zero);
3039 SDValue Lo = DAG.getNode(Opcode: PPCISD::Lo, DL, VT: PtrVT, N1: LoPart, N2: Zero);
3040
3041 // With PIC, the first instruction is actually "GR+hi(&G)".
3042 if (isPIC)
3043 Hi = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT,
3044 N1: DAG.getNode(Opcode: PPCISD::GlobalBaseReg, DL, VT: PtrVT), N2: Hi);
3045
3046 // Generate non-pic code that has direct accesses to the constant pool.
3047 // The address of the global is just (hi(&g)+lo(&g)).
3048 return DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: Hi, N2: Lo);
3049}
3050
3051static void setUsesTOCBasePtr(MachineFunction &MF) {
3052 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
3053 FuncInfo->setUsesTOCBasePtr();
3054}
3055
3056static void setUsesTOCBasePtr(SelectionDAG &DAG) {
3057 setUsesTOCBasePtr(DAG.getMachineFunction());
3058}
3059
3060SDValue PPCTargetLowering::getTOCEntry(SelectionDAG &DAG, const SDLoc &dl,
3061 SDValue GA) const {
3062 EVT VT = Subtarget.getScalarIntVT();
3063 SDValue Reg = Subtarget.isPPC64() ? DAG.getRegister(Reg: PPC::X2, VT)
3064 : Subtarget.isAIXABI()
3065 ? DAG.getRegister(Reg: PPC::R2, VT)
3066 : DAG.getNode(Opcode: PPCISD::GlobalBaseReg, DL: dl, VT);
3067 SDValue Ops[] = { GA, Reg };
3068 return DAG.getMemIntrinsicNode(
3069 Opcode: PPCISD::TOC_ENTRY, dl, VTList: DAG.getVTList(VT1: VT, VT2: MVT::Other), Ops, MemVT: VT,
3070 PtrInfo: MachinePointerInfo::getGOT(MF&: DAG.getMachineFunction()), Alignment: std::nullopt,
3071 Flags: MachineMemOperand::MOLoad);
3072}
3073
3074SDValue PPCTargetLowering::LowerConstantPool(SDValue Op,
3075 SelectionDAG &DAG) const {
3076 EVT PtrVT = Op.getValueType();
3077 ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Val&: Op);
3078 const Constant *C = CP->getConstVal();
3079
3080 // 64-bit SVR4 ABI and AIX ABI code are always position-independent.
3081 // The actual address of the GlobalValue is stored in the TOC.
3082 if (Subtarget.is64BitELFABI() || Subtarget.isAIXABI()) {
3083 if (Subtarget.isUsingPCRelativeCalls()) {
3084 SDLoc DL(CP);
3085 EVT Ty = getPointerTy(DL: DAG.getDataLayout());
3086 SDValue ConstPool = DAG.getTargetConstantPool(
3087 C, VT: Ty, Align: CP->getAlign(), Offset: CP->getOffset(), TargetFlags: PPCII::MO_PCREL_FLAG);
3088 return DAG.getNode(Opcode: PPCISD::MAT_PCREL_ADDR, DL, VT: Ty, Operand: ConstPool);
3089 }
3090 setUsesTOCBasePtr(DAG);
3091 SDValue GA = DAG.getTargetConstantPool(C, VT: PtrVT, Align: CP->getAlign(), Offset: 0);
3092 return getTOCEntry(DAG, dl: SDLoc(CP), GA);
3093 }
3094
3095 unsigned MOHiFlag, MOLoFlag;
3096 bool IsPIC = isPositionIndependent();
3097 getLabelAccessInfo(IsPIC, Subtarget, HiOpFlags&: MOHiFlag, LoOpFlags&: MOLoFlag);
3098
3099 if (IsPIC && Subtarget.isSVR4ABI()) {
3100 SDValue GA =
3101 DAG.getTargetConstantPool(C, VT: PtrVT, Align: CP->getAlign(), Offset: PPCII::MO_PIC_FLAG);
3102 return getTOCEntry(DAG, dl: SDLoc(CP), GA);
3103 }
3104
3105 SDValue CPIHi =
3106 DAG.getTargetConstantPool(C, VT: PtrVT, Align: CP->getAlign(), Offset: 0, TargetFlags: MOHiFlag);
3107 SDValue CPILo =
3108 DAG.getTargetConstantPool(C, VT: PtrVT, Align: CP->getAlign(), Offset: 0, TargetFlags: MOLoFlag);
3109 return LowerLabelRef(HiPart: CPIHi, LoPart: CPILo, isPIC: IsPIC, DAG);
3110}
3111
3112// For 64-bit PowerPC, prefer the more compact relative encodings.
3113// This trades 32 bits per jump table entry for one or two instructions
3114// on the jump site.
3115unsigned PPCTargetLowering::getJumpTableEncoding() const {
3116 if (isJumpTableRelative())
3117 return MachineJumpTableInfo::EK_LabelDifference32;
3118
3119 return TargetLowering::getJumpTableEncoding();
3120}
3121
3122bool PPCTargetLowering::isJumpTableRelative() const {
3123 if (UseAbsoluteJumpTables)
3124 return false;
3125 if (Subtarget.isPPC64() || Subtarget.isAIXABI())
3126 return true;
3127 return TargetLowering::isJumpTableRelative();
3128}
3129
3130SDValue PPCTargetLowering::getPICJumpTableRelocBase(SDValue Table,
3131 SelectionDAG &DAG) const {
3132 if (!Subtarget.isPPC64() || Subtarget.isAIXABI())
3133 return TargetLowering::getPICJumpTableRelocBase(Table, DAG);
3134
3135 switch (getTargetMachine().getCodeModel()) {
3136 case CodeModel::Small:
3137 case CodeModel::Medium:
3138 return TargetLowering::getPICJumpTableRelocBase(Table, DAG);
3139 default:
3140 return DAG.getNode(Opcode: PPCISD::GlobalBaseReg, DL: SDLoc(),
3141 VT: getPointerTy(DL: DAG.getDataLayout()));
3142 }
3143}
3144
3145const MCExpr *
3146PPCTargetLowering::getPICJumpTableRelocBaseExpr(const MachineFunction *MF,
3147 unsigned JTI,
3148 MCContext &Ctx) const {
3149 if (!Subtarget.isPPC64() || Subtarget.isAIXABI())
3150 return TargetLowering::getPICJumpTableRelocBaseExpr(MF, JTI, Ctx);
3151
3152 switch (getTargetMachine().getCodeModel()) {
3153 case CodeModel::Small:
3154 case CodeModel::Medium:
3155 return TargetLowering::getPICJumpTableRelocBaseExpr(MF, JTI, Ctx);
3156 default:
3157 return MCSymbolRefExpr::create(Symbol: MF->getPICBaseSymbol(), Ctx);
3158 }
3159}
3160
3161SDValue PPCTargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const {
3162 EVT PtrVT = Op.getValueType();
3163 JumpTableSDNode *JT = cast<JumpTableSDNode>(Val&: Op);
3164
3165 // isUsingPCRelativeCalls() returns true when PCRelative is enabled
3166 if (Subtarget.isUsingPCRelativeCalls()) {
3167 SDLoc DL(JT);
3168 EVT Ty = getPointerTy(DL: DAG.getDataLayout());
3169 SDValue GA =
3170 DAG.getTargetJumpTable(JTI: JT->getIndex(), VT: Ty, TargetFlags: PPCII::MO_PCREL_FLAG);
3171 SDValue MatAddr = DAG.getNode(Opcode: PPCISD::MAT_PCREL_ADDR, DL, VT: Ty, Operand: GA);
3172 return MatAddr;
3173 }
3174
3175 // 64-bit SVR4 ABI and AIX ABI code are always position-independent.
3176 // The actual address of the GlobalValue is stored in the TOC.
3177 if (Subtarget.is64BitELFABI() || Subtarget.isAIXABI()) {
3178 setUsesTOCBasePtr(DAG);
3179 SDValue GA = DAG.getTargetJumpTable(JTI: JT->getIndex(), VT: PtrVT);
3180 return getTOCEntry(DAG, dl: SDLoc(JT), GA);
3181 }
3182
3183 unsigned MOHiFlag, MOLoFlag;
3184 bool IsPIC = isPositionIndependent();
3185 getLabelAccessInfo(IsPIC, Subtarget, HiOpFlags&: MOHiFlag, LoOpFlags&: MOLoFlag);
3186
3187 if (IsPIC && Subtarget.isSVR4ABI()) {
3188 SDValue GA = DAG.getTargetJumpTable(JTI: JT->getIndex(), VT: PtrVT,
3189 TargetFlags: PPCII::MO_PIC_FLAG);
3190 return getTOCEntry(DAG, dl: SDLoc(GA), GA);
3191 }
3192
3193 SDValue JTIHi = DAG.getTargetJumpTable(JTI: JT->getIndex(), VT: PtrVT, TargetFlags: MOHiFlag);
3194 SDValue JTILo = DAG.getTargetJumpTable(JTI: JT->getIndex(), VT: PtrVT, TargetFlags: MOLoFlag);
3195 return LowerLabelRef(HiPart: JTIHi, LoPart: JTILo, isPIC: IsPIC, DAG);
3196}
3197
3198SDValue PPCTargetLowering::LowerBlockAddress(SDValue Op,
3199 SelectionDAG &DAG) const {
3200 EVT PtrVT = Op.getValueType();
3201 BlockAddressSDNode *BASDN = cast<BlockAddressSDNode>(Val&: Op);
3202 const BlockAddress *BA = BASDN->getBlockAddress();
3203
3204 // isUsingPCRelativeCalls() returns true when PCRelative is enabled
3205 if (Subtarget.isUsingPCRelativeCalls()) {
3206 SDLoc DL(BASDN);
3207 EVT Ty = getPointerTy(DL: DAG.getDataLayout());
3208 SDValue GA = DAG.getTargetBlockAddress(BA, VT: Ty, Offset: BASDN->getOffset(),
3209 TargetFlags: PPCII::MO_PCREL_FLAG);
3210 SDValue MatAddr = DAG.getNode(Opcode: PPCISD::MAT_PCREL_ADDR, DL, VT: Ty, Operand: GA);
3211 return MatAddr;
3212 }
3213
3214 // 64-bit SVR4 ABI and AIX ABI code are always position-independent.
3215 // The actual BlockAddress is stored in the TOC.
3216 if (Subtarget.is64BitELFABI() || Subtarget.isAIXABI()) {
3217 setUsesTOCBasePtr(DAG);
3218 SDValue GA = DAG.getTargetBlockAddress(BA, VT: PtrVT, Offset: BASDN->getOffset());
3219 return getTOCEntry(DAG, dl: SDLoc(BASDN), GA);
3220 }
3221
3222 // 32-bit position-independent ELF stores the BlockAddress in the .got.
3223 if (Subtarget.is32BitELFABI() && isPositionIndependent())
3224 return getTOCEntry(
3225 DAG, dl: SDLoc(BASDN),
3226 GA: DAG.getTargetBlockAddress(BA, VT: PtrVT, Offset: BASDN->getOffset()));
3227
3228 unsigned MOHiFlag, MOLoFlag;
3229 bool IsPIC = isPositionIndependent();
3230 getLabelAccessInfo(IsPIC, Subtarget, HiOpFlags&: MOHiFlag, LoOpFlags&: MOLoFlag);
3231 SDValue TgtBAHi = DAG.getTargetBlockAddress(BA, VT: PtrVT, Offset: 0, TargetFlags: MOHiFlag);
3232 SDValue TgtBALo = DAG.getTargetBlockAddress(BA, VT: PtrVT, Offset: 0, TargetFlags: MOLoFlag);
3233 return LowerLabelRef(HiPart: TgtBAHi, LoPart: TgtBALo, isPIC: IsPIC, DAG);
3234}
3235
3236SDValue PPCTargetLowering::LowerGlobalTLSAddress(SDValue Op,
3237 SelectionDAG &DAG) const {
3238 if (Subtarget.isAIXABI())
3239 return LowerGlobalTLSAddressAIX(Op, DAG);
3240
3241 return LowerGlobalTLSAddressLinux(Op, DAG);
3242}
3243
3244/// updateForAIXShLibTLSModelOpt - Helper to initialize TLS model opt settings,
3245/// and then apply the update.
3246static void updateForAIXShLibTLSModelOpt(TLSModel::Model &Model,
3247 SelectionDAG &DAG,
3248 const TargetMachine &TM) {
3249 // Initialize TLS model opt setting lazily:
3250 // (1) Use initial-exec for single TLS var references within current function.
3251 // (2) Use local-dynamic for multiple TLS var references within current
3252 // function.
3253 PPCFunctionInfo *FuncInfo =
3254 DAG.getMachineFunction().getInfo<PPCFunctionInfo>();
3255 if (!FuncInfo->isAIXFuncTLSModelOptInitDone()) {
3256 SmallPtrSet<const GlobalValue *, 8> TLSGV;
3257 // Iterate over all instructions within current function, collect all TLS
3258 // global variables (global variables taken as the first parameter to
3259 // Intrinsic::threadlocal_address).
3260 const Function &Func = DAG.getMachineFunction().getFunction();
3261 for (const BasicBlock &BB : Func)
3262 for (const Instruction &I : BB)
3263 if (I.getOpcode() == Instruction::Call)
3264 if (const CallInst *CI = dyn_cast<const CallInst>(Val: &I))
3265 if (Function *CF = CI->getCalledFunction())
3266 if (CF->isDeclaration() &&
3267 CF->getIntrinsicID() == Intrinsic::threadlocal_address)
3268 if (const GlobalValue *GV =
3269 dyn_cast<GlobalValue>(Val: I.getOperand(i: 0))) {
3270 TLSModel::Model GVModel = TM.getTLSModel(GV);
3271 if (GVModel == TLSModel::LocalDynamic)
3272 TLSGV.insert(Ptr: GV);
3273 }
3274
3275 unsigned TLSGVCnt = TLSGV.size();
3276 LLVM_DEBUG(dbgs() << format("LocalDynamic TLSGV count:%d\n", TLSGVCnt));
3277 if (TLSGVCnt <= PPCAIXTLSModelOptUseIEForLDLimit)
3278 FuncInfo->setAIXFuncUseTLSIEForLD();
3279 FuncInfo->setAIXFuncTLSModelOptInitDone();
3280 }
3281
3282 if (FuncInfo->isAIXFuncUseTLSIEForLD()) {
3283 LLVM_DEBUG(
3284 dbgs() << DAG.getMachineFunction().getName()
3285 << " function is using the TLS-IE model for TLS-LD access.\n");
3286 Model = TLSModel::InitialExec;
3287 }
3288}
3289
3290SDValue PPCTargetLowering::LowerGlobalTLSAddressAIX(SDValue Op,
3291 SelectionDAG &DAG) const {
3292 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Val&: Op);
3293
3294 if (DAG.getTarget().useEmulatedTLS())
3295 report_fatal_error(reason: "Emulated TLS is not yet supported on AIX");
3296
3297 SDLoc dl(GA);
3298 const GlobalValue *GV = GA->getGlobal();
3299 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
3300 bool Is64Bit = Subtarget.isPPC64();
3301 TLSModel::Model Model = getTargetMachine().getTLSModel(GV);
3302
3303 // Apply update to the TLS model.
3304 if (Subtarget.hasAIXShLibTLSModelOpt())
3305 updateForAIXShLibTLSModelOpt(Model, DAG, TM: getTargetMachine());
3306
3307 // TLS variables are accessed through TOC entries.
3308 // To support this, set the DAG to use the TOC base pointer.
3309 setUsesTOCBasePtr(DAG);
3310
3311 bool IsTLSLocalExecModel = Model == TLSModel::LocalExec;
3312
3313 if (IsTLSLocalExecModel || Model == TLSModel::InitialExec) {
3314 bool HasAIXSmallLocalExecTLS = Subtarget.hasAIXSmallLocalExecTLS();
3315 bool HasAIXSmallTLSGlobalAttr = false;
3316 SDValue VariableOffsetTGA =
3317 DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: 0, TargetFlags: PPCII::MO_TPREL_FLAG);
3318 SDValue VariableOffset = getTOCEntry(DAG, dl, GA: VariableOffsetTGA);
3319 SDValue TLSReg;
3320
3321 if (const GlobalVariable *GVar = dyn_cast<GlobalVariable>(Val: GV))
3322 if (GVar->hasAttribute(Kind: "aix-small-tls"))
3323 HasAIXSmallTLSGlobalAttr = true;
3324
3325 if (Is64Bit) {
3326 // For local-exec and initial-exec on AIX (64-bit), the sequence generated
3327 // involves a load of the variable offset (from the TOC), followed by an
3328 // add of the loaded variable offset to R13 (the thread pointer).
3329 // This code sequence looks like:
3330 // ld reg1,var[TC](2)
3331 // add reg2, reg1, r13 // r13 contains the thread pointer
3332 TLSReg = DAG.getRegister(Reg: PPC::X13, VT: MVT::i64);
3333
3334 // With the -maix-small-local-exec-tls option, or with the "aix-small-tls"
3335 // global variable attribute, produce a faster access sequence for
3336 // local-exec TLS variables where the offset from the TLS base is encoded
3337 // as an immediate operand.
3338 //
3339 // We only utilize the faster local-exec access sequence when the TLS
3340 // variable has a size within the policy limit. We treat types that are
3341 // not sized or are empty as being over the policy size limit.
3342 if ((HasAIXSmallLocalExecTLS || HasAIXSmallTLSGlobalAttr) &&
3343 IsTLSLocalExecModel) {
3344 Type *GVType = GV->getValueType();
3345 if (GVType->isSized() && !GVType->isEmptyTy() &&
3346 GV->getDataLayout().getTypeAllocSize(Ty: GVType) <=
3347 AIXSmallTlsPolicySizeLimit)
3348 return DAG.getNode(Opcode: PPCISD::Lo, DL: dl, VT: PtrVT, N1: VariableOffsetTGA, N2: TLSReg);
3349 }
3350 } else {
3351 // For local-exec and initial-exec on AIX (32-bit), the sequence generated
3352 // involves loading the variable offset from the TOC, generating a call to
3353 // .__get_tpointer to get the thread pointer (which will be in R3), and
3354 // adding the two together:
3355 // lwz reg1,var[TC](2)
3356 // bla .__get_tpointer
3357 // add reg2, reg1, r3
3358 TLSReg = DAG.getNode(Opcode: PPCISD::GET_TPOINTER, DL: dl, VT: PtrVT);
3359
3360 // We do not implement the 32-bit version of the faster access sequence
3361 // for local-exec that is controlled by the -maix-small-local-exec-tls
3362 // option, or the "aix-small-tls" global variable attribute.
3363 if (HasAIXSmallLocalExecTLS || HasAIXSmallTLSGlobalAttr)
3364 report_fatal_error(reason: "The small-local-exec TLS access sequence is "
3365 "currently only supported on AIX (64-bit mode).");
3366 }
3367 return DAG.getNode(Opcode: PPCISD::ADD_TLS, DL: dl, VT: PtrVT, N1: TLSReg, N2: VariableOffset);
3368 }
3369
3370 if (Model == TLSModel::LocalDynamic) {
3371 bool HasAIXSmallLocalDynamicTLS = Subtarget.hasAIXSmallLocalDynamicTLS();
3372
3373 // We do not implement the 32-bit version of the faster access sequence
3374 // for local-dynamic that is controlled by -maix-small-local-dynamic-tls.
3375 if (!Is64Bit && HasAIXSmallLocalDynamicTLS)
3376 report_fatal_error(reason: "The small-local-dynamic TLS access sequence is "
3377 "currently only supported on AIX (64-bit mode).");
3378
3379 // For local-dynamic on AIX, we need to generate one TOC entry for each
3380 // variable offset, and a single module-handle TOC entry for the entire
3381 // file.
3382
3383 SDValue VariableOffsetTGA =
3384 DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: 0, TargetFlags: PPCII::MO_TLSLD_FLAG);
3385 SDValue VariableOffset = getTOCEntry(DAG, dl, GA: VariableOffsetTGA);
3386
3387 Module *M = DAG.getMachineFunction().getFunction().getParent();
3388 GlobalVariable *TLSGV =
3389 dyn_cast_or_null<GlobalVariable>(Val: M->getOrInsertGlobal(
3390 Name: StringRef("_$TLSML"), Ty: PointerType::getUnqual(C&: *DAG.getContext())));
3391 TLSGV->setThreadLocalMode(GlobalVariable::LocalDynamicTLSModel);
3392 assert(TLSGV && "Not able to create GV for _$TLSML.");
3393 SDValue ModuleHandleTGA =
3394 DAG.getTargetGlobalAddress(GV: TLSGV, DL: dl, VT: PtrVT, offset: 0, TargetFlags: PPCII::MO_TLSLDM_FLAG);
3395 SDValue ModuleHandleTOC = getTOCEntry(DAG, dl, GA: ModuleHandleTGA);
3396 SDValue ModuleHandle =
3397 DAG.getNode(Opcode: PPCISD::TLSLD_AIX, DL: dl, VT: PtrVT, Operand: ModuleHandleTOC);
3398
3399 // With the -maix-small-local-dynamic-tls option, produce a faster access
3400 // sequence for local-dynamic TLS variables where the offset from the
3401 // module-handle is encoded as an immediate operand.
3402 //
3403 // We only utilize the faster local-dynamic access sequence when the TLS
3404 // variable has a size within the policy limit. We treat types that are
3405 // not sized or are empty as being over the policy size limit.
3406 if (HasAIXSmallLocalDynamicTLS) {
3407 Type *GVType = GV->getValueType();
3408 if (GVType->isSized() && !GVType->isEmptyTy() &&
3409 GV->getDataLayout().getTypeAllocSize(Ty: GVType) <=
3410 AIXSmallTlsPolicySizeLimit)
3411 return DAG.getNode(Opcode: PPCISD::Lo, DL: dl, VT: PtrVT, N1: VariableOffsetTGA,
3412 N2: ModuleHandle);
3413 }
3414
3415 return DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: ModuleHandle, N2: VariableOffset);
3416 }
3417
3418 // If Local- or Initial-exec or Local-dynamic is not possible or specified,
3419 // all GlobalTLSAddress nodes are lowered using the general-dynamic model. We
3420 // need to generate two TOC entries, one for the variable offset, one for the
3421 // region handle. The global address for the TOC entry of the region handle is
3422 // created with the MO_TLSGDM_FLAG flag and the global address for the TOC
3423 // entry of the variable offset is created with MO_TLSGD_FLAG.
3424 SDValue VariableOffsetTGA =
3425 DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: 0, TargetFlags: PPCII::MO_TLSGD_FLAG);
3426 SDValue RegionHandleTGA =
3427 DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: 0, TargetFlags: PPCII::MO_TLSGDM_FLAG);
3428 SDValue VariableOffset = getTOCEntry(DAG, dl, GA: VariableOffsetTGA);
3429 SDValue RegionHandle = getTOCEntry(DAG, dl, GA: RegionHandleTGA);
3430 return DAG.getNode(Opcode: PPCISD::TLSGD_AIX, DL: dl, VT: PtrVT, N1: VariableOffset,
3431 N2: RegionHandle);
3432}
3433
3434SDValue PPCTargetLowering::LowerGlobalTLSAddressLinux(SDValue Op,
3435 SelectionDAG &DAG) const {
3436 // FIXME: TLS addresses currently use medium model code sequences,
3437 // which is the most useful form. Eventually support for small and
3438 // large models could be added if users need it, at the cost of
3439 // additional complexity.
3440 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Val&: Op);
3441 if (DAG.getTarget().useEmulatedTLS())
3442 return LowerToTLSEmulatedModel(GA, DAG);
3443
3444 SDLoc dl(GA);
3445 const GlobalValue *GV = GA->getGlobal();
3446 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
3447 bool is64bit = Subtarget.isPPC64();
3448 const Module *M = DAG.getMachineFunction().getFunction().getParent();
3449 PICLevel::Level picLevel = M->getPICLevel();
3450
3451 const TargetMachine &TM = getTargetMachine();
3452 TLSModel::Model Model = TM.getTLSModel(GV);
3453
3454 if (Model == TLSModel::LocalExec) {
3455 if (Subtarget.isUsingPCRelativeCalls()) {
3456 SDValue TLSReg = DAG.getRegister(Reg: PPC::X13, VT: MVT::i64);
3457 SDValue TGA = DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: 0,
3458 TargetFlags: PPCII::MO_TPREL_PCREL_FLAG);
3459 SDValue MatAddr =
3460 DAG.getNode(Opcode: PPCISD::TLS_LOCAL_EXEC_MAT_ADDR, DL: dl, VT: PtrVT, Operand: TGA);
3461 return DAG.getNode(Opcode: PPCISD::ADD_TLS, DL: dl, VT: PtrVT, N1: TLSReg, N2: MatAddr);
3462 }
3463
3464 SDValue TGAHi = DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: 0,
3465 TargetFlags: PPCII::MO_TPREL_HA);
3466 SDValue TGALo = DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: 0,
3467 TargetFlags: PPCII::MO_TPREL_LO);
3468 SDValue TLSReg = is64bit ? DAG.getRegister(Reg: PPC::X13, VT: MVT::i64)
3469 : DAG.getRegister(Reg: PPC::R2, VT: MVT::i32);
3470
3471 SDValue Hi = DAG.getNode(Opcode: PPCISD::Hi, DL: dl, VT: PtrVT, N1: TGAHi, N2: TLSReg);
3472 return DAG.getNode(Opcode: PPCISD::Lo, DL: dl, VT: PtrVT, N1: TGALo, N2: Hi);
3473 }
3474
3475 if (Model == TLSModel::InitialExec) {
3476 bool IsPCRel = Subtarget.isUsingPCRelativeCalls();
3477 SDValue TGA = DAG.getTargetGlobalAddress(
3478 GV, DL: dl, VT: PtrVT, offset: 0, TargetFlags: IsPCRel ? PPCII::MO_GOT_TPREL_PCREL_FLAG : 0);
3479 SDValue TGATLS = DAG.getTargetGlobalAddress(
3480 GV, DL: dl, VT: PtrVT, offset: 0, TargetFlags: IsPCRel ? PPCII::MO_TLS_PCREL_FLAG : PPCII::MO_TLS);
3481 SDValue TPOffset;
3482 if (IsPCRel) {
3483 SDValue MatPCRel = DAG.getNode(Opcode: PPCISD::MAT_PCREL_ADDR, DL: dl, VT: PtrVT, Operand: TGA);
3484 TPOffset = DAG.getLoad(VT: MVT::i64, dl, Chain: DAG.getEntryNode(), Ptr: MatPCRel,
3485 PtrInfo: MachinePointerInfo());
3486 } else {
3487 SDValue GOTPtr;
3488 if (is64bit) {
3489 setUsesTOCBasePtr(DAG);
3490 SDValue GOTReg = DAG.getRegister(Reg: PPC::X2, VT: MVT::i64);
3491 GOTPtr =
3492 DAG.getNode(Opcode: PPCISD::ADDIS_GOT_TPREL_HA, DL: dl, VT: PtrVT, N1: GOTReg, N2: TGA);
3493 } else {
3494 if (!TM.isPositionIndependent())
3495 GOTPtr = DAG.getNode(Opcode: PPCISD::PPC32_GOT, DL: dl, VT: PtrVT);
3496 else if (picLevel == PICLevel::SmallPIC)
3497 GOTPtr = DAG.getNode(Opcode: PPCISD::GlobalBaseReg, DL: dl, VT: PtrVT);
3498 else
3499 GOTPtr = DAG.getNode(Opcode: PPCISD::PPC32_PICGOT, DL: dl, VT: PtrVT);
3500 }
3501 TPOffset = DAG.getNode(Opcode: PPCISD::LD_GOT_TPREL_L, DL: dl, VT: PtrVT, N1: TGA, N2: GOTPtr);
3502 }
3503 return DAG.getNode(Opcode: PPCISD::ADD_TLS, DL: dl, VT: PtrVT, N1: TPOffset, N2: TGATLS);
3504 }
3505
3506 if (Model == TLSModel::GeneralDynamic) {
3507 if (Subtarget.isUsingPCRelativeCalls()) {
3508 SDValue TGA = DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: 0,
3509 TargetFlags: PPCII::MO_GOT_TLSGD_PCREL_FLAG);
3510 return DAG.getNode(Opcode: PPCISD::TLS_DYNAMIC_MAT_PCREL_ADDR, DL: dl, VT: PtrVT, Operand: TGA);
3511 }
3512
3513 SDValue TGA = DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: 0, TargetFlags: 0);
3514 SDValue GOTPtr;
3515 if (is64bit) {
3516 setUsesTOCBasePtr(DAG);
3517 SDValue GOTReg = DAG.getRegister(Reg: PPC::X2, VT: MVT::i64);
3518 GOTPtr = DAG.getNode(Opcode: PPCISD::ADDIS_TLSGD_HA, DL: dl, VT: PtrVT,
3519 N1: GOTReg, N2: TGA);
3520 } else {
3521 if (picLevel == PICLevel::SmallPIC)
3522 GOTPtr = DAG.getNode(Opcode: PPCISD::GlobalBaseReg, DL: dl, VT: PtrVT);
3523 else
3524 GOTPtr = DAG.getNode(Opcode: PPCISD::PPC32_PICGOT, DL: dl, VT: PtrVT);
3525 }
3526 return DAG.getNode(Opcode: PPCISD::ADDI_TLSGD_L_ADDR, DL: dl, VT: PtrVT,
3527 N1: GOTPtr, N2: TGA, N3: TGA);
3528 }
3529
3530 if (Model == TLSModel::LocalDynamic) {
3531 if (Subtarget.isUsingPCRelativeCalls()) {
3532 SDValue TGA = DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: 0,
3533 TargetFlags: PPCII::MO_GOT_TLSLD_PCREL_FLAG);
3534 SDValue MatPCRel =
3535 DAG.getNode(Opcode: PPCISD::TLS_DYNAMIC_MAT_PCREL_ADDR, DL: dl, VT: PtrVT, Operand: TGA);
3536 return DAG.getNode(Opcode: PPCISD::PADDI_DTPREL, DL: dl, VT: PtrVT, N1: MatPCRel, N2: TGA);
3537 }
3538
3539 SDValue TGA = DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: 0, TargetFlags: 0);
3540 SDValue GOTPtr;
3541 if (is64bit) {
3542 setUsesTOCBasePtr(DAG);
3543 SDValue GOTReg = DAG.getRegister(Reg: PPC::X2, VT: MVT::i64);
3544 GOTPtr = DAG.getNode(Opcode: PPCISD::ADDIS_TLSLD_HA, DL: dl, VT: PtrVT,
3545 N1: GOTReg, N2: TGA);
3546 } else {
3547 if (picLevel == PICLevel::SmallPIC)
3548 GOTPtr = DAG.getNode(Opcode: PPCISD::GlobalBaseReg, DL: dl, VT: PtrVT);
3549 else
3550 GOTPtr = DAG.getNode(Opcode: PPCISD::PPC32_PICGOT, DL: dl, VT: PtrVT);
3551 }
3552 SDValue TLSAddr = DAG.getNode(Opcode: PPCISD::ADDI_TLSLD_L_ADDR, DL: dl,
3553 VT: PtrVT, N1: GOTPtr, N2: TGA, N3: TGA);
3554 SDValue DtvOffsetHi = DAG.getNode(Opcode: PPCISD::ADDIS_DTPREL_HA, DL: dl,
3555 VT: PtrVT, N1: TLSAddr, N2: TGA);
3556 return DAG.getNode(Opcode: PPCISD::ADDI_DTPREL_L, DL: dl, VT: PtrVT, N1: DtvOffsetHi, N2: TGA);
3557 }
3558
3559 llvm_unreachable("Unknown TLS model!");
3560}
3561
3562SDValue PPCTargetLowering::LowerGlobalAddress(SDValue Op,
3563 SelectionDAG &DAG) const {
3564 EVT PtrVT = Op.getValueType();
3565 GlobalAddressSDNode *GSDN = cast<GlobalAddressSDNode>(Val&: Op);
3566 SDLoc DL(GSDN);
3567 const GlobalValue *GV = GSDN->getGlobal();
3568
3569 // 64-bit SVR4 ABI & AIX ABI code is always position-independent.
3570 // The actual address of the GlobalValue is stored in the TOC.
3571 if (Subtarget.is64BitELFABI() || Subtarget.isAIXABI()) {
3572 if (Subtarget.isUsingPCRelativeCalls()) {
3573 EVT Ty = getPointerTy(DL: DAG.getDataLayout());
3574 if (isAccessedAsGotIndirect(N: Op)) {
3575 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, VT: Ty, offset: GSDN->getOffset(),
3576 TargetFlags: PPCII::MO_GOT_PCREL_FLAG);
3577 SDValue MatPCRel = DAG.getNode(Opcode: PPCISD::MAT_PCREL_ADDR, DL, VT: Ty, Operand: GA);
3578 SDValue Load = DAG.getLoad(VT: MVT::i64, dl: DL, Chain: DAG.getEntryNode(), Ptr: MatPCRel,
3579 PtrInfo: MachinePointerInfo());
3580 return Load;
3581 } else {
3582 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, VT: Ty, offset: GSDN->getOffset(),
3583 TargetFlags: PPCII::MO_PCREL_FLAG);
3584 return DAG.getNode(Opcode: PPCISD::MAT_PCREL_ADDR, DL, VT: Ty, Operand: GA);
3585 }
3586 }
3587 setUsesTOCBasePtr(DAG);
3588 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, VT: PtrVT, offset: GSDN->getOffset());
3589 return getTOCEntry(DAG, dl: DL, GA);
3590 }
3591
3592 unsigned MOHiFlag, MOLoFlag;
3593 bool IsPIC = isPositionIndependent();
3594 getLabelAccessInfo(IsPIC, Subtarget, HiOpFlags&: MOHiFlag, LoOpFlags&: MOLoFlag, GV);
3595
3596 if (IsPIC && Subtarget.isSVR4ABI()) {
3597 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, VT: PtrVT,
3598 offset: GSDN->getOffset(),
3599 TargetFlags: PPCII::MO_PIC_FLAG);
3600 return getTOCEntry(DAG, dl: DL, GA);
3601 }
3602
3603 SDValue GAHi =
3604 DAG.getTargetGlobalAddress(GV, DL, VT: PtrVT, offset: GSDN->getOffset(), TargetFlags: MOHiFlag);
3605 SDValue GALo =
3606 DAG.getTargetGlobalAddress(GV, DL, VT: PtrVT, offset: GSDN->getOffset(), TargetFlags: MOLoFlag);
3607
3608 return LowerLabelRef(HiPart: GAHi, LoPart: GALo, isPIC: IsPIC, DAG);
3609}
3610
3611SDValue PPCTargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
3612 bool IsStrict = Op->isStrictFPOpcode();
3613 const SDNodeFlags Flags = Op.getNode()->getFlags();
3614 ISD::CondCode CC =
3615 cast<CondCodeSDNode>(Val: Op.getOperand(i: IsStrict ? 3 : 2))->get();
3616 SDValue LHS = Op.getOperand(i: IsStrict ? 1 : 0);
3617 SDValue RHS = Op.getOperand(i: IsStrict ? 2 : 1);
3618 SDValue Chain = IsStrict ? Op.getOperand(i: 0) : SDValue();
3619 EVT LHSVT = LHS.getValueType();
3620 SDLoc dl(Op);
3621
3622 // Soften the setcc with libcall if it is fp128 or it is SPE and fp32/fp64.
3623 if (LHSVT == MVT::f128 ||
3624 (Subtarget.hasSPE() && (LHSVT == MVT::f32 || LHSVT == MVT::f64) &&
3625 (!Flags.hasNoNaNs() || !Flags.hasNoInfs()))) {
3626 assert(!Subtarget.hasP9Vector() &&
3627 "SETCC for f128 is already legal under Power9!");
3628 softenSetCCOperands(DAG, VT: LHSVT, NewLHS&: LHS, NewRHS&: RHS, CCCode&: CC, DL: dl, OldLHS: LHS, OldRHS: RHS, Chain,
3629 IsSignaling: Op->getOpcode() == ISD::STRICT_FSETCCS);
3630 if (RHS.getNode())
3631 LHS = DAG.getNode(Opcode: ISD::SETCC, DL: dl, VT: Op.getValueType(), N1: LHS, N2: RHS,
3632 N3: DAG.getCondCode(Cond: CC));
3633 if (IsStrict)
3634 return DAG.getMergeValues(Ops: {LHS, Chain}, dl);
3635 return LHS;
3636 } else if (LHSVT == MVT::f32 || LHSVT == MVT::f64) {
3637 return Op;
3638 }
3639
3640 assert(!IsStrict && "Don't know how to handle STRICT_FSETCC!");
3641
3642 if (Op.getValueType() == MVT::v2i64) {
3643 // When the operands themselves are v2i64 values, we need to do something
3644 // special because VSX has no underlying comparison operations for these.
3645 if (LHS.getValueType() == MVT::v2i64) {
3646 // Equality can be handled by casting to the legal type for Altivec
3647 // comparisons, everything else needs to be expanded.
3648 if (CC != ISD::SETEQ && CC != ISD::SETNE)
3649 return SDValue();
3650 SDValue SetCC32 = DAG.getSetCC(
3651 DL: dl, VT: MVT::v4i32, LHS: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v4i32, Operand: LHS),
3652 RHS: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v4i32, Operand: RHS), Cond: CC);
3653 int ShuffV[] = {1, 0, 3, 2};
3654 SDValue Shuff =
3655 DAG.getVectorShuffle(VT: MVT::v4i32, dl, N1: SetCC32, N2: SetCC32, Mask: ShuffV);
3656 return DAG.getBitcast(VT: MVT::v2i64,
3657 V: DAG.getNode(Opcode: CC == ISD::SETEQ ? ISD::AND : ISD::OR,
3658 DL: dl, VT: MVT::v4i32, N1: Shuff, N2: SetCC32));
3659 }
3660
3661 // We handle most of these in the usual way.
3662 return Op;
3663 }
3664
3665 // If we're comparing for equality to zero, expose the fact that this is
3666 // implemented as a ctlz/srl pair on ppc, so that the dag combiner can
3667 // fold the new nodes.
3668 if (SDValue V = lowerCmpEqZeroToCtlzSrl(Op, DAG))
3669 return V;
3670
3671 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: RHS)) {
3672 // Leave comparisons against 0 and -1 alone for now, since they're usually
3673 // optimized. FIXME: revisit this when we can custom lower all setcc
3674 // optimizations.
3675 if (C->isAllOnes() || C->isZero())
3676 return SDValue();
3677 }
3678
3679 // If we have an integer seteq/setne, turn it into a compare against zero
3680 // by xor'ing the rhs with the lhs, which is faster than setting a
3681 // condition register, reading it back out, and masking the correct bit. The
3682 // normal approach here uses sub to do this instead of xor. Using xor exposes
3683 // the result to other bit-twiddling opportunities.
3684 if (LHSVT.isInteger() && (CC == ISD::SETEQ || CC == ISD::SETNE)) {
3685 EVT VT = Op.getValueType();
3686 SDValue Sub = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: LHSVT, N1: LHS, N2: RHS);
3687 return DAG.getSetCC(DL: dl, VT, LHS: Sub, RHS: DAG.getConstant(Val: 0, DL: dl, VT: LHSVT), Cond: CC);
3688 }
3689 return SDValue();
3690}
3691
3692SDValue PPCTargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
3693 const SDNodeFlags Flags = Op->getFlags();
3694 SDValue Chain = Op.getOperand(i: 0);
3695 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 1))->get();
3696 SDValue LHS = Op.getOperand(i: 2);
3697 SDValue RHS = Op.getOperand(i: 3);
3698 SDValue Dest = Op.getOperand(i: 4);
3699 EVT LHSVT = LHS.getValueType();
3700 SDLoc dl(Op);
3701
3702 assert(Subtarget.hasSPE() && "LowerBR_CC used only for targets with SPE");
3703
3704 if ((LHSVT == MVT::f32 || LHSVT == MVT::f64) && Flags.hasNoNaNs() &&
3705 Flags.hasNoInfs())
3706 return Op;
3707
3708 softenSetCCOperands(DAG, VT: LHSVT, NewLHS&: LHS, NewRHS&: RHS, CCCode&: CC, DL: dl, OldLHS: LHS, OldRHS: RHS);
3709
3710 // If softenSetCCOperands returned a scalar, we need to compare the result
3711 // against zero to select between true and false values.
3712 if (!RHS) {
3713 RHS = DAG.getConstant(Val: 0, DL: dl, VT: LHSVT);
3714 CC = ISD::SETNE;
3715 }
3716
3717 return DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: Op.getValueType(), N1: Chain,
3718 N2: DAG.getCondCode(Cond: CC), N3: LHS, N4: RHS, N5: Dest);
3719}
3720
3721SDValue PPCTargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
3722 SDNode *Node = Op.getNode();
3723 EVT VT = Node->getValueType(ResNo: 0);
3724 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
3725 SDValue InChain = Node->getOperand(Num: 0);
3726 SDValue VAListPtr = Node->getOperand(Num: 1);
3727 const Value *SV = cast<SrcValueSDNode>(Val: Node->getOperand(Num: 2))->getValue();
3728 SDLoc dl(Node);
3729
3730 assert(!Subtarget.isPPC64() && "LowerVAARG is PPC32 only");
3731
3732 // gpr_index
3733 SDValue GprIndex = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT: MVT::i32, Chain: InChain,
3734 Ptr: VAListPtr, PtrInfo: MachinePointerInfo(SV), MemVT: MVT::i8);
3735 InChain = GprIndex.getValue(R: 1);
3736
3737 if (VT == MVT::i64) {
3738 // Check if GprIndex is even
3739 SDValue GprAnd = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32, N1: GprIndex,
3740 N2: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
3741 SDValue CC64 = DAG.getSetCC(DL: dl, VT: MVT::i32, LHS: GprAnd,
3742 RHS: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32), Cond: ISD::SETNE);
3743 SDValue GprIndexPlusOne = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::i32, N1: GprIndex,
3744 N2: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
3745 // Align GprIndex to be even if it isn't
3746 GprIndex = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: MVT::i32, N1: CC64, N2: GprIndexPlusOne,
3747 N3: GprIndex);
3748 }
3749
3750 // fpr index is 1 byte after gpr
3751 SDValue FprPtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: VAListPtr,
3752 N2: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
3753
3754 // fpr
3755 SDValue FprIndex = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT: MVT::i32, Chain: InChain,
3756 Ptr: FprPtr, PtrInfo: MachinePointerInfo(SV), MemVT: MVT::i8);
3757 InChain = FprIndex.getValue(R: 1);
3758
3759 SDValue RegSaveAreaPtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: VAListPtr,
3760 N2: DAG.getConstant(Val: 8, DL: dl, VT: MVT::i32));
3761
3762 SDValue OverflowAreaPtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: VAListPtr,
3763 N2: DAG.getConstant(Val: 4, DL: dl, VT: MVT::i32));
3764
3765 // areas
3766 SDValue OverflowArea =
3767 DAG.getLoad(VT: MVT::i32, dl, Chain: InChain, Ptr: OverflowAreaPtr, PtrInfo: MachinePointerInfo());
3768 InChain = OverflowArea.getValue(R: 1);
3769
3770 SDValue RegSaveArea =
3771 DAG.getLoad(VT: MVT::i32, dl, Chain: InChain, Ptr: RegSaveAreaPtr, PtrInfo: MachinePointerInfo());
3772 InChain = RegSaveArea.getValue(R: 1);
3773
3774 // select overflow_area if index > 8
3775 SDValue CC = DAG.getSetCC(DL: dl, VT: MVT::i32, LHS: VT.isInteger() ? GprIndex : FprIndex,
3776 RHS: DAG.getConstant(Val: 8, DL: dl, VT: MVT::i32), Cond: ISD::SETLT);
3777
3778 // adjustment constant gpr_index * 4/8
3779 SDValue RegConstant = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MVT::i32,
3780 N1: VT.isInteger() ? GprIndex : FprIndex,
3781 N2: DAG.getConstant(Val: VT.isInteger() ? 4 : 8, DL: dl,
3782 VT: MVT::i32));
3783
3784 // OurReg = RegSaveArea + RegConstant
3785 SDValue OurReg = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: RegSaveArea,
3786 N2: RegConstant);
3787
3788 // Floating types are 32 bytes into RegSaveArea
3789 if (VT.isFloatingPoint())
3790 OurReg = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: OurReg,
3791 N2: DAG.getConstant(Val: 32, DL: dl, VT: MVT::i32));
3792
3793 // increase {f,g}pr_index by 1 (or 2 if VT is i64)
3794 SDValue IndexPlus1 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::i32,
3795 N1: VT.isInteger() ? GprIndex : FprIndex,
3796 N2: DAG.getConstant(Val: VT == MVT::i64 ? 2 : 1, DL: dl,
3797 VT: MVT::i32));
3798
3799 InChain = DAG.getTruncStore(Chain: InChain, dl, Val: IndexPlus1,
3800 Ptr: VT.isInteger() ? VAListPtr : FprPtr,
3801 PtrInfo: MachinePointerInfo(SV), SVT: MVT::i8);
3802
3803 // determine if we should load from reg_save_area or overflow_area
3804 SDValue Result = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: PtrVT, N1: CC, N2: OurReg, N3: OverflowArea);
3805
3806 // increase overflow_area by 4/8 if gpr/fpr > 8
3807 SDValue OverflowAreaPlusN = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: OverflowArea,
3808 N2: DAG.getConstant(Val: VT.isInteger() ? 4 : 8,
3809 DL: dl, VT: MVT::i32));
3810
3811 OverflowArea = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: MVT::i32, N1: CC, N2: OverflowArea,
3812 N3: OverflowAreaPlusN);
3813
3814 InChain = DAG.getTruncStore(Chain: InChain, dl, Val: OverflowArea, Ptr: OverflowAreaPtr,
3815 PtrInfo: MachinePointerInfo(), SVT: MVT::i32);
3816
3817 return DAG.getLoad(VT, dl, Chain: InChain, Ptr: Result, PtrInfo: MachinePointerInfo());
3818}
3819
3820SDValue PPCTargetLowering::LowerVACOPY(SDValue Op, SelectionDAG &DAG) const {
3821 assert(!Subtarget.isPPC64() && "LowerVACOPY is PPC32 only");
3822
3823 // We have to copy the entire va_list struct:
3824 // 2*sizeof(char) + 2 Byte alignment + 2*sizeof(char*) = 12 Byte
3825 return DAG.getMemcpy(Chain: Op.getOperand(i: 0), dl: Op, Dst: Op.getOperand(i: 1), Src: Op.getOperand(i: 2),
3826 Size: DAG.getConstant(Val: 12, DL: SDLoc(Op), VT: MVT::i32), DstAlign: Align(8),
3827 SrcAlign: Align(8), isVol: false, AlwaysInline: true, /*CI=*/nullptr, OverrideTailCall: std::nullopt,
3828 DstPtrInfo: MachinePointerInfo(), SrcPtrInfo: MachinePointerInfo());
3829}
3830
3831SDValue PPCTargetLowering::LowerADJUST_TRAMPOLINE(SDValue Op,
3832 SelectionDAG &DAG) const {
3833 return Op.getOperand(i: 0);
3834}
3835
3836SDValue PPCTargetLowering::LowerINLINEASM(SDValue Op, SelectionDAG &DAG) const {
3837 MachineFunction &MF = DAG.getMachineFunction();
3838 PPCFunctionInfo &MFI = *MF.getInfo<PPCFunctionInfo>();
3839
3840 assert((Op.getOpcode() == ISD::INLINEASM ||
3841 Op.getOpcode() == ISD::INLINEASM_BR) &&
3842 "Expecting Inline ASM node.");
3843
3844 // If an LR store is already known to be required then there is not point in
3845 // checking this ASM as well.
3846 if (MFI.isLRStoreRequired())
3847 return Op;
3848
3849 // Inline ASM nodes have an optional last operand that is an incoming Flag of
3850 // type MVT::Glue. We want to ignore this last operand if that is the case.
3851 unsigned NumOps = Op.getNumOperands();
3852 if (Op.getOperand(i: NumOps - 1).getValueType() == MVT::Glue)
3853 --NumOps;
3854
3855 // Check all operands that may contain the LR.
3856 for (unsigned i = InlineAsm::Op_FirstOperand; i != NumOps;) {
3857 const InlineAsm::Flag Flags(Op.getConstantOperandVal(i));
3858 unsigned NumVals = Flags.getNumOperandRegisters();
3859 ++i; // Skip the ID value.
3860
3861 switch (Flags.getKind()) {
3862 default:
3863 llvm_unreachable("Bad flags!");
3864 case InlineAsm::Kind::RegUse:
3865 case InlineAsm::Kind::Imm:
3866 case InlineAsm::Kind::Mem:
3867 i += NumVals;
3868 break;
3869 case InlineAsm::Kind::Clobber:
3870 case InlineAsm::Kind::RegDef:
3871 case InlineAsm::Kind::RegDefEarlyClobber: {
3872 for (; NumVals; --NumVals, ++i) {
3873 Register Reg = cast<RegisterSDNode>(Val: Op.getOperand(i))->getReg();
3874 if (Reg != PPC::LR && Reg != PPC::LR8)
3875 continue;
3876 MFI.setLRStoreRequired();
3877 return Op;
3878 }
3879 break;
3880 }
3881 }
3882 }
3883
3884 return Op;
3885}
3886
3887SDValue PPCTargetLowering::LowerINIT_TRAMPOLINE(SDValue Op,
3888 SelectionDAG &DAG) const {
3889 SDValue Chain = Op.getOperand(i: 0);
3890 SDValue Trmp = Op.getOperand(i: 1); // trampoline
3891 SDValue FPtr = Op.getOperand(i: 2); // nested function
3892 SDValue Nest = Op.getOperand(i: 3); // 'nest' parameter value
3893 SDLoc dl(Op);
3894
3895 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
3896
3897 if (Subtarget.isAIXABI()) {
3898 // On AIX we create a trampoline descriptor by combining the
3899 // entry point and TOC from the global descriptor (FPtr) with the
3900 // nest argument as the environment pointer.
3901 uint64_t PointerSize = Subtarget.isPPC64() ? 8 : 4;
3902 MaybeAlign PointerAlign(PointerSize);
3903 auto MMOFlags = Subtarget.hasInvariantFunctionDescriptors()
3904 ? (MachineMemOperand::MODereferenceable |
3905 MachineMemOperand::MOInvariant)
3906 : MachineMemOperand::MONone;
3907
3908 uint64_t TOCPointerOffset = 1 * PointerSize;
3909 uint64_t EnvPointerOffset = 2 * PointerSize;
3910 SDValue SDTOCPtrOffset = DAG.getConstant(Val: TOCPointerOffset, DL: dl, VT: PtrVT);
3911 SDValue SDEnvPtrOffset = DAG.getConstant(Val: EnvPointerOffset, DL: dl, VT: PtrVT);
3912
3913 const Value *TrampolineAddr =
3914 cast<SrcValueSDNode>(Val: Op.getOperand(i: 4))->getValue();
3915 const Function *Func =
3916 cast<Function>(Val: cast<SrcValueSDNode>(Val: Op.getOperand(i: 5))->getValue());
3917
3918 SDValue OutChains[3];
3919
3920 // Copy the entry point address from the global descriptor to the
3921 // trampoline buffer.
3922 SDValue LoadEntryPoint =
3923 DAG.getLoad(VT: PtrVT, dl, Chain, Ptr: FPtr, PtrInfo: MachinePointerInfo(Func, 0),
3924 Alignment: PointerAlign, MMOFlags);
3925 SDValue EPLoadChain = LoadEntryPoint.getValue(R: 1);
3926 OutChains[0] = DAG.getStore(Chain: EPLoadChain, dl, Val: LoadEntryPoint, Ptr: Trmp,
3927 PtrInfo: MachinePointerInfo(TrampolineAddr, 0));
3928
3929 // Copy the TOC pointer from the global descriptor to the trampoline
3930 // buffer.
3931 SDValue TOCFromDescriptorPtr =
3932 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: FPtr, N2: SDTOCPtrOffset);
3933 SDValue TOCReg = DAG.getLoad(VT: PtrVT, dl, Chain, Ptr: TOCFromDescriptorPtr,
3934 PtrInfo: MachinePointerInfo(Func, TOCPointerOffset),
3935 Alignment: PointerAlign, MMOFlags);
3936 SDValue TrampolineTOCPointer =
3937 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: Trmp, N2: SDTOCPtrOffset);
3938 SDValue TOCLoadChain = TOCReg.getValue(R: 1);
3939 OutChains[1] =
3940 DAG.getStore(Chain: TOCLoadChain, dl, Val: TOCReg, Ptr: TrampolineTOCPointer,
3941 PtrInfo: MachinePointerInfo(TrampolineAddr, TOCPointerOffset));
3942
3943 // Store the nest argument into the environment pointer in the trampoline
3944 // buffer.
3945 SDValue EnvPointer = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: Trmp, N2: SDEnvPtrOffset);
3946 OutChains[2] =
3947 DAG.getStore(Chain, dl, Val: Nest, Ptr: EnvPointer,
3948 PtrInfo: MachinePointerInfo(TrampolineAddr, EnvPointerOffset));
3949
3950 SDValue TokenFactor =
3951 DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: OutChains);
3952 return TokenFactor;
3953 }
3954
3955 bool isPPC64 = (PtrVT == MVT::i64);
3956 Type *IntPtrTy = DAG.getDataLayout().getIntPtrType(C&: *DAG.getContext());
3957
3958 TargetLowering::ArgListTy Args;
3959 Args.emplace_back(args&: Trmp, args&: IntPtrTy);
3960 // TrampSize == (isPPC64 ? 48 : 40);
3961 Args.emplace_back(
3962 args: DAG.getConstant(Val: isPPC64 ? 48 : 40, DL: dl, VT: Subtarget.getScalarIntVT()),
3963 args&: IntPtrTy);
3964 Args.emplace_back(args&: FPtr, args&: IntPtrTy);
3965 Args.emplace_back(args&: Nest, args&: IntPtrTy);
3966
3967 // Lower to a call to __trampoline_setup(Trmp, TrampSize, FPtr, ctx_reg)
3968 TargetLowering::CallLoweringInfo CLI(DAG);
3969 CLI.setDebugLoc(dl).setChain(Chain).setLibCallee(
3970 CC: CallingConv::C, ResultType: Type::getVoidTy(C&: *DAG.getContext()),
3971 Target: DAG.getExternalSymbol(Sym: "__trampoline_setup", VT: PtrVT), ArgsList: std::move(Args));
3972
3973 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
3974 return CallResult.second;
3975}
3976
3977SDValue PPCTargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {
3978 MachineFunction &MF = DAG.getMachineFunction();
3979 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
3980 EVT PtrVT = getPointerTy(DL: MF.getDataLayout());
3981
3982 SDLoc dl(Op);
3983
3984 if (Subtarget.isPPC64() || Subtarget.isAIXABI()) {
3985 // vastart just stores the address of the VarArgsFrameIndex slot into the
3986 // memory location argument.
3987 SDValue FR = DAG.getFrameIndex(FI: FuncInfo->getVarArgsFrameIndex(), VT: PtrVT);
3988 const Value *SV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 2))->getValue();
3989 return DAG.getStore(Chain: Op.getOperand(i: 0), dl, Val: FR, Ptr: Op.getOperand(i: 1),
3990 PtrInfo: MachinePointerInfo(SV));
3991 }
3992
3993 // For the 32-bit SVR4 ABI we follow the layout of the va_list struct.
3994 // We suppose the given va_list is already allocated.
3995 //
3996 // typedef struct {
3997 // char gpr; /* index into the array of 8 GPRs
3998 // * stored in the register save area
3999 // * gpr=0 corresponds to r3,
4000 // * gpr=1 to r4, etc.
4001 // */
4002 // char fpr; /* index into the array of 8 FPRs
4003 // * stored in the register save area
4004 // * fpr=0 corresponds to f1,
4005 // * fpr=1 to f2, etc.
4006 // */
4007 // char *overflow_arg_area;
4008 // /* location on stack that holds
4009 // * the next overflow argument
4010 // */
4011 // char *reg_save_area;
4012 // /* where r3:r10 and f1:f8 (if saved)
4013 // * are stored
4014 // */
4015 // } va_list[1];
4016
4017 SDValue ArgGPR = DAG.getConstant(Val: FuncInfo->getVarArgsNumGPR(), DL: dl, VT: MVT::i32);
4018 SDValue ArgFPR = DAG.getConstant(Val: FuncInfo->getVarArgsNumFPR(), DL: dl, VT: MVT::i32);
4019 SDValue StackOffsetFI = DAG.getFrameIndex(FI: FuncInfo->getVarArgsStackOffset(),
4020 VT: PtrVT);
4021 SDValue FR = DAG.getFrameIndex(FI: FuncInfo->getVarArgsFrameIndex(),
4022 VT: PtrVT);
4023
4024 uint64_t FrameOffset = PtrVT.getSizeInBits()/8;
4025 SDValue ConstFrameOffset = DAG.getConstant(Val: FrameOffset, DL: dl, VT: PtrVT);
4026
4027 uint64_t StackOffset = PtrVT.getSizeInBits()/8 - 1;
4028 SDValue ConstStackOffset = DAG.getConstant(Val: StackOffset, DL: dl, VT: PtrVT);
4029
4030 uint64_t FPROffset = 1;
4031 SDValue ConstFPROffset = DAG.getConstant(Val: FPROffset, DL: dl, VT: PtrVT);
4032
4033 const Value *SV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 2))->getValue();
4034
4035 // Store first byte : number of int regs
4036 SDValue firstStore =
4037 DAG.getTruncStore(Chain: Op.getOperand(i: 0), dl, Val: ArgGPR, Ptr: Op.getOperand(i: 1),
4038 PtrInfo: MachinePointerInfo(SV), SVT: MVT::i8);
4039 uint64_t nextOffset = FPROffset;
4040 SDValue nextPtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: Op.getOperand(i: 1),
4041 N2: ConstFPROffset);
4042
4043 // Store second byte : number of float regs
4044 SDValue secondStore =
4045 DAG.getTruncStore(Chain: firstStore, dl, Val: ArgFPR, Ptr: nextPtr,
4046 PtrInfo: MachinePointerInfo(SV, nextOffset), SVT: MVT::i8);
4047 nextOffset += StackOffset;
4048 nextPtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: nextPtr, N2: ConstStackOffset);
4049
4050 // Store second word : arguments given on stack
4051 SDValue thirdStore = DAG.getStore(Chain: secondStore, dl, Val: StackOffsetFI, Ptr: nextPtr,
4052 PtrInfo: MachinePointerInfo(SV, nextOffset));
4053 nextOffset += FrameOffset;
4054 nextPtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: nextPtr, N2: ConstFrameOffset);
4055
4056 // Store third word : arguments given in registers
4057 return DAG.getStore(Chain: thirdStore, dl, Val: FR, Ptr: nextPtr,
4058 PtrInfo: MachinePointerInfo(SV, nextOffset));
4059}
4060
4061/// FPR - The set of FP registers that should be allocated for arguments
4062/// on Darwin and AIX.
4063static const MCPhysReg FPR[] = {PPC::F1, PPC::F2, PPC::F3, PPC::F4, PPC::F5,
4064 PPC::F6, PPC::F7, PPC::F8, PPC::F9, PPC::F10,
4065 PPC::F11, PPC::F12, PPC::F13};
4066
4067/// CalculateStackSlotSize - Calculates the size reserved for this argument on
4068/// the stack.
4069static unsigned CalculateStackSlotSize(EVT ArgVT, ISD::ArgFlagsTy Flags,
4070 unsigned PtrByteSize) {
4071 unsigned ArgSize = ArgVT.getStoreSize();
4072 if (Flags.isByVal())
4073 ArgSize = Flags.getByValSize();
4074
4075 // Round up to multiples of the pointer size, except for array members,
4076 // which are always packed.
4077 if (!Flags.isInConsecutiveRegs())
4078 ArgSize = ((ArgSize + PtrByteSize - 1)/PtrByteSize) * PtrByteSize;
4079
4080 return ArgSize;
4081}
4082
4083/// CalculateStackSlotAlignment - Calculates the alignment of this argument
4084/// on the stack.
4085static Align CalculateStackSlotAlignment(EVT ArgVT, EVT OrigVT,
4086 ISD::ArgFlagsTy Flags,
4087 unsigned PtrByteSize) {
4088 Align Alignment(PtrByteSize);
4089
4090 // Altivec parameters are padded to a 16 byte boundary.
4091 if (ArgVT == MVT::v4f32 || ArgVT == MVT::v4i32 ||
4092 ArgVT == MVT::v8i16 || ArgVT == MVT::v16i8 ||
4093 ArgVT == MVT::v2f64 || ArgVT == MVT::v2i64 ||
4094 ArgVT == MVT::v1i128 || ArgVT == MVT::f128)
4095 Alignment = Align(16);
4096
4097 // ByVal parameters are aligned as requested.
4098 if (Flags.isByVal()) {
4099 auto BVAlign = Flags.getNonZeroByValAlign();
4100 if (BVAlign > PtrByteSize) {
4101 if (BVAlign.value() % PtrByteSize != 0)
4102 llvm_unreachable(
4103 "ByVal alignment is not a multiple of the pointer size");
4104
4105 Alignment = BVAlign;
4106 }
4107 }
4108
4109 // Array members are always packed to their original alignment.
4110 if (Flags.isInConsecutiveRegs()) {
4111 // If the array member was split into multiple registers, the first
4112 // needs to be aligned to the size of the full type. (Except for
4113 // ppcf128, which is only aligned as its f64 components.)
4114 if (Flags.isSplit() && OrigVT != MVT::ppcf128)
4115 Alignment = Align(OrigVT.getStoreSize());
4116 else
4117 Alignment = Align(ArgVT.getStoreSize());
4118 }
4119
4120 return Alignment;
4121}
4122
4123/// CalculateStackSlotUsed - Return whether this argument will use its
4124/// stack slot (instead of being passed in registers). ArgOffset,
4125/// AvailableFPRs, and AvailableVRs must hold the current argument
4126/// position, and will be updated to account for this argument.
4127static bool CalculateStackSlotUsed(EVT ArgVT, EVT OrigVT, ISD::ArgFlagsTy Flags,
4128 unsigned PtrByteSize, unsigned LinkageSize,
4129 unsigned ParamAreaSize, unsigned &ArgOffset,
4130 unsigned &AvailableFPRs,
4131 unsigned &AvailableVRs) {
4132 bool UseMemory = false;
4133
4134 // Respect alignment of argument on the stack.
4135 Align Alignment =
4136 CalculateStackSlotAlignment(ArgVT, OrigVT, Flags, PtrByteSize);
4137 ArgOffset = alignTo(Size: ArgOffset, A: Alignment);
4138 // If there's no space left in the argument save area, we must
4139 // use memory (this check also catches zero-sized arguments).
4140 if (ArgOffset >= LinkageSize + ParamAreaSize)
4141 UseMemory = true;
4142
4143 // Allocate argument on the stack.
4144 ArgOffset += CalculateStackSlotSize(ArgVT, Flags, PtrByteSize);
4145 if (Flags.isInConsecutiveRegsLast())
4146 ArgOffset = ((ArgOffset + PtrByteSize - 1)/PtrByteSize) * PtrByteSize;
4147 // If we overran the argument save area, we must use memory
4148 // (this check catches arguments passed partially in memory)
4149 if (ArgOffset > LinkageSize + ParamAreaSize)
4150 UseMemory = true;
4151
4152 // However, if the argument is actually passed in an FPR or a VR,
4153 // we don't use memory after all.
4154 if (!Flags.isByVal()) {
4155 if (ArgVT == MVT::f32 || ArgVT == MVT::f64)
4156 if (AvailableFPRs > 0) {
4157 --AvailableFPRs;
4158 return false;
4159 }
4160 if (ArgVT == MVT::v4f32 || ArgVT == MVT::v4i32 ||
4161 ArgVT == MVT::v8i16 || ArgVT == MVT::v16i8 ||
4162 ArgVT == MVT::v2f64 || ArgVT == MVT::v2i64 ||
4163 ArgVT == MVT::v1i128 || ArgVT == MVT::f128)
4164 if (AvailableVRs > 0) {
4165 --AvailableVRs;
4166 return false;
4167 }
4168 }
4169
4170 return UseMemory;
4171}
4172
4173/// EnsureStackAlignment - Round stack frame size up from NumBytes to
4174/// ensure minimum alignment required for target.
4175static unsigned EnsureStackAlignment(const PPCFrameLowering *Lowering,
4176 unsigned NumBytes) {
4177 return alignTo(Size: NumBytes, A: Lowering->getStackAlign());
4178}
4179
4180SDValue PPCTargetLowering::LowerFormalArguments(
4181 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
4182 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
4183 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
4184 if (Subtarget.isAIXABI())
4185 return LowerFormalArguments_AIX(Chain, CallConv, isVarArg, Ins, dl, DAG,
4186 InVals);
4187 if (Subtarget.is64BitELFABI())
4188 return LowerFormalArguments_64SVR4(Chain, CallConv, isVarArg, Ins, dl, DAG,
4189 InVals);
4190 assert(Subtarget.is32BitELFABI());
4191 return LowerFormalArguments_32SVR4(Chain, CallConv, isVarArg, Ins, dl, DAG,
4192 InVals);
4193}
4194
4195SDValue PPCTargetLowering::LowerFormalArguments_32SVR4(
4196 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
4197 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
4198 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
4199
4200 // 32-bit SVR4 ABI Stack Frame Layout:
4201 // +-----------------------------------+
4202 // +--> | Back chain |
4203 // | +-----------------------------------+
4204 // | | Floating-point register save area |
4205 // | +-----------------------------------+
4206 // | | General register save area |
4207 // | +-----------------------------------+
4208 // | | CR save word |
4209 // | +-----------------------------------+
4210 // | | VRSAVE save word |
4211 // | +-----------------------------------+
4212 // | | Alignment padding |
4213 // | +-----------------------------------+
4214 // | | Vector register save area |
4215 // | +-----------------------------------+
4216 // | | Local variable space |
4217 // | +-----------------------------------+
4218 // | | Parameter list area |
4219 // | +-----------------------------------+
4220 // | | LR save word |
4221 // | +-----------------------------------+
4222 // SP--> +--- | Back chain |
4223 // +-----------------------------------+
4224 //
4225 // Specifications:
4226 // System V Application Binary Interface PowerPC Processor Supplement
4227 // AltiVec Technology Programming Interface Manual
4228
4229 MachineFunction &MF = DAG.getMachineFunction();
4230 MachineFrameInfo &MFI = MF.getFrameInfo();
4231 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
4232
4233 EVT PtrVT = getPointerTy(DL: MF.getDataLayout());
4234 // Potential tail calls could cause overwriting of argument stack slots.
4235 bool isImmutable = !(getTargetMachine().Options.GuaranteedTailCallOpt &&
4236 (CallConv == CallingConv::Fast));
4237 const Align PtrAlign(4);
4238
4239 // Assign locations to all of the incoming arguments.
4240 SmallVector<CCValAssign, 16> ArgLocs;
4241 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs,
4242 *DAG.getContext());
4243
4244 // Reserve space for the linkage area on the stack.
4245 unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize();
4246 CCInfo.AllocateStack(Size: LinkageSize, Alignment: PtrAlign);
4247 CCInfo.AnalyzeFormalArguments(Ins, Fn: CC_PPC32_SVR4);
4248
4249 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
4250 CCValAssign &VA = ArgLocs[i];
4251
4252 // Arguments stored in registers.
4253 if (VA.isRegLoc()) {
4254 const TargetRegisterClass *RC;
4255 EVT ValVT = VA.getValVT();
4256
4257 switch (ValVT.getSimpleVT().SimpleTy) {
4258 default:
4259 llvm_unreachable("ValVT not supported by formal arguments Lowering");
4260 case MVT::i1:
4261 case MVT::i32:
4262 RC = &PPC::GPRCRegClass;
4263 break;
4264 case MVT::f32:
4265 if (Subtarget.hasP8Vector())
4266 RC = &PPC::VSSRCRegClass;
4267 else if (Subtarget.hasSPE())
4268 RC = &PPC::GPRCRegClass;
4269 else
4270 RC = &PPC::F4RCRegClass;
4271 break;
4272 case MVT::f64:
4273 if (Subtarget.hasVSX())
4274 RC = &PPC::VSFRCRegClass;
4275 else if (Subtarget.hasSPE())
4276 // SPE passes doubles in GPR pairs.
4277 RC = &PPC::GPRCRegClass;
4278 else
4279 RC = &PPC::F8RCRegClass;
4280 break;
4281 case MVT::v16i8:
4282 case MVT::v8i16:
4283 case MVT::v4i32:
4284 RC = &PPC::VRRCRegClass;
4285 break;
4286 case MVT::v4f32:
4287 RC = &PPC::VRRCRegClass;
4288 break;
4289 case MVT::v2f64:
4290 case MVT::v2i64:
4291 RC = &PPC::VRRCRegClass;
4292 break;
4293 }
4294
4295 SDValue ArgValue;
4296 // Transform the arguments stored in physical registers into
4297 // virtual ones.
4298 if (VA.getLocVT() == MVT::f64 && Subtarget.hasSPE()) {
4299 assert(i + 1 < e && "No second half of double precision argument");
4300 Register RegLo = MF.addLiveIn(PReg: VA.getLocReg(), RC);
4301 Register RegHi = MF.addLiveIn(PReg: ArgLocs[++i].getLocReg(), RC);
4302 SDValue ArgValueLo = DAG.getCopyFromReg(Chain, dl, Reg: RegLo, VT: MVT::i32);
4303 SDValue ArgValueHi = DAG.getCopyFromReg(Chain, dl, Reg: RegHi, VT: MVT::i32);
4304 if (!Subtarget.isLittleEndian())
4305 std::swap (a&: ArgValueLo, b&: ArgValueHi);
4306 ArgValue = DAG.getNode(Opcode: PPCISD::BUILD_SPE64, DL: dl, VT: MVT::f64, N1: ArgValueLo,
4307 N2: ArgValueHi);
4308 } else {
4309 Register Reg = MF.addLiveIn(PReg: VA.getLocReg(), RC);
4310 ArgValue = DAG.getCopyFromReg(Chain, dl, Reg,
4311 VT: ValVT == MVT::i1 ? MVT::i32 : ValVT);
4312 if (ValVT == MVT::i1)
4313 ArgValue = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i1, Operand: ArgValue);
4314 }
4315
4316 InVals.push_back(Elt: ArgValue);
4317 } else {
4318 // Argument stored in memory.
4319 assert(VA.isMemLoc());
4320
4321 // Get the extended size of the argument type in stack
4322 unsigned ArgSize = VA.getLocVT().getStoreSize();
4323 // Get the actual size of the argument type
4324 unsigned ObjSize = VA.getValVT().getStoreSize();
4325 unsigned ArgOffset = VA.getLocMemOffset();
4326 // Stack objects in PPC32 are right justified.
4327 ArgOffset += ArgSize - ObjSize;
4328 int FI = MFI.CreateFixedObject(Size: ArgSize, SPOffset: ArgOffset, IsImmutable: isImmutable);
4329
4330 // Create load nodes to retrieve arguments from the stack.
4331 SDValue FIN = DAG.getFrameIndex(FI, VT: PtrVT);
4332 InVals.push_back(
4333 Elt: DAG.getLoad(VT: VA.getValVT(), dl, Chain, Ptr: FIN, PtrInfo: MachinePointerInfo()));
4334 }
4335 }
4336
4337 // Assign locations to all of the incoming aggregate by value arguments.
4338 // Aggregates passed by value are stored in the local variable space of the
4339 // caller's stack frame, right above the parameter list area.
4340 SmallVector<CCValAssign, 16> ByValArgLocs;
4341 CCState CCByValInfo(CallConv, isVarArg, DAG.getMachineFunction(),
4342 ByValArgLocs, *DAG.getContext());
4343
4344 // Reserve stack space for the allocations in CCInfo.
4345 CCByValInfo.AllocateStack(Size: CCInfo.getStackSize(), Alignment: PtrAlign);
4346
4347 CCByValInfo.AnalyzeFormalArguments(Ins, Fn: CC_PPC32_SVR4_ByVal);
4348
4349 // Area that is at least reserved in the caller of this function.
4350 unsigned MinReservedArea = CCByValInfo.getStackSize();
4351 MinReservedArea = std::max(a: MinReservedArea, b: LinkageSize);
4352
4353 // Set the size that is at least reserved in caller of this function. Tail
4354 // call optimized function's reserved stack space needs to be aligned so that
4355 // taking the difference between two stack areas will result in an aligned
4356 // stack.
4357 MinReservedArea =
4358 EnsureStackAlignment(Lowering: Subtarget.getFrameLowering(), NumBytes: MinReservedArea);
4359 FuncInfo->setMinReservedArea(MinReservedArea);
4360
4361 SmallVector<SDValue, 8> MemOps;
4362
4363 // If the function takes variable number of arguments, make a frame index for
4364 // the start of the first vararg value... for expansion of llvm.va_start.
4365 if (isVarArg) {
4366 static const MCPhysReg GPArgRegs[] = {
4367 PPC::R3, PPC::R4, PPC::R5, PPC::R6,
4368 PPC::R7, PPC::R8, PPC::R9, PPC::R10,
4369 };
4370 const unsigned NumGPArgRegs = std::size(GPArgRegs);
4371
4372 static const MCPhysReg FPArgRegs[] = {
4373 PPC::F1, PPC::F2, PPC::F3, PPC::F4, PPC::F5, PPC::F6, PPC::F7,
4374 PPC::F8
4375 };
4376 unsigned NumFPArgRegs = std::size(FPArgRegs);
4377
4378 if (useSoftFloat() || hasSPE())
4379 NumFPArgRegs = 0;
4380
4381 FuncInfo->setVarArgsNumGPR(CCInfo.getFirstUnallocated(Regs: GPArgRegs));
4382 FuncInfo->setVarArgsNumFPR(CCInfo.getFirstUnallocated(Regs: FPArgRegs));
4383
4384 // Make room for NumGPArgRegs and NumFPArgRegs.
4385 int Depth = NumGPArgRegs * PtrVT.getSizeInBits()/8 +
4386 NumFPArgRegs * MVT(MVT::f64).getSizeInBits()/8;
4387
4388 FuncInfo->setVarArgsStackOffset(MFI.CreateFixedObject(
4389 Size: PtrVT.getSizeInBits() / 8, SPOffset: CCInfo.getStackSize(), IsImmutable: true));
4390
4391 FuncInfo->setVarArgsFrameIndex(
4392 MFI.CreateStackObject(Size: Depth, Alignment: Align(8), isSpillSlot: false));
4393 SDValue FIN = DAG.getFrameIndex(FI: FuncInfo->getVarArgsFrameIndex(), VT: PtrVT);
4394
4395 // The fixed integer arguments of a variadic function are stored to the
4396 // VarArgsFrameIndex on the stack so that they may be loaded by
4397 // dereferencing the result of va_next.
4398 for (MCPhysReg GPArgReg : GPArgRegs) {
4399 // Get an existing live-in vreg, or add a new one.
4400 Register VReg = MF.getRegInfo().getLiveInVirtReg(PReg: GPArgReg);
4401 if (!VReg)
4402 VReg = MF.addLiveIn(PReg: GPArgReg, RC: &PPC::GPRCRegClass);
4403
4404 SDValue Val = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: PtrVT);
4405 SDValue Store =
4406 DAG.getStore(Chain: Val.getValue(R: 1), dl, Val, Ptr: FIN, PtrInfo: MachinePointerInfo());
4407 MemOps.push_back(Elt: Store);
4408 // Increment the address by four for the next argument to store
4409 SDValue PtrOff = DAG.getConstant(Val: PtrVT.getSizeInBits()/8, DL: dl, VT: PtrVT);
4410 FIN = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrOff.getValueType(), N1: FIN, N2: PtrOff);
4411 }
4412
4413 // FIXME 32-bit SVR4: We only need to save FP argument registers if CR bit 6
4414 // is set.
4415 // The double arguments are stored to the VarArgsFrameIndex
4416 // on the stack.
4417 for (unsigned FPRIndex = 0; FPRIndex != NumFPArgRegs; ++FPRIndex) {
4418 // Get an existing live-in vreg, or add a new one.
4419 Register VReg = MF.getRegInfo().getLiveInVirtReg(PReg: FPArgRegs[FPRIndex]);
4420 if (!VReg)
4421 VReg = MF.addLiveIn(PReg: FPArgRegs[FPRIndex], RC: &PPC::F8RCRegClass);
4422
4423 SDValue Val = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: MVT::f64);
4424 SDValue Store =
4425 DAG.getStore(Chain: Val.getValue(R: 1), dl, Val, Ptr: FIN, PtrInfo: MachinePointerInfo());
4426 MemOps.push_back(Elt: Store);
4427 // Increment the address by eight for the next argument to store
4428 SDValue PtrOff = DAG.getConstant(Val: MVT(MVT::f64).getSizeInBits()/8, DL: dl,
4429 VT: PtrVT);
4430 FIN = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrOff.getValueType(), N1: FIN, N2: PtrOff);
4431 }
4432 }
4433
4434 if (!MemOps.empty())
4435 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: MemOps);
4436
4437 return Chain;
4438}
4439
4440// PPC64 passes i8, i16, and i32 values in i64 registers. Promote
4441// value to MVT::i64 and then truncate to the correct register size.
4442SDValue PPCTargetLowering::extendArgForPPC64(ISD::ArgFlagsTy Flags,
4443 EVT ObjectVT, SelectionDAG &DAG,
4444 SDValue ArgVal,
4445 const SDLoc &dl) const {
4446 if (Flags.isSExt())
4447 ArgVal = DAG.getNode(Opcode: ISD::AssertSext, DL: dl, VT: MVT::i64, N1: ArgVal,
4448 N2: DAG.getValueType(ObjectVT));
4449 else if (Flags.isZExt())
4450 ArgVal = DAG.getNode(Opcode: ISD::AssertZext, DL: dl, VT: MVT::i64, N1: ArgVal,
4451 N2: DAG.getValueType(ObjectVT));
4452
4453 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: ObjectVT, Operand: ArgVal);
4454}
4455
4456SDValue PPCTargetLowering::LowerFormalArguments_64SVR4(
4457 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
4458 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
4459 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
4460 // TODO: add description of PPC stack frame format, or at least some docs.
4461 //
4462 bool isELFv2ABI = Subtarget.isELFv2ABI();
4463 bool isLittleEndian = Subtarget.isLittleEndian();
4464 MachineFunction &MF = DAG.getMachineFunction();
4465 MachineFrameInfo &MFI = MF.getFrameInfo();
4466 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
4467
4468 assert(!(CallConv == CallingConv::Fast && isVarArg) &&
4469 "fastcc not supported on varargs functions");
4470
4471 EVT PtrVT = getPointerTy(DL: MF.getDataLayout());
4472 // Potential tail calls could cause overwriting of argument stack slots.
4473 bool isImmutable = !(getTargetMachine().Options.GuaranteedTailCallOpt &&
4474 (CallConv == CallingConv::Fast));
4475 unsigned PtrByteSize = 8;
4476 unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize();
4477
4478 static const MCPhysReg GPR[] = {
4479 PPC::X3, PPC::X4, PPC::X5, PPC::X6,
4480 PPC::X7, PPC::X8, PPC::X9, PPC::X10,
4481 };
4482 static const MCPhysReg VR[] = {
4483 PPC::V2, PPC::V3, PPC::V4, PPC::V5, PPC::V6, PPC::V7, PPC::V8,
4484 PPC::V9, PPC::V10, PPC::V11, PPC::V12, PPC::V13
4485 };
4486
4487 const unsigned Num_GPR_Regs = std::size(GPR);
4488 const unsigned Num_FPR_Regs = useSoftFloat() ? 0 : 13;
4489 const unsigned Num_VR_Regs = std::size(VR);
4490
4491 // Do a first pass over the arguments to determine whether the ABI
4492 // guarantees that our caller has allocated the parameter save area
4493 // on its stack frame. In the ELFv1 ABI, this is always the case;
4494 // in the ELFv2 ABI, it is true if this is a vararg function or if
4495 // any parameter is located in a stack slot.
4496
4497 bool HasParameterArea = !isELFv2ABI || isVarArg;
4498 unsigned ParamAreaSize = Num_GPR_Regs * PtrByteSize;
4499 unsigned NumBytes = LinkageSize;
4500 unsigned AvailableFPRs = Num_FPR_Regs;
4501 unsigned AvailableVRs = Num_VR_Regs;
4502 for (const ISD::InputArg &In : Ins) {
4503 if (In.Flags.isNest())
4504 continue;
4505
4506 if (CalculateStackSlotUsed(ArgVT: In.VT, OrigVT: In.ArgVT, Flags: In.Flags, PtrByteSize,
4507 LinkageSize, ParamAreaSize, ArgOffset&: NumBytes,
4508 AvailableFPRs, AvailableVRs))
4509 HasParameterArea = true;
4510 }
4511
4512 // Add DAG nodes to load the arguments or copy them out of registers. On
4513 // entry to a function on PPC, the arguments start after the linkage area,
4514 // although the first ones are often in registers.
4515
4516 unsigned ArgOffset = LinkageSize;
4517 unsigned GPR_idx = 0, FPR_idx = 0, VR_idx = 0;
4518 SmallVector<SDValue, 8> MemOps;
4519 Function::const_arg_iterator FuncArg = MF.getFunction().arg_begin();
4520 unsigned CurArgIdx = 0;
4521 for (unsigned ArgNo = 0, e = Ins.size(); ArgNo != e; ++ArgNo) {
4522 SDValue ArgVal;
4523 bool needsLoad = false;
4524 EVT ObjectVT = Ins[ArgNo].VT;
4525 EVT OrigVT = Ins[ArgNo].ArgVT;
4526 unsigned ObjSize = ObjectVT.getStoreSize();
4527 unsigned ArgSize = ObjSize;
4528 ISD::ArgFlagsTy Flags = Ins[ArgNo].Flags;
4529 if (Ins[ArgNo].isOrigArg()) {
4530 std::advance(i&: FuncArg, n: Ins[ArgNo].getOrigArgIndex() - CurArgIdx);
4531 CurArgIdx = Ins[ArgNo].getOrigArgIndex();
4532 }
4533 // We re-align the argument offset for each argument, except when using the
4534 // fast calling convention, when we need to make sure we do that only when
4535 // we'll actually use a stack slot.
4536 unsigned CurArgOffset;
4537 Align Alignment;
4538 auto ComputeArgOffset = [&]() {
4539 /* Respect alignment of argument on the stack. */
4540 Alignment =
4541 CalculateStackSlotAlignment(ArgVT: ObjectVT, OrigVT, Flags, PtrByteSize);
4542 ArgOffset = alignTo(Size: ArgOffset, A: Alignment);
4543 CurArgOffset = ArgOffset;
4544 };
4545
4546 if (CallConv != CallingConv::Fast) {
4547 ComputeArgOffset();
4548
4549 /* Compute GPR index associated with argument offset. */
4550 GPR_idx = (ArgOffset - LinkageSize) / PtrByteSize;
4551 GPR_idx = std::min(a: GPR_idx, b: Num_GPR_Regs);
4552 }
4553
4554 // FIXME the codegen can be much improved in some cases.
4555 // We do not have to keep everything in memory.
4556 if (Flags.isByVal()) {
4557 assert(Ins[ArgNo].isOrigArg() && "Byval arguments cannot be implicit");
4558
4559 if (CallConv == CallingConv::Fast)
4560 ComputeArgOffset();
4561
4562 // ObjSize is the true size, ArgSize rounded up to multiple of registers.
4563 ObjSize = Flags.getByValSize();
4564 ArgSize = ((ObjSize + PtrByteSize - 1)/PtrByteSize) * PtrByteSize;
4565 // Empty aggregate parameters do not take up registers. Examples:
4566 // struct { } a;
4567 // union { } b;
4568 // int c[0];
4569 // etc. However, we have to provide a place-holder in InVals, so
4570 // pretend we have an 8-byte item at the current address for that
4571 // purpose.
4572 if (!ObjSize) {
4573 int FI = MFI.CreateFixedObject(Size: PtrByteSize, SPOffset: ArgOffset, IsImmutable: true);
4574 SDValue FIN = DAG.getFrameIndex(FI, VT: PtrVT);
4575 InVals.push_back(Elt: FIN);
4576 continue;
4577 }
4578
4579 // Create a stack object covering all stack doublewords occupied
4580 // by the argument. If the argument is (fully or partially) on
4581 // the stack, or if the argument is fully in registers but the
4582 // caller has allocated the parameter save anyway, we can refer
4583 // directly to the caller's stack frame. Otherwise, create a
4584 // local copy in our own frame.
4585 int FI;
4586 if (HasParameterArea ||
4587 ArgSize + ArgOffset > LinkageSize + Num_GPR_Regs * PtrByteSize)
4588 FI = MFI.CreateFixedObject(Size: ArgSize, SPOffset: ArgOffset, IsImmutable: false, isAliased: true);
4589 else
4590 FI = MFI.CreateStackObject(Size: ArgSize, Alignment, isSpillSlot: false);
4591 SDValue FIN = DAG.getFrameIndex(FI, VT: PtrVT);
4592
4593 // Handle aggregates smaller than 8 bytes.
4594 if (ObjSize < PtrByteSize) {
4595 // The value of the object is its address, which differs from the
4596 // address of the enclosing doubleword on big-endian systems.
4597 SDValue Arg = FIN;
4598 if (!isLittleEndian) {
4599 SDValue ArgOff = DAG.getConstant(Val: PtrByteSize - ObjSize, DL: dl, VT: PtrVT);
4600 Arg = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ArgOff.getValueType(), N1: Arg, N2: ArgOff);
4601 }
4602 InVals.push_back(Elt: Arg);
4603
4604 if (GPR_idx != Num_GPR_Regs) {
4605 Register VReg = MF.addLiveIn(PReg: GPR[GPR_idx++], RC: &PPC::G8RCRegClass);
4606 FuncInfo->addLiveInAttr(VReg, Flags);
4607 SDValue Val = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: PtrVT);
4608 EVT ObjType = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ObjSize * 8);
4609 SDValue Store =
4610 DAG.getTruncStore(Chain: Val.getValue(R: 1), dl, Val, Ptr: Arg,
4611 PtrInfo: MachinePointerInfo(&*FuncArg), SVT: ObjType);
4612 MemOps.push_back(Elt: Store);
4613 }
4614 // Whether we copied from a register or not, advance the offset
4615 // into the parameter save area by a full doubleword.
4616 ArgOffset += PtrByteSize;
4617 continue;
4618 }
4619
4620 // The value of the object is its address, which is the address of
4621 // its first stack doubleword.
4622 InVals.push_back(Elt: FIN);
4623
4624 // Store whatever pieces of the object are in registers to memory.
4625 for (unsigned j = 0; j < ArgSize; j += PtrByteSize) {
4626 if (GPR_idx == Num_GPR_Regs)
4627 break;
4628
4629 Register VReg = MF.addLiveIn(PReg: GPR[GPR_idx], RC: &PPC::G8RCRegClass);
4630 FuncInfo->addLiveInAttr(VReg, Flags);
4631 SDValue Val = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: PtrVT);
4632 SDValue Addr = FIN;
4633 if (j) {
4634 SDValue Off = DAG.getConstant(Val: j, DL: dl, VT: PtrVT);
4635 Addr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: Off.getValueType(), N1: Addr, N2: Off);
4636 }
4637 unsigned StoreSizeInBits = std::min(a: PtrByteSize, b: (ObjSize - j)) * 8;
4638 EVT ObjType = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: StoreSizeInBits);
4639 SDValue Store =
4640 DAG.getTruncStore(Chain: Val.getValue(R: 1), dl, Val, Ptr: Addr,
4641 PtrInfo: MachinePointerInfo(&*FuncArg, j), SVT: ObjType);
4642 MemOps.push_back(Elt: Store);
4643 ++GPR_idx;
4644 }
4645 ArgOffset += ArgSize;
4646 continue;
4647 }
4648
4649 switch (ObjectVT.getSimpleVT().SimpleTy) {
4650 default: llvm_unreachable("Unhandled argument type!");
4651 case MVT::i1:
4652 case MVT::i32:
4653 case MVT::i64:
4654 if (Flags.isNest()) {
4655 // The 'nest' parameter, if any, is passed in R11.
4656 Register VReg = MF.addLiveIn(PReg: PPC::X11, RC: &PPC::G8RCRegClass);
4657 ArgVal = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: MVT::i64);
4658
4659 if (ObjectVT == MVT::i32 || ObjectVT == MVT::i1)
4660 ArgVal = extendArgForPPC64(Flags, ObjectVT, DAG, ArgVal, dl);
4661
4662 break;
4663 }
4664
4665 // These can be scalar arguments or elements of an integer array type
4666 // passed directly. Clang may use those instead of "byval" aggregate
4667 // types to avoid forcing arguments to memory unnecessarily.
4668 if (GPR_idx != Num_GPR_Regs) {
4669 Register VReg = MF.addLiveIn(PReg: GPR[GPR_idx++], RC: &PPC::G8RCRegClass);
4670 FuncInfo->addLiveInAttr(VReg, Flags);
4671 ArgVal = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: MVT::i64);
4672
4673 if (ObjectVT == MVT::i32 || ObjectVT == MVT::i1)
4674 // PPC64 passes i8, i16, and i32 values in i64 registers. Promote
4675 // value to MVT::i64 and then truncate to the correct register size.
4676 ArgVal = extendArgForPPC64(Flags, ObjectVT, DAG, ArgVal, dl);
4677 } else {
4678 if (CallConv == CallingConv::Fast)
4679 ComputeArgOffset();
4680
4681 needsLoad = true;
4682 ArgSize = PtrByteSize;
4683 }
4684 if (CallConv != CallingConv::Fast || needsLoad)
4685 ArgOffset += 8;
4686 break;
4687
4688 case MVT::f32:
4689 case MVT::f64:
4690 // These can be scalar arguments or elements of a float array type
4691 // passed directly. The latter are used to implement ELFv2 homogenous
4692 // float aggregates.
4693 if (FPR_idx != Num_FPR_Regs) {
4694 unsigned VReg;
4695
4696 if (ObjectVT == MVT::f32)
4697 VReg = MF.addLiveIn(PReg: FPR[FPR_idx],
4698 RC: Subtarget.hasP8Vector()
4699 ? &PPC::VSSRCRegClass
4700 : &PPC::F4RCRegClass);
4701 else
4702 VReg = MF.addLiveIn(PReg: FPR[FPR_idx], RC: Subtarget.hasVSX()
4703 ? &PPC::VSFRCRegClass
4704 : &PPC::F8RCRegClass);
4705
4706 ArgVal = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: ObjectVT);
4707 ++FPR_idx;
4708 } else if (GPR_idx != Num_GPR_Regs && CallConv != CallingConv::Fast) {
4709 // FIXME: We may want to re-enable this for CallingConv::Fast on the P8
4710 // once we support fp <-> gpr moves.
4711
4712 // This can only ever happen in the presence of f32 array types,
4713 // since otherwise we never run out of FPRs before running out
4714 // of GPRs.
4715 Register VReg = MF.addLiveIn(PReg: GPR[GPR_idx++], RC: &PPC::G8RCRegClass);
4716 FuncInfo->addLiveInAttr(VReg, Flags);
4717 ArgVal = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: MVT::i64);
4718
4719 if (ObjectVT == MVT::f32) {
4720 if ((ArgOffset % PtrByteSize) == (isLittleEndian ? 4 : 0))
4721 ArgVal = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i64, N1: ArgVal,
4722 N2: DAG.getConstant(Val: 32, DL: dl, VT: MVT::i32));
4723 ArgVal = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i32, Operand: ArgVal);
4724 }
4725
4726 ArgVal = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: ObjectVT, Operand: ArgVal);
4727 } else {
4728 if (CallConv == CallingConv::Fast)
4729 ComputeArgOffset();
4730
4731 needsLoad = true;
4732 }
4733
4734 // When passing an array of floats, the array occupies consecutive
4735 // space in the argument area; only round up to the next doubleword
4736 // at the end of the array. Otherwise, each float takes 8 bytes.
4737 if (CallConv != CallingConv::Fast || needsLoad) {
4738 ArgSize = Flags.isInConsecutiveRegs() ? ObjSize : PtrByteSize;
4739 ArgOffset += ArgSize;
4740 if (Flags.isInConsecutiveRegsLast())
4741 ArgOffset = ((ArgOffset + PtrByteSize - 1)/PtrByteSize) * PtrByteSize;
4742 }
4743 break;
4744 case MVT::v4f32:
4745 case MVT::v4i32:
4746 case MVT::v8i16:
4747 case MVT::v16i8:
4748 case MVT::v2f64:
4749 case MVT::v2i64:
4750 case MVT::v1i128:
4751 case MVT::f128:
4752 // These can be scalar arguments or elements of a vector array type
4753 // passed directly. The latter are used to implement ELFv2 homogenous
4754 // vector aggregates.
4755 if (VR_idx != Num_VR_Regs) {
4756 Register VReg = MF.addLiveIn(PReg: VR[VR_idx], RC: &PPC::VRRCRegClass);
4757 ArgVal = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: ObjectVT);
4758 ++VR_idx;
4759 } else {
4760 if (CallConv == CallingConv::Fast)
4761 ComputeArgOffset();
4762 needsLoad = true;
4763 }
4764 if (CallConv != CallingConv::Fast || needsLoad)
4765 ArgOffset += 16;
4766 break;
4767 }
4768
4769 // We need to load the argument to a virtual register if we determined
4770 // above that we ran out of physical registers of the appropriate type.
4771 if (needsLoad) {
4772 if (ObjSize < ArgSize && !isLittleEndian)
4773 CurArgOffset += ArgSize - ObjSize;
4774 int FI = MFI.CreateFixedObject(Size: ObjSize, SPOffset: CurArgOffset, IsImmutable: isImmutable);
4775 SDValue FIN = DAG.getFrameIndex(FI, VT: PtrVT);
4776 ArgVal = DAG.getLoad(VT: ObjectVT, dl, Chain, Ptr: FIN, PtrInfo: MachinePointerInfo());
4777 }
4778
4779 InVals.push_back(Elt: ArgVal);
4780 }
4781
4782 // Area that is at least reserved in the caller of this function.
4783 unsigned MinReservedArea;
4784 if (HasParameterArea)
4785 MinReservedArea = std::max(a: ArgOffset, b: LinkageSize + 8 * PtrByteSize);
4786 else
4787 MinReservedArea = LinkageSize;
4788
4789 // Set the size that is at least reserved in caller of this function. Tail
4790 // call optimized functions' reserved stack space needs to be aligned so that
4791 // taking the difference between two stack areas will result in an aligned
4792 // stack.
4793 MinReservedArea =
4794 EnsureStackAlignment(Lowering: Subtarget.getFrameLowering(), NumBytes: MinReservedArea);
4795 FuncInfo->setMinReservedArea(MinReservedArea);
4796
4797 // If the function takes variable number of arguments, make a frame index for
4798 // the start of the first vararg value... for expansion of llvm.va_start.
4799 // On ELFv2ABI spec, it writes:
4800 // C programs that are intended to be *portable* across different compilers
4801 // and architectures must use the header file <stdarg.h> to deal with variable
4802 // argument lists.
4803 if (isVarArg && MFI.hasVAStart()) {
4804 int Depth = ArgOffset;
4805
4806 FuncInfo->setVarArgsFrameIndex(
4807 MFI.CreateFixedObject(Size: PtrByteSize, SPOffset: Depth, IsImmutable: true));
4808 SDValue FIN = DAG.getFrameIndex(FI: FuncInfo->getVarArgsFrameIndex(), VT: PtrVT);
4809
4810 // If this function is vararg, store any remaining integer argument regs
4811 // to their spots on the stack so that they may be loaded by dereferencing
4812 // the result of va_next.
4813 for (GPR_idx = (ArgOffset - LinkageSize) / PtrByteSize;
4814 GPR_idx < Num_GPR_Regs; ++GPR_idx) {
4815 Register VReg = MF.addLiveIn(PReg: GPR[GPR_idx], RC: &PPC::G8RCRegClass);
4816 SDValue Val = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: PtrVT);
4817 SDValue Store =
4818 DAG.getStore(Chain: Val.getValue(R: 1), dl, Val, Ptr: FIN, PtrInfo: MachinePointerInfo());
4819 MemOps.push_back(Elt: Store);
4820 // Increment the address by four for the next argument to store
4821 SDValue PtrOff = DAG.getConstant(Val: PtrByteSize, DL: dl, VT: PtrVT);
4822 FIN = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrOff.getValueType(), N1: FIN, N2: PtrOff);
4823 }
4824 }
4825
4826 if (!MemOps.empty())
4827 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: MemOps);
4828
4829 return Chain;
4830}
4831
4832/// CalculateTailCallSPDiff - Get the amount the stack pointer has to be
4833/// adjusted to accommodate the arguments for the tailcall.
4834static int CalculateTailCallSPDiff(SelectionDAG& DAG, bool isTailCall,
4835 unsigned ParamSize) {
4836
4837 if (!isTailCall) return 0;
4838
4839 PPCFunctionInfo *FI = DAG.getMachineFunction().getInfo<PPCFunctionInfo>();
4840 unsigned CallerMinReservedArea = FI->getMinReservedArea();
4841 int SPDiff = (int)CallerMinReservedArea - (int)ParamSize;
4842 // Remember only if the new adjustment is bigger.
4843 if (SPDiff < FI->getTailCallSPDelta())
4844 FI->setTailCallSPDelta(SPDiff);
4845
4846 return SPDiff;
4847}
4848
4849static bool isFunctionGlobalAddress(const GlobalValue *CalleeGV);
4850
4851static bool callsShareTOCBase(const Function *Caller,
4852 const GlobalValue *CalleeGV,
4853 const TargetMachine &TM) {
4854 // It does not make sense to call callsShareTOCBase() with a caller that
4855 // is PC Relative since PC Relative callers do not have a TOC.
4856#ifndef NDEBUG
4857 const PPCSubtarget *STICaller = &TM.getSubtarget<PPCSubtarget>(*Caller);
4858 assert(!STICaller->isUsingPCRelativeCalls() &&
4859 "PC Relative callers do not have a TOC and cannot share a TOC Base");
4860#endif
4861
4862 // Callee is either a GlobalAddress or an ExternalSymbol. ExternalSymbols
4863 // don't have enough information to determine if the caller and callee share
4864 // the same TOC base, so we have to pessimistically assume they don't for
4865 // correctness.
4866 if (!CalleeGV)
4867 return false;
4868
4869 // If the callee is preemptable, then the static linker will use a plt-stub
4870 // which saves the toc to the stack, and needs a nop after the call
4871 // instruction to convert to a toc-restore.
4872 if (!TM.shouldAssumeDSOLocal(GV: CalleeGV))
4873 return false;
4874
4875 // Functions with PC Relative enabled may clobber the TOC in the same DSO.
4876 // We may need a TOC restore in the situation where the caller requires a
4877 // valid TOC but the callee is PC Relative and does not.
4878 const Function *F = dyn_cast<Function>(Val: CalleeGV);
4879 const GlobalAlias *Alias = dyn_cast<GlobalAlias>(Val: CalleeGV);
4880
4881 // If we have an Alias we can try to get the function from there.
4882 if (Alias) {
4883 const GlobalObject *GlobalObj = Alias->getAliaseeObject();
4884 F = dyn_cast<Function>(Val: GlobalObj);
4885 }
4886
4887 // If we still have no valid function pointer we do not have enough
4888 // information to determine if the callee uses PC Relative calls so we must
4889 // assume that it does.
4890 if (!F)
4891 return false;
4892
4893 // If the callee uses PC Relative we cannot guarantee that the callee won't
4894 // clobber the TOC of the caller and so we must assume that the two
4895 // functions do not share a TOC base.
4896 const PPCSubtarget *STICallee = &TM.getSubtarget<PPCSubtarget>(F: *F);
4897 if (STICallee->isUsingPCRelativeCalls())
4898 return false;
4899
4900 // If the GV is not a strong definition then we need to assume it can be
4901 // replaced by another function at link time. The function that replaces
4902 // it may not share the same TOC as the caller since the callee may be
4903 // replaced by a PC Relative version of the same function.
4904 if (!CalleeGV->isStrongDefinitionForLinker())
4905 return false;
4906
4907 // The medium and large code models are expected to provide a sufficiently
4908 // large TOC to provide all data addressing needs of a module with a
4909 // single TOC.
4910 if (CodeModel::Medium == TM.getCodeModel() ||
4911 CodeModel::Large == TM.getCodeModel())
4912 return true;
4913
4914 // Any explicitly-specified sections and section prefixes must also match.
4915 // Also, if we're using -ffunction-sections, then each function is always in
4916 // a different section (the same is true for COMDAT functions).
4917 if (TM.getFunctionSections() || CalleeGV->hasComdat() ||
4918 Caller->hasComdat() || CalleeGV->getSection() != Caller->getSection())
4919 return false;
4920 if (const auto *F = dyn_cast<Function>(Val: CalleeGV)) {
4921 if (F->getSectionPrefix() != Caller->getSectionPrefix())
4922 return false;
4923 }
4924
4925 return true;
4926}
4927
4928static bool
4929needStackSlotPassParameters(const PPCSubtarget &Subtarget,
4930 const SmallVectorImpl<ISD::OutputArg> &Outs) {
4931 assert(Subtarget.is64BitELFABI());
4932
4933 const unsigned PtrByteSize = 8;
4934 const unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize();
4935
4936 static const MCPhysReg GPR[] = {
4937 PPC::X3, PPC::X4, PPC::X5, PPC::X6,
4938 PPC::X7, PPC::X8, PPC::X9, PPC::X10,
4939 };
4940 static const MCPhysReg VR[] = {
4941 PPC::V2, PPC::V3, PPC::V4, PPC::V5, PPC::V6, PPC::V7, PPC::V8,
4942 PPC::V9, PPC::V10, PPC::V11, PPC::V12, PPC::V13
4943 };
4944
4945 const unsigned NumGPRs = std::size(GPR);
4946 const unsigned NumFPRs = 13;
4947 const unsigned NumVRs = std::size(VR);
4948 const unsigned ParamAreaSize = NumGPRs * PtrByteSize;
4949
4950 unsigned NumBytes = LinkageSize;
4951 unsigned AvailableFPRs = NumFPRs;
4952 unsigned AvailableVRs = NumVRs;
4953
4954 for (const ISD::OutputArg& Param : Outs) {
4955 if (Param.Flags.isNest()) continue;
4956
4957 if (CalculateStackSlotUsed(ArgVT: Param.VT, OrigVT: Param.ArgVT, Flags: Param.Flags, PtrByteSize,
4958 LinkageSize, ParamAreaSize, ArgOffset&: NumBytes,
4959 AvailableFPRs, AvailableVRs))
4960 return true;
4961 }
4962 return false;
4963}
4964
4965static bool hasSameArgumentList(const Function *CallerFn, const CallBase &CB) {
4966 if (CB.arg_size() != CallerFn->arg_size())
4967 return false;
4968
4969 auto CalleeArgIter = CB.arg_begin();
4970 auto CalleeArgEnd = CB.arg_end();
4971 Function::const_arg_iterator CallerArgIter = CallerFn->arg_begin();
4972
4973 for (; CalleeArgIter != CalleeArgEnd; ++CalleeArgIter, ++CallerArgIter) {
4974 const Value* CalleeArg = *CalleeArgIter;
4975 const Value* CallerArg = &(*CallerArgIter);
4976 if (CalleeArg == CallerArg)
4977 continue;
4978
4979 // e.g. @caller([4 x i64] %a, [4 x i64] %b) {
4980 // tail call @callee([4 x i64] undef, [4 x i64] %b)
4981 // }
4982 // 1st argument of callee is undef and has the same type as caller.
4983 if (CalleeArg->getType() == CallerArg->getType() &&
4984 isa<UndefValue>(Val: CalleeArg))
4985 continue;
4986
4987 return false;
4988 }
4989
4990 return true;
4991}
4992
4993// Returns true if TCO is possible between the callers and callees
4994// calling conventions.
4995static bool
4996areCallingConvEligibleForTCO_64SVR4(CallingConv::ID CallerCC,
4997 CallingConv::ID CalleeCC) {
4998 // Tail calls are possible with fastcc and ccc.
4999 auto isTailCallableCC = [] (CallingConv::ID CC){
5000 return CC == CallingConv::C || CC == CallingConv::Fast;
5001 };
5002 if (!isTailCallableCC(CallerCC) || !isTailCallableCC(CalleeCC))
5003 return false;
5004
5005 // We can safely tail call both fastcc and ccc callees from a c calling
5006 // convention caller. If the caller is fastcc, we may have less stack space
5007 // than a non-fastcc caller with the same signature so disable tail-calls in
5008 // that case.
5009 return CallerCC == CallingConv::C || CallerCC == CalleeCC;
5010}
5011
5012bool PPCTargetLowering::IsEligibleForTailCallOptimization_64SVR4(
5013 const GlobalValue *CalleeGV, CallingConv::ID CalleeCC,
5014 CallingConv::ID CallerCC, const CallBase *CB, bool isVarArg,
5015 const SmallVectorImpl<ISD::OutputArg> &Outs,
5016 const SmallVectorImpl<ISD::InputArg> &Ins, const Function *CallerFunc,
5017 bool isCalleeExternalSymbol) const {
5018 bool TailCallOpt = getTargetMachine().Options.GuaranteedTailCallOpt;
5019
5020 if (DisableSCO && !TailCallOpt) return false;
5021
5022 // Variadic argument functions are not supported.
5023 if (isVarArg) return false;
5024
5025 // Check that the calling conventions are compatible for tco.
5026 if (!areCallingConvEligibleForTCO_64SVR4(CallerCC, CalleeCC))
5027 return false;
5028
5029 // Caller contains any byval parameter is not supported.
5030 if (any_of(Range: Ins, P: [](const ISD::InputArg &IA) { return IA.Flags.isByVal(); }))
5031 return false;
5032
5033 // Callee contains any byval parameter is not supported, too.
5034 // Note: This is a quick work around, because in some cases, e.g.
5035 // caller's stack size > callee's stack size, we are still able to apply
5036 // sibling call optimization. For example, gcc is able to do SCO for caller1
5037 // in the following example, but not for caller2.
5038 // struct test {
5039 // long int a;
5040 // char ary[56];
5041 // } gTest;
5042 // __attribute__((noinline)) int callee(struct test v, struct test *b) {
5043 // b->a = v.a;
5044 // return 0;
5045 // }
5046 // void caller1(struct test a, struct test c, struct test *b) {
5047 // callee(gTest, b); }
5048 // void caller2(struct test *b) { callee(gTest, b); }
5049 if (any_of(Range: Outs, P: [](const ISD::OutputArg& OA) { return OA.Flags.isByVal(); }))
5050 return false;
5051
5052 // If callee and caller use different calling conventions, we cannot pass
5053 // parameters on stack since offsets for the parameter area may be different.
5054 if (CallerCC != CalleeCC && needStackSlotPassParameters(Subtarget, Outs))
5055 return false;
5056
5057 // All variants of 64-bit ELF ABIs without PC-Relative addressing require that
5058 // the caller and callee share the same TOC for TCO/SCO. If the caller and
5059 // callee potentially have different TOC bases then we cannot tail call since
5060 // we need to restore the TOC pointer after the call.
5061 // ref: https://bugzilla.mozilla.org/show_bug.cgi?id=973977
5062 // We cannot guarantee this for indirect calls or calls to external functions.
5063 // When PC-Relative addressing is used, the concept of the TOC is no longer
5064 // applicable so this check is not required.
5065 // Check first for indirect calls.
5066 if (!Subtarget.isUsingPCRelativeCalls() &&
5067 !isFunctionGlobalAddress(CalleeGV) && !isCalleeExternalSymbol)
5068 return false;
5069
5070 // Check if we share the TOC base.
5071 if (!Subtarget.isUsingPCRelativeCalls() &&
5072 !callsShareTOCBase(Caller: CallerFunc, CalleeGV, TM: getTargetMachine()))
5073 return false;
5074
5075 // TCO allows altering callee ABI, so we don't have to check further.
5076 if (CalleeCC == CallingConv::Fast && TailCallOpt)
5077 return true;
5078
5079 if (DisableSCO) return false;
5080
5081 // If callee use the same argument list that caller is using, then we can
5082 // apply SCO on this case. If it is not, then we need to check if callee needs
5083 // stack for passing arguments.
5084 // PC Relative tail calls may not have a CallBase.
5085 // If there is no CallBase we cannot verify if we have the same argument
5086 // list so assume that we don't have the same argument list.
5087 if (CB && !hasSameArgumentList(CallerFn: CallerFunc, CB: *CB) &&
5088 needStackSlotPassParameters(Subtarget, Outs))
5089 return false;
5090 else if (!CB && needStackSlotPassParameters(Subtarget, Outs))
5091 return false;
5092
5093 return true;
5094}
5095
5096/// IsEligibleForTailCallOptimization - Check whether the call is eligible
5097/// for tail call optimization. Targets which want to do tail call
5098/// optimization should implement this function.
5099bool PPCTargetLowering::IsEligibleForTailCallOptimization(
5100 const GlobalValue *CalleeGV, CallingConv::ID CalleeCC,
5101 CallingConv::ID CallerCC, bool isVarArg,
5102 const SmallVectorImpl<ISD::InputArg> &Ins) const {
5103 if (!getTargetMachine().Options.GuaranteedTailCallOpt)
5104 return false;
5105
5106 // Variable argument functions are not supported.
5107 if (isVarArg)
5108 return false;
5109
5110 if (CalleeCC == CallingConv::Fast && CallerCC == CalleeCC) {
5111 // Functions containing by val parameters are not supported.
5112 if (any_of(Range: Ins, P: [](const ISD::InputArg &IA) { return IA.Flags.isByVal(); }))
5113 return false;
5114
5115 // Non-PIC/GOT tail calls are supported.
5116 if (getTargetMachine().getRelocationModel() != Reloc::PIC_)
5117 return true;
5118
5119 // At the moment we can only do local tail calls (in same module, hidden
5120 // or protected) if we are generating PIC.
5121 if (CalleeGV)
5122 return CalleeGV->hasHiddenVisibility() ||
5123 CalleeGV->hasProtectedVisibility();
5124 }
5125
5126 return false;
5127}
5128
5129/// isCallCompatibleAddress - Return the immediate to use if the specified
5130/// 32-bit value is representable in the immediate field of a BxA instruction.
5131static SDNode *isBLACompatibleAddress(SDValue Op, SelectionDAG &DAG) {
5132 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: Op);
5133 if (!C) return nullptr;
5134
5135 int Addr = C->getZExtValue();
5136 if ((Addr & 3) != 0 || // Low 2 bits are implicitly zero.
5137 SignExtend32<26>(X: Addr) != Addr)
5138 return nullptr; // Top 6 bits have to be sext of immediate.
5139
5140 return DAG
5141 .getSignedConstant(
5142 Val: (int)C->getZExtValue() >> 2, DL: SDLoc(Op),
5143 VT: DAG.getTargetLoweringInfo().getPointerTy(DL: DAG.getDataLayout()))
5144 .getNode();
5145}
5146
5147namespace {
5148
5149struct TailCallArgumentInfo {
5150 SDValue Arg;
5151 SDValue FrameIdxOp;
5152 int FrameIdx = 0;
5153
5154 TailCallArgumentInfo() = default;
5155};
5156
5157} // end anonymous namespace
5158
5159/// StoreTailCallArgumentsToStackSlot - Stores arguments to their stack slot.
5160static void StoreTailCallArgumentsToStackSlot(
5161 SelectionDAG &DAG, SDValue Chain,
5162 const SmallVectorImpl<TailCallArgumentInfo> &TailCallArgs,
5163 SmallVectorImpl<SDValue> &MemOpChains, const SDLoc &dl) {
5164 for (unsigned i = 0, e = TailCallArgs.size(); i != e; ++i) {
5165 SDValue Arg = TailCallArgs[i].Arg;
5166 SDValue FIN = TailCallArgs[i].FrameIdxOp;
5167 int FI = TailCallArgs[i].FrameIdx;
5168 // Store relative to framepointer.
5169 MemOpChains.push_back(Elt: DAG.getStore(
5170 Chain, dl, Val: Arg, Ptr: FIN,
5171 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI)));
5172 }
5173}
5174
5175/// EmitTailCallStoreFPAndRetAddr - Move the frame pointer and return address to
5176/// the appropriate stack slot for the tail call optimized function call.
5177static SDValue EmitTailCallStoreFPAndRetAddr(SelectionDAG &DAG, SDValue Chain,
5178 SDValue OldRetAddr, SDValue OldFP,
5179 int SPDiff, const SDLoc &dl) {
5180 if (SPDiff) {
5181 // Calculate the new stack slot for the return address.
5182 MachineFunction &MF = DAG.getMachineFunction();
5183 const PPCSubtarget &Subtarget = MF.getSubtarget<PPCSubtarget>();
5184 const PPCFrameLowering *FL = Subtarget.getFrameLowering();
5185 int SlotSize = Subtarget.isPPC64() ? 8 : 4;
5186 int NewRetAddrLoc = SPDiff + FL->getReturnSaveOffset();
5187 int NewRetAddr = MF.getFrameInfo().CreateFixedObject(Size: SlotSize,
5188 SPOffset: NewRetAddrLoc, IsImmutable: true);
5189 SDValue NewRetAddrFrIdx =
5190 DAG.getFrameIndex(FI: NewRetAddr, VT: Subtarget.getScalarIntVT());
5191 Chain = DAG.getStore(Chain, dl, Val: OldRetAddr, Ptr: NewRetAddrFrIdx,
5192 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: NewRetAddr));
5193 }
5194 return Chain;
5195}
5196
5197/// CalculateTailCallArgDest - Remember Argument for later processing. Calculate
5198/// the position of the argument.
5199static void CalculateTailCallArgDest(
5200 SelectionDAG &DAG, MachineFunction &MF, bool IsPPC64, SDValue Arg,
5201 int SPDiff, unsigned ArgOffset,
5202 SmallVectorImpl<TailCallArgumentInfo> &TailCallArguments) {
5203 int Offset = ArgOffset + SPDiff;
5204 uint32_t OpSize = (Arg.getValueSizeInBits() + 7) / 8;
5205 int FI = MF.getFrameInfo().CreateFixedObject(Size: OpSize, SPOffset: Offset, IsImmutable: true);
5206 EVT VT = IsPPC64 ? MVT::i64 : MVT::i32;
5207 SDValue FIN = DAG.getFrameIndex(FI, VT);
5208 TailCallArgumentInfo Info;
5209 Info.Arg = Arg;
5210 Info.FrameIdxOp = FIN;
5211 Info.FrameIdx = FI;
5212 TailCallArguments.push_back(Elt: Info);
5213}
5214
5215/// EmitTCFPAndRetAddrLoad - Emit load from frame pointer and return address
5216/// stack slot. Returns the chain as result and the loaded frame pointers in
5217/// LROpOut/FPOpout. Used when tail calling.
5218SDValue PPCTargetLowering::EmitTailCallLoadFPAndRetAddr(
5219 SelectionDAG &DAG, int SPDiff, SDValue Chain, SDValue &LROpOut,
5220 SDValue &FPOpOut, const SDLoc &dl) const {
5221 if (SPDiff) {
5222 // Load the LR and FP stack slot for later adjusting.
5223 LROpOut = getReturnAddrFrameIndex(DAG);
5224 LROpOut = DAG.getLoad(VT: Subtarget.getScalarIntVT(), dl, Chain, Ptr: LROpOut,
5225 PtrInfo: MachinePointerInfo());
5226 Chain = SDValue(LROpOut.getNode(), 1);
5227 }
5228 return Chain;
5229}
5230
5231/// CreateCopyOfByValArgument - Make a copy of an aggregate at address specified
5232/// by "Src" to address "Dst" of size "Size". Alignment information is
5233/// specified by the specific parameter attribute. The copy will be passed as
5234/// a byval function parameter.
5235/// Sometimes what we are copying is the end of a larger object, the part that
5236/// does not fit in registers.
5237static SDValue CreateCopyOfByValArgument(SDValue Src, SDValue Dst,
5238 SDValue Chain, ISD::ArgFlagsTy Flags,
5239 SelectionDAG &DAG, const SDLoc &dl) {
5240 SDValue SizeNode = DAG.getConstant(Val: Flags.getByValSize(), DL: dl, VT: MVT::i32);
5241 Align Alignment = Flags.getNonZeroByValAlign();
5242 return DAG.getMemcpy(
5243 Chain, dl, Dst, Src, Size: SizeNode, DstAlign: Alignment, SrcAlign: Alignment, isVol: false, AlwaysInline: false,
5244 /*CI=*/nullptr, OverrideTailCall: std::nullopt, DstPtrInfo: MachinePointerInfo(), SrcPtrInfo: MachinePointerInfo());
5245}
5246
5247/// LowerMemOpCallTo - Store the argument to the stack or remember it in case of
5248/// tail calls.
5249static void LowerMemOpCallTo(
5250 SelectionDAG &DAG, MachineFunction &MF, SDValue Chain, SDValue Arg,
5251 SDValue PtrOff, int SPDiff, unsigned ArgOffset, bool isPPC64,
5252 bool isTailCall, bool isVector, SmallVectorImpl<SDValue> &MemOpChains,
5253 SmallVectorImpl<TailCallArgumentInfo> &TailCallArguments, const SDLoc &dl) {
5254 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DL: DAG.getDataLayout());
5255 if (!isTailCall) {
5256 if (isVector) {
5257 SDValue StackPtr;
5258 if (isPPC64)
5259 StackPtr = DAG.getRegister(Reg: PPC::X1, VT: MVT::i64);
5260 else
5261 StackPtr = DAG.getRegister(Reg: PPC::R1, VT: MVT::i32);
5262 PtrOff = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: StackPtr,
5263 N2: DAG.getConstant(Val: ArgOffset, DL: dl, VT: PtrVT));
5264 }
5265 MemOpChains.push_back(
5266 Elt: DAG.getStore(Chain, dl, Val: Arg, Ptr: PtrOff, PtrInfo: MachinePointerInfo()));
5267 // Calculate and remember argument location.
5268 } else
5269 CalculateTailCallArgDest(DAG, MF, IsPPC64: isPPC64, Arg, SPDiff, ArgOffset,
5270 TailCallArguments);
5271}
5272
5273static void
5274PrepareTailCall(SelectionDAG &DAG, SDValue &InGlue, SDValue &Chain,
5275 const SDLoc &dl, int SPDiff, unsigned NumBytes, SDValue LROp,
5276 SDValue FPOp,
5277 SmallVectorImpl<TailCallArgumentInfo> &TailCallArguments) {
5278 // Emit a sequence of copyto/copyfrom virtual registers for arguments that
5279 // might overwrite each other in case of tail call optimization.
5280 SmallVector<SDValue, 8> MemOpChains2;
5281 // Do not flag preceding copytoreg stuff together with the following stuff.
5282 InGlue = SDValue();
5283 StoreTailCallArgumentsToStackSlot(DAG, Chain, TailCallArgs: TailCallArguments,
5284 MemOpChains&: MemOpChains2, dl);
5285 if (!MemOpChains2.empty())
5286 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: MemOpChains2);
5287
5288 // Store the return address to the appropriate stack slot.
5289 Chain = EmitTailCallStoreFPAndRetAddr(DAG, Chain, OldRetAddr: LROp, OldFP: FPOp, SPDiff, dl);
5290
5291 // Emit callseq_end just before tailcall node.
5292 Chain = DAG.getCALLSEQ_END(Chain, Size1: NumBytes, Size2: 0, Glue: InGlue, DL: dl);
5293 InGlue = Chain.getValue(R: 1);
5294}
5295
5296// Is this global address that of a function that can be called by name? (as
5297// opposed to something that must hold a descriptor for an indirect call).
5298static bool isFunctionGlobalAddress(const GlobalValue *GV) {
5299 if (GV) {
5300 if (GV->isThreadLocal())
5301 return false;
5302
5303 return GV->getValueType()->isFunctionTy();
5304 }
5305
5306 return false;
5307}
5308
5309SDValue PPCTargetLowering::LowerCallResult(
5310 SDValue Chain, SDValue InGlue, CallingConv::ID CallConv, bool isVarArg,
5311 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
5312 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
5313 SmallVector<CCValAssign, 16> RVLocs;
5314 CCState CCRetInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
5315 *DAG.getContext());
5316
5317 CCRetInfo.AnalyzeCallResult(
5318 Ins, Fn: (Subtarget.isSVR4ABI() && CallConv == CallingConv::Cold)
5319 ? RetCC_PPC_Cold
5320 : RetCC_PPC);
5321
5322 // Copy all of the result registers out of their specified physreg.
5323 for (unsigned i = 0, e = RVLocs.size(); i != e; ++i) {
5324 CCValAssign &VA = RVLocs[i];
5325 assert(VA.isRegLoc() && "Can only return in registers!");
5326
5327 SDValue Val;
5328
5329 if (Subtarget.hasSPE() && VA.getLocVT() == MVT::f64) {
5330 SDValue Lo = DAG.getCopyFromReg(Chain, dl, Reg: VA.getLocReg(), VT: MVT::i32,
5331 Glue: InGlue);
5332 Chain = Lo.getValue(R: 1);
5333 InGlue = Lo.getValue(R: 2);
5334 VA = RVLocs[++i]; // skip ahead to next loc
5335 SDValue Hi = DAG.getCopyFromReg(Chain, dl, Reg: VA.getLocReg(), VT: MVT::i32,
5336 Glue: InGlue);
5337 Chain = Hi.getValue(R: 1);
5338 InGlue = Hi.getValue(R: 2);
5339 if (!Subtarget.isLittleEndian())
5340 std::swap (a&: Lo, b&: Hi);
5341 Val = DAG.getNode(Opcode: PPCISD::BUILD_SPE64, DL: dl, VT: MVT::f64, N1: Lo, N2: Hi);
5342 } else {
5343 Val = DAG.getCopyFromReg(Chain, dl,
5344 Reg: VA.getLocReg(), VT: VA.getLocVT(), Glue: InGlue);
5345 Chain = Val.getValue(R: 1);
5346 InGlue = Val.getValue(R: 2);
5347 }
5348
5349 switch (VA.getLocInfo()) {
5350 default: llvm_unreachable("Unknown loc info!");
5351 case CCValAssign::Full: break;
5352 case CCValAssign::AExt:
5353 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: VA.getValVT(), Operand: Val);
5354 break;
5355 case CCValAssign::ZExt:
5356 Val = DAG.getNode(Opcode: ISD::AssertZext, DL: dl, VT: VA.getLocVT(), N1: Val,
5357 N2: DAG.getValueType(VA.getValVT()));
5358 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: VA.getValVT(), Operand: Val);
5359 break;
5360 case CCValAssign::SExt:
5361 Val = DAG.getNode(Opcode: ISD::AssertSext, DL: dl, VT: VA.getLocVT(), N1: Val,
5362 N2: DAG.getValueType(VA.getValVT()));
5363 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: VA.getValVT(), Operand: Val);
5364 break;
5365 }
5366
5367 InVals.push_back(Elt: Val);
5368 }
5369
5370 return Chain;
5371}
5372
5373static bool isIndirectCall(const SDValue &Callee, SelectionDAG &DAG,
5374 const PPCSubtarget &Subtarget, bool isPatchPoint) {
5375 auto *G = dyn_cast<GlobalAddressSDNode>(Val: Callee);
5376 const GlobalValue *GV = G ? G->getGlobal() : nullptr;
5377
5378 // PatchPoint calls are not indirect.
5379 if (isPatchPoint)
5380 return false;
5381
5382 if (isFunctionGlobalAddress(GV) || isa<ExternalSymbolSDNode>(Val: Callee))
5383 return false;
5384
5385 // Darwin, and 32-bit ELF can use a BLA. The descriptor based ABIs can not
5386 // becuase the immediate function pointer points to a descriptor instead of
5387 // a function entry point. The ELFv2 ABI cannot use a BLA because the function
5388 // pointer immediate points to the global entry point, while the BLA would
5389 // need to jump to the local entry point (see rL211174).
5390 if (!Subtarget.usesFunctionDescriptors() && !Subtarget.isELFv2ABI() &&
5391 isBLACompatibleAddress(Op: Callee, DAG))
5392 return false;
5393
5394 return true;
5395}
5396
5397// AIX and 64-bit ELF ABIs w/o PCRel require a TOC save/restore around calls.
5398static inline bool isTOCSaveRestoreRequired(const PPCSubtarget &Subtarget) {
5399 return Subtarget.isAIXABI() ||
5400 (Subtarget.is64BitELFABI() && !Subtarget.isUsingPCRelativeCalls());
5401}
5402
5403static unsigned getCallOpcode(PPCTargetLowering::CallFlags CFlags,
5404 const Function &Caller, const SDValue &Callee,
5405 const PPCSubtarget &Subtarget,
5406 const TargetMachine &TM,
5407 bool IsStrictFPCall = false) {
5408 if (CFlags.IsTailCall)
5409 return PPCISD::TC_RETURN;
5410
5411 unsigned RetOpc = 0;
5412 // This is a call through a function pointer.
5413 if (CFlags.IsIndirect) {
5414 // AIX and the 64-bit ELF ABIs need to maintain the TOC pointer accross
5415 // indirect calls. The save of the caller's TOC pointer to the stack will be
5416 // inserted into the DAG as part of call lowering. The restore of the TOC
5417 // pointer is modeled by using a pseudo instruction for the call opcode that
5418 // represents the 2 instruction sequence of an indirect branch and link,
5419 // immediately followed by a load of the TOC pointer from the stack save
5420 // slot into gpr2. For 64-bit ELFv2 ABI with PCRel, do not restore the TOC
5421 // as it is not saved or used.
5422 if (Subtarget.usePointerGlueHelper())
5423 RetOpc = PPCISD::BL_LOAD_TOC;
5424 else
5425 RetOpc = isTOCSaveRestoreRequired(Subtarget) ? PPCISD::BCTRL_LOAD_TOC
5426 : PPCISD::BCTRL;
5427 } else if (Subtarget.isUsingPCRelativeCalls()) {
5428 assert(Subtarget.is64BitELFABI() && "PC Relative is only on ELF ABI.");
5429 RetOpc = PPCISD::CALL_NOTOC;
5430 } else if (Subtarget.isAIXABI() || Subtarget.is64BitELFABI()) {
5431 // The ABIs that maintain a TOC pointer accross calls need to have a nop
5432 // immediately following the call instruction if the caller and callee may
5433 // have different TOC bases. At link time if the linker determines the calls
5434 // may not share a TOC base, the call is redirected to a trampoline inserted
5435 // by the linker. The trampoline will (among other things) save the callers
5436 // TOC pointer at an ABI designated offset in the linkage area and the
5437 // linker will rewrite the nop to be a load of the TOC pointer from the
5438 // linkage area into gpr2.
5439 auto *G = dyn_cast<GlobalAddressSDNode>(Val: Callee);
5440 const GlobalValue *GV = G ? G->getGlobal() : nullptr;
5441 RetOpc =
5442 callsShareTOCBase(Caller: &Caller, CalleeGV: GV, TM) ? PPCISD::CALL : PPCISD::CALL_NOP;
5443 } else
5444 RetOpc = PPCISD::CALL;
5445 if (IsStrictFPCall) {
5446 switch (RetOpc) {
5447 default:
5448 llvm_unreachable("Unknown call opcode");
5449 case PPCISD::BCTRL_LOAD_TOC:
5450 RetOpc = PPCISD::BCTRL_LOAD_TOC_RM;
5451 break;
5452 case PPCISD::BCTRL:
5453 RetOpc = PPCISD::BCTRL_RM;
5454 break;
5455 case PPCISD::BL_LOAD_TOC:
5456 RetOpc = PPCISD::BL_LOAD_TOC_RM;
5457 break;
5458 case PPCISD::CALL_NOTOC:
5459 RetOpc = PPCISD::CALL_NOTOC_RM;
5460 break;
5461 case PPCISD::CALL:
5462 RetOpc = PPCISD::CALL_RM;
5463 break;
5464 case PPCISD::CALL_NOP:
5465 RetOpc = PPCISD::CALL_NOP_RM;
5466 break;
5467 }
5468 }
5469 return RetOpc;
5470}
5471
5472static SDValue transformCallee(const SDValue &Callee, SelectionDAG &DAG,
5473 const SDLoc &dl, const PPCSubtarget &Subtarget) {
5474 if (!Subtarget.usesFunctionDescriptors() && !Subtarget.isELFv2ABI())
5475 if (SDNode *Dest = isBLACompatibleAddress(Op: Callee, DAG))
5476 return SDValue(Dest, 0);
5477
5478 // Returns true if the callee is local, and false otherwise.
5479 auto isLocalCallee = [&]() {
5480 const GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Val: Callee);
5481 const GlobalValue *GV = G ? G->getGlobal() : nullptr;
5482
5483 return DAG.getTarget().shouldAssumeDSOLocal(GV) &&
5484 !isa_and_nonnull<GlobalIFunc>(Val: GV);
5485 };
5486
5487 // The PLT is only used in 32-bit ELF PIC mode. Attempting to use the PLT in
5488 // a static relocation model causes some versions of GNU LD (2.17.50, at
5489 // least) to force BSS-PLT, instead of secure-PLT, even if all objects are
5490 // built with secure-PLT.
5491 bool UsePlt =
5492 Subtarget.is32BitELFABI() && !isLocalCallee() &&
5493 Subtarget.getTargetMachine().getRelocationModel() == Reloc::PIC_;
5494
5495 const auto getAIXFuncEntryPointSymbolSDNode = [&](const GlobalValue *GV) {
5496 const TargetMachine &TM = Subtarget.getTargetMachine();
5497 const TargetLoweringObjectFile *TLOF = TM.getObjFileLowering();
5498 auto *S =
5499 static_cast<MCSymbolXCOFF *>(TLOF->getFunctionEntryPointSymbol(Func: GV, TM));
5500
5501 MVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DL: DAG.getDataLayout());
5502 return DAG.getMCSymbol(Sym: S, VT: PtrVT);
5503 };
5504
5505 auto *G = dyn_cast<GlobalAddressSDNode>(Val: Callee);
5506 const GlobalValue *GV = G ? G->getGlobal() : nullptr;
5507 if (isFunctionGlobalAddress(GV)) {
5508 const GlobalValue *GV = cast<GlobalAddressSDNode>(Val: Callee)->getGlobal();
5509
5510 if (Subtarget.isAIXABI()) {
5511 return getAIXFuncEntryPointSymbolSDNode(GV);
5512 }
5513 return DAG.getTargetGlobalAddress(GV, DL: dl, VT: Callee.getValueType(), offset: 0,
5514 TargetFlags: UsePlt ? PPCII::MO_PLT : 0);
5515 }
5516
5517 if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Val: Callee)) {
5518 const char *SymName = S->getSymbol();
5519 if (Subtarget.isAIXABI()) {
5520 // If there exists a user-declared function whose name is the same as the
5521 // ExternalSymbol's, then we pick up the user-declared version.
5522 const Module *Mod = DAG.getMachineFunction().getFunction().getParent();
5523 if (const Function *F =
5524 dyn_cast_or_null<Function>(Val: Mod->getNamedValue(Name: SymName)))
5525 return getAIXFuncEntryPointSymbolSDNode(F);
5526
5527 // On AIX, direct function calls reference the symbol for the function's
5528 // entry point, which is named by prepending a "." before the function's
5529 // C-linkage name. A Qualname is returned here because an external
5530 // function entry point is a csect with XTY_ER property.
5531 const auto getExternalFunctionEntryPointSymbol = [&](StringRef SymName) {
5532 auto &Context = DAG.getMachineFunction().getContext();
5533 MCSectionXCOFF *Sec = Context.getXCOFFSection(
5534 Section: (Twine(".") + Twine(SymName)).str(), K: SectionKind::getMetadata(),
5535 CsectProp: XCOFF::CsectProperties(XCOFF::XMC_PR, XCOFF::XTY_ER));
5536 return Sec->getQualNameSymbol();
5537 };
5538
5539 SymName = getExternalFunctionEntryPointSymbol(SymName)->getName().data();
5540 }
5541 return DAG.getTargetExternalSymbol(Sym: SymName, VT: Callee.getValueType(),
5542 TargetFlags: UsePlt ? PPCII::MO_PLT : 0);
5543 }
5544
5545 // No transformation needed.
5546 assert(Callee.getNode() && "What no callee?");
5547 return Callee;
5548}
5549
5550static SDValue getOutputChainFromCallSeq(SDValue CallSeqStart) {
5551 assert(CallSeqStart.getOpcode() == ISD::CALLSEQ_START &&
5552 "Expected a CALLSEQ_STARTSDNode.");
5553
5554 // The last operand is the chain, except when the node has glue. If the node
5555 // has glue, then the last operand is the glue, and the chain is the second
5556 // last operand.
5557 SDValue LastValue = CallSeqStart.getValue(R: CallSeqStart->getNumValues() - 1);
5558 if (LastValue.getValueType() != MVT::Glue)
5559 return LastValue;
5560
5561 return CallSeqStart.getValue(R: CallSeqStart->getNumValues() - 2);
5562}
5563
5564// Creates the node that moves a functions address into the count register
5565// to prepare for an indirect call instruction.
5566static void prepareIndirectCall(SelectionDAG &DAG, SDValue &Callee,
5567 SDValue &Glue, SDValue &Chain,
5568 const SDLoc &dl) {
5569 SDValue MTCTROps[] = {Chain, Callee, Glue};
5570 EVT ReturnTypes[] = {MVT::Other, MVT::Glue};
5571 Chain = DAG.getNode(Opcode: PPCISD::MTCTR, DL: dl, ResultTys: ReturnTypes,
5572 Ops: ArrayRef(MTCTROps, Glue.getNode() ? 3 : 2));
5573 // The glue is the second value produced.
5574 Glue = Chain.getValue(R: 1);
5575}
5576
5577static void prepareDescriptorIndirectCall(SelectionDAG &DAG, SDValue &Callee,
5578 SDValue &Glue, SDValue &Chain,
5579 SDValue CallSeqStart,
5580 const CallBase *CB, const SDLoc &dl,
5581 bool hasNest,
5582 const PPCSubtarget &Subtarget) {
5583 // Function pointers in the 64-bit SVR4 ABI do not point to the function
5584 // entry point, but to the function descriptor (the function entry point
5585 // address is part of the function descriptor though).
5586 // The function descriptor is a three doubleword structure with the
5587 // following fields: function entry point, TOC base address and
5588 // environment pointer.
5589 // Thus for a call through a function pointer, the following actions need
5590 // to be performed:
5591 // 1. Save the TOC of the caller in the TOC save area of its stack
5592 // frame (this is done in LowerCall_Darwin() or LowerCall_64SVR4()).
5593 // 2. Load the address of the function entry point from the function
5594 // descriptor.
5595 // 3. Load the TOC of the callee from the function descriptor into r2.
5596 // 4. Load the environment pointer from the function descriptor into
5597 // r11.
5598 // 5. Branch to the function entry point address.
5599 // 6. On return of the callee, the TOC of the caller needs to be
5600 // restored (this is done in FinishCall()).
5601 //
5602 // The loads are scheduled at the beginning of the call sequence, and the
5603 // register copies are flagged together to ensure that no other
5604 // operations can be scheduled in between. E.g. without flagging the
5605 // copies together, a TOC access in the caller could be scheduled between
5606 // the assignment of the callee TOC and the branch to the callee, which leads
5607 // to incorrect code.
5608
5609 // Start by loading the function address from the descriptor.
5610 SDValue LDChain = getOutputChainFromCallSeq(CallSeqStart);
5611 auto MMOFlags = Subtarget.hasInvariantFunctionDescriptors()
5612 ? (MachineMemOperand::MODereferenceable |
5613 MachineMemOperand::MOInvariant)
5614 : MachineMemOperand::MONone;
5615
5616 MachinePointerInfo MPI(CB ? CB->getCalledOperand() : nullptr);
5617
5618 // Registers used in building the DAG.
5619 const MCRegister EnvPtrReg = Subtarget.getEnvironmentPointerRegister();
5620 const MCRegister TOCReg = Subtarget.getTOCPointerRegister();
5621
5622 // Offsets of descriptor members.
5623 const unsigned TOCAnchorOffset = Subtarget.descriptorTOCAnchorOffset();
5624 const unsigned EnvPtrOffset = Subtarget.descriptorEnvironmentPointerOffset();
5625
5626 const MVT RegVT = Subtarget.getScalarIntVT();
5627 const Align Alignment = Subtarget.isPPC64() ? Align(8) : Align(4);
5628
5629 // One load for the functions entry point address.
5630 SDValue LoadFuncPtr = DAG.getLoad(VT: RegVT, dl, Chain: LDChain, Ptr: Callee, PtrInfo: MPI,
5631 Alignment, MMOFlags);
5632
5633 // One for loading the TOC anchor for the module that contains the called
5634 // function.
5635 SDValue TOCOff = DAG.getIntPtrConstant(Val: TOCAnchorOffset, DL: dl);
5636 SDValue AddTOC = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: RegVT, N1: Callee, N2: TOCOff);
5637 SDValue TOCPtr =
5638 DAG.getLoad(VT: RegVT, dl, Chain: LDChain, Ptr: AddTOC,
5639 PtrInfo: MPI.getWithOffset(O: TOCAnchorOffset), Alignment, MMOFlags);
5640
5641 // One for loading the environment pointer.
5642 SDValue PtrOff = DAG.getIntPtrConstant(Val: EnvPtrOffset, DL: dl);
5643 SDValue AddPtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: RegVT, N1: Callee, N2: PtrOff);
5644 SDValue LoadEnvPtr =
5645 DAG.getLoad(VT: RegVT, dl, Chain: LDChain, Ptr: AddPtr,
5646 PtrInfo: MPI.getWithOffset(O: EnvPtrOffset), Alignment, MMOFlags);
5647
5648
5649 // Then copy the newly loaded TOC anchor to the TOC pointer.
5650 SDValue TOCVal = DAG.getCopyToReg(Chain, dl, Reg: TOCReg, N: TOCPtr, Glue);
5651 Chain = TOCVal.getValue(R: 0);
5652 Glue = TOCVal.getValue(R: 1);
5653
5654 // If the function call has an explicit 'nest' parameter, it takes the
5655 // place of the environment pointer.
5656 assert((!hasNest || !Subtarget.isAIXABI()) &&
5657 "Nest parameter is not supported on AIX.");
5658 if (!hasNest) {
5659 SDValue EnvVal = DAG.getCopyToReg(Chain, dl, Reg: EnvPtrReg, N: LoadEnvPtr, Glue);
5660 Chain = EnvVal.getValue(R: 0);
5661 Glue = EnvVal.getValue(R: 1);
5662 }
5663
5664 // The rest of the indirect call sequence is the same as the non-descriptor
5665 // DAG.
5666 prepareIndirectCall(DAG, Callee&: LoadFuncPtr, Glue, Chain, dl);
5667}
5668
5669static void prepareOutOfLineGlueCall(SelectionDAG &DAG, SDValue &Callee,
5670 SDValue &Glue, SDValue &Chain,
5671 SDValue CallSeqStart, const CallBase *CB,
5672 const SDLoc &dl, bool hasNest,
5673 const PPCSubtarget &Subtarget) {
5674 // On AIX there is a feature ("out of line glue code") which uses a special
5675 // trampoline function ._ptrgl to do the indirect call. If this option is
5676 // enabled we instead simply load the address of the descriptor into gpr11,
5677 // with the arguments in the 'normal' registers and branch to the ._ptrgl
5678 // stub.
5679 const MCRegister PtrGlueReg = Subtarget.getGlueCodeDescriptorRegister();
5680 SDValue MoveToPhysicalReg =
5681 DAG.getCopyToReg(Chain, dl, Reg: PtrGlueReg, N: Callee, Glue);
5682 Chain = MoveToPhysicalReg.getValue(R: 0);
5683 Glue = MoveToPhysicalReg.getValue(R: 1);
5684}
5685
5686static void
5687buildCallOperands(SmallVectorImpl<SDValue> &Ops,
5688 PPCTargetLowering::CallFlags CFlags, const SDLoc &dl,
5689 SelectionDAG &DAG,
5690 SmallVector<std::pair<unsigned, SDValue>, 8> &RegsToPass,
5691 SDValue Glue, SDValue Chain, SDValue &Callee, int SPDiff,
5692 const PPCSubtarget &Subtarget) {
5693 const bool IsPPC64 = Subtarget.isPPC64();
5694 // MVT for a general purpose register.
5695 const MVT RegVT = Subtarget.getScalarIntVT();
5696
5697 // First operand is always the chain.
5698 Ops.push_back(Elt: Chain);
5699
5700 // If it's a direct call pass the callee as the second operand.
5701 if (!CFlags.IsIndirect)
5702 Ops.push_back(Elt: Callee);
5703 else if (Subtarget.usePointerGlueHelper()) {
5704 Ops.push_back(Elt: Callee);
5705 // Add the register used to pass the descriptor address.
5706 Ops.push_back(
5707 Elt: DAG.getRegister(Reg: Subtarget.getGlueCodeDescriptorRegister(), VT: RegVT));
5708 } else {
5709 assert(!CFlags.IsPatchPoint && "Patch point calls are not indirect.");
5710
5711 // For the TOC based ABIs, we have saved the TOC pointer to the linkage area
5712 // on the stack (this would have been done in `LowerCall_64SVR4` or
5713 // `LowerCall_AIX`). The call instruction is a pseudo instruction that
5714 // represents both the indirect branch and a load that restores the TOC
5715 // pointer from the linkage area. The operand for the TOC restore is an add
5716 // of the TOC save offset to the stack pointer. This must be the second
5717 // operand: after the chain input but before any other variadic arguments.
5718 // For 64-bit ELFv2 ABI with PCRel, do not restore the TOC as it is not
5719 // saved or used.
5720 if (isTOCSaveRestoreRequired(Subtarget)) {
5721 const MCRegister StackPtrReg = Subtarget.getStackPointerRegister();
5722
5723 SDValue StackPtr = DAG.getRegister(Reg: StackPtrReg, VT: RegVT);
5724 unsigned TOCSaveOffset = Subtarget.getFrameLowering()->getTOCSaveOffset();
5725 SDValue TOCOff = DAG.getIntPtrConstant(Val: TOCSaveOffset, DL: dl);
5726 SDValue AddTOC = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: RegVT, N1: StackPtr, N2: TOCOff);
5727 Ops.push_back(Elt: AddTOC);
5728 }
5729
5730 // Add the register used for the environment pointer.
5731 if (Subtarget.usesFunctionDescriptors() && !CFlags.HasNest)
5732 Ops.push_back(Elt: DAG.getRegister(Reg: Subtarget.getEnvironmentPointerRegister(),
5733 VT: RegVT));
5734
5735
5736 // Add CTR register as callee so a bctr can be emitted later.
5737 if (CFlags.IsTailCall)
5738 Ops.push_back(Elt: DAG.getRegister(Reg: IsPPC64 ? PPC::CTR8 : PPC::CTR, VT: RegVT));
5739 }
5740
5741 // If this is a tail call add stack pointer delta.
5742 if (CFlags.IsTailCall)
5743 Ops.push_back(Elt: DAG.getConstant(Val: SPDiff, DL: dl, VT: MVT::i32));
5744
5745 // Add argument registers to the end of the list so that they are known live
5746 // into the call.
5747 for (const auto &[Reg, N] : RegsToPass)
5748 Ops.push_back(Elt: DAG.getRegister(Reg, VT: N.getValueType()));
5749
5750 // We cannot add R2/X2 as an operand here for PATCHPOINT, because there is
5751 // no way to mark dependencies as implicit here.
5752 // We will add the R2/X2 dependency in EmitInstrWithCustomInserter.
5753 if ((Subtarget.is64BitELFABI() || Subtarget.isAIXABI()) &&
5754 !CFlags.IsPatchPoint && !Subtarget.isUsingPCRelativeCalls())
5755 Ops.push_back(Elt: DAG.getRegister(Reg: Subtarget.getTOCPointerRegister(), VT: RegVT));
5756
5757 // Add implicit use of CR bit 6 for 32-bit SVR4 vararg calls
5758 if (CFlags.IsVarArg && Subtarget.is32BitELFABI())
5759 Ops.push_back(Elt: DAG.getRegister(Reg: PPC::CR1EQ, VT: MVT::i32));
5760
5761 // Add a register mask operand representing the call-preserved registers.
5762 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
5763 const uint32_t *Mask =
5764 TRI->getCallPreservedMask(MF: DAG.getMachineFunction(), CFlags.CallConv);
5765 assert(Mask && "Missing call preserved mask for calling convention");
5766 Ops.push_back(Elt: DAG.getRegisterMask(RegMask: Mask));
5767
5768 // If the glue is valid, it is the last operand.
5769 if (Glue.getNode())
5770 Ops.push_back(Elt: Glue);
5771}
5772
5773SDValue PPCTargetLowering::FinishCall(
5774 CallFlags CFlags, const SDLoc &dl, SelectionDAG &DAG,
5775 SmallVector<std::pair<unsigned, SDValue>, 8> &RegsToPass, SDValue Glue,
5776 SDValue Chain, SDValue CallSeqStart, SDValue &Callee, int SPDiff,
5777 unsigned NumBytes, const SmallVectorImpl<ISD::InputArg> &Ins,
5778 SmallVectorImpl<SDValue> &InVals, const CallBase *CB) const {
5779
5780 if ((Subtarget.is64BitELFABI() && !Subtarget.isUsingPCRelativeCalls()) ||
5781 Subtarget.isAIXABI())
5782 setUsesTOCBasePtr(DAG);
5783
5784 unsigned CallOpc =
5785 getCallOpcode(CFlags, Caller: DAG.getMachineFunction().getFunction(), Callee,
5786 Subtarget, TM: DAG.getTarget(), IsStrictFPCall: CB ? CB->isStrictFP() : false);
5787
5788 if (!CFlags.IsIndirect)
5789 Callee = transformCallee(Callee, DAG, dl, Subtarget);
5790 else if (Subtarget.usesFunctionDescriptors()) {
5791 if (Subtarget.usePointerGlueHelper()) {
5792 prepareOutOfLineGlueCall(DAG, Callee, Glue, Chain, CallSeqStart, CB, dl,
5793 hasNest: CFlags.HasNest, Subtarget);
5794 SDValue PtrGlueCallee =
5795 DAG.getExternalSymbol(Sym: "_ptrgl", VT: getPointerTy(DL: DAG.getDataLayout()));
5796 Callee = transformCallee(Callee: PtrGlueCallee, DAG, dl, Subtarget);
5797 } else {
5798 prepareDescriptorIndirectCall(DAG, Callee, Glue, Chain, CallSeqStart, CB,
5799 dl, hasNest: CFlags.HasNest, Subtarget);
5800 }
5801 } else {
5802 prepareIndirectCall(DAG, Callee, Glue, Chain, dl);
5803 }
5804
5805 // Build the operand list for the call instruction.
5806 SmallVector<SDValue, 8> Ops;
5807 buildCallOperands(Ops, CFlags, dl, DAG, RegsToPass, Glue, Chain, Callee,
5808 SPDiff, Subtarget);
5809
5810 // Emit tail call.
5811 if (CFlags.IsTailCall) {
5812 // Indirect tail call when using PC Relative calls do not have the same
5813 // constraints.
5814 assert(((Callee.getOpcode() == ISD::Register &&
5815 cast<RegisterSDNode>(Callee)->getReg() == PPC::CTR) ||
5816 Callee.getOpcode() == ISD::TargetExternalSymbol ||
5817 Callee.getOpcode() == ISD::TargetGlobalAddress ||
5818 isa<ConstantSDNode>(Callee) ||
5819 (CFlags.IsIndirect && Subtarget.isUsingPCRelativeCalls())) &&
5820 "Expecting a global address, external symbol, absolute value, "
5821 "register or an indirect tail call when PC Relative calls are "
5822 "used.");
5823 // PC Relative calls also use TC_RETURN as the way to mark tail calls.
5824 assert(CallOpc == PPCISD::TC_RETURN &&
5825 "Unexpected call opcode for a tail call.");
5826 DAG.getMachineFunction().getFrameInfo().setHasTailCall();
5827 SDValue Ret = DAG.getNode(Opcode: CallOpc, DL: dl, VT: MVT::Other, Ops);
5828 DAG.addNoMergeSiteInfo(Node: Ret.getNode(), NoMerge: CFlags.NoMerge);
5829 return Ret;
5830 }
5831
5832 std::array<EVT, 2> ReturnTypes = {._M_elems: {MVT::Other, MVT::Glue}};
5833 Chain = DAG.getNode(Opcode: CallOpc, DL: dl, ResultTys: ReturnTypes, Ops);
5834 DAG.addNoMergeSiteInfo(Node: Chain.getNode(), NoMerge: CFlags.NoMerge);
5835 Glue = Chain.getValue(R: 1);
5836
5837 // When performing tail call optimization the callee pops its arguments off
5838 // the stack. Account for this here so these bytes can be pushed back on in
5839 // PPCFrameLowering::eliminateCallFramePseudoInstr.
5840 int BytesCalleePops = (CFlags.CallConv == CallingConv::Fast &&
5841 getTargetMachine().Options.GuaranteedTailCallOpt)
5842 ? NumBytes
5843 : 0;
5844
5845 Chain = DAG.getCALLSEQ_END(Chain, Size1: NumBytes, Size2: BytesCalleePops, Glue, DL: dl);
5846 Glue = Chain.getValue(R: 1);
5847
5848 return LowerCallResult(Chain, InGlue: Glue, CallConv: CFlags.CallConv, isVarArg: CFlags.IsVarArg, Ins, dl,
5849 DAG, InVals);
5850}
5851
5852bool PPCTargetLowering::supportsTailCallFor(const CallBase *CB) const {
5853 CallingConv::ID CalleeCC = CB->getCallingConv();
5854 const Function *CallerFunc = CB->getCaller();
5855 CallingConv::ID CallerCC = CallerFunc->getCallingConv();
5856 const Function *CalleeFunc = CB->getCalledFunction();
5857 if (!CalleeFunc)
5858 return false;
5859 const GlobalValue *CalleeGV = dyn_cast<GlobalValue>(Val: CalleeFunc);
5860
5861 SmallVector<ISD::OutputArg, 2> Outs;
5862 SmallVector<ISD::InputArg, 2> Ins;
5863
5864 GetReturnInfo(CC: CalleeCC, ReturnType: CalleeFunc->getReturnType(),
5865 attr: CalleeFunc->getAttributes(), Outs, TLI: *this,
5866 DL: CalleeFunc->getDataLayout());
5867
5868 return isEligibleForTCO(CalleeGV, CalleeCC, CallerCC, CB,
5869 isVarArg: CalleeFunc->isVarArg(), Outs, Ins, CallerFunc,
5870 isCalleeExternalSymbol: false /*isCalleeExternalSymbol*/);
5871}
5872
5873bool PPCTargetLowering::isEligibleForTCO(
5874 const GlobalValue *CalleeGV, CallingConv::ID CalleeCC,
5875 CallingConv::ID CallerCC, const CallBase *CB, bool isVarArg,
5876 const SmallVectorImpl<ISD::OutputArg> &Outs,
5877 const SmallVectorImpl<ISD::InputArg> &Ins, const Function *CallerFunc,
5878 bool isCalleeExternalSymbol) const {
5879 if (Subtarget.useLongCalls() && !(CB && CB->isMustTailCall()))
5880 return false;
5881
5882 if (Subtarget.isSVR4ABI() && Subtarget.isPPC64())
5883 return IsEligibleForTailCallOptimization_64SVR4(
5884 CalleeGV, CalleeCC, CallerCC, CB, isVarArg, Outs, Ins, CallerFunc,
5885 isCalleeExternalSymbol);
5886 else
5887 return IsEligibleForTailCallOptimization(CalleeGV, CalleeCC, CallerCC,
5888 isVarArg, Ins);
5889}
5890
5891SDValue
5892PPCTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
5893 SmallVectorImpl<SDValue> &InVals) const {
5894 SelectionDAG &DAG = CLI.DAG;
5895 SDLoc &dl = CLI.DL;
5896 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
5897 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
5898 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
5899 SDValue Chain = CLI.Chain;
5900 SDValue Callee = CLI.Callee;
5901 bool &isTailCall = CLI.IsTailCall;
5902 CallingConv::ID CallConv = CLI.CallConv;
5903 bool isVarArg = CLI.IsVarArg;
5904 bool isPatchPoint = CLI.IsPatchPoint;
5905 const CallBase *CB = CLI.CB;
5906
5907 if (isTailCall) {
5908 MachineFunction &MF = DAG.getMachineFunction();
5909 CallingConv::ID CallerCC = MF.getFunction().getCallingConv();
5910 auto *G = dyn_cast<GlobalAddressSDNode>(Val&: Callee);
5911 const GlobalValue *GV = G ? G->getGlobal() : nullptr;
5912 bool IsCalleeExternalSymbol = isa<ExternalSymbolSDNode>(Val: Callee);
5913
5914 isTailCall =
5915 isEligibleForTCO(CalleeGV: GV, CalleeCC: CallConv, CallerCC, CB, isVarArg, Outs, Ins,
5916 CallerFunc: &(MF.getFunction()), isCalleeExternalSymbol: IsCalleeExternalSymbol);
5917 if (isTailCall) {
5918 ++NumTailCalls;
5919 if (!getTargetMachine().Options.GuaranteedTailCallOpt)
5920 ++NumSiblingCalls;
5921
5922 // PC Relative calls no longer guarantee that the callee is a Global
5923 // Address Node. The callee could be an indirect tail call in which
5924 // case the SDValue for the callee could be a load (to load the address
5925 // of a function pointer) or it may be a register copy (to move the
5926 // address of the callee from a function parameter into a virtual
5927 // register). It may also be an ExternalSymbolSDNode (ex memcopy).
5928 assert((Subtarget.isUsingPCRelativeCalls() ||
5929 isa<GlobalAddressSDNode>(Callee)) &&
5930 "Callee should be an llvm::Function object.");
5931
5932 LLVM_DEBUG(dbgs() << "TCO caller: " << DAG.getMachineFunction().getName()
5933 << "\nTCO callee: ");
5934 LLVM_DEBUG(Callee.dump());
5935 }
5936 }
5937
5938 if (!isTailCall && CB && CB->isMustTailCall())
5939 report_fatal_error(reason: "failed to perform tail call elimination on a call "
5940 "site marked musttail");
5941
5942 // When long calls (i.e. indirect calls) are always used, calls are always
5943 // made via function pointer. If we have a function name, first translate it
5944 // into a pointer.
5945 if (Subtarget.useLongCalls() && isa<GlobalAddressSDNode>(Val: Callee) &&
5946 !isTailCall)
5947 Callee = LowerGlobalAddress(Op: Callee, DAG);
5948
5949 CallFlags CFlags(
5950 CallConv, isTailCall, isVarArg, isPatchPoint,
5951 isIndirectCall(Callee, DAG, Subtarget, isPatchPoint),
5952 // hasNest
5953 Subtarget.is64BitELFABI() &&
5954 any_of(Range&: Outs, P: [](ISD::OutputArg Arg) { return Arg.Flags.isNest(); }),
5955 CLI.NoMerge);
5956
5957 if (Subtarget.isAIXABI())
5958 return LowerCall_AIX(Chain, Callee, CFlags, Outs, OutVals, Ins, dl, DAG,
5959 InVals, CB);
5960
5961 assert(Subtarget.isSVR4ABI());
5962 if (Subtarget.isPPC64())
5963 return LowerCall_64SVR4(Chain, Callee, CFlags, Outs, OutVals, Ins, dl, DAG,
5964 InVals, CB);
5965 return LowerCall_32SVR4(Chain, Callee, CFlags, Outs, OutVals, Ins, dl, DAG,
5966 InVals, CB);
5967}
5968
5969SDValue PPCTargetLowering::LowerCall_32SVR4(
5970 SDValue Chain, SDValue Callee, CallFlags CFlags,
5971 const SmallVectorImpl<ISD::OutputArg> &Outs,
5972 const SmallVectorImpl<SDValue> &OutVals,
5973 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
5974 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals,
5975 const CallBase *CB) const {
5976 // See PPCTargetLowering::LowerFormalArguments_32SVR4() for a description
5977 // of the 32-bit SVR4 ABI stack frame layout.
5978
5979 const CallingConv::ID CallConv = CFlags.CallConv;
5980 const bool IsVarArg = CFlags.IsVarArg;
5981 const bool IsTailCall = CFlags.IsTailCall;
5982
5983 assert((CallConv == CallingConv::C ||
5984 CallConv == CallingConv::Cold ||
5985 CallConv == CallingConv::Fast) && "Unknown calling convention!");
5986
5987 const Align PtrAlign(4);
5988
5989 MachineFunction &MF = DAG.getMachineFunction();
5990
5991 // Mark this function as potentially containing a function that contains a
5992 // tail call. As a consequence the frame pointer will be used for dynamicalloc
5993 // and restoring the callers stack pointer in this functions epilog. This is
5994 // done because by tail calling the called function might overwrite the value
5995 // in this function's (MF) stack pointer stack slot 0(SP).
5996 if (getTargetMachine().Options.GuaranteedTailCallOpt &&
5997 CallConv == CallingConv::Fast)
5998 MF.getInfo<PPCFunctionInfo>()->setHasFastCall();
5999
6000 // Count how many bytes are to be pushed on the stack, including the linkage
6001 // area, parameter list area and the part of the local variable space which
6002 // contains copies of aggregates which are passed by value.
6003
6004 // Assign locations to all of the outgoing arguments.
6005 SmallVector<CCValAssign, 16> ArgLocs;
6006 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
6007
6008 // Reserve space for the linkage area on the stack.
6009 CCInfo.AllocateStack(Size: Subtarget.getFrameLowering()->getLinkageSize(),
6010 Alignment: PtrAlign);
6011
6012 if (IsVarArg) {
6013 // Handle fixed and variable vector arguments differently.
6014 // Fixed vector arguments go into registers as long as registers are
6015 // available. Variable vector arguments always go into memory.
6016 unsigned NumArgs = Outs.size();
6017
6018 for (unsigned i = 0; i != NumArgs; ++i) {
6019 MVT ArgVT = Outs[i].VT;
6020 ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
6021 bool Result;
6022
6023 if (!ArgFlags.isVarArg()) {
6024 Result = CC_PPC32_SVR4(ValNo: i, ValVT: ArgVT, LocVT: ArgVT, LocInfo: CCValAssign::Full, ArgFlags,
6025 OrigTy: Outs[i].OrigTy, State&: CCInfo);
6026 } else {
6027 Result = CC_PPC32_SVR4_VarArg(ValNo: i, ValVT: ArgVT, LocVT: ArgVT, LocInfo: CCValAssign::Full,
6028 ArgFlags, OrigTy: Outs[i].OrigTy, State&: CCInfo);
6029 }
6030
6031 if (Result) {
6032#ifndef NDEBUG
6033 errs() << "Call operand #" << i << " has unhandled type "
6034 << ArgVT << "\n";
6035#endif
6036 llvm_unreachable(nullptr);
6037 }
6038 }
6039 } else {
6040 // All arguments are treated the same.
6041 CCInfo.AnalyzeCallOperands(Outs, Fn: CC_PPC32_SVR4);
6042 }
6043
6044 // Assign locations to all of the outgoing aggregate by value arguments.
6045 SmallVector<CCValAssign, 16> ByValArgLocs;
6046 CCState CCByValInfo(CallConv, IsVarArg, MF, ByValArgLocs, *DAG.getContext());
6047
6048 // Reserve stack space for the allocations in CCInfo.
6049 CCByValInfo.AllocateStack(Size: CCInfo.getStackSize(), Alignment: PtrAlign);
6050
6051 CCByValInfo.AnalyzeCallOperands(Outs, Fn: CC_PPC32_SVR4_ByVal);
6052
6053 // Size of the linkage area, parameter list area and the part of the local
6054 // space variable where copies of aggregates which are passed by value are
6055 // stored.
6056 unsigned NumBytes = CCByValInfo.getStackSize();
6057
6058 // Calculate by how many bytes the stack has to be adjusted in case of tail
6059 // call optimization.
6060 int SPDiff = CalculateTailCallSPDiff(DAG, isTailCall: IsTailCall, ParamSize: NumBytes);
6061
6062 // Adjust the stack pointer for the new arguments...
6063 // These operations are automatically eliminated by the prolog/epilog pass
6064 Chain = DAG.getCALLSEQ_START(Chain, InSize: NumBytes, OutSize: 0, DL: dl);
6065 SDValue CallSeqStart = Chain;
6066
6067 // Load the return address and frame pointer so it can be moved somewhere else
6068 // later.
6069 SDValue LROp, FPOp;
6070 Chain = EmitTailCallLoadFPAndRetAddr(DAG, SPDiff, Chain, LROpOut&: LROp, FPOpOut&: FPOp, dl);
6071
6072 // Set up a copy of the stack pointer for use loading and storing any
6073 // arguments that may not fit in the registers available for argument
6074 // passing.
6075 SDValue StackPtr = DAG.getRegister(Reg: PPC::R1, VT: MVT::i32);
6076
6077 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
6078 SmallVector<TailCallArgumentInfo, 8> TailCallArguments;
6079 SmallVector<SDValue, 8> MemOpChains;
6080
6081 bool seenFloatArg = false;
6082 // Walk the register/memloc assignments, inserting copies/loads.
6083 // i - Tracks the index into the list of registers allocated for the call
6084 // RealArgIdx - Tracks the index into the list of actual function arguments
6085 // j - Tracks the index into the list of byval arguments
6086 for (unsigned i = 0, RealArgIdx = 0, j = 0, e = ArgLocs.size();
6087 i != e;
6088 ++i, ++RealArgIdx) {
6089 CCValAssign &VA = ArgLocs[i];
6090 SDValue Arg = OutVals[RealArgIdx];
6091 ISD::ArgFlagsTy Flags = Outs[RealArgIdx].Flags;
6092
6093 if (Flags.isByVal()) {
6094 // Argument is an aggregate which is passed by value, thus we need to
6095 // create a copy of it in the local variable space of the current stack
6096 // frame (which is the stack frame of the caller) and pass the address of
6097 // this copy to the callee.
6098 assert((j < ByValArgLocs.size()) && "Index out of bounds!");
6099 CCValAssign &ByValVA = ByValArgLocs[j++];
6100 assert((VA.getValNo() == ByValVA.getValNo()) && "ValNo mismatch!");
6101
6102 // Memory reserved in the local variable space of the callers stack frame.
6103 unsigned LocMemOffset = ByValVA.getLocMemOffset();
6104
6105 SDValue PtrOff = DAG.getIntPtrConstant(Val: LocMemOffset, DL: dl);
6106 PtrOff = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: getPointerTy(DL: MF.getDataLayout()),
6107 N1: StackPtr, N2: PtrOff);
6108
6109 // Create a copy of the argument in the local area of the current
6110 // stack frame.
6111 SDValue MemcpyCall =
6112 CreateCopyOfByValArgument(Src: Arg, Dst: PtrOff,
6113 Chain: CallSeqStart.getNode()->getOperand(Num: 0),
6114 Flags, DAG, dl);
6115
6116 // This must go outside the CALLSEQ_START..END.
6117 SDValue NewCallSeqStart = DAG.getCALLSEQ_START(Chain: MemcpyCall, InSize: NumBytes, OutSize: 0,
6118 DL: SDLoc(MemcpyCall));
6119 DAG.ReplaceAllUsesWith(From: CallSeqStart.getNode(),
6120 To: NewCallSeqStart.getNode());
6121 Chain = CallSeqStart = NewCallSeqStart;
6122
6123 // Pass the address of the aggregate copy on the stack either in a
6124 // physical register or in the parameter list area of the current stack
6125 // frame to the callee.
6126 Arg = PtrOff;
6127 }
6128
6129 // When useCRBits() is true, there can be i1 arguments.
6130 // It is because getRegisterType(MVT::i1) => MVT::i1,
6131 // and for other integer types getRegisterType() => MVT::i32.
6132 // Extend i1 and ensure callee will get i32.
6133 if (Arg.getValueType() == MVT::i1)
6134 Arg = DAG.getNode(Opcode: Flags.isSExt() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
6135 DL: dl, VT: MVT::i32, Operand: Arg);
6136
6137 if (VA.isRegLoc()) {
6138 seenFloatArg |= VA.getLocVT().isFloatingPoint();
6139 // Put argument in a physical register.
6140 if (Subtarget.hasSPE() && Arg.getValueType() == MVT::f64) {
6141 bool IsLE = Subtarget.isLittleEndian();
6142 SDValue SVal = DAG.getNode(Opcode: PPCISD::EXTRACT_SPE, DL: dl, VT: MVT::i32, N1: Arg,
6143 N2: DAG.getIntPtrConstant(Val: IsLE ? 0 : 1, DL: dl));
6144 RegsToPass.push_back(Elt: std::make_pair(x: VA.getLocReg(), y: SVal.getValue(R: 0)));
6145 SVal = DAG.getNode(Opcode: PPCISD::EXTRACT_SPE, DL: dl, VT: MVT::i32, N1: Arg,
6146 N2: DAG.getIntPtrConstant(Val: IsLE ? 1 : 0, DL: dl));
6147 RegsToPass.push_back(Elt: std::make_pair(x: ArgLocs[++i].getLocReg(),
6148 y: SVal.getValue(R: 0)));
6149 } else
6150 RegsToPass.push_back(Elt: std::make_pair(x: VA.getLocReg(), y&: Arg));
6151 } else {
6152 // Put argument in the parameter list area of the current stack frame.
6153 assert(VA.isMemLoc());
6154 unsigned LocMemOffset = VA.getLocMemOffset();
6155
6156 if (!IsTailCall) {
6157 SDValue PtrOff = DAG.getIntPtrConstant(Val: LocMemOffset, DL: dl);
6158 PtrOff = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: getPointerTy(DL: MF.getDataLayout()),
6159 N1: StackPtr, N2: PtrOff);
6160
6161 MemOpChains.push_back(
6162 Elt: DAG.getStore(Chain, dl, Val: Arg, Ptr: PtrOff, PtrInfo: MachinePointerInfo()));
6163 } else {
6164 // Calculate and remember argument location.
6165 CalculateTailCallArgDest(DAG, MF, IsPPC64: false, Arg, SPDiff, ArgOffset: LocMemOffset,
6166 TailCallArguments);
6167 }
6168 }
6169 }
6170
6171 if (!MemOpChains.empty())
6172 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: MemOpChains);
6173
6174 // Build a sequence of copy-to-reg nodes chained together with token chain
6175 // and flag operands which copy the outgoing args into the appropriate regs.
6176 SDValue InGlue;
6177 for (const auto &[Reg, N] : RegsToPass) {
6178 Chain = DAG.getCopyToReg(Chain, dl, Reg, N, Glue: InGlue);
6179 InGlue = Chain.getValue(R: 1);
6180 }
6181
6182 // Set CR bit 6 to true if this is a vararg call with floating args passed in
6183 // registers.
6184 if (IsVarArg) {
6185 SDVTList VTs = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
6186 SDValue Ops[] = { Chain, InGlue };
6187
6188 Chain = DAG.getNode(Opcode: seenFloatArg ? PPCISD::CR6SET : PPCISD::CR6UNSET, DL: dl,
6189 VTList: VTs, Ops: ArrayRef(Ops, InGlue.getNode() ? 2 : 1));
6190
6191 InGlue = Chain.getValue(R: 1);
6192 }
6193
6194 if (IsTailCall)
6195 PrepareTailCall(DAG, InGlue, Chain, dl, SPDiff, NumBytes, LROp, FPOp,
6196 TailCallArguments);
6197
6198 return FinishCall(CFlags, dl, DAG, RegsToPass, Glue: InGlue, Chain, CallSeqStart,
6199 Callee, SPDiff, NumBytes, Ins, InVals, CB);
6200}
6201
6202// Copy an argument into memory, being careful to do this outside the
6203// call sequence for the call to which the argument belongs.
6204SDValue PPCTargetLowering::createMemcpyOutsideCallSeq(
6205 SDValue Arg, SDValue PtrOff, SDValue CallSeqStart, ISD::ArgFlagsTy Flags,
6206 SelectionDAG &DAG, const SDLoc &dl) const {
6207 SDValue MemcpyCall = CreateCopyOfByValArgument(Src: Arg, Dst: PtrOff,
6208 Chain: CallSeqStart.getNode()->getOperand(Num: 0),
6209 Flags, DAG, dl);
6210 // The MEMCPY must go outside the CALLSEQ_START..END.
6211 int64_t FrameSize = CallSeqStart.getConstantOperandVal(i: 1);
6212 SDValue NewCallSeqStart = DAG.getCALLSEQ_START(Chain: MemcpyCall, InSize: FrameSize, OutSize: 0,
6213 DL: SDLoc(MemcpyCall));
6214 DAG.ReplaceAllUsesWith(From: CallSeqStart.getNode(),
6215 To: NewCallSeqStart.getNode());
6216 return NewCallSeqStart;
6217}
6218
6219SDValue PPCTargetLowering::LowerCall_64SVR4(
6220 SDValue Chain, SDValue Callee, CallFlags CFlags,
6221 const SmallVectorImpl<ISD::OutputArg> &Outs,
6222 const SmallVectorImpl<SDValue> &OutVals,
6223 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
6224 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals,
6225 const CallBase *CB) const {
6226 bool isELFv2ABI = Subtarget.isELFv2ABI();
6227 bool isLittleEndian = Subtarget.isLittleEndian();
6228 unsigned NumOps = Outs.size();
6229 bool IsSibCall = false;
6230 bool IsFastCall = CFlags.CallConv == CallingConv::Fast;
6231
6232 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
6233 unsigned PtrByteSize = 8;
6234
6235 MachineFunction &MF = DAG.getMachineFunction();
6236
6237 if (CFlags.IsTailCall && !getTargetMachine().Options.GuaranteedTailCallOpt)
6238 IsSibCall = true;
6239
6240 // Mark this function as potentially containing a function that contains a
6241 // tail call. As a consequence the frame pointer will be used for dynamicalloc
6242 // and restoring the callers stack pointer in this functions epilog. This is
6243 // done because by tail calling the called function might overwrite the value
6244 // in this function's (MF) stack pointer stack slot 0(SP).
6245 if (getTargetMachine().Options.GuaranteedTailCallOpt && IsFastCall)
6246 MF.getInfo<PPCFunctionInfo>()->setHasFastCall();
6247
6248 assert(!(IsFastCall && CFlags.IsVarArg) &&
6249 "fastcc not supported on varargs functions");
6250
6251 // Count how many bytes are to be pushed on the stack, including the linkage
6252 // area, and parameter passing area. On ELFv1, the linkage area is 48 bytes
6253 // reserved space for [SP][CR][LR][2 x unused][TOC]; on ELFv2, the linkage
6254 // area is 32 bytes reserved space for [SP][CR][LR][TOC].
6255 unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize();
6256 unsigned NumBytes = LinkageSize;
6257 unsigned GPR_idx = 0, FPR_idx = 0, VR_idx = 0;
6258
6259 static const MCPhysReg GPR[] = {
6260 PPC::X3, PPC::X4, PPC::X5, PPC::X6,
6261 PPC::X7, PPC::X8, PPC::X9, PPC::X10,
6262 };
6263 static const MCPhysReg VR[] = {
6264 PPC::V2, PPC::V3, PPC::V4, PPC::V5, PPC::V6, PPC::V7, PPC::V8,
6265 PPC::V9, PPC::V10, PPC::V11, PPC::V12, PPC::V13
6266 };
6267
6268 const unsigned NumGPRs = std::size(GPR);
6269 const unsigned NumFPRs = useSoftFloat() ? 0 : 13;
6270 const unsigned NumVRs = std::size(VR);
6271
6272 // On ELFv2, we can avoid allocating the parameter area if all the arguments
6273 // can be passed to the callee in registers.
6274 // For the fast calling convention, there is another check below.
6275 // Note: We should keep consistent with LowerFormalArguments_64SVR4()
6276 bool HasParameterArea = !isELFv2ABI || CFlags.IsVarArg || IsFastCall;
6277 if (!HasParameterArea) {
6278 unsigned ParamAreaSize = NumGPRs * PtrByteSize;
6279 unsigned AvailableFPRs = NumFPRs;
6280 unsigned AvailableVRs = NumVRs;
6281 unsigned NumBytesTmp = NumBytes;
6282 for (unsigned i = 0; i != NumOps; ++i) {
6283 if (Outs[i].Flags.isNest()) continue;
6284 if (CalculateStackSlotUsed(ArgVT: Outs[i].VT, OrigVT: Outs[i].ArgVT, Flags: Outs[i].Flags,
6285 PtrByteSize, LinkageSize, ParamAreaSize,
6286 ArgOffset&: NumBytesTmp, AvailableFPRs, AvailableVRs))
6287 HasParameterArea = true;
6288 }
6289 }
6290
6291 // When using the fast calling convention, we don't provide backing for
6292 // arguments that will be in registers.
6293 unsigned NumGPRsUsed = 0, NumFPRsUsed = 0, NumVRsUsed = 0;
6294
6295 // Avoid allocating parameter area for fastcc functions if all the arguments
6296 // can be passed in the registers.
6297 if (IsFastCall)
6298 HasParameterArea = false;
6299
6300 // Add up all the space actually used.
6301 for (unsigned i = 0; i != NumOps; ++i) {
6302 ISD::ArgFlagsTy Flags = Outs[i].Flags;
6303 EVT ArgVT = Outs[i].VT;
6304 EVT OrigVT = Outs[i].ArgVT;
6305
6306 if (Flags.isNest())
6307 continue;
6308
6309 if (IsFastCall) {
6310 if (Flags.isByVal()) {
6311 NumGPRsUsed += (Flags.getByValSize()+7)/8;
6312 if (NumGPRsUsed > NumGPRs)
6313 HasParameterArea = true;
6314 } else {
6315 switch (ArgVT.getSimpleVT().SimpleTy) {
6316 default: llvm_unreachable("Unexpected ValueType for argument!");
6317 case MVT::i1:
6318 case MVT::i32:
6319 case MVT::i64:
6320 if (++NumGPRsUsed <= NumGPRs)
6321 continue;
6322 break;
6323 case MVT::v4i32:
6324 case MVT::v8i16:
6325 case MVT::v16i8:
6326 case MVT::v2f64:
6327 case MVT::v2i64:
6328 case MVT::v1i128:
6329 case MVT::f128:
6330 if (++NumVRsUsed <= NumVRs)
6331 continue;
6332 break;
6333 case MVT::v4f32:
6334 if (++NumVRsUsed <= NumVRs)
6335 continue;
6336 break;
6337 case MVT::f32:
6338 case MVT::f64:
6339 if (++NumFPRsUsed <= NumFPRs)
6340 continue;
6341 break;
6342 }
6343 HasParameterArea = true;
6344 }
6345 }
6346
6347 /* Respect alignment of argument on the stack. */
6348 auto Alignement =
6349 CalculateStackSlotAlignment(ArgVT, OrigVT, Flags, PtrByteSize);
6350 NumBytes = alignTo(Size: NumBytes, A: Alignement);
6351
6352 NumBytes += CalculateStackSlotSize(ArgVT, Flags, PtrByteSize);
6353 if (Flags.isInConsecutiveRegsLast())
6354 NumBytes = ((NumBytes + PtrByteSize - 1)/PtrByteSize) * PtrByteSize;
6355 }
6356
6357 unsigned NumBytesActuallyUsed = NumBytes;
6358
6359 // In the old ELFv1 ABI,
6360 // the prolog code of the callee may store up to 8 GPR argument registers to
6361 // the stack, allowing va_start to index over them in memory if its varargs.
6362 // Because we cannot tell if this is needed on the caller side, we have to
6363 // conservatively assume that it is needed. As such, make sure we have at
6364 // least enough stack space for the caller to store the 8 GPRs.
6365 // In the ELFv2 ABI, we allocate the parameter area iff a callee
6366 // really requires memory operands, e.g. a vararg function.
6367 if (HasParameterArea)
6368 NumBytes = std::max(a: NumBytes, b: LinkageSize + 8 * PtrByteSize);
6369 else
6370 NumBytes = LinkageSize;
6371
6372 // Tail call needs the stack to be aligned.
6373 if (getTargetMachine().Options.GuaranteedTailCallOpt && IsFastCall)
6374 NumBytes = EnsureStackAlignment(Lowering: Subtarget.getFrameLowering(), NumBytes);
6375
6376 int SPDiff = 0;
6377
6378 // Calculate by how many bytes the stack has to be adjusted in case of tail
6379 // call optimization.
6380 if (!IsSibCall)
6381 SPDiff = CalculateTailCallSPDiff(DAG, isTailCall: CFlags.IsTailCall, ParamSize: NumBytes);
6382
6383 // To protect arguments on the stack from being clobbered in a tail call,
6384 // force all the loads to happen before doing any other lowering.
6385 if (CFlags.IsTailCall)
6386 Chain = DAG.getStackArgumentTokenFactor(Chain);
6387
6388 // Adjust the stack pointer for the new arguments...
6389 // These operations are automatically eliminated by the prolog/epilog pass
6390 if (!IsSibCall)
6391 Chain = DAG.getCALLSEQ_START(Chain, InSize: NumBytes, OutSize: 0, DL: dl);
6392 SDValue CallSeqStart = Chain;
6393
6394 // Load the return address and frame pointer so it can be move somewhere else
6395 // later.
6396 SDValue LROp, FPOp;
6397 Chain = EmitTailCallLoadFPAndRetAddr(DAG, SPDiff, Chain, LROpOut&: LROp, FPOpOut&: FPOp, dl);
6398
6399 // Set up a copy of the stack pointer for use loading and storing any
6400 // arguments that may not fit in the registers available for argument
6401 // passing.
6402 SDValue StackPtr = DAG.getRegister(Reg: PPC::X1, VT: MVT::i64);
6403
6404 // Figure out which arguments are going to go in registers, and which in
6405 // memory. Also, if this is a vararg function, floating point operations
6406 // must be stored to our stack, and loaded into integer regs as well, if
6407 // any integer regs are available for argument passing.
6408 unsigned ArgOffset = LinkageSize;
6409
6410 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
6411 SmallVector<TailCallArgumentInfo, 8> TailCallArguments;
6412
6413 SmallVector<SDValue, 8> MemOpChains;
6414 for (unsigned i = 0; i != NumOps; ++i) {
6415 SDValue Arg = OutVals[i];
6416 ISD::ArgFlagsTy Flags = Outs[i].Flags;
6417 EVT ArgVT = Outs[i].VT;
6418 EVT OrigVT = Outs[i].ArgVT;
6419
6420 // PtrOff will be used to store the current argument to the stack if a
6421 // register cannot be found for it.
6422 SDValue PtrOff;
6423
6424 // We re-align the argument offset for each argument, except when using the
6425 // fast calling convention, when we need to make sure we do that only when
6426 // we'll actually use a stack slot.
6427 auto ComputePtrOff = [&]() {
6428 /* Respect alignment of argument on the stack. */
6429 auto Alignment =
6430 CalculateStackSlotAlignment(ArgVT, OrigVT, Flags, PtrByteSize);
6431 ArgOffset = alignTo(Size: ArgOffset, A: Alignment);
6432
6433 PtrOff = DAG.getConstant(Val: ArgOffset, DL: dl, VT: StackPtr.getValueType());
6434
6435 PtrOff = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: StackPtr, N2: PtrOff);
6436 };
6437
6438 if (!IsFastCall) {
6439 ComputePtrOff();
6440
6441 /* Compute GPR index associated with argument offset. */
6442 GPR_idx = (ArgOffset - LinkageSize) / PtrByteSize;
6443 GPR_idx = std::min(a: GPR_idx, b: NumGPRs);
6444 }
6445
6446 // Promote integers to 64-bit values.
6447 if (Arg.getValueType() == MVT::i32 || Arg.getValueType() == MVT::i1) {
6448 // FIXME: Should this use ANY_EXTEND if neither sext nor zext?
6449 unsigned ExtOp = Flags.isSExt() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
6450 Arg = DAG.getNode(Opcode: ExtOp, DL: dl, VT: MVT::i64, Operand: Arg);
6451 }
6452
6453 // FIXME memcpy is used way more than necessary. Correctness first.
6454 // Note: "by value" is code for passing a structure by value, not
6455 // basic types.
6456 if (Flags.isByVal()) {
6457 // Note: Size includes alignment padding, so
6458 // struct x { short a; char b; }
6459 // will have Size = 4. With #pragma pack(1), it will have Size = 3.
6460 // These are the proper values we need for right-justifying the
6461 // aggregate in a parameter register.
6462 unsigned Size = Flags.getByValSize();
6463
6464 // An empty aggregate parameter takes up no storage and no
6465 // registers.
6466 if (Size == 0)
6467 continue;
6468
6469 if (IsFastCall)
6470 ComputePtrOff();
6471
6472 // All aggregates smaller than 8 bytes must be passed right-justified.
6473 if (Size==1 || Size==2 || Size==4) {
6474 EVT VT = (Size==1) ? MVT::i8 : ((Size==2) ? MVT::i16 : MVT::i32);
6475 if (GPR_idx != NumGPRs) {
6476 SDValue Load = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl, VT: PtrVT, Chain, Ptr: Arg,
6477 PtrInfo: MachinePointerInfo(), MemVT: VT);
6478 MemOpChains.push_back(Elt: Load.getValue(R: 1));
6479 RegsToPass.push_back(Elt: std::make_pair(x: GPR[GPR_idx++], y&: Load));
6480
6481 ArgOffset += PtrByteSize;
6482 continue;
6483 }
6484 }
6485
6486 if (GPR_idx == NumGPRs && Size < 8) {
6487 SDValue AddPtr = PtrOff;
6488 if (!isLittleEndian) {
6489 SDValue Const = DAG.getConstant(Val: PtrByteSize - Size, DL: dl,
6490 VT: PtrOff.getValueType());
6491 AddPtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: PtrOff, N2: Const);
6492 }
6493 Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, PtrOff: AddPtr,
6494 CallSeqStart,
6495 Flags, DAG, dl);
6496 ArgOffset += PtrByteSize;
6497 continue;
6498 }
6499 // Copy the object to parameter save area if it can not be entirely passed
6500 // by registers.
6501 // FIXME: we only need to copy the parts which need to be passed in
6502 // parameter save area. For the parts passed by registers, we don't need
6503 // to copy them to the stack although we need to allocate space for them
6504 // in parameter save area.
6505 if ((NumGPRs - GPR_idx) * PtrByteSize < Size)
6506 Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, PtrOff,
6507 CallSeqStart,
6508 Flags, DAG, dl);
6509
6510 // When a register is available, pass a small aggregate right-justified.
6511 if (Size < 8 && GPR_idx != NumGPRs) {
6512 // The easiest way to get this right-justified in a register
6513 // is to copy the structure into the rightmost portion of a
6514 // local variable slot, then load the whole slot into the
6515 // register.
6516 // FIXME: The memcpy seems to produce pretty awful code for
6517 // small aggregates, particularly for packed ones.
6518 // FIXME: It would be preferable to use the slot in the
6519 // parameter save area instead of a new local variable.
6520 SDValue AddPtr = PtrOff;
6521 if (!isLittleEndian) {
6522 SDValue Const = DAG.getConstant(Val: 8 - Size, DL: dl, VT: PtrOff.getValueType());
6523 AddPtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: PtrOff, N2: Const);
6524 }
6525 Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, PtrOff: AddPtr,
6526 CallSeqStart,
6527 Flags, DAG, dl);
6528
6529 // Load the slot into the register.
6530 SDValue Load =
6531 DAG.getLoad(VT: PtrVT, dl, Chain, Ptr: PtrOff, PtrInfo: MachinePointerInfo());
6532 MemOpChains.push_back(Elt: Load.getValue(R: 1));
6533 RegsToPass.push_back(Elt: std::make_pair(x: GPR[GPR_idx++], y&: Load));
6534
6535 // Done with this argument.
6536 ArgOffset += PtrByteSize;
6537 continue;
6538 }
6539
6540 // For aggregates larger than PtrByteSize, copy the pieces of the
6541 // object that fit into registers from the parameter save area.
6542 for (unsigned j=0; j<Size; j+=PtrByteSize) {
6543 SDValue Const = DAG.getConstant(Val: j, DL: dl, VT: PtrOff.getValueType());
6544 SDValue AddArg = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: Arg, N2: Const);
6545 if (GPR_idx != NumGPRs) {
6546 unsigned LoadSizeInBits = std::min(a: PtrByteSize, b: (Size - j)) * 8;
6547 EVT ObjType = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: LoadSizeInBits);
6548 SDValue Load = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl, VT: PtrVT, Chain, Ptr: AddArg,
6549 PtrInfo: MachinePointerInfo(), MemVT: ObjType);
6550
6551 MemOpChains.push_back(Elt: Load.getValue(R: 1));
6552 RegsToPass.push_back(Elt: std::make_pair(x: GPR[GPR_idx++], y&: Load));
6553 ArgOffset += PtrByteSize;
6554 } else {
6555 ArgOffset += ((Size - j + PtrByteSize-1)/PtrByteSize)*PtrByteSize;
6556 break;
6557 }
6558 }
6559 continue;
6560 }
6561
6562 switch (Arg.getSimpleValueType().SimpleTy) {
6563 default: llvm_unreachable("Unexpected ValueType for argument!");
6564 case MVT::i1:
6565 case MVT::i32:
6566 case MVT::i64:
6567 if (Flags.isNest()) {
6568 // The 'nest' parameter, if any, is passed in R11.
6569 RegsToPass.push_back(Elt: std::make_pair(x: PPC::X11, y&: Arg));
6570 break;
6571 }
6572
6573 // These can be scalar arguments or elements of an integer array type
6574 // passed directly. Clang may use those instead of "byval" aggregate
6575 // types to avoid forcing arguments to memory unnecessarily.
6576 if (GPR_idx != NumGPRs) {
6577 RegsToPass.push_back(Elt: std::make_pair(x: GPR[GPR_idx++], y&: Arg));
6578 } else {
6579 if (IsFastCall)
6580 ComputePtrOff();
6581
6582 assert(HasParameterArea &&
6583 "Parameter area must exist to pass an argument in memory.");
6584 LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset,
6585 isPPC64: true, isTailCall: CFlags.IsTailCall, isVector: false, MemOpChains,
6586 TailCallArguments, dl);
6587 if (IsFastCall)
6588 ArgOffset += PtrByteSize;
6589 }
6590 if (!IsFastCall)
6591 ArgOffset += PtrByteSize;
6592 break;
6593 case MVT::f32:
6594 case MVT::f64: {
6595 // These can be scalar arguments or elements of a float array type
6596 // passed directly. The latter are used to implement ELFv2 homogenous
6597 // float aggregates.
6598
6599 // Named arguments go into FPRs first, and once they overflow, the
6600 // remaining arguments go into GPRs and then the parameter save area.
6601 // Unnamed arguments for vararg functions always go to GPRs and
6602 // then the parameter save area. For now, put all arguments to vararg
6603 // routines always in both locations (FPR *and* GPR or stack slot).
6604 bool NeedGPROrStack = CFlags.IsVarArg || FPR_idx == NumFPRs;
6605 bool NeededLoad = false;
6606
6607 // First load the argument into the next available FPR.
6608 if (FPR_idx != NumFPRs)
6609 RegsToPass.push_back(Elt: std::make_pair(x: FPR[FPR_idx++], y&: Arg));
6610
6611 // Next, load the argument into GPR or stack slot if needed.
6612 if (!NeedGPROrStack)
6613 ;
6614 else if (GPR_idx != NumGPRs && !IsFastCall) {
6615 // FIXME: We may want to re-enable this for CallingConv::Fast on the P8
6616 // once we support fp <-> gpr moves.
6617
6618 // In the non-vararg case, this can only ever happen in the
6619 // presence of f32 array types, since otherwise we never run
6620 // out of FPRs before running out of GPRs.
6621 SDValue ArgVal;
6622
6623 // Double values are always passed in a single GPR.
6624 if (Arg.getValueType() != MVT::f32) {
6625 ArgVal = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i64, Operand: Arg);
6626
6627 // Non-array float values are extended and passed in a GPR.
6628 } else if (!Flags.isInConsecutiveRegs()) {
6629 ArgVal = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: Arg);
6630 ArgVal = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: MVT::i64, Operand: ArgVal);
6631
6632 // If we have an array of floats, we collect every odd element
6633 // together with its predecessor into one GPR.
6634 } else if (ArgOffset % PtrByteSize != 0) {
6635 SDValue Lo, Hi;
6636 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: OutVals[i - 1]);
6637 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: Arg);
6638 if (!isLittleEndian)
6639 std::swap(a&: Lo, b&: Hi);
6640 ArgVal = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: MVT::i64, N1: Lo, N2: Hi);
6641
6642 // The final element, if even, goes into the first half of a GPR.
6643 } else if (Flags.isInConsecutiveRegsLast()) {
6644 ArgVal = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: Arg);
6645 ArgVal = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: MVT::i64, Operand: ArgVal);
6646 if (!isLittleEndian)
6647 ArgVal = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: MVT::i64, N1: ArgVal,
6648 N2: DAG.getConstant(Val: 32, DL: dl, VT: MVT::i32));
6649
6650 // Non-final even elements are skipped; they will be handled
6651 // together the with subsequent argument on the next go-around.
6652 } else
6653 ArgVal = SDValue();
6654
6655 if (ArgVal.getNode())
6656 RegsToPass.push_back(Elt: std::make_pair(x: GPR[GPR_idx++], y&: ArgVal));
6657 } else {
6658 if (IsFastCall)
6659 ComputePtrOff();
6660
6661 // Single-precision floating-point values are mapped to the
6662 // second (rightmost) word of the stack doubleword.
6663 if (Arg.getValueType() == MVT::f32 &&
6664 !isLittleEndian && !Flags.isInConsecutiveRegs()) {
6665 SDValue ConstFour = DAG.getConstant(Val: 4, DL: dl, VT: PtrOff.getValueType());
6666 PtrOff = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: PtrOff, N2: ConstFour);
6667 }
6668
6669 assert(HasParameterArea &&
6670 "Parameter area must exist to pass an argument in memory.");
6671 LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset,
6672 isPPC64: true, isTailCall: CFlags.IsTailCall, isVector: false, MemOpChains,
6673 TailCallArguments, dl);
6674
6675 NeededLoad = true;
6676 }
6677 // When passing an array of floats, the array occupies consecutive
6678 // space in the argument area; only round up to the next doubleword
6679 // at the end of the array. Otherwise, each float takes 8 bytes.
6680 if (!IsFastCall || NeededLoad) {
6681 ArgOffset += (Arg.getValueType() == MVT::f32 &&
6682 Flags.isInConsecutiveRegs()) ? 4 : 8;
6683 if (Flags.isInConsecutiveRegsLast())
6684 ArgOffset = ((ArgOffset + PtrByteSize - 1)/PtrByteSize) * PtrByteSize;
6685 }
6686 break;
6687 }
6688 case MVT::v4f32:
6689 case MVT::v4i32:
6690 case MVT::v8i16:
6691 case MVT::v16i8:
6692 case MVT::v2f64:
6693 case MVT::v2i64:
6694 case MVT::v1i128:
6695 case MVT::f128:
6696 // These can be scalar arguments or elements of a vector array type
6697 // passed directly. The latter are used to implement ELFv2 homogenous
6698 // vector aggregates.
6699
6700 // For a varargs call, named arguments go into VRs or on the stack as
6701 // usual; unnamed arguments always go to the stack or the corresponding
6702 // GPRs when within range. For now, we always put the value in both
6703 // locations (or even all three).
6704 if (CFlags.IsVarArg) {
6705 assert(HasParameterArea &&
6706 "Parameter area must exist if we have a varargs call.");
6707 // We could elide this store in the case where the object fits
6708 // entirely in R registers. Maybe later.
6709 SDValue Store =
6710 DAG.getStore(Chain, dl, Val: Arg, Ptr: PtrOff, PtrInfo: MachinePointerInfo());
6711 MemOpChains.push_back(Elt: Store);
6712 if (VR_idx != NumVRs) {
6713 SDValue Load =
6714 DAG.getLoad(VT: MVT::v4f32, dl, Chain: Store, Ptr: PtrOff, PtrInfo: MachinePointerInfo());
6715 MemOpChains.push_back(Elt: Load.getValue(R: 1));
6716 RegsToPass.push_back(Elt: std::make_pair(x: VR[VR_idx++], y&: Load));
6717 }
6718 ArgOffset += 16;
6719 for (unsigned i=0; i<16; i+=PtrByteSize) {
6720 if (GPR_idx == NumGPRs)
6721 break;
6722 SDValue Ix = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: PtrOff,
6723 N2: DAG.getConstant(Val: i, DL: dl, VT: PtrVT));
6724 SDValue Load =
6725 DAG.getLoad(VT: PtrVT, dl, Chain: Store, Ptr: Ix, PtrInfo: MachinePointerInfo());
6726 MemOpChains.push_back(Elt: Load.getValue(R: 1));
6727 RegsToPass.push_back(Elt: std::make_pair(x: GPR[GPR_idx++], y&: Load));
6728 }
6729 break;
6730 }
6731
6732 // Non-varargs Altivec params go into VRs or on the stack.
6733 if (VR_idx != NumVRs) {
6734 RegsToPass.push_back(Elt: std::make_pair(x: VR[VR_idx++], y&: Arg));
6735 } else {
6736 if (IsFastCall)
6737 ComputePtrOff();
6738
6739 assert(HasParameterArea &&
6740 "Parameter area must exist to pass an argument in memory.");
6741 LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset,
6742 isPPC64: true, isTailCall: CFlags.IsTailCall, isVector: true, MemOpChains,
6743 TailCallArguments, dl);
6744 if (IsFastCall)
6745 ArgOffset += 16;
6746 }
6747
6748 if (!IsFastCall)
6749 ArgOffset += 16;
6750 break;
6751 }
6752 }
6753
6754 assert((!HasParameterArea || NumBytesActuallyUsed == ArgOffset) &&
6755 "mismatch in size of parameter area");
6756 (void)NumBytesActuallyUsed;
6757
6758 if (!MemOpChains.empty())
6759 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: MemOpChains);
6760
6761 // Check if this is an indirect call (MTCTR/BCTRL).
6762 // See prepareDescriptorIndirectCall and buildCallOperands for more
6763 // information about calls through function pointers in the 64-bit SVR4 ABI.
6764 if (CFlags.IsIndirect) {
6765 // For 64-bit ELFv2 ABI with PCRel, do not save the TOC of the
6766 // caller in the TOC save area.
6767 if (isTOCSaveRestoreRequired(Subtarget)) {
6768 assert(!CFlags.IsTailCall && "Indirect tails calls not supported");
6769 // Load r2 into a virtual register and store it to the TOC save area.
6770 setUsesTOCBasePtr(DAG);
6771 SDValue Val = DAG.getCopyFromReg(Chain, dl, Reg: PPC::X2, VT: MVT::i64);
6772 // TOC save area offset.
6773 unsigned TOCSaveOffset = Subtarget.getFrameLowering()->getTOCSaveOffset();
6774 SDValue PtrOff = DAG.getIntPtrConstant(Val: TOCSaveOffset, DL: dl);
6775 SDValue AddPtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: StackPtr, N2: PtrOff);
6776 Chain = DAG.getStore(Chain: Val.getValue(R: 1), dl, Val, Ptr: AddPtr,
6777 PtrInfo: MachinePointerInfo::getStack(
6778 MF&: DAG.getMachineFunction(), Offset: TOCSaveOffset));
6779 }
6780 // In the ELFv2 ABI, R12 must contain the address of an indirect callee.
6781 // This does not mean the MTCTR instruction must use R12; it's easier
6782 // to model this as an extra parameter, so do that.
6783 if (isELFv2ABI && !CFlags.IsPatchPoint)
6784 RegsToPass.push_back(Elt: std::make_pair(x: (unsigned)PPC::X12, y&: Callee));
6785 }
6786
6787 // Build a sequence of copy-to-reg nodes chained together with token chain
6788 // and flag operands which copy the outgoing args into the appropriate regs.
6789 SDValue InGlue;
6790 for (const auto &[Reg, N] : RegsToPass) {
6791 Chain = DAG.getCopyToReg(Chain, dl, Reg, N, Glue: InGlue);
6792 InGlue = Chain.getValue(R: 1);
6793 }
6794
6795 if (CFlags.IsTailCall && !IsSibCall)
6796 PrepareTailCall(DAG, InGlue, Chain, dl, SPDiff, NumBytes, LROp, FPOp,
6797 TailCallArguments);
6798
6799 return FinishCall(CFlags, dl, DAG, RegsToPass, Glue: InGlue, Chain, CallSeqStart,
6800 Callee, SPDiff, NumBytes, Ins, InVals, CB);
6801}
6802
6803// Returns true when the shadow of a general purpose argument register
6804// in the parameter save area is aligned to at least 'RequiredAlign'.
6805static bool isGPRShadowAligned(MCPhysReg Reg, Align RequiredAlign) {
6806 assert(RequiredAlign.value() <= 16 &&
6807 "Required alignment greater than stack alignment.");
6808 switch (Reg) {
6809 default:
6810 report_fatal_error(reason: "called on invalid register.");
6811 case PPC::R5:
6812 case PPC::R9:
6813 case PPC::X3:
6814 case PPC::X5:
6815 case PPC::X7:
6816 case PPC::X9:
6817 // These registers are 16 byte aligned which is the most strict aligment
6818 // we can support.
6819 return true;
6820 case PPC::R3:
6821 case PPC::R7:
6822 case PPC::X4:
6823 case PPC::X6:
6824 case PPC::X8:
6825 case PPC::X10:
6826 // The shadow of these registers in the PSA is 8 byte aligned.
6827 return RequiredAlign <= 8;
6828 case PPC::R4:
6829 case PPC::R6:
6830 case PPC::R8:
6831 case PPC::R10:
6832 return RequiredAlign <= 4;
6833 }
6834}
6835
6836static bool CC_AIX(unsigned ValNo, MVT ValVT, MVT LocVT,
6837 CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
6838 Type *OrigTy, CCState &State) {
6839 const PPCSubtarget &Subtarget = static_cast<const PPCSubtarget &>(
6840 State.getMachineFunction().getSubtarget());
6841 const bool IsPPC64 = Subtarget.isPPC64();
6842 const unsigned PtrSize = IsPPC64 ? 8 : 4;
6843 const Align PtrAlign(PtrSize);
6844 const Align StackAlign(16);
6845 const MVT RegVT = Subtarget.getScalarIntVT();
6846
6847 if (ValVT == MVT::f128)
6848 report_fatal_error(reason: "f128 is unimplemented on AIX.");
6849
6850 static const MCPhysReg GPR_32[] = {// 32-bit registers.
6851 PPC::R3, PPC::R4, PPC::R5, PPC::R6,
6852 PPC::R7, PPC::R8, PPC::R9, PPC::R10};
6853 static const MCPhysReg GPR_64[] = {// 64-bit registers.
6854 PPC::X3, PPC::X4, PPC::X5, PPC::X6,
6855 PPC::X7, PPC::X8, PPC::X9, PPC::X10};
6856
6857 static const MCPhysReg VR[] = {// Vector registers.
6858 PPC::V2, PPC::V3, PPC::V4, PPC::V5,
6859 PPC::V6, PPC::V7, PPC::V8, PPC::V9,
6860 PPC::V10, PPC::V11, PPC::V12, PPC::V13};
6861
6862 const ArrayRef<MCPhysReg> GPRs = IsPPC64 ? GPR_64 : GPR_32;
6863
6864 if (ArgFlags.isNest()) {
6865 MCRegister EnvReg = State.AllocateReg(Reg: IsPPC64 ? PPC::X11 : PPC::R11);
6866 if (!EnvReg)
6867 report_fatal_error(reason: "More then one nest argument.");
6868 State.addLoc(V: CCValAssign::getReg(ValNo, ValVT, Reg: EnvReg, LocVT: RegVT, HTP: LocInfo));
6869 return false;
6870 }
6871
6872 if (ArgFlags.isByVal()) {
6873 const Align ByValAlign(ArgFlags.getNonZeroByValAlign());
6874 if (ByValAlign > StackAlign)
6875 report_fatal_error(reason: "Pass-by-value arguments with alignment greater than "
6876 "16 are not supported.");
6877
6878 const unsigned ByValSize = ArgFlags.getByValSize();
6879 const Align ObjAlign = ByValAlign > PtrAlign ? ByValAlign : PtrAlign;
6880
6881 // An empty aggregate parameter takes up no storage and no registers,
6882 // but needs a MemLoc for a stack slot for the formal arguments side.
6883 if (ByValSize == 0) {
6884 State.addLoc(V: CCValAssign::getMem(ValNo, ValVT: MVT::INVALID_SIMPLE_VALUE_TYPE,
6885 Offset: State.getStackSize(), LocVT: RegVT, HTP: LocInfo));
6886 return false;
6887 }
6888
6889 // Shadow allocate any registers that are not properly aligned.
6890 unsigned NextReg = State.getFirstUnallocated(Regs: GPRs);
6891 while (NextReg != GPRs.size() &&
6892 !isGPRShadowAligned(Reg: GPRs[NextReg], RequiredAlign: ObjAlign)) {
6893 // Shadow allocate next registers since its aligment is not strict enough.
6894 MCRegister Reg = State.AllocateReg(Regs: GPRs);
6895 // Allocate the stack space shadowed by said register.
6896 State.AllocateStack(Size: PtrSize, Alignment: PtrAlign);
6897 assert(Reg && "Alocating register unexpectedly failed.");
6898 (void)Reg;
6899 NextReg = State.getFirstUnallocated(Regs: GPRs);
6900 }
6901
6902 const unsigned StackSize = alignTo(Size: ByValSize, A: ObjAlign);
6903 unsigned Offset = State.AllocateStack(Size: StackSize, Alignment: ObjAlign);
6904 for (const unsigned E = Offset + StackSize; Offset < E; Offset += PtrSize) {
6905 if (MCRegister Reg = State.AllocateReg(Regs: GPRs))
6906 State.addLoc(V: CCValAssign::getReg(ValNo, ValVT, Reg, LocVT: RegVT, HTP: LocInfo));
6907 else {
6908 State.addLoc(V: CCValAssign::getMem(ValNo, ValVT: MVT::INVALID_SIMPLE_VALUE_TYPE,
6909 Offset, LocVT: MVT::INVALID_SIMPLE_VALUE_TYPE,
6910 HTP: LocInfo));
6911 break;
6912 }
6913 }
6914 return false;
6915 }
6916
6917 // Arguments always reserve parameter save area.
6918 switch (ValVT.SimpleTy) {
6919 default:
6920 report_fatal_error(reason: "Unhandled value type for argument.");
6921 case MVT::i64:
6922 // i64 arguments should have been split to i32 for PPC32.
6923 assert(IsPPC64 && "PPC32 should have split i64 values.");
6924 [[fallthrough]];
6925 case MVT::i1:
6926 case MVT::i32: {
6927 const unsigned Offset = State.AllocateStack(Size: PtrSize, Alignment: PtrAlign);
6928 // AIX integer arguments are always passed in register width.
6929 if (ValVT.getFixedSizeInBits() < RegVT.getFixedSizeInBits())
6930 LocInfo = ArgFlags.isSExt() ? CCValAssign::LocInfo::SExt
6931 : CCValAssign::LocInfo::ZExt;
6932 if (MCRegister Reg = State.AllocateReg(Regs: GPRs))
6933 State.addLoc(V: CCValAssign::getReg(ValNo, ValVT, Reg, LocVT: RegVT, HTP: LocInfo));
6934 else
6935 State.addLoc(V: CCValAssign::getMem(ValNo, ValVT, Offset, LocVT: RegVT, HTP: LocInfo));
6936
6937 return false;
6938 }
6939 case MVT::f32:
6940 case MVT::f64: {
6941 // Parameter save area (PSA) is reserved even if the float passes in fpr.
6942 const unsigned StoreSize = LocVT.getStoreSize();
6943 // Floats are always 4-byte aligned in the PSA on AIX.
6944 // This includes f64 in 64-bit mode for ABI compatibility.
6945 const unsigned Offset =
6946 State.AllocateStack(Size: IsPPC64 ? 8 : StoreSize, Alignment: Align(4));
6947 MCRegister FReg = State.AllocateReg(Regs: FPR);
6948 if (FReg)
6949 State.addLoc(V: CCValAssign::getReg(ValNo, ValVT, Reg: FReg, LocVT, HTP: LocInfo));
6950
6951 // Reserve and initialize GPRs or initialize the PSA as required.
6952 for (unsigned I = 0; I < StoreSize; I += PtrSize) {
6953 if (MCRegister Reg = State.AllocateReg(Regs: GPRs)) {
6954 assert(FReg && "An FPR should be available when a GPR is reserved.");
6955 if (State.isVarArg()) {
6956 // Successfully reserved GPRs are only initialized for vararg calls.
6957 // Custom handling is required for:
6958 // f64 in PPC32 needs to be split into 2 GPRs.
6959 // f32 in PPC64 needs to occupy only lower 32 bits of 64-bit GPR.
6960 State.addLoc(
6961 V: CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT: RegVT, HTP: LocInfo));
6962 }
6963 } else {
6964 // If there are insufficient GPRs, the PSA needs to be initialized.
6965 // Initialization occurs even if an FPR was initialized for
6966 // compatibility with the AIX XL compiler. The full memory for the
6967 // argument will be initialized even if a prior word is saved in GPR.
6968 // A custom memLoc is used when the argument also passes in FPR so
6969 // that the callee handling can skip over it easily.
6970 State.addLoc(
6971 V: FReg ? CCValAssign::getCustomMem(ValNo, ValVT, Offset, LocVT,
6972 HTP: LocInfo)
6973 : CCValAssign::getMem(ValNo, ValVT, Offset, LocVT, HTP: LocInfo));
6974 break;
6975 }
6976 }
6977
6978 return false;
6979 }
6980 case MVT::v4f32:
6981 case MVT::v4i32:
6982 case MVT::v8i16:
6983 case MVT::v16i8:
6984 case MVT::v2i64:
6985 case MVT::v2f64:
6986 case MVT::v1i128: {
6987 const unsigned VecSize = 16;
6988 const Align VecAlign(VecSize);
6989
6990 if (!State.isVarArg()) {
6991 // If there are vector registers remaining we don't consume any stack
6992 // space.
6993 if (MCRegister VReg = State.AllocateReg(Regs: VR)) {
6994 State.addLoc(V: CCValAssign::getReg(ValNo, ValVT, Reg: VReg, LocVT, HTP: LocInfo));
6995 return false;
6996 }
6997 // Vectors passed on the stack do not shadow GPRs or FPRs even though they
6998 // might be allocated in the portion of the PSA that is shadowed by the
6999 // GPRs.
7000 const unsigned Offset = State.AllocateStack(Size: VecSize, Alignment: VecAlign);
7001 State.addLoc(V: CCValAssign::getMem(ValNo, ValVT, Offset, LocVT, HTP: LocInfo));
7002 return false;
7003 }
7004
7005 unsigned NextRegIndex = State.getFirstUnallocated(Regs: GPRs);
7006 // Burn any underaligned registers and their shadowed stack space until
7007 // we reach the required alignment.
7008 while (NextRegIndex != GPRs.size() &&
7009 !isGPRShadowAligned(Reg: GPRs[NextRegIndex], RequiredAlign: VecAlign)) {
7010 // Shadow allocate register and its stack shadow.
7011 MCRegister Reg = State.AllocateReg(Regs: GPRs);
7012 State.AllocateStack(Size: PtrSize, Alignment: PtrAlign);
7013 assert(Reg && "Allocating register unexpectedly failed.");
7014 (void)Reg;
7015 NextRegIndex = State.getFirstUnallocated(Regs: GPRs);
7016 }
7017
7018 // Vectors that are passed as fixed arguments are handled differently.
7019 // They are passed in VRs if any are available (unlike arguments passed
7020 // through ellipses) and shadow GPRs (unlike arguments to non-vaarg
7021 // functions)
7022 if (!ArgFlags.isVarArg()) {
7023 if (MCRegister VReg = State.AllocateReg(Regs: VR)) {
7024 State.addLoc(V: CCValAssign::getReg(ValNo, ValVT, Reg: VReg, LocVT, HTP: LocInfo));
7025 // Shadow allocate GPRs and stack space even though we pass in a VR.
7026 for (unsigned I = 0; I != VecSize; I += PtrSize)
7027 State.AllocateReg(Regs: GPRs);
7028 State.AllocateStack(Size: VecSize, Alignment: VecAlign);
7029 return false;
7030 }
7031 // No vector registers remain so pass on the stack.
7032 const unsigned Offset = State.AllocateStack(Size: VecSize, Alignment: VecAlign);
7033 State.addLoc(V: CCValAssign::getMem(ValNo, ValVT, Offset, LocVT, HTP: LocInfo));
7034 return false;
7035 }
7036
7037 // If all GPRS are consumed then we pass the argument fully on the stack.
7038 if (NextRegIndex == GPRs.size()) {
7039 const unsigned Offset = State.AllocateStack(Size: VecSize, Alignment: VecAlign);
7040 State.addLoc(V: CCValAssign::getMem(ValNo, ValVT, Offset, LocVT, HTP: LocInfo));
7041 return false;
7042 }
7043
7044 // Corner case for 32-bit codegen. We have 2 registers to pass the first
7045 // half of the argument, and then need to pass the remaining half on the
7046 // stack.
7047 if (GPRs[NextRegIndex] == PPC::R9) {
7048 const unsigned Offset = State.AllocateStack(Size: VecSize, Alignment: VecAlign);
7049 State.addLoc(
7050 V: CCValAssign::getCustomMem(ValNo, ValVT, Offset, LocVT, HTP: LocInfo));
7051
7052 const MCRegister FirstReg = State.AllocateReg(Reg: PPC::R9);
7053 const MCRegister SecondReg = State.AllocateReg(Reg: PPC::R10);
7054 assert(FirstReg && SecondReg &&
7055 "Allocating R9 or R10 unexpectedly failed.");
7056 State.addLoc(
7057 V: CCValAssign::getCustomReg(ValNo, ValVT, Reg: FirstReg, LocVT: RegVT, HTP: LocInfo));
7058 State.addLoc(
7059 V: CCValAssign::getCustomReg(ValNo, ValVT, Reg: SecondReg, LocVT: RegVT, HTP: LocInfo));
7060 return false;
7061 }
7062
7063 // We have enough GPRs to fully pass the vector argument, and we have
7064 // already consumed any underaligned registers. Start with the custom
7065 // MemLoc and then the custom RegLocs.
7066 const unsigned Offset = State.AllocateStack(Size: VecSize, Alignment: VecAlign);
7067 State.addLoc(
7068 V: CCValAssign::getCustomMem(ValNo, ValVT, Offset, LocVT, HTP: LocInfo));
7069 for (unsigned I = 0; I != VecSize; I += PtrSize) {
7070 const MCRegister Reg = State.AllocateReg(Regs: GPRs);
7071 assert(Reg && "Failed to allocated register for vararg vector argument");
7072 State.addLoc(
7073 V: CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT: RegVT, HTP: LocInfo));
7074 }
7075 return false;
7076 }
7077 }
7078 return true;
7079}
7080
7081// So far, this function is only used by LowerFormalArguments_AIX()
7082static const TargetRegisterClass *getRegClassForSVT(MVT::SimpleValueType SVT,
7083 bool IsPPC64,
7084 bool HasP8Vector,
7085 bool HasVSX) {
7086 assert((IsPPC64 || SVT != MVT::i64) &&
7087 "i64 should have been split for 32-bit codegen.");
7088
7089 switch (SVT) {
7090 default:
7091 report_fatal_error(reason: "Unexpected value type for formal argument");
7092 case MVT::i1:
7093 case MVT::i32:
7094 case MVT::i64:
7095 return IsPPC64 ? &PPC::G8RCRegClass : &PPC::GPRCRegClass;
7096 case MVT::f32:
7097 return HasP8Vector ? &PPC::VSSRCRegClass : &PPC::F4RCRegClass;
7098 case MVT::f64:
7099 return HasVSX ? &PPC::VSFRCRegClass : &PPC::F8RCRegClass;
7100 case MVT::v4f32:
7101 case MVT::v4i32:
7102 case MVT::v8i16:
7103 case MVT::v16i8:
7104 case MVT::v2i64:
7105 case MVT::v2f64:
7106 case MVT::v1i128:
7107 return &PPC::VRRCRegClass;
7108 }
7109}
7110
7111static SDValue truncateScalarIntegerArg(ISD::ArgFlagsTy Flags, EVT ValVT,
7112 SelectionDAG &DAG, SDValue ArgValue,
7113 MVT LocVT, const SDLoc &dl) {
7114 assert(ValVT.isScalarInteger() && LocVT.isScalarInteger());
7115 assert(ValVT.getFixedSizeInBits() < LocVT.getFixedSizeInBits());
7116
7117 if (Flags.isSExt())
7118 ArgValue = DAG.getNode(Opcode: ISD::AssertSext, DL: dl, VT: LocVT, N1: ArgValue,
7119 N2: DAG.getValueType(ValVT));
7120 else if (Flags.isZExt())
7121 ArgValue = DAG.getNode(Opcode: ISD::AssertZext, DL: dl, VT: LocVT, N1: ArgValue,
7122 N2: DAG.getValueType(ValVT));
7123
7124 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: ValVT, Operand: ArgValue);
7125}
7126
7127static unsigned mapArgRegToOffsetAIX(unsigned Reg, const PPCFrameLowering *FL) {
7128 const unsigned LASize = FL->getLinkageSize();
7129
7130 if (PPC::GPRCRegClass.contains(Reg)) {
7131 assert(Reg >= PPC::R3 && Reg <= PPC::R10 &&
7132 "Reg must be a valid argument register!");
7133 return LASize + 4 * (Reg - PPC::R3);
7134 }
7135
7136 if (PPC::G8RCRegClass.contains(Reg)) {
7137 assert(Reg >= PPC::X3 && Reg <= PPC::X10 &&
7138 "Reg must be a valid argument register!");
7139 return LASize + 8 * (Reg - PPC::X3);
7140 }
7141
7142 llvm_unreachable("Only general purpose registers expected.");
7143}
7144
7145// AIX ABI Stack Frame Layout:
7146//
7147// Low Memory +--------------------------------------------+
7148// SP +---> | Back chain | ---+
7149// | +--------------------------------------------+ |
7150// | | Saved Condition Register | |
7151// | +--------------------------------------------+ |
7152// | | Saved Linkage Register | |
7153// | +--------------------------------------------+ | Linkage Area
7154// | | Reserved for compilers | |
7155// | +--------------------------------------------+ |
7156// | | Reserved for binders | |
7157// | +--------------------------------------------+ |
7158// | | Saved TOC pointer | ---+
7159// | +--------------------------------------------+
7160// | | Parameter save area |
7161// | +--------------------------------------------+
7162// | | Alloca space |
7163// | +--------------------------------------------+
7164// | | Local variable space |
7165// | +--------------------------------------------+
7166// | | Float/int conversion temporary |
7167// | +--------------------------------------------+
7168// | | Save area for AltiVec registers |
7169// | +--------------------------------------------+
7170// | | AltiVec alignment padding |
7171// | +--------------------------------------------+
7172// | | Save area for VRSAVE register |
7173// | +--------------------------------------------+
7174// | | Save area for General Purpose registers |
7175// | +--------------------------------------------+
7176// | | Save area for Floating Point registers |
7177// | +--------------------------------------------+
7178// +---- | Back chain |
7179// High Memory +--------------------------------------------+
7180//
7181// Specifications:
7182// AIX 7.2 Assembler Language Reference
7183// Subroutine linkage convention
7184
7185SDValue PPCTargetLowering::LowerFormalArguments_AIX(
7186 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
7187 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
7188 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
7189
7190 assert((CallConv == CallingConv::C || CallConv == CallingConv::Cold ||
7191 CallConv == CallingConv::Fast) &&
7192 "Unexpected calling convention!");
7193
7194 if (getTargetMachine().Options.GuaranteedTailCallOpt)
7195 report_fatal_error(reason: "Tail call support is unimplemented on AIX.");
7196
7197 if (useSoftFloat())
7198 report_fatal_error(reason: "Soft float support is unimplemented on AIX.");
7199
7200 const PPCSubtarget &Subtarget = DAG.getSubtarget<PPCSubtarget>();
7201
7202 const bool IsPPC64 = Subtarget.isPPC64();
7203 const unsigned PtrByteSize = IsPPC64 ? 8 : 4;
7204
7205 // Assign locations to all of the incoming arguments.
7206 SmallVector<CCValAssign, 16> ArgLocs;
7207 MachineFunction &MF = DAG.getMachineFunction();
7208 MachineFrameInfo &MFI = MF.getFrameInfo();
7209 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
7210 CCState CCInfo(CallConv, isVarArg, MF, ArgLocs, *DAG.getContext());
7211
7212 const EVT PtrVT = getPointerTy(DL: MF.getDataLayout());
7213 // Reserve space for the linkage area on the stack.
7214 const unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize();
7215 CCInfo.AllocateStack(Size: LinkageSize, Alignment: Align(PtrByteSize));
7216 uint64_t SaveStackPos = CCInfo.getStackSize();
7217 bool SaveParams = MF.getFunction().hasFnAttribute(Kind: "save-reg-params");
7218 CCInfo.AnalyzeFormalArguments(Ins, Fn: CC_AIX);
7219
7220 SmallVector<SDValue, 8> MemOps;
7221
7222 for (size_t I = 0, End = ArgLocs.size(); I != End; /* No increment here */) {
7223 CCValAssign &VA = ArgLocs[I++];
7224 MVT LocVT = VA.getLocVT();
7225 MVT ValVT = VA.getValVT();
7226 ISD::ArgFlagsTy Flags = Ins[VA.getValNo()].Flags;
7227
7228 EVT ArgVT = Ins[VA.getValNo()].ArgVT;
7229 bool ArgSignExt = Ins[VA.getValNo()].Flags.isSExt();
7230 // For compatibility with the AIX XL compiler, the float args in the
7231 // parameter save area are initialized even if the argument is available
7232 // in register. The caller is required to initialize both the register
7233 // and memory, however, the callee can choose to expect it in either.
7234 // The memloc is dismissed here because the argument is retrieved from
7235 // the register.
7236 if (VA.isMemLoc() && VA.needsCustom() && ValVT.isFloatingPoint())
7237 continue;
7238
7239 if (SaveParams && VA.isRegLoc() && !Flags.isByVal() && !VA.needsCustom()) {
7240 const TargetRegisterClass *RegClass = getRegClassForSVT(
7241 SVT: LocVT.SimpleTy, IsPPC64, HasP8Vector: Subtarget.hasP8Vector(), HasVSX: Subtarget.hasVSX());
7242 // On PPC64, debugger assumes extended 8-byte values are stored from GPR.
7243 MVT SaveVT = RegClass == &PPC::G8RCRegClass ? MVT::i64 : LocVT;
7244 const Register VReg = MF.addLiveIn(PReg: VA.getLocReg(), RC: RegClass);
7245 SDValue Parm = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: SaveVT);
7246 int FI = MFI.CreateFixedObject(Size: SaveVT.getStoreSize(), SPOffset: SaveStackPos, IsImmutable: true);
7247 SDValue FIN = DAG.getFrameIndex(FI, VT: PtrVT);
7248 SDValue StoreReg = DAG.getStore(Chain, dl, Val: Parm, Ptr: FIN,
7249 PtrInfo: MachinePointerInfo(), Alignment: Align(PtrByteSize));
7250 SaveStackPos = alignTo(Value: SaveStackPos + SaveVT.getStoreSize(), Align: PtrByteSize);
7251 MemOps.push_back(Elt: StoreReg);
7252 }
7253
7254 if (SaveParams && (VA.isMemLoc() || Flags.isByVal()) && !VA.needsCustom()) {
7255 unsigned StoreSize =
7256 Flags.isByVal() ? Flags.getByValSize() : LocVT.getStoreSize();
7257 SaveStackPos = alignTo(Value: SaveStackPos + StoreSize, Align: PtrByteSize);
7258 }
7259
7260 auto HandleMemLoc = [&]() {
7261 const unsigned LocSize = LocVT.getStoreSize();
7262 const unsigned ValSize = ValVT.getStoreSize();
7263 assert((ValSize <= LocSize) &&
7264 "Object size is larger than size of MemLoc");
7265 int CurArgOffset = VA.getLocMemOffset();
7266 // Objects are right-justified because AIX is big-endian.
7267 if (LocSize > ValSize)
7268 CurArgOffset += LocSize - ValSize;
7269 // Potential tail calls could cause overwriting of argument stack slots.
7270 const bool IsImmutable =
7271 !(getTargetMachine().Options.GuaranteedTailCallOpt &&
7272 (CallConv == CallingConv::Fast));
7273 int FI = MFI.CreateFixedObject(Size: ValSize, SPOffset: CurArgOffset, IsImmutable);
7274 SDValue FIN = DAG.getFrameIndex(FI, VT: PtrVT);
7275 SDValue ArgValue =
7276 DAG.getLoad(VT: ValVT, dl, Chain, Ptr: FIN, PtrInfo: MachinePointerInfo());
7277
7278 // While the ABI specifies the argument type is (sign or zero) extended
7279 // out to register width, not all code is compliant. We truncate and
7280 // re-extend to be more forgiving of these callers when the argument type
7281 // is smaller than register width.
7282 if (!ArgVT.isVector() && !ValVT.isVector() && ArgVT.isInteger() &&
7283 ValVT.isInteger() &&
7284 ArgVT.getScalarSizeInBits() < ValVT.getScalarSizeInBits()) {
7285 // It is possible to have either real integer values
7286 // or integers that were not originally integers.
7287 // In the latter case, these could have came from structs,
7288 // and these integers would not have an extend on the parameter.
7289 // Since these types of integers do not have an extend specified
7290 // in the first place, the type of extend that we do should not matter.
7291 EVT TruncatedArgVT = ArgVT.isSimple() && ArgVT.getSimpleVT() == MVT::i1
7292 ? MVT::i8
7293 : ArgVT;
7294 SDValue ArgValueTrunc =
7295 DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: TruncatedArgVT, Operand: ArgValue);
7296 SDValue ArgValueExt =
7297 ArgSignExt ? DAG.getSExtOrTrunc(Op: ArgValueTrunc, DL: dl, VT: ValVT)
7298 : DAG.getZExtOrTrunc(Op: ArgValueTrunc, DL: dl, VT: ValVT);
7299 InVals.push_back(Elt: ArgValueExt);
7300 } else {
7301 InVals.push_back(Elt: ArgValue);
7302 }
7303 };
7304
7305 // Vector arguments to VaArg functions are passed both on the stack, and
7306 // in any available GPRs. Load the value from the stack and add the GPRs
7307 // as live ins.
7308 if (VA.isMemLoc() && VA.needsCustom()) {
7309 assert(ValVT.isVector() && "Unexpected Custom MemLoc type.");
7310 assert(isVarArg && "Only use custom memloc for vararg.");
7311 // ValNo of the custom MemLoc, so we can compare it to the ValNo of the
7312 // matching custom RegLocs.
7313 const unsigned OriginalValNo = VA.getValNo();
7314 (void)OriginalValNo;
7315
7316 auto HandleCustomVecRegLoc = [&]() {
7317 assert(I != End && ArgLocs[I].isRegLoc() && ArgLocs[I].needsCustom() &&
7318 "Missing custom RegLoc.");
7319 VA = ArgLocs[I++];
7320 assert(VA.getValVT().isVector() &&
7321 "Unexpected Val type for custom RegLoc.");
7322 assert(VA.getValNo() == OriginalValNo &&
7323 "ValNo mismatch between custom MemLoc and RegLoc.");
7324 MVT::SimpleValueType SVT = VA.getLocVT().SimpleTy;
7325 MF.addLiveIn(PReg: VA.getLocReg(),
7326 RC: getRegClassForSVT(SVT, IsPPC64, HasP8Vector: Subtarget.hasP8Vector(),
7327 HasVSX: Subtarget.hasVSX()));
7328 };
7329
7330 HandleMemLoc();
7331 // In 64-bit there will be exactly 2 custom RegLocs that follow, and in
7332 // in 32-bit there will be 2 custom RegLocs if we are passing in R9 and
7333 // R10.
7334 HandleCustomVecRegLoc();
7335 HandleCustomVecRegLoc();
7336
7337 // If we are targeting 32-bit, there might be 2 extra custom RegLocs if
7338 // we passed the vector in R5, R6, R7 and R8.
7339 if (I != End && ArgLocs[I].isRegLoc() && ArgLocs[I].needsCustom()) {
7340 assert(!IsPPC64 &&
7341 "Only 2 custom RegLocs expected for 64-bit codegen.");
7342 HandleCustomVecRegLoc();
7343 HandleCustomVecRegLoc();
7344 }
7345
7346 continue;
7347 }
7348
7349 if (VA.isRegLoc()) {
7350 if (VA.getValVT().isScalarInteger())
7351 FuncInfo->appendParameterType(Type: PPCFunctionInfo::FixedType);
7352 else if (VA.getValVT().isFloatingPoint() && !VA.getValVT().isVector()) {
7353 switch (VA.getValVT().SimpleTy) {
7354 default:
7355 report_fatal_error(reason: "Unhandled value type for argument.");
7356 case MVT::f32:
7357 FuncInfo->appendParameterType(Type: PPCFunctionInfo::ShortFloatingPoint);
7358 break;
7359 case MVT::f64:
7360 FuncInfo->appendParameterType(Type: PPCFunctionInfo::LongFloatingPoint);
7361 break;
7362 }
7363 } else if (VA.getValVT().isVector()) {
7364 switch (VA.getValVT().SimpleTy) {
7365 default:
7366 report_fatal_error(reason: "Unhandled value type for argument.");
7367 case MVT::v16i8:
7368 FuncInfo->appendParameterType(Type: PPCFunctionInfo::VectorChar);
7369 break;
7370 case MVT::v8i16:
7371 FuncInfo->appendParameterType(Type: PPCFunctionInfo::VectorShort);
7372 break;
7373 case MVT::v4i32:
7374 case MVT::v2i64:
7375 case MVT::v1i128:
7376 FuncInfo->appendParameterType(Type: PPCFunctionInfo::VectorInt);
7377 break;
7378 case MVT::v4f32:
7379 case MVT::v2f64:
7380 FuncInfo->appendParameterType(Type: PPCFunctionInfo::VectorFloat);
7381 break;
7382 }
7383 }
7384 }
7385
7386 if (Flags.isByVal() && VA.isMemLoc()) {
7387 const unsigned Size =
7388 alignTo(Value: Flags.getByValSize() ? Flags.getByValSize() : PtrByteSize,
7389 Align: PtrByteSize);
7390 const int FI = MF.getFrameInfo().CreateFixedObject(
7391 Size, SPOffset: VA.getLocMemOffset(), /* IsImmutable */ false,
7392 /* IsAliased */ isAliased: true);
7393 SDValue FIN = DAG.getFrameIndex(FI, VT: PtrVT);
7394 InVals.push_back(Elt: FIN);
7395
7396 continue;
7397 }
7398
7399 if (Flags.isByVal()) {
7400 assert(VA.isRegLoc() && "MemLocs should already be handled.");
7401
7402 const MCPhysReg ArgReg = VA.getLocReg();
7403 const PPCFrameLowering *FL = Subtarget.getFrameLowering();
7404
7405 const unsigned StackSize = alignTo(Value: Flags.getByValSize(), Align: PtrByteSize);
7406 const int FI = MF.getFrameInfo().CreateFixedObject(
7407 Size: StackSize, SPOffset: mapArgRegToOffsetAIX(Reg: ArgReg, FL), /* IsImmutable */ false,
7408 /* IsAliased */ isAliased: true);
7409 SDValue FIN = DAG.getFrameIndex(FI, VT: PtrVT);
7410 InVals.push_back(Elt: FIN);
7411
7412 // Add live ins for all the RegLocs for the same ByVal.
7413 const TargetRegisterClass *RegClass =
7414 IsPPC64 ? &PPC::G8RCRegClass : &PPC::GPRCRegClass;
7415
7416 auto HandleRegLoc = [&, RegClass, LocVT](const MCPhysReg PhysReg,
7417 unsigned Offset) {
7418 const Register VReg = MF.addLiveIn(PReg: PhysReg, RC: RegClass);
7419 // Since the callers side has left justified the aggregate in the
7420 // register, we can simply store the entire register into the stack
7421 // slot.
7422 SDValue CopyFrom = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: LocVT);
7423 // The store to the fixedstack object is needed becuase accessing a
7424 // field of the ByVal will use a gep and load. Ideally we will optimize
7425 // to extracting the value from the register directly, and elide the
7426 // stores when the arguments address is not taken, but that will need to
7427 // be future work.
7428 SDValue Store = DAG.getStore(
7429 Chain: CopyFrom.getValue(R: 1), dl, Val: CopyFrom,
7430 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: FIN, Offset: TypeSize::getFixed(ExactSize: Offset)),
7431 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI, Offset));
7432
7433 MemOps.push_back(Elt: Store);
7434 };
7435
7436 unsigned Offset = 0;
7437 HandleRegLoc(VA.getLocReg(), Offset);
7438 Offset += PtrByteSize;
7439 for (; Offset != StackSize && ArgLocs[I].isRegLoc();
7440 Offset += PtrByteSize) {
7441 assert(ArgLocs[I].getValNo() == VA.getValNo() &&
7442 "RegLocs should be for ByVal argument.");
7443
7444 const CCValAssign RL = ArgLocs[I++];
7445 HandleRegLoc(RL.getLocReg(), Offset);
7446 FuncInfo->appendParameterType(Type: PPCFunctionInfo::FixedType);
7447 }
7448
7449 if (Offset != StackSize) {
7450 assert(ArgLocs[I].getValNo() == VA.getValNo() &&
7451 "Expected MemLoc for remaining bytes.");
7452 assert(ArgLocs[I].isMemLoc() && "Expected MemLoc for remaining bytes.");
7453 // Consume the MemLoc.The InVal has already been emitted, so nothing
7454 // more needs to be done.
7455 ++I;
7456 }
7457
7458 continue;
7459 }
7460
7461 if (VA.isRegLoc() && !VA.needsCustom()) {
7462 MVT::SimpleValueType SVT = ValVT.SimpleTy;
7463 Register VReg =
7464 MF.addLiveIn(PReg: VA.getLocReg(),
7465 RC: getRegClassForSVT(SVT, IsPPC64, HasP8Vector: Subtarget.hasP8Vector(),
7466 HasVSX: Subtarget.hasVSX()));
7467 SDValue ArgValue = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: LocVT);
7468 if (ValVT.isScalarInteger() &&
7469 (ValVT.getFixedSizeInBits() < LocVT.getFixedSizeInBits())) {
7470 ArgValue =
7471 truncateScalarIntegerArg(Flags, ValVT, DAG, ArgValue, LocVT, dl);
7472 }
7473 InVals.push_back(Elt: ArgValue);
7474 continue;
7475 }
7476 if (VA.isMemLoc()) {
7477 HandleMemLoc();
7478 continue;
7479 }
7480 }
7481
7482 // On AIX a minimum of 8 words is saved to the parameter save area.
7483 const unsigned MinParameterSaveArea = 8 * PtrByteSize;
7484 // Area that is at least reserved in the caller of this function.
7485 unsigned CallerReservedArea = std::max<unsigned>(
7486 a: CCInfo.getStackSize(), b: LinkageSize + MinParameterSaveArea);
7487
7488 // Set the size that is at least reserved in caller of this function. Tail
7489 // call optimized function's reserved stack space needs to be aligned so
7490 // that taking the difference between two stack areas will result in an
7491 // aligned stack.
7492 CallerReservedArea =
7493 EnsureStackAlignment(Lowering: Subtarget.getFrameLowering(), NumBytes: CallerReservedArea);
7494 FuncInfo->setMinReservedArea(CallerReservedArea);
7495
7496 if (isVarArg) {
7497 int VAListIndex = 0;
7498 // If any of the optional arguments are passed in register then the fixed
7499 // stack object we spill into is not immutable. Create a fixed stack object
7500 // that overlaps the remainder of the parameter save area.
7501 if (CCInfo.getStackSize() < (LinkageSize + MinParameterSaveArea)) {
7502 unsigned FixedStackSize =
7503 LinkageSize + MinParameterSaveArea - CCInfo.getStackSize();
7504 VAListIndex =
7505 MFI.CreateFixedObject(Size: FixedStackSize, SPOffset: CCInfo.getStackSize(),
7506 /* IsImmutable */ false, /* IsAliased */ isAliased: true);
7507 } else {
7508 // All the arguments passed through ellipses are on the stack. Create a
7509 // dummy fixed stack object the same size as a pointer since we don't
7510 // know the actual size.
7511 VAListIndex =
7512 MFI.CreateFixedObject(Size: PtrByteSize, SPOffset: CCInfo.getStackSize(),
7513 /* IsImmutable */ true, /* IsAliased */ isAliased: true);
7514 }
7515
7516 FuncInfo->setVarArgsFrameIndex(VAListIndex);
7517 SDValue FIN = DAG.getFrameIndex(FI: VAListIndex, VT: PtrVT);
7518
7519 static const MCPhysReg GPR_32[] = {PPC::R3, PPC::R4, PPC::R5, PPC::R6,
7520 PPC::R7, PPC::R8, PPC::R9, PPC::R10};
7521
7522 static const MCPhysReg GPR_64[] = {PPC::X3, PPC::X4, PPC::X5, PPC::X6,
7523 PPC::X7, PPC::X8, PPC::X9, PPC::X10};
7524 const unsigned NumGPArgRegs = std::size(IsPPC64 ? GPR_64 : GPR_32);
7525
7526 // The fixed integer arguments of a variadic function are stored to the
7527 // VarArgsFrameIndex on the stack so that they may be loaded by
7528 // dereferencing the result of va_next.
7529 for (unsigned
7530 GPRIndex = (CCInfo.getStackSize() - LinkageSize) / PtrByteSize,
7531 Offset = 0;
7532 GPRIndex < NumGPArgRegs; ++GPRIndex, Offset += PtrByteSize) {
7533
7534 const Register VReg =
7535 IsPPC64 ? MF.addLiveIn(PReg: GPR_64[GPRIndex], RC: &PPC::G8RCRegClass)
7536 : MF.addLiveIn(PReg: GPR_32[GPRIndex], RC: &PPC::GPRCRegClass);
7537
7538 SDValue Val = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: PtrVT);
7539 MachinePointerInfo MPI =
7540 MachinePointerInfo::getFixedStack(MF, FI: VAListIndex, Offset);
7541 SDValue Store = DAG.getStore(Chain: Val.getValue(R: 1), dl, Val, Ptr: FIN, PtrInfo: MPI);
7542 MemOps.push_back(Elt: Store);
7543 // Increment the address for the next argument to store.
7544 SDValue PtrOff = DAG.getConstant(Val: PtrByteSize, DL: dl, VT: PtrVT);
7545 FIN = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrOff.getValueType(), N1: FIN, N2: PtrOff);
7546 }
7547 }
7548
7549 if (!MemOps.empty())
7550 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: MemOps);
7551
7552 return Chain;
7553}
7554
7555SDValue PPCTargetLowering::LowerCall_AIX(
7556 SDValue Chain, SDValue Callee, CallFlags CFlags,
7557 const SmallVectorImpl<ISD::OutputArg> &Outs,
7558 const SmallVectorImpl<SDValue> &OutVals,
7559 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
7560 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals,
7561 const CallBase *CB) const {
7562 // See PPCTargetLowering::LowerFormalArguments_AIX() for a description of the
7563 // AIX ABI stack frame layout.
7564
7565 assert((CFlags.CallConv == CallingConv::C ||
7566 CFlags.CallConv == CallingConv::Cold ||
7567 CFlags.CallConv == CallingConv::Fast) &&
7568 "Unexpected calling convention!");
7569
7570 if (CFlags.IsPatchPoint)
7571 report_fatal_error(reason: "This call type is unimplemented on AIX.");
7572
7573 const PPCSubtarget &Subtarget = DAG.getSubtarget<PPCSubtarget>();
7574
7575 MachineFunction &MF = DAG.getMachineFunction();
7576 SmallVector<CCValAssign, 16> ArgLocs;
7577 CCState CCInfo(CFlags.CallConv, CFlags.IsVarArg, MF, ArgLocs,
7578 *DAG.getContext());
7579
7580 // Reserve space for the linkage save area (LSA) on the stack.
7581 // In both PPC32 and PPC64 there are 6 reserved slots in the LSA:
7582 // [SP][CR][LR][2 x reserved][TOC].
7583 // The LSA is 24 bytes (6x4) in PPC32 and 48 bytes (6x8) in PPC64.
7584 const unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize();
7585 const bool IsPPC64 = Subtarget.isPPC64();
7586 const EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
7587 const unsigned PtrByteSize = IsPPC64 ? 8 : 4;
7588 CCInfo.AllocateStack(Size: LinkageSize, Alignment: Align(PtrByteSize));
7589 CCInfo.AnalyzeCallOperands(Outs, Fn: CC_AIX);
7590
7591 // The prolog code of the callee may store up to 8 GPR argument registers to
7592 // the stack, allowing va_start to index over them in memory if the callee
7593 // is variadic.
7594 // Because we cannot tell if this is needed on the caller side, we have to
7595 // conservatively assume that it is needed. As such, make sure we have at
7596 // least enough stack space for the caller to store the 8 GPRs.
7597 const unsigned MinParameterSaveAreaSize = 8 * PtrByteSize;
7598 const unsigned NumBytes = std::max<unsigned>(
7599 a: LinkageSize + MinParameterSaveAreaSize, b: CCInfo.getStackSize());
7600
7601 // Adjust the stack pointer for the new arguments...
7602 // These operations are automatically eliminated by the prolog/epilog pass.
7603 Chain = DAG.getCALLSEQ_START(Chain, InSize: NumBytes, OutSize: 0, DL: dl);
7604 SDValue CallSeqStart = Chain;
7605
7606 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
7607 SmallVector<SDValue, 8> MemOpChains;
7608
7609 // Set up a copy of the stack pointer for loading and storing any
7610 // arguments that may not fit in the registers available for argument
7611 // passing.
7612 const SDValue StackPtr = IsPPC64 ? DAG.getRegister(Reg: PPC::X1, VT: MVT::i64)
7613 : DAG.getRegister(Reg: PPC::R1, VT: MVT::i32);
7614
7615 for (unsigned I = 0, E = ArgLocs.size(); I != E;) {
7616 const unsigned ValNo = ArgLocs[I].getValNo();
7617 SDValue Arg = OutVals[ValNo];
7618 ISD::ArgFlagsTy Flags = Outs[ValNo].Flags;
7619
7620 if (Flags.isByVal()) {
7621 const unsigned ByValSize = Flags.getByValSize();
7622
7623 // Nothing to do for zero-sized ByVals on the caller side.
7624 if (!ByValSize) {
7625 ++I;
7626 continue;
7627 }
7628
7629 auto GetLoad = [&](EVT VT, unsigned LoadOffset) {
7630 return DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT: PtrVT, Chain,
7631 Ptr: (LoadOffset != 0)
7632 ? DAG.getObjectPtrOffset(
7633 SL: dl, Ptr: Arg, Offset: TypeSize::getFixed(ExactSize: LoadOffset))
7634 : Arg,
7635 PtrInfo: MachinePointerInfo(), MemVT: VT);
7636 };
7637
7638 unsigned LoadOffset = 0;
7639
7640 // Initialize registers, which are fully occupied by the by-val argument.
7641 while (LoadOffset + PtrByteSize <= ByValSize && ArgLocs[I].isRegLoc()) {
7642 SDValue Load = GetLoad(PtrVT, LoadOffset);
7643 MemOpChains.push_back(Elt: Load.getValue(R: 1));
7644 LoadOffset += PtrByteSize;
7645 const CCValAssign &ByValVA = ArgLocs[I++];
7646 assert(ByValVA.getValNo() == ValNo &&
7647 "Unexpected location for pass-by-value argument.");
7648 RegsToPass.push_back(Elt: std::make_pair(x: ByValVA.getLocReg(), y&: Load));
7649 }
7650
7651 if (LoadOffset == ByValSize)
7652 continue;
7653
7654 // There must be one more loc to handle the remainder.
7655 assert(ArgLocs[I].getValNo() == ValNo &&
7656 "Expected additional location for by-value argument.");
7657
7658 if (ArgLocs[I].isMemLoc()) {
7659 assert(LoadOffset < ByValSize && "Unexpected memloc for by-val arg.");
7660 const CCValAssign &ByValVA = ArgLocs[I++];
7661 ISD::ArgFlagsTy MemcpyFlags = Flags;
7662 // Only memcpy the bytes that don't pass in register.
7663 MemcpyFlags.setByValSize(ByValSize - LoadOffset);
7664 Chain = CallSeqStart = createMemcpyOutsideCallSeq(
7665 Arg: (LoadOffset != 0) ? DAG.getObjectPtrOffset(
7666 SL: dl, Ptr: Arg, Offset: TypeSize::getFixed(ExactSize: LoadOffset))
7667 : Arg,
7668 PtrOff: DAG.getObjectPtrOffset(
7669 SL: dl, Ptr: StackPtr, Offset: TypeSize::getFixed(ExactSize: ByValVA.getLocMemOffset())),
7670 CallSeqStart, Flags: MemcpyFlags, DAG, dl);
7671 continue;
7672 }
7673
7674 // Initialize the final register residue.
7675 // Any residue that occupies the final by-val arg register must be
7676 // left-justified on AIX. Loads must be a power-of-2 size and cannot be
7677 // larger than the ByValSize. For example: a 7 byte by-val arg requires 4,
7678 // 2 and 1 byte loads.
7679 const unsigned ResidueBytes = ByValSize % PtrByteSize;
7680 assert(ResidueBytes != 0 && LoadOffset + PtrByteSize > ByValSize &&
7681 "Unexpected register residue for by-value argument.");
7682 SDValue ResidueVal;
7683 for (unsigned Bytes = 0; Bytes != ResidueBytes;) {
7684 const unsigned N = llvm::bit_floor(Value: ResidueBytes - Bytes);
7685 const MVT VT =
7686 N == 1 ? MVT::i8
7687 : ((N == 2) ? MVT::i16 : (N == 4 ? MVT::i32 : MVT::i64));
7688 SDValue Load = GetLoad(VT, LoadOffset);
7689 MemOpChains.push_back(Elt: Load.getValue(R: 1));
7690 LoadOffset += N;
7691 Bytes += N;
7692
7693 // By-val arguments are passed left-justfied in register.
7694 // Every load here needs to be shifted, otherwise a full register load
7695 // should have been used.
7696 assert(PtrVT.getSimpleVT().getSizeInBits() > (Bytes * 8) &&
7697 "Unexpected load emitted during handling of pass-by-value "
7698 "argument.");
7699 unsigned NumSHLBits = PtrVT.getSimpleVT().getSizeInBits() - (Bytes * 8);
7700 EVT ShiftAmountTy =
7701 getShiftAmountTy(LHSTy: Load->getValueType(ResNo: 0), DL: DAG.getDataLayout());
7702 SDValue SHLAmt = DAG.getConstant(Val: NumSHLBits, DL: dl, VT: ShiftAmountTy);
7703 SDValue ShiftedLoad =
7704 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: Load.getValueType(), N1: Load, N2: SHLAmt);
7705 ResidueVal = ResidueVal ? DAG.getNode(Opcode: ISD::OR, DL: dl, VT: PtrVT, N1: ResidueVal,
7706 N2: ShiftedLoad)
7707 : ShiftedLoad;
7708 }
7709
7710 const CCValAssign &ByValVA = ArgLocs[I++];
7711 RegsToPass.push_back(Elt: std::make_pair(x: ByValVA.getLocReg(), y&: ResidueVal));
7712 continue;
7713 }
7714
7715 CCValAssign &VA = ArgLocs[I++];
7716 const MVT LocVT = VA.getLocVT();
7717 const MVT ValVT = VA.getValVT();
7718
7719 switch (VA.getLocInfo()) {
7720 default:
7721 report_fatal_error(reason: "Unexpected argument extension type.");
7722 case CCValAssign::Full:
7723 break;
7724 case CCValAssign::ZExt:
7725 Arg = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: Arg);
7726 break;
7727 case CCValAssign::SExt:
7728 Arg = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: Arg);
7729 break;
7730 }
7731
7732 if (VA.isRegLoc() && !VA.needsCustom()) {
7733 RegsToPass.push_back(Elt: std::make_pair(x: VA.getLocReg(), y&: Arg));
7734 continue;
7735 }
7736
7737 // Vector arguments passed to VarArg functions need custom handling when
7738 // they are passed (at least partially) in GPRs.
7739 if (VA.isMemLoc() && VA.needsCustom() && ValVT.isVector()) {
7740 assert(CFlags.IsVarArg && "Custom MemLocs only used for Vector args.");
7741 // Store value to its stack slot.
7742 SDValue PtrOff =
7743 DAG.getConstant(Val: VA.getLocMemOffset(), DL: dl, VT: StackPtr.getValueType());
7744 PtrOff = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: StackPtr, N2: PtrOff);
7745 SDValue Store =
7746 DAG.getStore(Chain, dl, Val: Arg, Ptr: PtrOff, PtrInfo: MachinePointerInfo());
7747 MemOpChains.push_back(Elt: Store);
7748 const unsigned OriginalValNo = VA.getValNo();
7749 // Then load the GPRs from the stack
7750 unsigned LoadOffset = 0;
7751 auto HandleCustomVecRegLoc = [&]() {
7752 assert(I != E && "Unexpected end of CCvalAssigns.");
7753 assert(ArgLocs[I].isRegLoc() && ArgLocs[I].needsCustom() &&
7754 "Expected custom RegLoc.");
7755 CCValAssign RegVA = ArgLocs[I++];
7756 assert(RegVA.getValNo() == OriginalValNo &&
7757 "Custom MemLoc ValNo and custom RegLoc ValNo must match.");
7758 SDValue Add = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: PtrOff,
7759 N2: DAG.getConstant(Val: LoadOffset, DL: dl, VT: PtrVT));
7760 SDValue Load = DAG.getLoad(VT: PtrVT, dl, Chain: Store, Ptr: Add, PtrInfo: MachinePointerInfo());
7761 MemOpChains.push_back(Elt: Load.getValue(R: 1));
7762 RegsToPass.push_back(Elt: std::make_pair(x: RegVA.getLocReg(), y&: Load));
7763 LoadOffset += PtrByteSize;
7764 };
7765
7766 // In 64-bit there will be exactly 2 custom RegLocs that follow, and in
7767 // in 32-bit there will be 2 custom RegLocs if we are passing in R9 and
7768 // R10.
7769 HandleCustomVecRegLoc();
7770 HandleCustomVecRegLoc();
7771
7772 if (I != E && ArgLocs[I].isRegLoc() && ArgLocs[I].needsCustom() &&
7773 ArgLocs[I].getValNo() == OriginalValNo) {
7774 assert(!IsPPC64 &&
7775 "Only 2 custom RegLocs expected for 64-bit codegen.");
7776 HandleCustomVecRegLoc();
7777 HandleCustomVecRegLoc();
7778 }
7779
7780 continue;
7781 }
7782
7783 if (VA.isMemLoc()) {
7784 SDValue PtrOff =
7785 DAG.getConstant(Val: VA.getLocMemOffset(), DL: dl, VT: StackPtr.getValueType());
7786 PtrOff = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: StackPtr, N2: PtrOff);
7787 MemOpChains.push_back(
7788 Elt: DAG.getStore(Chain, dl, Val: Arg, Ptr: PtrOff,
7789 PtrInfo: MachinePointerInfo::getStack(MF, Offset: VA.getLocMemOffset()),
7790 Alignment: Subtarget.getFrameLowering()->getStackAlign()));
7791
7792 continue;
7793 }
7794
7795 if (!ValVT.isFloatingPoint())
7796 report_fatal_error(
7797 reason: "Unexpected register handling for calling convention.");
7798
7799 // Custom handling is used for GPR initializations for vararg float
7800 // arguments.
7801 assert(VA.isRegLoc() && VA.needsCustom() && CFlags.IsVarArg &&
7802 LocVT.isInteger() &&
7803 "Custom register handling only expected for VarArg.");
7804
7805 SDValue ArgAsInt =
7806 DAG.getBitcast(VT: MVT::getIntegerVT(BitWidth: ValVT.getSizeInBits()), V: Arg);
7807
7808 if (Arg.getValueType().getStoreSize() == LocVT.getStoreSize())
7809 // f32 in 32-bit GPR
7810 // f64 in 64-bit GPR
7811 RegsToPass.push_back(Elt: std::make_pair(x: VA.getLocReg(), y&: ArgAsInt));
7812 else if (Arg.getValueType().getFixedSizeInBits() <
7813 LocVT.getFixedSizeInBits())
7814 // f32 in 64-bit GPR.
7815 RegsToPass.push_back(Elt: std::make_pair(
7816 x: VA.getLocReg(), y: DAG.getZExtOrTrunc(Op: ArgAsInt, DL: dl, VT: LocVT)));
7817 else {
7818 // f64 in two 32-bit GPRs
7819 // The 2 GPRs are marked custom and expected to be adjacent in ArgLocs.
7820 assert(Arg.getValueType() == MVT::f64 && CFlags.IsVarArg && !IsPPC64 &&
7821 "Unexpected custom register for argument!");
7822 CCValAssign &GPR1 = VA;
7823 SDValue MSWAsI64 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i64, N1: ArgAsInt,
7824 N2: DAG.getConstant(Val: 32, DL: dl, VT: MVT::i8));
7825 RegsToPass.push_back(Elt: std::make_pair(
7826 x: GPR1.getLocReg(), y: DAG.getZExtOrTrunc(Op: MSWAsI64, DL: dl, VT: MVT::i32)));
7827
7828 if (I != E) {
7829 // If only 1 GPR was available, there will only be one custom GPR and
7830 // the argument will also pass in memory.
7831 CCValAssign &PeekArg = ArgLocs[I];
7832 if (PeekArg.isRegLoc() && PeekArg.getValNo() == PeekArg.getValNo()) {
7833 assert(PeekArg.needsCustom() && "A second custom GPR is expected.");
7834 CCValAssign &GPR2 = ArgLocs[I++];
7835 RegsToPass.push_back(Elt: std::make_pair(
7836 x: GPR2.getLocReg(), y: DAG.getZExtOrTrunc(Op: ArgAsInt, DL: dl, VT: MVT::i32)));
7837 }
7838 }
7839 }
7840 }
7841
7842 if (!MemOpChains.empty())
7843 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: MemOpChains);
7844
7845 // For indirect calls, we need to save the TOC base to the stack for
7846 // restoration after the call.
7847 if (CFlags.IsIndirect && !Subtarget.usePointerGlueHelper()) {
7848 assert(!CFlags.IsTailCall && "Indirect tail-calls not supported.");
7849 const MCRegister TOCBaseReg = Subtarget.getTOCPointerRegister();
7850 const MCRegister StackPtrReg = Subtarget.getStackPointerRegister();
7851 const MVT PtrVT = Subtarget.getScalarIntVT();
7852 const unsigned TOCSaveOffset =
7853 Subtarget.getFrameLowering()->getTOCSaveOffset();
7854
7855 setUsesTOCBasePtr(DAG);
7856 SDValue Val = DAG.getCopyFromReg(Chain, dl, Reg: TOCBaseReg, VT: PtrVT);
7857 SDValue PtrOff = DAG.getIntPtrConstant(Val: TOCSaveOffset, DL: dl);
7858 SDValue StackPtr = DAG.getRegister(Reg: StackPtrReg, VT: PtrVT);
7859 SDValue AddPtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: StackPtr, N2: PtrOff);
7860 Chain = DAG.getStore(
7861 Chain: Val.getValue(R: 1), dl, Val, Ptr: AddPtr,
7862 PtrInfo: MachinePointerInfo::getStack(MF&: DAG.getMachineFunction(), Offset: TOCSaveOffset));
7863 }
7864
7865 // Build a sequence of copy-to-reg nodes chained together with token chain
7866 // and flag operands which copy the outgoing args into the appropriate regs.
7867 SDValue InGlue;
7868 for (auto Reg : RegsToPass) {
7869 Chain = DAG.getCopyToReg(Chain, dl, Reg: Reg.first, N: Reg.second, Glue: InGlue);
7870 InGlue = Chain.getValue(R: 1);
7871 }
7872
7873 const int SPDiff = 0;
7874 return FinishCall(CFlags, dl, DAG, RegsToPass, Glue: InGlue, Chain, CallSeqStart,
7875 Callee, SPDiff, NumBytes, Ins, InVals, CB);
7876}
7877
7878bool
7879PPCTargetLowering::CanLowerReturn(CallingConv::ID CallConv,
7880 MachineFunction &MF, bool isVarArg,
7881 const SmallVectorImpl<ISD::OutputArg> &Outs,
7882 LLVMContext &Context,
7883 const Type *RetTy) const {
7884 SmallVector<CCValAssign, 16> RVLocs;
7885 CCState CCInfo(CallConv, isVarArg, MF, RVLocs, Context);
7886 return CCInfo.CheckReturn(
7887 Outs, Fn: (Subtarget.isSVR4ABI() && CallConv == CallingConv::Cold)
7888 ? RetCC_PPC_Cold
7889 : RetCC_PPC);
7890}
7891
7892SDValue
7893PPCTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
7894 bool isVarArg,
7895 const SmallVectorImpl<ISD::OutputArg> &Outs,
7896 const SmallVectorImpl<SDValue> &OutVals,
7897 const SDLoc &dl, SelectionDAG &DAG) const {
7898 SmallVector<CCValAssign, 16> RVLocs;
7899 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
7900 *DAG.getContext());
7901 CCInfo.AnalyzeReturn(Outs,
7902 Fn: (Subtarget.isSVR4ABI() && CallConv == CallingConv::Cold)
7903 ? RetCC_PPC_Cold
7904 : RetCC_PPC);
7905
7906 SDValue Glue;
7907 SmallVector<SDValue, 4> RetOps(1, Chain);
7908
7909 // Copy the result values into the output registers.
7910 for (unsigned i = 0, RealResIdx = 0; i != RVLocs.size(); ++i, ++RealResIdx) {
7911 CCValAssign &VA = RVLocs[i];
7912 assert(VA.isRegLoc() && "Can only return in registers!");
7913
7914 SDValue Arg = OutVals[RealResIdx];
7915
7916 switch (VA.getLocInfo()) {
7917 default: llvm_unreachable("Unknown loc info!");
7918 case CCValAssign::Full: break;
7919 case CCValAssign::AExt:
7920 Arg = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: Arg);
7921 break;
7922 case CCValAssign::ZExt:
7923 Arg = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: Arg);
7924 break;
7925 case CCValAssign::SExt:
7926 Arg = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: Arg);
7927 break;
7928 }
7929 if (Subtarget.hasSPE() && VA.getLocVT() == MVT::f64) {
7930 bool isLittleEndian = Subtarget.isLittleEndian();
7931 // Legalize ret f64 -> ret 2 x i32.
7932 SDValue SVal =
7933 DAG.getNode(Opcode: PPCISD::EXTRACT_SPE, DL: dl, VT: MVT::i32, N1: Arg,
7934 N2: DAG.getIntPtrConstant(Val: isLittleEndian ? 0 : 1, DL: dl));
7935 Chain = DAG.getCopyToReg(Chain, dl, Reg: VA.getLocReg(), N: SVal, Glue);
7936 RetOps.push_back(Elt: DAG.getRegister(Reg: VA.getLocReg(), VT: VA.getLocVT()));
7937 SVal = DAG.getNode(Opcode: PPCISD::EXTRACT_SPE, DL: dl, VT: MVT::i32, N1: Arg,
7938 N2: DAG.getIntPtrConstant(Val: isLittleEndian ? 1 : 0, DL: dl));
7939 Glue = Chain.getValue(R: 1);
7940 VA = RVLocs[++i]; // skip ahead to next loc
7941 Chain = DAG.getCopyToReg(Chain, dl, Reg: VA.getLocReg(), N: SVal, Glue);
7942 } else
7943 Chain = DAG.getCopyToReg(Chain, dl, Reg: VA.getLocReg(), N: Arg, Glue);
7944 Glue = Chain.getValue(R: 1);
7945 RetOps.push_back(Elt: DAG.getRegister(Reg: VA.getLocReg(), VT: VA.getLocVT()));
7946 }
7947
7948 RetOps[0] = Chain; // Update chain.
7949
7950 // Add the glue if we have it.
7951 if (Glue.getNode())
7952 RetOps.push_back(Elt: Glue);
7953
7954 return DAG.getNode(Opcode: PPCISD::RET_GLUE, DL: dl, VT: MVT::Other, Ops: RetOps);
7955}
7956
7957SDValue
7958PPCTargetLowering::LowerGET_DYNAMIC_AREA_OFFSET(SDValue Op,
7959 SelectionDAG &DAG) const {
7960 SDLoc dl(Op);
7961
7962 // Get the correct type for integers.
7963 EVT IntVT = Op.getValueType();
7964
7965 // Get the inputs.
7966 SDValue Chain = Op.getOperand(i: 0);
7967 SDValue FPSIdx = getFramePointerFrameIndex(DAG);
7968 // Build a DYNAREAOFFSET node.
7969 SDValue Ops[2] = {Chain, FPSIdx};
7970 SDVTList VTs = DAG.getVTList(VT: IntVT);
7971 return DAG.getNode(Opcode: PPCISD::DYNAREAOFFSET, DL: dl, VTList: VTs, Ops);
7972}
7973
7974SDValue PPCTargetLowering::LowerSTACKRESTORE(SDValue Op,
7975 SelectionDAG &DAG) const {
7976 // When we pop the dynamic allocation we need to restore the SP link.
7977 SDLoc dl(Op);
7978
7979 // Get the correct type for pointers.
7980 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
7981
7982 // Construct the stack pointer operand.
7983 bool isPPC64 = Subtarget.isPPC64();
7984 unsigned SP = isPPC64 ? PPC::X1 : PPC::R1;
7985 SDValue StackPtr = DAG.getRegister(Reg: SP, VT: PtrVT);
7986
7987 // Get the operands for the STACKRESTORE.
7988 SDValue Chain = Op.getOperand(i: 0);
7989 SDValue SaveSP = Op.getOperand(i: 1);
7990
7991 // Load the old link SP.
7992 SDValue LoadLinkSP =
7993 DAG.getLoad(VT: PtrVT, dl, Chain, Ptr: StackPtr, PtrInfo: MachinePointerInfo());
7994
7995 // Restore the stack pointer.
7996 Chain = DAG.getCopyToReg(Chain: LoadLinkSP.getValue(R: 1), dl, Reg: SP, N: SaveSP);
7997
7998 // Store the old link SP.
7999 return DAG.getStore(Chain, dl, Val: LoadLinkSP, Ptr: StackPtr, PtrInfo: MachinePointerInfo());
8000}
8001
8002SDValue PPCTargetLowering::getReturnAddrFrameIndex(SelectionDAG &DAG) const {
8003 MachineFunction &MF = DAG.getMachineFunction();
8004 bool isPPC64 = Subtarget.isPPC64();
8005 EVT PtrVT = getPointerTy(DL: MF.getDataLayout());
8006
8007 // Get current frame pointer save index. The users of this index will be
8008 // primarily DYNALLOC instructions.
8009 PPCFunctionInfo *FI = MF.getInfo<PPCFunctionInfo>();
8010 int RASI = FI->getReturnAddrSaveIndex();
8011
8012 // If the frame pointer save index hasn't been defined yet.
8013 if (!RASI) {
8014 // Find out what the fix offset of the frame pointer save area.
8015 int LROffset = Subtarget.getFrameLowering()->getReturnSaveOffset();
8016 // Allocate the frame index for frame pointer save area.
8017 RASI = MF.getFrameInfo().CreateFixedObject(Size: isPPC64? 8 : 4, SPOffset: LROffset, IsImmutable: false);
8018 // Save the result.
8019 FI->setReturnAddrSaveIndex(RASI);
8020 }
8021 return DAG.getFrameIndex(FI: RASI, VT: PtrVT);
8022}
8023
8024SDValue
8025PPCTargetLowering::getFramePointerFrameIndex(SelectionDAG & DAG) const {
8026 MachineFunction &MF = DAG.getMachineFunction();
8027 bool isPPC64 = Subtarget.isPPC64();
8028 EVT PtrVT = getPointerTy(DL: MF.getDataLayout());
8029
8030 // Get current frame pointer save index. The users of this index will be
8031 // primarily DYNALLOC instructions.
8032 PPCFunctionInfo *FI = MF.getInfo<PPCFunctionInfo>();
8033 int FPSI = FI->getFramePointerSaveIndex();
8034
8035 // If the frame pointer save index hasn't been defined yet.
8036 if (!FPSI) {
8037 // Find out what the fix offset of the frame pointer save area.
8038 int FPOffset = Subtarget.getFrameLowering()->getFramePointerSaveOffset();
8039 // Allocate the frame index for frame pointer save area.
8040 FPSI = MF.getFrameInfo().CreateFixedObject(Size: isPPC64? 8 : 4, SPOffset: FPOffset, IsImmutable: true);
8041 // Save the result.
8042 FI->setFramePointerSaveIndex(FPSI);
8043 }
8044 return DAG.getFrameIndex(FI: FPSI, VT: PtrVT);
8045}
8046
8047SDValue PPCTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
8048 SelectionDAG &DAG) const {
8049 MachineFunction &MF = DAG.getMachineFunction();
8050 // Get the inputs.
8051 SDValue Chain = Op.getOperand(i: 0);
8052 SDValue Size = Op.getOperand(i: 1);
8053 SDLoc dl(Op);
8054
8055 // Get the correct type for pointers.
8056 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
8057 // Negate the size.
8058 SDValue NegSize = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: PtrVT,
8059 N1: DAG.getConstant(Val: 0, DL: dl, VT: PtrVT), N2: Size);
8060 // Construct a node for the frame pointer save index.
8061 SDValue FPSIdx = getFramePointerFrameIndex(DAG);
8062 SDValue Ops[3] = { Chain, NegSize, FPSIdx };
8063 SDVTList VTs = DAG.getVTList(VT1: PtrVT, VT2: MVT::Other);
8064 if (hasInlineStackProbe(MF))
8065 return DAG.getNode(Opcode: PPCISD::PROBED_ALLOCA, DL: dl, VTList: VTs, Ops);
8066 return DAG.getNode(Opcode: PPCISD::DYNALLOC, DL: dl, VTList: VTs, Ops);
8067}
8068
8069SDValue PPCTargetLowering::LowerEH_DWARF_CFA(SDValue Op,
8070 SelectionDAG &DAG) const {
8071 MachineFunction &MF = DAG.getMachineFunction();
8072
8073 bool isPPC64 = Subtarget.isPPC64();
8074 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
8075
8076 int FI = MF.getFrameInfo().CreateFixedObject(Size: isPPC64 ? 8 : 4, SPOffset: 0, IsImmutable: false);
8077 return DAG.getFrameIndex(FI, VT: PtrVT);
8078}
8079
8080SDValue PPCTargetLowering::lowerEH_SJLJ_SETJMP(SDValue Op,
8081 SelectionDAG &DAG) const {
8082 SDLoc DL(Op);
8083 return DAG.getNode(Opcode: PPCISD::EH_SJLJ_SETJMP, DL,
8084 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other),
8085 N1: Op.getOperand(i: 0), N2: Op.getOperand(i: 1));
8086}
8087
8088SDValue PPCTargetLowering::lowerEH_SJLJ_LONGJMP(SDValue Op,
8089 SelectionDAG &DAG) const {
8090 SDLoc DL(Op);
8091 return DAG.getNode(Opcode: PPCISD::EH_SJLJ_LONGJMP, DL, VT: MVT::Other,
8092 N1: Op.getOperand(i: 0), N2: Op.getOperand(i: 1));
8093}
8094
8095SDValue PPCTargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
8096 if (Op.getValueType().isVector())
8097 return LowerVectorLoad(Op, DAG);
8098
8099 assert(Op.getValueType() == MVT::i1 &&
8100 "Custom lowering only for i1 loads");
8101
8102 // First, load 8 bits into 32 bits, then truncate to 1 bit.
8103
8104 SDLoc dl(Op);
8105 LoadSDNode *LD = cast<LoadSDNode>(Val&: Op);
8106
8107 SDValue Chain = LD->getChain();
8108 SDValue BasePtr = LD->getBasePtr();
8109 MachineMemOperand *MMO = LD->getMemOperand();
8110
8111 SDValue NewLD =
8112 DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl, VT: getPointerTy(DL: DAG.getDataLayout()), Chain,
8113 Ptr: BasePtr, MemVT: MVT::i8, MMO);
8114 SDValue Result = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i1, Operand: NewLD);
8115
8116 SDValue Ops[] = { Result, SDValue(NewLD.getNode(), 1) };
8117 return DAG.getMergeValues(Ops, dl);
8118}
8119
8120SDValue PPCTargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
8121 if (Op.getOperand(i: 1).getValueType().isVector())
8122 return LowerVectorStore(Op, DAG);
8123
8124 assert(Op.getOperand(1).getValueType() == MVT::i1 &&
8125 "Custom lowering only for i1 stores");
8126
8127 // First, zero extend to 32 bits, then use a truncating store to 8 bits.
8128
8129 SDLoc dl(Op);
8130 StoreSDNode *ST = cast<StoreSDNode>(Val&: Op);
8131
8132 SDValue Chain = ST->getChain();
8133 SDValue BasePtr = ST->getBasePtr();
8134 SDValue Value = ST->getValue();
8135 MachineMemOperand *MMO = ST->getMemOperand();
8136
8137 Value = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: getPointerTy(DL: DAG.getDataLayout()),
8138 Operand: Value);
8139 return DAG.getTruncStore(Chain, dl, Val: Value, Ptr: BasePtr, SVT: MVT::i8, MMO);
8140}
8141
8142// FIXME: Remove this once the ANDI glue bug is fixed:
8143SDValue PPCTargetLowering::LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const {
8144 assert(Op.getValueType() == MVT::i1 &&
8145 "Custom lowering only for i1 results");
8146
8147 SDLoc DL(Op);
8148 return DAG.getNode(Opcode: PPCISD::ANDI_rec_1_GT_BIT, DL, VT: MVT::i1, Operand: Op.getOperand(i: 0));
8149}
8150
8151SDValue PPCTargetLowering::LowerTRUNCATEVector(SDValue Op,
8152 SelectionDAG &DAG) const {
8153
8154 // Implements a vector truncate that fits in a vector register as a shuffle.
8155 // We want to legalize vector truncates down to where the source fits in
8156 // a vector register (and target is therefore smaller than vector register
8157 // size). At that point legalization will try to custom lower the sub-legal
8158 // result and get here - where we can contain the truncate as a single target
8159 // operation.
8160
8161 // For example a trunc <2 x i16> to <2 x i8> could be visualized as follows:
8162 // <MSB1|LSB1, MSB2|LSB2> to <LSB1, LSB2>
8163 //
8164 // We will implement it for big-endian ordering as this (where x denotes
8165 // undefined):
8166 // < MSB1|LSB1, MSB2|LSB2, uu, uu, uu, uu, uu, uu> to
8167 // < LSB1, LSB2, u, u, u, u, u, u, u, u, u, u, u, u, u, u>
8168 //
8169 // The same operation in little-endian ordering will be:
8170 // <uu, uu, uu, uu, uu, uu, LSB2|MSB2, LSB1|MSB1> to
8171 // <u, u, u, u, u, u, u, u, u, u, u, u, u, u, LSB2, LSB1>
8172
8173 EVT TrgVT = Op.getValueType();
8174 assert(TrgVT.isVector() && "Vector type expected.");
8175 unsigned TrgNumElts = TrgVT.getVectorNumElements();
8176 EVT EltVT = TrgVT.getVectorElementType();
8177 if (!isOperationCustom(Op: Op.getOpcode(), VT: TrgVT) ||
8178 TrgVT.getSizeInBits() > 128 || !isPowerOf2_32(Value: TrgNumElts) ||
8179 !llvm::has_single_bit<uint32_t>(Value: EltVT.getSizeInBits()))
8180 return SDValue();
8181
8182 SDValue N1 = Op.getOperand(i: 0);
8183 EVT SrcVT = N1.getValueType();
8184 unsigned SrcSize = SrcVT.getSizeInBits();
8185 if (SrcSize > 256 || !isPowerOf2_32(Value: SrcVT.getVectorNumElements()) ||
8186 !llvm::has_single_bit<uint32_t>(
8187 Value: SrcVT.getVectorElementType().getSizeInBits()))
8188 return SDValue();
8189 if (SrcSize == 256 && SrcVT.getVectorNumElements() < 2)
8190 return SDValue();
8191
8192 unsigned WideNumElts = 128 / EltVT.getSizeInBits();
8193 EVT WideVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, NumElements: WideNumElts);
8194
8195 SDLoc DL(Op);
8196 SDValue Op1, Op2;
8197 if (SrcSize == 256) {
8198 EVT VecIdxTy = getVectorIdxTy(DL: DAG.getDataLayout());
8199 EVT SplitVT =
8200 N1.getValueType().getHalfNumVectorElementsVT(Context&: *DAG.getContext());
8201 unsigned SplitNumElts = SplitVT.getVectorNumElements();
8202 Op1 = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: SplitVT, N1,
8203 N2: DAG.getConstant(Val: 0, DL, VT: VecIdxTy));
8204 Op2 = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: SplitVT, N1,
8205 N2: DAG.getConstant(Val: SplitNumElts, DL, VT: VecIdxTy));
8206 }
8207 else {
8208 Op1 = SrcSize == 128 ? N1 : widenVec(DAG, Vec: N1, dl: DL);
8209 Op2 = DAG.getUNDEF(VT: WideVT);
8210 }
8211
8212 // First list the elements we want to keep.
8213 unsigned SizeMult = SrcSize / TrgVT.getSizeInBits();
8214 SmallVector<int, 16> ShuffV;
8215 if (Subtarget.isLittleEndian())
8216 for (unsigned i = 0; i < TrgNumElts; ++i)
8217 ShuffV.push_back(Elt: i * SizeMult);
8218 else
8219 for (unsigned i = 1; i <= TrgNumElts; ++i)
8220 ShuffV.push_back(Elt: i * SizeMult - 1);
8221
8222 // Populate the remaining elements with undefs.
8223 for (unsigned i = TrgNumElts; i < WideNumElts; ++i)
8224 // ShuffV.push_back(i + WideNumElts);
8225 ShuffV.push_back(Elt: WideNumElts + 1);
8226
8227 Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: WideVT, Operand: Op1);
8228 Op2 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: WideVT, Operand: Op2);
8229 return DAG.getVectorShuffle(VT: WideVT, dl: DL, N1: Op1, N2: Op2, Mask: ShuffV);
8230}
8231
8232/// LowerSELECT_CC - Lower floating point select_cc's into fsel instruction when
8233/// possible.
8234SDValue PPCTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
8235 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 4))->get();
8236 EVT ResVT = Op.getValueType();
8237 EVT CmpVT = Op.getOperand(i: 0).getValueType();
8238 SDValue LHS = Op.getOperand(i: 0), RHS = Op.getOperand(i: 1);
8239 SDValue TV = Op.getOperand(i: 2), FV = Op.getOperand(i: 3);
8240 SDLoc dl(Op);
8241
8242 // Without power9-vector, we don't have native instruction for f128 comparison.
8243 // Following transformation to libcall is needed for setcc:
8244 // select_cc lhs, rhs, tv, fv, cc -> select_cc (setcc cc, x, y), 0, tv, fv, NE
8245 if (!Subtarget.hasP9Vector() && CmpVT == MVT::f128) {
8246 SDValue Z = DAG.getSetCC(
8247 DL: dl, VT: getSetCCResultType(DL: DAG.getDataLayout(), C&: *DAG.getContext(), VT: CmpVT),
8248 LHS, RHS, Cond: CC);
8249 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: Z.getValueType());
8250 return DAG.getSelectCC(DL: dl, LHS: Z, RHS: Zero, True: TV, False: FV, Cond: ISD::SETNE);
8251 }
8252
8253 // Not FP, or using SPE? Not a fsel.
8254 if (!CmpVT.isFloatingPoint() || !TV.getValueType().isFloatingPoint() ||
8255 Subtarget.hasSPE())
8256 return Op;
8257
8258 SDNodeFlags Flags = Op.getNode()->getFlags();
8259
8260 // We have xsmaxc[dq]p/xsminc[dq]p which are OK to emit even in the
8261 // presence of infinities.
8262 if (Subtarget.hasP9Vector() && LHS == TV && RHS == FV) {
8263 switch (CC) {
8264 default:
8265 break;
8266 case ISD::SETOGT:
8267 case ISD::SETGT:
8268 return DAG.getNode(Opcode: PPCISD::XSMAXC, DL: dl, VT: Op.getValueType(), N1: LHS, N2: RHS);
8269 case ISD::SETOLT:
8270 case ISD::SETLT:
8271 return DAG.getNode(Opcode: PPCISD::XSMINC, DL: dl, VT: Op.getValueType(), N1: LHS, N2: RHS);
8272 }
8273 }
8274
8275 // We might be able to do better than this under some circumstances, but in
8276 // general, fsel-based lowering of select is a finite-math-only optimization.
8277 // For more information, see section F.3 of the 2.06 ISA specification.
8278 // With ISA 3.0
8279 if (!Flags.hasNoInfs() || !Flags.hasNoNaNs() || ResVT == MVT::f128)
8280 return Op;
8281
8282 // If the RHS of the comparison is a 0.0, we don't need to do the
8283 // subtraction at all.
8284 SDValue Sel1;
8285 if (isFloatingPointZero(Op: RHS))
8286 switch (CC) {
8287 default: break; // SETUO etc aren't handled by fsel.
8288 case ISD::SETNE:
8289 std::swap(a&: TV, b&: FV);
8290 [[fallthrough]];
8291 case ISD::SETEQ:
8292 if (LHS.getValueType() == MVT::f32) // Comparison is always 64-bits
8293 LHS = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: LHS);
8294 Sel1 = DAG.getNode(Opcode: PPCISD::FSEL, DL: dl, VT: ResVT, N1: LHS, N2: TV, N3: FV);
8295 if (Sel1.getValueType() == MVT::f32) // Comparison is always 64-bits
8296 Sel1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: Sel1);
8297 return DAG.getNode(Opcode: PPCISD::FSEL, DL: dl, VT: ResVT,
8298 N1: DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT: MVT::f64, Operand: LHS), N2: Sel1, N3: FV);
8299 case ISD::SETULT:
8300 case ISD::SETLT:
8301 std::swap(a&: TV, b&: FV); // fsel is natively setge, swap operands for setlt
8302 [[fallthrough]];
8303 case ISD::SETOGE:
8304 case ISD::SETGE:
8305 if (LHS.getValueType() == MVT::f32) // Comparison is always 64-bits
8306 LHS = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: LHS);
8307 return DAG.getNode(Opcode: PPCISD::FSEL, DL: dl, VT: ResVT, N1: LHS, N2: TV, N3: FV);
8308 case ISD::SETUGT:
8309 case ISD::SETGT:
8310 std::swap(a&: TV, b&: FV); // fsel is natively setge, swap operands for setlt
8311 [[fallthrough]];
8312 case ISD::SETOLE:
8313 case ISD::SETLE:
8314 if (LHS.getValueType() == MVT::f32) // Comparison is always 64-bits
8315 LHS = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: LHS);
8316 return DAG.getNode(Opcode: PPCISD::FSEL, DL: dl, VT: ResVT,
8317 N1: DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT: MVT::f64, Operand: LHS), N2: TV, N3: FV);
8318 }
8319
8320 SDValue Cmp;
8321 switch (CC) {
8322 default: break; // SETUO etc aren't handled by fsel.
8323 case ISD::SETNE:
8324 std::swap(a&: TV, b&: FV);
8325 [[fallthrough]];
8326 case ISD::SETEQ:
8327 Cmp = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: CmpVT, N1: LHS, N2: RHS, Flags);
8328 if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits
8329 Cmp = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: Cmp);
8330 Sel1 = DAG.getNode(Opcode: PPCISD::FSEL, DL: dl, VT: ResVT, N1: Cmp, N2: TV, N3: FV);
8331 if (Sel1.getValueType() == MVT::f32) // Comparison is always 64-bits
8332 Sel1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: Sel1);
8333 return DAG.getNode(Opcode: PPCISD::FSEL, DL: dl, VT: ResVT,
8334 N1: DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT: MVT::f64, Operand: Cmp), N2: Sel1, N3: FV);
8335 case ISD::SETULT:
8336 case ISD::SETLT:
8337 Cmp = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: CmpVT, N1: LHS, N2: RHS, Flags);
8338 if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits
8339 Cmp = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: Cmp);
8340 return DAG.getNode(Opcode: PPCISD::FSEL, DL: dl, VT: ResVT, N1: Cmp, N2: FV, N3: TV);
8341 case ISD::SETOGE:
8342 case ISD::SETGE:
8343 Cmp = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: CmpVT, N1: LHS, N2: RHS, Flags);
8344 if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits
8345 Cmp = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: Cmp);
8346 return DAG.getNode(Opcode: PPCISD::FSEL, DL: dl, VT: ResVT, N1: Cmp, N2: TV, N3: FV);
8347 case ISD::SETUGT:
8348 case ISD::SETGT:
8349 Cmp = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: CmpVT, N1: RHS, N2: LHS, Flags);
8350 if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits
8351 Cmp = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: Cmp);
8352 return DAG.getNode(Opcode: PPCISD::FSEL, DL: dl, VT: ResVT, N1: Cmp, N2: FV, N3: TV);
8353 case ISD::SETOLE:
8354 case ISD::SETLE:
8355 Cmp = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: CmpVT, N1: RHS, N2: LHS, Flags);
8356 if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits
8357 Cmp = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: Cmp);
8358 return DAG.getNode(Opcode: PPCISD::FSEL, DL: dl, VT: ResVT, N1: Cmp, N2: TV, N3: FV);
8359 }
8360 return Op;
8361}
8362
8363static unsigned getPPCStrictOpcode(unsigned Opc) {
8364 switch (Opc) {
8365 default:
8366 llvm_unreachable("No strict version of this opcode!");
8367 case PPCISD::FCTIDZ:
8368 return PPCISD::STRICT_FCTIDZ;
8369 case PPCISD::FCTIWZ:
8370 return PPCISD::STRICT_FCTIWZ;
8371 case PPCISD::FCTIDUZ:
8372 return PPCISD::STRICT_FCTIDUZ;
8373 case PPCISD::FCTIWUZ:
8374 return PPCISD::STRICT_FCTIWUZ;
8375 case PPCISD::FCFID:
8376 return PPCISD::STRICT_FCFID;
8377 case PPCISD::FCFIDU:
8378 return PPCISD::STRICT_FCFIDU;
8379 case PPCISD::FCFIDS:
8380 return PPCISD::STRICT_FCFIDS;
8381 case PPCISD::FCFIDUS:
8382 return PPCISD::STRICT_FCFIDUS;
8383 }
8384}
8385
8386static SDValue convertFPToInt(SDValue Op, SelectionDAG &DAG,
8387 const PPCSubtarget &Subtarget) {
8388 SDLoc dl(Op);
8389 bool IsStrict = Op->isStrictFPOpcode();
8390 bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT ||
8391 Op.getOpcode() == ISD::STRICT_FP_TO_SINT;
8392
8393 // TODO: Any other flags to propagate?
8394 SDNodeFlags Flags;
8395 Flags.setNoFPExcept(Op->getFlags().hasNoFPExcept());
8396
8397 // For strict nodes, source is the second operand.
8398 SDValue Src = Op.getOperand(i: IsStrict ? 1 : 0);
8399 SDValue Chain = IsStrict ? Op.getOperand(i: 0) : SDValue();
8400 MVT DestTy = Op.getSimpleValueType();
8401 assert(Src.getValueType().isFloatingPoint() &&
8402 (DestTy == MVT::i8 || DestTy == MVT::i16 || DestTy == MVT::i32 ||
8403 DestTy == MVT::i64) &&
8404 "Invalid FP_TO_INT types");
8405 if (Src.getValueType() == MVT::f32) {
8406 if (IsStrict) {
8407 Src =
8408 DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl,
8409 VTList: DAG.getVTList(VT1: MVT::f64, VT2: MVT::Other), Ops: {Chain, Src}, Flags);
8410 Chain = Src.getValue(R: 1);
8411 } else
8412 Src = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: Src);
8413 }
8414 if ((DestTy == MVT::i8 || DestTy == MVT::i16) && Subtarget.hasP9Vector())
8415 DestTy = Subtarget.getScalarIntVT();
8416 unsigned Opc = ISD::DELETED_NODE;
8417 switch (DestTy.SimpleTy) {
8418 default: llvm_unreachable("Unhandled FP_TO_INT type in custom expander!");
8419 case MVT::i32:
8420 Opc = IsSigned ? PPCISD::FCTIWZ
8421 : (Subtarget.hasFPCVT() ? PPCISD::FCTIWUZ : PPCISD::FCTIDZ);
8422 break;
8423 case MVT::i64:
8424 assert((IsSigned || Subtarget.hasFPCVT()) &&
8425 "i64 FP_TO_UINT is supported only with FPCVT");
8426 Opc = IsSigned ? PPCISD::FCTIDZ : PPCISD::FCTIDUZ;
8427 }
8428 EVT ConvTy = Src.getValueType() == MVT::f128 ? MVT::f128 : MVT::f64;
8429 SDValue Conv;
8430 if (IsStrict) {
8431 Opc = getPPCStrictOpcode(Opc);
8432 Conv = DAG.getNode(Opcode: Opc, DL: dl, VTList: DAG.getVTList(VT1: ConvTy, VT2: MVT::Other), Ops: {Chain, Src},
8433 Flags);
8434 } else {
8435 Conv = DAG.getNode(Opcode: Opc, DL: dl, VT: ConvTy, Operand: Src);
8436 }
8437 return Conv;
8438}
8439
8440void PPCTargetLowering::LowerFP_TO_INTForReuse(SDValue Op, ReuseLoadInfo &RLI,
8441 SelectionDAG &DAG,
8442 const SDLoc &dl) const {
8443 SDValue Tmp = convertFPToInt(Op, DAG, Subtarget);
8444 bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT ||
8445 Op.getOpcode() == ISD::STRICT_FP_TO_SINT;
8446 bool IsStrict = Op->isStrictFPOpcode();
8447
8448 // Convert the FP value to an int value through memory.
8449 bool i32Stack = Op.getValueType() == MVT::i32 && Subtarget.hasSTFIWX() &&
8450 (IsSigned || Subtarget.hasFPCVT());
8451 SDValue FIPtr = DAG.CreateStackTemporary(VT: i32Stack ? MVT::i32 : MVT::f64);
8452 int FI = cast<FrameIndexSDNode>(Val&: FIPtr)->getIndex();
8453 MachinePointerInfo MPI =
8454 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI);
8455
8456 // Emit a store to the stack slot.
8457 SDValue Chain = IsStrict ? Tmp.getValue(R: 1) : DAG.getEntryNode();
8458 Align Alignment(DAG.getEVTAlign(MemoryVT: Tmp.getValueType()));
8459 if (i32Stack) {
8460 MachineFunction &MF = DAG.getMachineFunction();
8461 Alignment = Align(4);
8462 MachineMemOperand *MMO =
8463 MF.getMachineMemOperand(PtrInfo: MPI, F: MachineMemOperand::MOStore, Size: 4, BaseAlignment: Alignment);
8464 SDValue Ops[] = { Chain, Tmp, FIPtr };
8465 Chain = DAG.getMemIntrinsicNode(Opcode: PPCISD::STFIWX, dl,
8466 VTList: DAG.getVTList(VT: MVT::Other), Ops, MemVT: MVT::i32, MMO);
8467 } else
8468 Chain = DAG.getStore(Chain, dl, Val: Tmp, Ptr: FIPtr, PtrInfo: MPI, Alignment);
8469
8470 // Result is a load from the stack slot. If loading 4 bytes, make sure to
8471 // add in a bias on big endian.
8472 if (Op.getValueType() == MVT::i32 && !i32Stack &&
8473 !Subtarget.isLittleEndian()) {
8474 FIPtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: FIPtr.getValueType(), N1: FIPtr,
8475 N2: DAG.getConstant(Val: 4, DL: dl, VT: FIPtr.getValueType()));
8476 MPI = MPI.getWithOffset(O: 4);
8477 }
8478
8479 RLI.Chain = Chain;
8480 RLI.Ptr = FIPtr;
8481 RLI.MPI = MPI;
8482 RLI.Alignment = Alignment;
8483}
8484
8485/// Custom lowers floating point to integer conversions to use
8486/// the direct move instructions available in ISA 2.07 to avoid the
8487/// need for load/store combinations.
8488SDValue PPCTargetLowering::LowerFP_TO_INTDirectMove(SDValue Op,
8489 SelectionDAG &DAG,
8490 const SDLoc &dl) const {
8491 SDValue Conv = convertFPToInt(Op, DAG, Subtarget);
8492 SDValue Mov = DAG.getNode(Opcode: PPCISD::MFVSR, DL: dl, VT: Op.getValueType(), Operand: Conv);
8493 if (Op->isStrictFPOpcode())
8494 return DAG.getMergeValues(Ops: {Mov, Conv.getValue(R: 1)}, dl);
8495 else
8496 return Mov;
8497}
8498
8499SDValue PPCTargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG,
8500 const SDLoc &dl) const {
8501 bool IsStrict = Op->isStrictFPOpcode();
8502 bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT ||
8503 Op.getOpcode() == ISD::STRICT_FP_TO_SINT;
8504 SDValue Src = Op.getOperand(i: IsStrict ? 1 : 0);
8505 EVT SrcVT = Src.getValueType();
8506 EVT DstVT = Op.getValueType();
8507
8508 // FP to INT conversions are legal for f128.
8509 if (SrcVT == MVT::f128)
8510 return Subtarget.hasP9Vector() ? Op : SDValue();
8511
8512 // Expand ppcf128 to i32 by hand for the benefit of llvm-gcc bootstrap on
8513 // PPC (the libcall is not available).
8514 if (SrcVT == MVT::ppcf128) {
8515 if (DstVT == MVT::i32) {
8516 // TODO: Conservatively pass only nofpexcept flag here. Need to check and
8517 // set other fast-math flags to FP operations in both strict and
8518 // non-strict cases. (FP_TO_SINT, FSUB)
8519 SDNodeFlags Flags;
8520 Flags.setNoFPExcept(Op->getFlags().hasNoFPExcept());
8521
8522 if (IsSigned) {
8523 SDValue Lo, Hi;
8524 std::tie(args&: Lo, args&: Hi) = DAG.SplitScalar(N: Src, DL: dl, LoVT: MVT::f64, HiVT: MVT::f64);
8525
8526 // Add the two halves of the long double in round-to-zero mode, and use
8527 // a smaller FP_TO_SINT.
8528 if (IsStrict) {
8529 SDValue Res = DAG.getNode(Opcode: PPCISD::STRICT_FADDRTZ, DL: dl,
8530 VTList: DAG.getVTList(VT1: MVT::f64, VT2: MVT::Other),
8531 Ops: {Op.getOperand(i: 0), Lo, Hi}, Flags);
8532 return DAG.getNode(Opcode: ISD::STRICT_FP_TO_SINT, DL: dl,
8533 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other),
8534 Ops: {Res.getValue(R: 1), Res}, Flags);
8535 } else {
8536 SDValue Res = DAG.getNode(Opcode: PPCISD::FADDRTZ, DL: dl, VT: MVT::f64, N1: Lo, N2: Hi);
8537 return DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: dl, VT: MVT::i32, Operand: Res);
8538 }
8539 } else {
8540 const uint64_t TwoE31[] = {0x41e0000000000000LL, 0};
8541 APFloat APF = APFloat(APFloat::PPCDoubleDouble(), APInt(128, TwoE31));
8542 SDValue Cst = DAG.getConstantFP(Val: APF, DL: dl, VT: SrcVT);
8543 SDValue SignMask = DAG.getConstant(Val: 0x80000000, DL: dl, VT: DstVT);
8544 if (IsStrict) {
8545 // Sel = Src < 0x80000000
8546 // FltOfs = select Sel, 0.0, 0x80000000
8547 // IntOfs = select Sel, 0, 0x80000000
8548 // Result = fp_to_sint(Src - FltOfs) ^ IntOfs
8549 SDValue Chain = Op.getOperand(i: 0);
8550 EVT SetCCVT =
8551 getSetCCResultType(DL: DAG.getDataLayout(), C&: *DAG.getContext(), VT: SrcVT);
8552 EVT DstSetCCVT =
8553 getSetCCResultType(DL: DAG.getDataLayout(), C&: *DAG.getContext(), VT: DstVT);
8554 SDValue Sel = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Src, RHS: Cst, Cond: ISD::SETLT,
8555 Chain, IsSignaling: true);
8556 Chain = Sel.getValue(R: 1);
8557
8558 SDValue FltOfs = DAG.getSelect(
8559 DL: dl, VT: SrcVT, Cond: Sel, LHS: DAG.getConstantFP(Val: 0.0, DL: dl, VT: SrcVT), RHS: Cst);
8560 Sel = DAG.getBoolExtOrTrunc(Op: Sel, SL: dl, VT: DstSetCCVT, OpVT: DstVT);
8561
8562 SDValue Val = DAG.getNode(Opcode: ISD::STRICT_FSUB, DL: dl,
8563 VTList: DAG.getVTList(VT1: SrcVT, VT2: MVT::Other),
8564 Ops: {Chain, Src, FltOfs}, Flags);
8565 Chain = Val.getValue(R: 1);
8566 SDValue SInt = DAG.getNode(Opcode: ISD::STRICT_FP_TO_SINT, DL: dl,
8567 VTList: DAG.getVTList(VT1: DstVT, VT2: MVT::Other),
8568 Ops: {Chain, Val}, Flags);
8569 Chain = SInt.getValue(R: 1);
8570 SDValue IntOfs = DAG.getSelect(
8571 DL: dl, VT: DstVT, Cond: Sel, LHS: DAG.getConstant(Val: 0, DL: dl, VT: DstVT), RHS: SignMask);
8572 SDValue Result = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: DstVT, N1: SInt, N2: IntOfs);
8573 return DAG.getMergeValues(Ops: {Result, Chain}, dl);
8574 } else {
8575 // X>=2^31 ? (int)(X-2^31)+0x80000000 : (int)X
8576 // FIXME: generated code sucks.
8577 SDValue True = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::ppcf128, N1: Src, N2: Cst);
8578 True = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: dl, VT: MVT::i32, Operand: True);
8579 True = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::i32, N1: True, N2: SignMask);
8580 SDValue False = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: dl, VT: MVT::i32, Operand: Src);
8581 return DAG.getSelectCC(DL: dl, LHS: Src, RHS: Cst, True, False, Cond: ISD::SETGE);
8582 }
8583 }
8584 }
8585
8586 return SDValue();
8587 }
8588
8589 if (Subtarget.hasDirectMove() && Subtarget.isPPC64())
8590 return LowerFP_TO_INTDirectMove(Op, DAG, dl);
8591
8592 ReuseLoadInfo RLI;
8593 LowerFP_TO_INTForReuse(Op, RLI, DAG, dl);
8594
8595 return DAG.getLoad(VT: Op.getValueType(), dl, Chain: RLI.Chain, Ptr: RLI.Ptr, PtrInfo: RLI.MPI,
8596 Alignment: RLI.Alignment, MMOFlags: RLI.MMOFlags(),
8597 Metadata: MMOMetadata(RLI.AAInfo, RLI.Ranges));
8598}
8599
8600// We're trying to insert a regular store, S, and then a load, L. If the
8601// incoming value, O, is a load, we might just be able to have our load use the
8602// address used by O. However, we don't know if anything else will store to
8603// that address before we can load from it. To prevent this situation, we need
8604// to insert our load, L, into the chain as a peer of O. To do this, we give L
8605// the same chain operand as O, we create a token factor from the chain results
8606// of O and L, and we replace all uses of O's chain result with that token
8607// factor (this last part is handled by makeEquivalentMemoryOrdering).
8608bool PPCTargetLowering::canReuseLoadAddress(SDValue Op, EVT MemVT,
8609 ReuseLoadInfo &RLI,
8610 SelectionDAG &DAG,
8611 ISD::LoadExtType ET) const {
8612 // Conservatively skip reusing for constrained FP nodes.
8613 if (Op->isStrictFPOpcode())
8614 return false;
8615
8616 SDLoc dl(Op);
8617 bool ValidFPToUint = Op.getOpcode() == ISD::FP_TO_UINT &&
8618 (Subtarget.hasFPCVT() || Op.getValueType() == MVT::i32);
8619 if (ET == ISD::NON_EXTLOAD &&
8620 (ValidFPToUint || Op.getOpcode() == ISD::FP_TO_SINT) &&
8621 isOperationLegalOrCustom(Op: Op.getOpcode(),
8622 VT: Op.getOperand(i: 0).getValueType())) {
8623
8624 LowerFP_TO_INTForReuse(Op, RLI, DAG, dl);
8625 return true;
8626 }
8627
8628 LoadSDNode *LD = dyn_cast<LoadSDNode>(Val&: Op);
8629 if (!LD || LD->getExtensionType() != ET || LD->isVolatile() ||
8630 LD->isNonTemporal())
8631 return false;
8632 if (LD->getMemoryVT() != MemVT)
8633 return false;
8634
8635 // If the result of the load is an illegal type, then we can't build a
8636 // valid chain for reuse since the legalised loads and token factor node that
8637 // ties the legalised loads together uses a different output chain then the
8638 // illegal load.
8639 if (!isTypeLegal(VT: LD->getValueType(ResNo: 0)))
8640 return false;
8641
8642 RLI.Ptr = LD->getBasePtr();
8643 if (LD->isIndexed() && !LD->getOffset().isUndef()) {
8644 assert(LD->getAddressingMode() == ISD::PRE_INC &&
8645 "Non-pre-inc AM on PPC?");
8646 RLI.Ptr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: RLI.Ptr.getValueType(), N1: RLI.Ptr,
8647 N2: LD->getOffset());
8648 }
8649
8650 RLI.Chain = LD->getChain();
8651 RLI.MPI = LD->getPointerInfo();
8652 RLI.IsDereferenceable = LD->isDereferenceable();
8653 RLI.IsInvariant = LD->isInvariant();
8654 RLI.Alignment = LD->getAlign();
8655 RLI.AAInfo = LD->getAAInfo();
8656 RLI.Ranges = LD->getRanges();
8657
8658 RLI.ResChain = SDValue(LD, LD->isIndexed() ? 2 : 1);
8659 return true;
8660}
8661
8662/// Analyze profitability of direct move
8663/// prefer float load to int load plus direct move
8664/// when there is no integer use of int load
8665bool PPCTargetLowering::directMoveIsProfitable(const SDValue &Op) const {
8666 SDNode *Origin = Op.getOperand(i: Op->isStrictFPOpcode() ? 1 : 0).getNode();
8667 if (Origin->getOpcode() != ISD::LOAD)
8668 return true;
8669
8670 // If there is no LXSIBZX/LXSIHZX, like Power8,
8671 // prefer direct move if the memory size is 1 or 2 bytes.
8672 MachineMemOperand *MMO = cast<LoadSDNode>(Val: Origin)->getMemOperand();
8673 if (!Subtarget.hasP9Vector() &&
8674 (!MMO->getSize().hasValue() || MMO->getSize().getValue() <= 2))
8675 return true;
8676
8677 for (SDUse &Use : Origin->uses()) {
8678
8679 // Only look at the users of the loaded value.
8680 if (Use.getResNo() != 0)
8681 continue;
8682
8683 SDNode *User = Use.getUser();
8684 if (User->getOpcode() != ISD::SINT_TO_FP &&
8685 User->getOpcode() != ISD::UINT_TO_FP &&
8686 User->getOpcode() != ISD::STRICT_SINT_TO_FP &&
8687 User->getOpcode() != ISD::STRICT_UINT_TO_FP)
8688 return true;
8689 }
8690
8691 return false;
8692}
8693
8694static SDValue convertIntToFP(SDValue Op, SDValue Src, SelectionDAG &DAG,
8695 const PPCSubtarget &Subtarget,
8696 SDValue Chain = SDValue()) {
8697 bool IsSigned = Op.getOpcode() == ISD::SINT_TO_FP ||
8698 Op.getOpcode() == ISD::STRICT_SINT_TO_FP;
8699 SDLoc dl(Op);
8700
8701 // TODO: Any other flags to propagate?
8702 SDNodeFlags Flags;
8703 Flags.setNoFPExcept(Op->getFlags().hasNoFPExcept());
8704
8705 // If we have FCFIDS, then use it when converting to single-precision.
8706 // Otherwise, convert to double-precision and then round.
8707 bool IsSingle = Op.getValueType() == MVT::f32 && Subtarget.hasFPCVT();
8708 unsigned ConvOpc = IsSingle ? (IsSigned ? PPCISD::FCFIDS : PPCISD::FCFIDUS)
8709 : (IsSigned ? PPCISD::FCFID : PPCISD::FCFIDU);
8710 EVT ConvTy = IsSingle ? MVT::f32 : MVT::f64;
8711 if (Op->isStrictFPOpcode()) {
8712 if (!Chain)
8713 Chain = Op.getOperand(i: 0);
8714 return DAG.getNode(Opcode: getPPCStrictOpcode(Opc: ConvOpc), DL: dl,
8715 VTList: DAG.getVTList(VT1: ConvTy, VT2: MVT::Other), Ops: {Chain, Src}, Flags);
8716 } else
8717 return DAG.getNode(Opcode: ConvOpc, DL: dl, VT: ConvTy, Operand: Src);
8718}
8719
8720/// Custom lowers integer to floating point conversions to use
8721/// the direct move instructions available in ISA 2.07 to avoid the
8722/// need for load/store combinations.
8723SDValue PPCTargetLowering::LowerINT_TO_FPDirectMove(SDValue Op,
8724 SelectionDAG &DAG,
8725 const SDLoc &dl) const {
8726 assert((Op.getValueType() == MVT::f32 ||
8727 Op.getValueType() == MVT::f64) &&
8728 "Invalid floating point type as target of conversion");
8729 assert(Subtarget.hasFPCVT() &&
8730 "Int to FP conversions with direct moves require FPCVT");
8731 SDValue Src = Op.getOperand(i: Op->isStrictFPOpcode() ? 1 : 0);
8732 bool WordInt = Src.getSimpleValueType().SimpleTy == MVT::i32;
8733 bool Signed = Op.getOpcode() == ISD::SINT_TO_FP ||
8734 Op.getOpcode() == ISD::STRICT_SINT_TO_FP;
8735 unsigned MovOpc = (WordInt && !Signed) ? PPCISD::MTVSRZ : PPCISD::MTVSRA;
8736 SDValue Mov = DAG.getNode(Opcode: MovOpc, DL: dl, VT: MVT::f64, Operand: Src);
8737 return convertIntToFP(Op, Src: Mov, DAG, Subtarget);
8738}
8739
8740static SDValue widenVec(SelectionDAG &DAG, SDValue Vec, const SDLoc &dl) {
8741
8742 EVT VecVT = Vec.getValueType();
8743 assert(VecVT.isVector() && "Expected a vector type.");
8744 assert(VecVT.getSizeInBits() < 128 && "Vector is already full width.");
8745
8746 EVT EltVT = VecVT.getVectorElementType();
8747 unsigned WideNumElts = 128 / EltVT.getSizeInBits();
8748 EVT WideVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, NumElements: WideNumElts);
8749
8750 unsigned NumConcat = WideNumElts / VecVT.getVectorNumElements();
8751 SmallVector<SDValue, 16> Ops(NumConcat);
8752 Ops[0] = Vec;
8753 SDValue UndefVec = DAG.getUNDEF(VT: VecVT);
8754 for (unsigned i = 1; i < NumConcat; ++i)
8755 Ops[i] = UndefVec;
8756
8757 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WideVT, Ops);
8758}
8759
8760SDValue PPCTargetLowering::LowerINT_TO_FPVector(SDValue Op, SelectionDAG &DAG,
8761 const SDLoc &dl) const {
8762 bool IsStrict = Op->isStrictFPOpcode();
8763 unsigned Opc = Op.getOpcode();
8764 SDValue Src = Op.getOperand(i: IsStrict ? 1 : 0);
8765 assert((Opc == ISD::UINT_TO_FP || Opc == ISD::SINT_TO_FP ||
8766 Opc == ISD::STRICT_UINT_TO_FP || Opc == ISD::STRICT_SINT_TO_FP) &&
8767 "Unexpected conversion type");
8768 assert((Op.getValueType() == MVT::v2f64 || Op.getValueType() == MVT::v4f32) &&
8769 "Supports conversions to v2f64/v4f32 only.");
8770
8771 // TODO: Any other flags to propagate?
8772 SDNodeFlags Flags;
8773 Flags.setNoFPExcept(Op->getFlags().hasNoFPExcept());
8774
8775 bool SignedConv = Opc == ISD::SINT_TO_FP || Opc == ISD::STRICT_SINT_TO_FP;
8776 bool FourEltRes = Op.getValueType() == MVT::v4f32;
8777
8778 SDValue Wide = widenVec(DAG, Vec: Src, dl);
8779 EVT WideVT = Wide.getValueType();
8780 unsigned WideNumElts = WideVT.getVectorNumElements();
8781 MVT IntermediateVT = FourEltRes ? MVT::v4i32 : MVT::v2i64;
8782
8783 SmallVector<int, 16> ShuffV;
8784 for (unsigned i = 0; i < WideNumElts; ++i)
8785 ShuffV.push_back(Elt: i + WideNumElts);
8786
8787 int Stride = FourEltRes ? WideNumElts / 4 : WideNumElts / 2;
8788 int SaveElts = FourEltRes ? 4 : 2;
8789 if (Subtarget.isLittleEndian())
8790 for (int i = 0; i < SaveElts; i++)
8791 ShuffV[i * Stride] = i;
8792 else
8793 for (int i = 1; i <= SaveElts; i++)
8794 ShuffV[i * Stride - 1] = i - 1;
8795
8796 SDValue ShuffleSrc2 =
8797 SignedConv ? DAG.getUNDEF(VT: WideVT) : DAG.getConstant(Val: 0, DL: dl, VT: WideVT);
8798 SDValue Arrange = DAG.getVectorShuffle(VT: WideVT, dl, N1: Wide, N2: ShuffleSrc2, Mask: ShuffV);
8799
8800 SDValue Extend;
8801 if (SignedConv) {
8802 Arrange = DAG.getBitcast(VT: IntermediateVT, V: Arrange);
8803 EVT ExtVT = Src.getValueType();
8804 if (Subtarget.hasP9Altivec())
8805 ExtVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WideVT.getVectorElementType(),
8806 NumElements: IntermediateVT.getVectorNumElements());
8807
8808 Extend = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl, VT: IntermediateVT, N1: Arrange,
8809 N2: DAG.getValueType(ExtVT));
8810 } else
8811 Extend = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: IntermediateVT, Operand: Arrange);
8812
8813 if (IsStrict)
8814 return DAG.getNode(Opcode: Opc, DL: dl, VTList: DAG.getVTList(VT1: Op.getValueType(), VT2: MVT::Other),
8815 Ops: {Op.getOperand(i: 0), Extend}, Flags);
8816
8817 return DAG.getNode(Opcode: Opc, DL: dl, VT: Op.getValueType(), Operand: Extend);
8818}
8819
8820SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op,
8821 SelectionDAG &DAG) const {
8822 SDLoc dl(Op);
8823 bool IsSigned = Op.getOpcode() == ISD::SINT_TO_FP ||
8824 Op.getOpcode() == ISD::STRICT_SINT_TO_FP;
8825 bool IsStrict = Op->isStrictFPOpcode();
8826 SDValue Src = Op.getOperand(i: IsStrict ? 1 : 0);
8827 SDValue Chain = IsStrict ? Op.getOperand(i: 0) : DAG.getEntryNode();
8828
8829 // TODO: Any other flags to propagate?
8830 SDNodeFlags Flags;
8831 Flags.setNoFPExcept(Op->getFlags().hasNoFPExcept());
8832
8833 EVT InVT = Src.getValueType();
8834 EVT OutVT = Op.getValueType();
8835 if (OutVT.isVector() && OutVT.isFloatingPoint() &&
8836 isOperationCustom(Op: Op.getOpcode(), VT: InVT))
8837 return LowerINT_TO_FPVector(Op, DAG, dl);
8838
8839 // Conversions to f128 are legal.
8840 if (Op.getValueType() == MVT::f128)
8841 return Subtarget.hasP9Vector() ? Op : SDValue();
8842
8843 // Don't handle ppc_fp128 here; let it be lowered to a libcall.
8844 if (Op.getValueType() != MVT::f32 && Op.getValueType() != MVT::f64)
8845 return SDValue();
8846
8847 if (Src.getValueType() == MVT::i1) {
8848 SDValue Sel = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: Op.getValueType(), N1: Src,
8849 N2: DAG.getConstantFP(Val: 1.0, DL: dl, VT: Op.getValueType()),
8850 N3: DAG.getConstantFP(Val: 0.0, DL: dl, VT: Op.getValueType()));
8851 if (IsStrict)
8852 return DAG.getMergeValues(Ops: {Sel, Chain}, dl);
8853 else
8854 return Sel;
8855 }
8856
8857 // If we have direct moves, we can do all the conversion, skip the store/load
8858 // however, without FPCVT we can't do most conversions.
8859 if (Subtarget.hasDirectMove() && directMoveIsProfitable(Op) &&
8860 Subtarget.isPPC64() && Subtarget.hasFPCVT())
8861 return LowerINT_TO_FPDirectMove(Op, DAG, dl);
8862
8863 assert((IsSigned || Subtarget.hasFPCVT()) &&
8864 "UINT_TO_FP is supported only with FPCVT");
8865
8866 if (Src.getValueType() == MVT::i64) {
8867 SDValue SINT = Src;
8868 // When converting to single-precision, we actually need to convert
8869 // to double-precision first and then round to single-precision.
8870 // To avoid double-rounding effects during that operation, we have
8871 // to prepare the input operand. Bits that might be truncated when
8872 // converting to double-precision are replaced by a bit that won't
8873 // be lost at this stage, but is below the single-precision rounding
8874 // position.
8875 //
8876 // However, if afn is in effect, accept double
8877 // rounding to avoid the extra overhead.
8878 // FIXME: Currently INT_TO_FP can't support fast math flags because
8879 // of nneg flag, thus Op->getFlags().hasApproximateFuncs() is always
8880 // false.
8881 if (Op.getValueType() == MVT::f32 && !Subtarget.hasFPCVT() &&
8882 !Op->getFlags().hasApproximateFuncs()) {
8883
8884 // Twiddle input to make sure the low 11 bits are zero. (If this
8885 // is the case, we are guaranteed the value will fit into the 53 bit
8886 // mantissa of an IEEE double-precision value without rounding.)
8887 // If any of those low 11 bits were not zero originally, make sure
8888 // bit 12 (value 2048) is set instead, so that the final rounding
8889 // to single-precision gets the correct result.
8890 SDValue Round = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i64,
8891 N1: SINT, N2: DAG.getConstant(Val: 2047, DL: dl, VT: MVT::i64));
8892 Round = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::i64,
8893 N1: Round, N2: DAG.getConstant(Val: 2047, DL: dl, VT: MVT::i64));
8894 Round = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: MVT::i64, N1: Round, N2: SINT);
8895 Round = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i64, N1: Round,
8896 N2: DAG.getSignedConstant(Val: -2048, DL: dl, VT: MVT::i64));
8897
8898 // However, we cannot use that value unconditionally: if the magnitude
8899 // of the input value is small, the bit-twiddling we did above might
8900 // end up visibly changing the output. Fortunately, in that case, we
8901 // don't need to twiddle bits since the original input will convert
8902 // exactly to double-precision floating-point already. Therefore,
8903 // construct a conditional to use the original value if the top 11
8904 // bits are all sign-bit copies, and use the rounded value computed
8905 // above otherwise.
8906 SDValue Cond = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT: MVT::i64,
8907 N1: SINT, N2: DAG.getConstant(Val: 53, DL: dl, VT: MVT::i32));
8908 Cond = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::i64,
8909 N1: Cond, N2: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i64));
8910 Cond = DAG.getSetCC(
8911 DL: dl,
8912 VT: getSetCCResultType(DL: DAG.getDataLayout(), C&: *DAG.getContext(), VT: MVT::i64),
8913 LHS: Cond, RHS: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i64), Cond: ISD::SETUGT);
8914
8915 SINT = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: MVT::i64, N1: Cond, N2: Round, N3: SINT);
8916 }
8917
8918 ReuseLoadInfo RLI;
8919 SDValue Bits;
8920
8921 MachineFunction &MF = DAG.getMachineFunction();
8922 if (canReuseLoadAddress(Op: SINT, MemVT: MVT::i64, RLI, DAG)) {
8923 // Drop range metadata, as this metadata becomes invalid for f64 bit
8924 // reinterpretation of i64 values.
8925 Bits =
8926 DAG.getLoad(VT: MVT::f64, dl, Chain: RLI.Chain, Ptr: RLI.Ptr, PtrInfo: RLI.MPI, Alignment: RLI.Alignment,
8927 MMOFlags: RLI.MMOFlags(), Metadata: MMOMetadata(RLI.AAInfo));
8928 if (RLI.ResChain)
8929 DAG.makeEquivalentMemoryOrdering(OldChain: RLI.ResChain, NewMemOpChain: Bits.getValue(R: 1));
8930 } else if (Subtarget.hasLFIWAX() &&
8931 canReuseLoadAddress(Op: SINT, MemVT: MVT::i32, RLI, DAG, ET: ISD::SEXTLOAD)) {
8932 MachineMemOperand *MMO = MF.getMachineMemOperand(
8933 PtrInfo: RLI.MPI, F: MachineMemOperand::MOLoad, Size: 4, BaseAlignment: RLI.Alignment,
8934 Metadata: MMOMetadata(RLI.AAInfo, RLI.Ranges));
8935 SDValue Ops[] = { RLI.Chain, RLI.Ptr };
8936 Bits = DAG.getMemIntrinsicNode(Opcode: PPCISD::LFIWAX, dl,
8937 VTList: DAG.getVTList(VT1: MVT::f64, VT2: MVT::Other),
8938 Ops, MemVT: MVT::i32, MMO);
8939 if (RLI.ResChain)
8940 DAG.makeEquivalentMemoryOrdering(OldChain: RLI.ResChain, NewMemOpChain: Bits.getValue(R: 1));
8941 } else if (Subtarget.hasFPCVT() &&
8942 canReuseLoadAddress(Op: SINT, MemVT: MVT::i32, RLI, DAG, ET: ISD::ZEXTLOAD)) {
8943 MachineMemOperand *MMO = MF.getMachineMemOperand(
8944 PtrInfo: RLI.MPI, F: MachineMemOperand::MOLoad, Size: 4, BaseAlignment: RLI.Alignment,
8945 Metadata: MMOMetadata(RLI.AAInfo, RLI.Ranges));
8946 SDValue Ops[] = { RLI.Chain, RLI.Ptr };
8947 Bits = DAG.getMemIntrinsicNode(Opcode: PPCISD::LFIWZX, dl,
8948 VTList: DAG.getVTList(VT1: MVT::f64, VT2: MVT::Other),
8949 Ops, MemVT: MVT::i32, MMO);
8950 if (RLI.ResChain)
8951 DAG.makeEquivalentMemoryOrdering(OldChain: RLI.ResChain, NewMemOpChain: Bits.getValue(R: 1));
8952 } else if (((Subtarget.hasLFIWAX() &&
8953 SINT.getOpcode() == ISD::SIGN_EXTEND) ||
8954 (Subtarget.hasFPCVT() &&
8955 SINT.getOpcode() == ISD::ZERO_EXTEND)) &&
8956 SINT.getOperand(i: 0).getValueType() == MVT::i32) {
8957 MachineFrameInfo &MFI = MF.getFrameInfo();
8958 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
8959
8960 int FrameIdx = MFI.CreateStackObject(Size: 4, Alignment: Align(4), isSpillSlot: false);
8961 SDValue FIdx = DAG.getFrameIndex(FI: FrameIdx, VT: PtrVT);
8962
8963 SDValue Store = DAG.getStore(Chain, dl, Val: SINT.getOperand(i: 0), Ptr: FIdx,
8964 PtrInfo: MachinePointerInfo::getFixedStack(
8965 MF&: DAG.getMachineFunction(), FI: FrameIdx));
8966 Chain = Store;
8967
8968 assert(cast<StoreSDNode>(Store)->getMemoryVT() == MVT::i32 &&
8969 "Expected an i32 store");
8970
8971 RLI.Ptr = FIdx;
8972 RLI.Chain = Chain;
8973 RLI.MPI =
8974 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: FrameIdx);
8975 RLI.Alignment = Align(4);
8976
8977 MachineMemOperand *MMO = MF.getMachineMemOperand(
8978 PtrInfo: RLI.MPI, F: MachineMemOperand::MOLoad, Size: 4, BaseAlignment: RLI.Alignment,
8979 Metadata: MMOMetadata(RLI.AAInfo, RLI.Ranges));
8980 SDValue Ops[] = { RLI.Chain, RLI.Ptr };
8981 Bits = DAG.getMemIntrinsicNode(Opcode: SINT.getOpcode() == ISD::ZERO_EXTEND ?
8982 PPCISD::LFIWZX : PPCISD::LFIWAX,
8983 dl, VTList: DAG.getVTList(VT1: MVT::f64, VT2: MVT::Other),
8984 Ops, MemVT: MVT::i32, MMO);
8985 Chain = Bits.getValue(R: 1);
8986 } else
8987 Bits = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::f64, Operand: SINT);
8988
8989 SDValue FP = convertIntToFP(Op, Src: Bits, DAG, Subtarget, Chain);
8990 if (IsStrict)
8991 Chain = FP.getValue(R: 1);
8992
8993 if (Op.getValueType() == MVT::f32 && !Subtarget.hasFPCVT()) {
8994 if (IsStrict)
8995 FP = DAG.getNode(
8996 Opcode: ISD::STRICT_FP_ROUND, DL: dl, VTList: DAG.getVTList(VT1: MVT::f32, VT2: MVT::Other),
8997 Ops: {Chain, FP, DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)},
8998 Flags);
8999 else
9000 FP = DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: MVT::f32, N1: FP,
9001 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
9002 }
9003 return FP;
9004 }
9005
9006 assert(Src.getValueType() == MVT::i32 &&
9007 "Unhandled INT_TO_FP type in custom expander!");
9008 // Since we only generate this in 64-bit mode, we can take advantage of
9009 // 64-bit registers. In particular, sign extend the input value into the
9010 // 64-bit register with extsw, store the WHOLE 64-bit value into the stack
9011 // then lfd it and fcfid it.
9012 MachineFunction &MF = DAG.getMachineFunction();
9013 MachineFrameInfo &MFI = MF.getFrameInfo();
9014 EVT PtrVT = getPointerTy(DL: MF.getDataLayout());
9015
9016 SDValue Ld;
9017 if (Subtarget.hasLFIWAX() || Subtarget.hasFPCVT()) {
9018 ReuseLoadInfo RLI;
9019 bool ReusingLoad;
9020 if (!(ReusingLoad = canReuseLoadAddress(Op: Src, MemVT: MVT::i32, RLI, DAG))) {
9021 int FrameIdx = MFI.CreateStackObject(Size: 4, Alignment: Align(4), isSpillSlot: false);
9022 SDValue FIdx = DAG.getFrameIndex(FI: FrameIdx, VT: PtrVT);
9023
9024 SDValue Store = DAG.getStore(Chain, dl, Val: Src, Ptr: FIdx,
9025 PtrInfo: MachinePointerInfo::getFixedStack(
9026 MF&: DAG.getMachineFunction(), FI: FrameIdx));
9027 Chain = Store;
9028
9029 assert(cast<StoreSDNode>(Store)->getMemoryVT() == MVT::i32 &&
9030 "Expected an i32 store");
9031
9032 RLI.Ptr = FIdx;
9033 RLI.Chain = Chain;
9034 RLI.MPI =
9035 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: FrameIdx);
9036 RLI.Alignment = Align(4);
9037 }
9038
9039 MachineMemOperand *MMO = MF.getMachineMemOperand(
9040 PtrInfo: RLI.MPI, F: MachineMemOperand::MOLoad, Size: 4, BaseAlignment: RLI.Alignment,
9041 Metadata: MMOMetadata(RLI.AAInfo, RLI.Ranges));
9042 SDValue Ops[] = { RLI.Chain, RLI.Ptr };
9043 Ld = DAG.getMemIntrinsicNode(Opcode: IsSigned ? PPCISD::LFIWAX : PPCISD::LFIWZX, dl,
9044 VTList: DAG.getVTList(VT1: MVT::f64, VT2: MVT::Other), Ops,
9045 MemVT: MVT::i32, MMO);
9046 Chain = Ld.getValue(R: 1);
9047 if (ReusingLoad && RLI.ResChain) {
9048 DAG.makeEquivalentMemoryOrdering(OldChain: RLI.ResChain, NewMemOpChain: Ld.getValue(R: 1));
9049 }
9050 } else {
9051 assert(Subtarget.isPPC64() &&
9052 "i32->FP without LFIWAX supported only on PPC64");
9053
9054 int FrameIdx = MFI.CreateStackObject(Size: 8, Alignment: Align(8), isSpillSlot: false);
9055 SDValue FIdx = DAG.getFrameIndex(FI: FrameIdx, VT: PtrVT);
9056
9057 SDValue Ext64 = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: MVT::i64, Operand: Src);
9058
9059 // STD the extended value into the stack slot.
9060 SDValue Store = DAG.getStore(
9061 Chain, dl, Val: Ext64, Ptr: FIdx,
9062 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: FrameIdx));
9063 Chain = Store;
9064
9065 // Load the value as a double.
9066 Ld = DAG.getLoad(
9067 VT: MVT::f64, dl, Chain, Ptr: FIdx,
9068 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: FrameIdx));
9069 Chain = Ld.getValue(R: 1);
9070 }
9071
9072 // FCFID it and return it.
9073 SDValue FP = convertIntToFP(Op, Src: Ld, DAG, Subtarget, Chain);
9074 if (IsStrict)
9075 Chain = FP.getValue(R: 1);
9076 if (Op.getValueType() == MVT::f32 && !Subtarget.hasFPCVT()) {
9077 if (IsStrict)
9078 FP = DAG.getNode(
9079 Opcode: ISD::STRICT_FP_ROUND, DL: dl, VTList: DAG.getVTList(VT1: MVT::f32, VT2: MVT::Other),
9080 Ops: {Chain, FP, DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)}, Flags);
9081 else
9082 FP = DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: MVT::f32, N1: FP,
9083 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
9084 }
9085 return FP;
9086}
9087
9088SDValue PPCTargetLowering::LowerSET_ROUNDING(SDValue Op,
9089 SelectionDAG &DAG) const {
9090 SDLoc Dl(Op);
9091 MachineFunction &MF = DAG.getMachineFunction();
9092 EVT PtrVT = getPointerTy(DL: MF.getDataLayout());
9093 SDValue Chain = Op.getOperand(i: 0);
9094
9095 // If requested mode is constant, just use simpler mtfsb/mffscrni
9096 if (auto *CVal = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1))) {
9097 uint64_t Mode = CVal->getZExtValue();
9098 assert(Mode < 4 && "Unsupported rounding mode!");
9099 unsigned InternalRnd = Mode ^ (~(Mode >> 1) & 1);
9100 if (Subtarget.isISA3_0())
9101 return SDValue(
9102 DAG.getMachineNode(
9103 Opcode: PPC::MFFSCRNI, dl: Dl, ResultTys: {MVT::f64, MVT::Other},
9104 Ops: {DAG.getConstant(Val: InternalRnd, DL: Dl, VT: MVT::i32, isTarget: true), Chain}),
9105 1);
9106 SDNode *SetHi = DAG.getMachineNode(
9107 Opcode: (InternalRnd & 2) ? PPC::MTFSB1 : PPC::MTFSB0, dl: Dl, VT: MVT::Other,
9108 Ops: {DAG.getConstant(Val: 30, DL: Dl, VT: MVT::i32, isTarget: true), Chain});
9109 SDNode *SetLo = DAG.getMachineNode(
9110 Opcode: (InternalRnd & 1) ? PPC::MTFSB1 : PPC::MTFSB0, dl: Dl, VT: MVT::Other,
9111 Ops: {DAG.getConstant(Val: 31, DL: Dl, VT: MVT::i32, isTarget: true), SDValue(SetHi, 0)});
9112 return SDValue(SetLo, 0);
9113 }
9114
9115 // Use x ^ (~(x >> 1) & 1) to transform LLVM rounding mode to Power format.
9116 SDValue One = DAG.getConstant(Val: 1, DL: Dl, VT: MVT::i32);
9117 SDValue SrcFlag = DAG.getNode(Opcode: ISD::AND, DL: Dl, VT: MVT::i32, N1: Op.getOperand(i: 1),
9118 N2: DAG.getConstant(Val: 3, DL: Dl, VT: MVT::i32));
9119 SDValue DstFlag = DAG.getNode(
9120 Opcode: ISD::XOR, DL: Dl, VT: MVT::i32, N1: SrcFlag,
9121 N2: DAG.getNode(Opcode: ISD::AND, DL: Dl, VT: MVT::i32,
9122 N1: DAG.getNOT(DL: Dl,
9123 Val: DAG.getNode(Opcode: ISD::SRL, DL: Dl, VT: MVT::i32, N1: SrcFlag, N2: One),
9124 VT: MVT::i32),
9125 N2: One));
9126 // For Power9, there's faster mffscrn, and we don't need to read FPSCR
9127 SDValue MFFS;
9128 if (!Subtarget.isISA3_0()) {
9129 MFFS = DAG.getNode(Opcode: PPCISD::MFFS, DL: Dl, ResultTys: {MVT::f64, MVT::Other}, Ops: Chain);
9130 Chain = MFFS.getValue(R: 1);
9131 }
9132 SDValue NewFPSCR;
9133 if (Subtarget.isPPC64()) {
9134 if (Subtarget.isISA3_0()) {
9135 NewFPSCR = DAG.getAnyExtOrTrunc(Op: DstFlag, DL: Dl, VT: MVT::i64);
9136 } else {
9137 // Set the last two bits (rounding mode) of bitcasted FPSCR.
9138 SDNode *InsertRN = DAG.getMachineNode(
9139 Opcode: PPC::RLDIMI, dl: Dl, VT: MVT::i64,
9140 Ops: {DAG.getNode(Opcode: ISD::BITCAST, DL: Dl, VT: MVT::i64, Operand: MFFS),
9141 DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: Dl, VT: MVT::i64, Operand: DstFlag),
9142 DAG.getTargetConstant(Val: 0, DL: Dl, VT: MVT::i32),
9143 DAG.getTargetConstant(Val: 62, DL: Dl, VT: MVT::i32)});
9144 NewFPSCR = SDValue(InsertRN, 0);
9145 }
9146 NewFPSCR = DAG.getNode(Opcode: ISD::BITCAST, DL: Dl, VT: MVT::f64, Operand: NewFPSCR);
9147 } else {
9148 // In 32-bit mode, store f64, load and update the lower half.
9149 int SSFI = MF.getFrameInfo().CreateStackObject(Size: 8, Alignment: Align(8), isSpillSlot: false);
9150 SDValue StackSlot = DAG.getFrameIndex(FI: SSFI, VT: PtrVT);
9151 SDValue Addr = Subtarget.isLittleEndian()
9152 ? StackSlot
9153 : DAG.getNode(Opcode: ISD::ADD, DL: Dl, VT: PtrVT, N1: StackSlot,
9154 N2: DAG.getConstant(Val: 4, DL: Dl, VT: PtrVT));
9155 if (Subtarget.isISA3_0()) {
9156 Chain = DAG.getStore(Chain, dl: Dl, Val: DstFlag, Ptr: Addr, PtrInfo: MachinePointerInfo());
9157 } else {
9158 Chain = DAG.getStore(Chain, dl: Dl, Val: MFFS, Ptr: StackSlot, PtrInfo: MachinePointerInfo());
9159 SDValue Tmp =
9160 DAG.getLoad(VT: MVT::i32, dl: Dl, Chain, Ptr: Addr, PtrInfo: MachinePointerInfo());
9161 Chain = Tmp.getValue(R: 1);
9162 Tmp = SDValue(DAG.getMachineNode(
9163 Opcode: PPC::RLWIMI, dl: Dl, VT: MVT::i32,
9164 Ops: {Tmp, DstFlag, DAG.getTargetConstant(Val: 0, DL: Dl, VT: MVT::i32),
9165 DAG.getTargetConstant(Val: 30, DL: Dl, VT: MVT::i32),
9166 DAG.getTargetConstant(Val: 31, DL: Dl, VT: MVT::i32)}),
9167 0);
9168 Chain = DAG.getStore(Chain, dl: Dl, Val: Tmp, Ptr: Addr, PtrInfo: MachinePointerInfo());
9169 }
9170 NewFPSCR =
9171 DAG.getLoad(VT: MVT::f64, dl: Dl, Chain, Ptr: StackSlot, PtrInfo: MachinePointerInfo());
9172 Chain = NewFPSCR.getValue(R: 1);
9173 }
9174 if (Subtarget.isISA3_0())
9175 return SDValue(DAG.getMachineNode(Opcode: PPC::MFFSCRN, dl: Dl, ResultTys: {MVT::f64, MVT::Other},
9176 Ops: {NewFPSCR, Chain}),
9177 1);
9178 SDValue Zero = DAG.getConstant(Val: 0, DL: Dl, VT: MVT::i32, isTarget: true);
9179 SDNode *MTFSF = DAG.getMachineNode(
9180 Opcode: PPC::MTFSF, dl: Dl, VT: MVT::Other,
9181 Ops: {DAG.getConstant(Val: 255, DL: Dl, VT: MVT::i32, isTarget: true), NewFPSCR, Zero, Zero, Chain});
9182 return SDValue(MTFSF, 0);
9183}
9184
9185SDValue PPCTargetLowering::LowerGET_ROUNDING(SDValue Op,
9186 SelectionDAG &DAG) const {
9187 SDLoc dl(Op);
9188 /*
9189 The rounding mode is in bits 30:31 of FPSR, and has the following
9190 settings:
9191 00 Round to nearest
9192 01 Round to 0
9193 10 Round to +inf
9194 11 Round to -inf
9195
9196 GET_ROUNDING, on the other hand, expects the following:
9197 -1 Undefined
9198 0 Round to 0
9199 1 Round to nearest
9200 2 Round to +inf
9201 3 Round to -inf
9202
9203 To perform the conversion, we do:
9204 ((FPSCR & 0x3) ^ ((~FPSCR & 0x3) >> 1))
9205 */
9206
9207 MachineFunction &MF = DAG.getMachineFunction();
9208 EVT VT = Op.getValueType();
9209 EVT PtrVT = getPointerTy(DL: MF.getDataLayout());
9210
9211 // Save FP Control Word to register
9212 SDValue Chain = Op.getOperand(i: 0);
9213 SDValue MFFS = DAG.getNode(Opcode: PPCISD::MFFS, DL: dl, ResultTys: {MVT::f64, MVT::Other}, Ops: Chain);
9214 Chain = MFFS.getValue(R: 1);
9215
9216 SDValue CWD;
9217 if (isTypeLegal(VT: MVT::i64)) {
9218 CWD = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i32,
9219 Operand: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i64, Operand: MFFS));
9220 } else {
9221 // Save FP register to stack slot
9222 int SSFI = MF.getFrameInfo().CreateStackObject(Size: 8, Alignment: Align(8), isSpillSlot: false);
9223 SDValue StackSlot = DAG.getFrameIndex(FI: SSFI, VT: PtrVT);
9224 Chain = DAG.getStore(Chain, dl, Val: MFFS, Ptr: StackSlot, PtrInfo: MachinePointerInfo());
9225
9226 // Load FP Control Word from low 32 bits of stack slot.
9227 assert(hasBigEndianPartOrdering(MVT::i64, MF.getDataLayout()) &&
9228 "Stack slot adjustment is valid only on big endian subtargets!");
9229 SDValue Four = DAG.getConstant(Val: 4, DL: dl, VT: PtrVT);
9230 SDValue Addr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: StackSlot, N2: Four);
9231 CWD = DAG.getLoad(VT: MVT::i32, dl, Chain, Ptr: Addr, PtrInfo: MachinePointerInfo());
9232 Chain = CWD.getValue(R: 1);
9233 }
9234
9235 // Transform as necessary
9236 SDValue CWD1 =
9237 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32,
9238 N1: CWD, N2: DAG.getConstant(Val: 3, DL: dl, VT: MVT::i32));
9239 SDValue CWD2 =
9240 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i32,
9241 N1: DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32,
9242 N1: DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: MVT::i32,
9243 N1: CWD, N2: DAG.getConstant(Val: 3, DL: dl, VT: MVT::i32)),
9244 N2: DAG.getConstant(Val: 3, DL: dl, VT: MVT::i32)),
9245 N2: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
9246
9247 SDValue RetVal =
9248 DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: MVT::i32, N1: CWD1, N2: CWD2);
9249
9250 RetVal =
9251 DAG.getNode(Opcode: (VT.getSizeInBits() < 16 ? ISD::TRUNCATE : ISD::ZERO_EXTEND),
9252 DL: dl, VT, Operand: RetVal);
9253
9254 return DAG.getMergeValues(Ops: {RetVal, Chain}, dl);
9255}
9256
9257SDValue PPCTargetLowering::LowerSHL_PARTS(SDValue Op, SelectionDAG &DAG) const {
9258 EVT VT = Op.getValueType();
9259 uint64_t BitWidth = VT.getSizeInBits();
9260 SDLoc dl(Op);
9261 assert(Op.getNumOperands() == 3 &&
9262 VT == Op.getOperand(1).getValueType() &&
9263 "Unexpected SHL!");
9264
9265 // Expand into a bunch of logical ops. Note that these ops
9266 // depend on the PPC behavior for oversized shift amounts.
9267 SDValue Lo = Op.getOperand(i: 0);
9268 SDValue Hi = Op.getOperand(i: 1);
9269 SDValue Amt = Op.getOperand(i: 2);
9270 EVT AmtVT = Amt.getValueType();
9271
9272 SDValue Tmp1 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: AmtVT,
9273 N1: DAG.getConstant(Val: BitWidth, DL: dl, VT: AmtVT), N2: Amt);
9274 SDValue Tmp2 = DAG.getNode(Opcode: PPCISD::SHL, DL: dl, VT, N1: Hi, N2: Amt);
9275 SDValue Tmp3 = DAG.getNode(Opcode: PPCISD::SRL, DL: dl, VT, N1: Lo, N2: Tmp1);
9276 SDValue Tmp4 = DAG.getNode(Opcode: ISD::OR , DL: dl, VT, N1: Tmp2, N2: Tmp3);
9277 SDValue Tmp5 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: AmtVT, N1: Amt,
9278 N2: DAG.getSignedConstant(Val: -BitWidth, DL: dl, VT: AmtVT));
9279 SDValue Tmp6 = DAG.getNode(Opcode: PPCISD::SHL, DL: dl, VT, N1: Lo, N2: Tmp5);
9280 SDValue OutHi = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp4, N2: Tmp6);
9281 SDValue OutLo = DAG.getNode(Opcode: PPCISD::SHL, DL: dl, VT, N1: Lo, N2: Amt);
9282 SDValue OutOps[] = { OutLo, OutHi };
9283 return DAG.getMergeValues(Ops: OutOps, dl);
9284}
9285
9286SDValue PPCTargetLowering::LowerSRL_PARTS(SDValue Op, SelectionDAG &DAG) const {
9287 EVT VT = Op.getValueType();
9288 SDLoc dl(Op);
9289 uint64_t BitWidth = VT.getSizeInBits();
9290 assert(Op.getNumOperands() == 3 &&
9291 VT == Op.getOperand(1).getValueType() &&
9292 "Unexpected SRL!");
9293
9294 // Expand into a bunch of logical ops. Note that these ops
9295 // depend on the PPC behavior for oversized shift amounts.
9296 SDValue Lo = Op.getOperand(i: 0);
9297 SDValue Hi = Op.getOperand(i: 1);
9298 SDValue Amt = Op.getOperand(i: 2);
9299 EVT AmtVT = Amt.getValueType();
9300
9301 SDValue Tmp1 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: AmtVT,
9302 N1: DAG.getConstant(Val: BitWidth, DL: dl, VT: AmtVT), N2: Amt);
9303 SDValue Tmp2 = DAG.getNode(Opcode: PPCISD::SRL, DL: dl, VT, N1: Lo, N2: Amt);
9304 SDValue Tmp3 = DAG.getNode(Opcode: PPCISD::SHL, DL: dl, VT, N1: Hi, N2: Tmp1);
9305 SDValue Tmp4 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp2, N2: Tmp3);
9306 SDValue Tmp5 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: AmtVT, N1: Amt,
9307 N2: DAG.getSignedConstant(Val: -BitWidth, DL: dl, VT: AmtVT));
9308 SDValue Tmp6 = DAG.getNode(Opcode: PPCISD::SRL, DL: dl, VT, N1: Hi, N2: Tmp5);
9309 SDValue OutLo = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp4, N2: Tmp6);
9310 SDValue OutHi = DAG.getNode(Opcode: PPCISD::SRL, DL: dl, VT, N1: Hi, N2: Amt);
9311 SDValue OutOps[] = { OutLo, OutHi };
9312 return DAG.getMergeValues(Ops: OutOps, dl);
9313}
9314
9315SDValue PPCTargetLowering::LowerSRA_PARTS(SDValue Op, SelectionDAG &DAG) const {
9316 SDLoc dl(Op);
9317 EVT VT = Op.getValueType();
9318 uint64_t BitWidth = VT.getSizeInBits();
9319 assert(Op.getNumOperands() == 3 &&
9320 VT == Op.getOperand(1).getValueType() &&
9321 "Unexpected SRA!");
9322
9323 // Expand into a bunch of logical ops, followed by a select_cc.
9324 SDValue Lo = Op.getOperand(i: 0);
9325 SDValue Hi = Op.getOperand(i: 1);
9326 SDValue Amt = Op.getOperand(i: 2);
9327 EVT AmtVT = Amt.getValueType();
9328
9329 SDValue Tmp1 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: AmtVT,
9330 N1: DAG.getConstant(Val: BitWidth, DL: dl, VT: AmtVT), N2: Amt);
9331 SDValue Tmp2 = DAG.getNode(Opcode: PPCISD::SRL, DL: dl, VT, N1: Lo, N2: Amt);
9332 SDValue Tmp3 = DAG.getNode(Opcode: PPCISD::SHL, DL: dl, VT, N1: Hi, N2: Tmp1);
9333 SDValue Tmp4 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp2, N2: Tmp3);
9334 SDValue Tmp5 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: AmtVT, N1: Amt,
9335 N2: DAG.getSignedConstant(Val: -BitWidth, DL: dl, VT: AmtVT));
9336 SDValue Tmp6 = DAG.getNode(Opcode: PPCISD::SRA, DL: dl, VT, N1: Hi, N2: Tmp5);
9337 SDValue OutHi = DAG.getNode(Opcode: PPCISD::SRA, DL: dl, VT, N1: Hi, N2: Amt);
9338 SDValue OutLo = DAG.getSelectCC(DL: dl, LHS: Tmp5, RHS: DAG.getConstant(Val: 0, DL: dl, VT: AmtVT),
9339 True: Tmp4, False: Tmp6, Cond: ISD::SETLE);
9340 SDValue OutOps[] = { OutLo, OutHi };
9341 return DAG.getMergeValues(Ops: OutOps, dl);
9342}
9343
9344SDValue PPCTargetLowering::LowerFunnelShift(SDValue Op,
9345 SelectionDAG &DAG) const {
9346 SDLoc dl(Op);
9347 EVT VT = Op.getValueType();
9348 unsigned BitWidth = VT.getSizeInBits();
9349
9350 bool IsFSHL = Op.getOpcode() == ISD::FSHL;
9351 SDValue X = Op.getOperand(i: 0);
9352 SDValue Y = Op.getOperand(i: 1);
9353 SDValue Z = Op.getOperand(i: 2);
9354 EVT AmtVT = Z.getValueType();
9355
9356 // fshl: (X << (Z % BW)) | (Y >> (BW - (Z % BW)))
9357 // fshr: (X << (BW - (Z % BW))) | (Y >> (Z % BW))
9358 // This is simpler than TargetLowering::expandFunnelShift because we can rely
9359 // on PowerPC shift by BW being well defined.
9360 Z = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: AmtVT, N1: Z,
9361 N2: DAG.getConstant(Val: BitWidth - 1, DL: dl, VT: AmtVT));
9362 SDValue SubZ =
9363 DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: AmtVT, N1: DAG.getConstant(Val: BitWidth, DL: dl, VT: AmtVT), N2: Z);
9364 X = DAG.getNode(Opcode: PPCISD::SHL, DL: dl, VT, N1: X, N2: IsFSHL ? Z : SubZ);
9365 Y = DAG.getNode(Opcode: PPCISD::SRL, DL: dl, VT, N1: Y, N2: IsFSHL ? SubZ : Z);
9366 return DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: X, N2: Y);
9367}
9368
9369//===----------------------------------------------------------------------===//
9370// Vector related lowering.
9371//
9372
9373/// getCanonicalConstSplat - Build a canonical splat immediate of Val with an
9374/// element size of SplatSize. Cast the result to VT.
9375static SDValue getCanonicalConstSplat(uint64_t Val, unsigned SplatSize, EVT VT,
9376 SelectionDAG &DAG, const SDLoc &dl) {
9377 static const MVT VTys[] = { // canonical VT to use for each size.
9378 MVT::v16i8, MVT::v8i16, MVT::Other, MVT::v4i32
9379 };
9380
9381 EVT ReqVT = VT != MVT::Other ? VT : VTys[SplatSize-1];
9382
9383 // For a splat with all ones, turn it to vspltisb 0xFF to canonicalize.
9384 if (Val == ((1LLU << (SplatSize * 8)) - 1)) {
9385 SplatSize = 1;
9386 Val = 0xFF;
9387 }
9388
9389 EVT CanonicalVT = VTys[SplatSize-1];
9390
9391 // Build a canonical splat for this value.
9392 // Explicitly truncate APInt here, as this API is used with a mix of
9393 // signed and unsigned values.
9394 return DAG.getBitcast(
9395 VT: ReqVT,
9396 V: DAG.getConstant(Val: APInt(64, Val).trunc(width: SplatSize * 8), DL: dl, VT: CanonicalVT));
9397}
9398
9399/// BuildIntrinsicOp - Return a unary operator intrinsic node with the
9400/// specified intrinsic ID.
9401static SDValue BuildIntrinsicOp(unsigned IID, SDValue Op, SelectionDAG &DAG,
9402 const SDLoc &dl, EVT DestVT = MVT::Other) {
9403 if (DestVT == MVT::Other) DestVT = Op.getValueType();
9404 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: DestVT,
9405 N1: DAG.getConstant(Val: IID, DL: dl, VT: MVT::i32), N2: Op);
9406}
9407
9408/// BuildIntrinsicOp - Return a binary operator intrinsic node with the
9409/// specified intrinsic ID.
9410static SDValue BuildIntrinsicOp(unsigned IID, SDValue LHS, SDValue RHS,
9411 SelectionDAG &DAG, const SDLoc &dl,
9412 EVT DestVT = MVT::Other) {
9413 if (DestVT == MVT::Other) DestVT = LHS.getValueType();
9414 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: DestVT,
9415 N1: DAG.getConstant(Val: IID, DL: dl, VT: MVT::i32), N2: LHS, N3: RHS);
9416}
9417
9418/// BuildIntrinsicOp - Return a ternary operator intrinsic node with the
9419/// specified intrinsic ID.
9420static SDValue BuildIntrinsicOp(unsigned IID, SDValue Op0, SDValue Op1,
9421 SDValue Op2, SelectionDAG &DAG, const SDLoc &dl,
9422 EVT DestVT = MVT::Other) {
9423 if (DestVT == MVT::Other) DestVT = Op0.getValueType();
9424 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: DestVT,
9425 N1: DAG.getConstant(Val: IID, DL: dl, VT: MVT::i32), N2: Op0, N3: Op1, N4: Op2);
9426}
9427
9428/// BuildVSLDOI - Return a VECTOR_SHUFFLE that is a vsldoi of the specified
9429/// amount. The result has the specified value type.
9430static SDValue BuildVSLDOI(SDValue LHS, SDValue RHS, unsigned Amt, EVT VT,
9431 SelectionDAG &DAG, const SDLoc &dl) {
9432 // Force LHS/RHS to be the right type.
9433 LHS = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: LHS);
9434 RHS = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: RHS);
9435
9436 int Ops[16];
9437 for (unsigned i = 0; i != 16; ++i)
9438 Ops[i] = i + Amt;
9439 SDValue T = DAG.getVectorShuffle(VT: MVT::v16i8, dl, N1: LHS, N2: RHS, Mask: Ops);
9440 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: T);
9441}
9442
9443/// Do we have an efficient pattern in a .td file for this node?
9444///
9445/// \param V - pointer to the BuildVectorSDNode being matched
9446/// \param HasDirectMove - does this subtarget have VSR <-> GPR direct moves?
9447///
9448/// There are some patterns where it is beneficial to keep a BUILD_VECTOR
9449/// node as a BUILD_VECTOR node rather than expanding it. The patterns where
9450/// the opposite is true (expansion is beneficial) are:
9451/// - The node builds a vector out of integers that are not 32 or 64-bits
9452/// - The node builds a vector out of constants
9453/// - The node is a "load-and-splat"
9454/// In all other cases, we will choose to keep the BUILD_VECTOR.
9455static bool haveEfficientBuildVectorPattern(BuildVectorSDNode *V,
9456 bool HasDirectMove,
9457 bool HasP8Vector) {
9458 EVT VecVT = V->getValueType(ResNo: 0);
9459 bool RightType = VecVT == MVT::v2f64 ||
9460 (HasP8Vector && VecVT == MVT::v4f32) ||
9461 (HasDirectMove && (VecVT == MVT::v2i64 || VecVT == MVT::v4i32));
9462 if (!RightType)
9463 return false;
9464
9465 bool IsSplat = true;
9466 bool IsLoad = false;
9467 SDValue Op0 = V->getOperand(Num: 0);
9468
9469 // This function is called in a block that confirms the node is not a constant
9470 // splat. So a constant BUILD_VECTOR here means the vector is built out of
9471 // different constants.
9472 if (V->isConstant())
9473 return false;
9474 for (int i = 0, e = V->getNumOperands(); i < e; ++i) {
9475 if (V->getOperand(Num: i).isUndef())
9476 return false;
9477 // We want to expand nodes that represent load-and-splat even if the
9478 // loaded value is a floating point truncation or conversion to int.
9479 if (V->getOperand(Num: i).getOpcode() == ISD::LOAD ||
9480 (V->getOperand(Num: i).getOpcode() == ISD::FP_ROUND &&
9481 V->getOperand(Num: i).getOperand(i: 0).getOpcode() == ISD::LOAD) ||
9482 (V->getOperand(Num: i).getOpcode() == ISD::FP_TO_SINT &&
9483 V->getOperand(Num: i).getOperand(i: 0).getOpcode() == ISD::LOAD) ||
9484 (V->getOperand(Num: i).getOpcode() == ISD::FP_TO_UINT &&
9485 V->getOperand(Num: i).getOperand(i: 0).getOpcode() == ISD::LOAD))
9486 IsLoad = true;
9487 // If the operands are different or the input is not a load and has more
9488 // uses than just this BV node, then it isn't a splat.
9489 if (V->getOperand(Num: i) != Op0 ||
9490 (!IsLoad && !V->isOnlyUserOf(N: V->getOperand(Num: i).getNode())))
9491 IsSplat = false;
9492 }
9493 return !(IsSplat && IsLoad);
9494}
9495
9496// Lower BITCAST(f128, (build_pair i64, i64)) to BUILD_FP128.
9497SDValue PPCTargetLowering::LowerBITCAST(SDValue Op, SelectionDAG &DAG) const {
9498
9499 SDLoc dl(Op);
9500 SDValue Op0 = Op->getOperand(Num: 0);
9501
9502 if (!Subtarget.isPPC64() || (Op0.getOpcode() != ISD::BUILD_PAIR) ||
9503 (Op.getValueType() != MVT::f128))
9504 return SDValue();
9505
9506 SDValue Lo = Op0.getOperand(i: 0);
9507 SDValue Hi = Op0.getOperand(i: 1);
9508 if ((Lo.getValueType() != MVT::i64) || (Hi.getValueType() != MVT::i64))
9509 return SDValue();
9510
9511 if (!Subtarget.isLittleEndian())
9512 std::swap(a&: Lo, b&: Hi);
9513
9514 return DAG.getNode(Opcode: PPCISD::BUILD_FP128, DL: dl, VT: MVT::f128, N1: Lo, N2: Hi);
9515}
9516
9517static const SDValue *getNormalLoadInput(const SDValue &Op, bool &IsPermuted) {
9518 const SDValue *InputLoad = &Op;
9519 while (InputLoad->getOpcode() == ISD::BITCAST)
9520 InputLoad = &InputLoad->getOperand(i: 0);
9521 if (InputLoad->getOpcode() == ISD::SCALAR_TO_VECTOR ||
9522 InputLoad->getOpcode() == PPCISD::SCALAR_TO_VECTOR_PERMUTED) {
9523 IsPermuted = InputLoad->getOpcode() == PPCISD::SCALAR_TO_VECTOR_PERMUTED;
9524 InputLoad = &InputLoad->getOperand(i: 0);
9525 }
9526 if (InputLoad->getOpcode() != ISD::LOAD)
9527 return nullptr;
9528 LoadSDNode *LD = cast<LoadSDNode>(Val: *InputLoad);
9529 return ISD::isNormalLoad(N: LD) ? InputLoad : nullptr;
9530}
9531
9532// Convert the argument APFloat to a single precision APFloat if there is no
9533// loss in information during the conversion to single precision APFloat and the
9534// resulting number is not a denormal number. Return true if successful.
9535bool llvm::convertToNonDenormSingle(APFloat &ArgAPFloat) {
9536 APFloat APFloatToConvert = ArgAPFloat;
9537 bool LosesInfo = true;
9538 APFloatToConvert.convert(ToSemantics: APFloat::IEEEsingle(), RM: APFloat::rmNearestTiesToEven,
9539 losesInfo: &LosesInfo);
9540 bool Success = (!LosesInfo && !APFloatToConvert.isDenormal());
9541 if (Success)
9542 ArgAPFloat = APFloatToConvert;
9543 return Success;
9544}
9545
9546// Bitcast the argument APInt to a double and convert it to a single precision
9547// APFloat, bitcast the APFloat to an APInt and assign it to the original
9548// argument if there is no loss in information during the conversion from
9549// double to single precision APFloat and the resulting number is not a denormal
9550// number. Return true if successful.
9551bool llvm::convertToNonDenormSingle(APInt &ArgAPInt) {
9552 double DpValue = ArgAPInt.bitsToDouble();
9553 APFloat APFloatDp(DpValue);
9554 bool Success = convertToNonDenormSingle(ArgAPFloat&: APFloatDp);
9555 if (Success)
9556 ArgAPInt = APFloatDp.bitcastToAPInt();
9557 return Success;
9558}
9559
9560// Nondestructive check for convertTonNonDenormSingle.
9561bool llvm::checkConvertToNonDenormSingle(APFloat &ArgAPFloat) {
9562 // Only convert if it loses info, since XXSPLTIDP should
9563 // handle the other case.
9564 APFloat APFloatToConvert = ArgAPFloat;
9565 bool LosesInfo = true;
9566 APFloatToConvert.convert(ToSemantics: APFloat::IEEEsingle(), RM: APFloat::rmNearestTiesToEven,
9567 losesInfo: &LosesInfo);
9568
9569 return (!LosesInfo && !APFloatToConvert.isDenormal());
9570}
9571
9572static bool isValidSplatLoad(const PPCSubtarget &Subtarget, const SDValue &Op,
9573 unsigned &Opcode) {
9574 LoadSDNode *InputNode = dyn_cast<LoadSDNode>(Val: Op.getOperand(i: 0));
9575 if (!InputNode || !Subtarget.hasVSX() || !ISD::isUNINDEXEDLoad(N: InputNode))
9576 return false;
9577
9578 EVT Ty = Op->getValueType(ResNo: 0);
9579 // For v2f64, v4f32 and v4i32 types, we require the load to be non-extending
9580 // as we cannot handle extending loads for these types.
9581 if ((Ty == MVT::v2f64 || Ty == MVT::v4f32 || Ty == MVT::v4i32) &&
9582 ISD::isNON_EXTLoad(N: InputNode))
9583 return true;
9584
9585 EVT MemVT = InputNode->getMemoryVT();
9586 // For v8i16 and v16i8 types, extending loads can be handled as long as the
9587 // memory VT is the same vector element VT type.
9588 // The loads feeding into the v8i16 and v16i8 types will be extending because
9589 // scalar i8/i16 are not legal types.
9590 if ((Ty == MVT::v8i16 || Ty == MVT::v16i8) && ISD::isEXTLoad(N: InputNode) &&
9591 (MemVT == Ty.getVectorElementType()))
9592 return true;
9593
9594 if (Ty == MVT::v2i64) {
9595 // Check the extend type, when the input type is i32, and the output vector
9596 // type is v2i64.
9597 if (MemVT == MVT::i32) {
9598 if (ISD::isZEXTLoad(N: InputNode))
9599 Opcode = PPCISD::ZEXT_LD_SPLAT;
9600 if (ISD::isSEXTLoad(N: InputNode))
9601 Opcode = PPCISD::SEXT_LD_SPLAT;
9602 }
9603 return true;
9604 }
9605 return false;
9606}
9607
9608bool isValidMtVsrBmi(APInt &BitMask, BuildVectorSDNode &BVN,
9609 bool IsLittleEndian) {
9610 assert(BVN.getNumOperands() > 0 && "Unexpected 0-size build vector");
9611
9612 BitMask.clearAllBits();
9613 EVT VT = BVN.getValueType(ResNo: 0);
9614 unsigned VTSize = VT.getSizeInBits();
9615 APInt ConstValue(VTSize, 0);
9616
9617 unsigned EltWidth = VT.getScalarSizeInBits();
9618
9619 unsigned BitPos = 0;
9620 for (auto OpVal : BVN.op_values()) {
9621 auto *CN = dyn_cast<ConstantSDNode>(Val&: OpVal);
9622
9623 if (!CN)
9624 return false;
9625 // The elements in a vector register are ordered in reverse byte order
9626 // between little-endian and big-endian modes.
9627 ConstValue.insertBits(SubBits: CN->getAPIntValue().zextOrTrunc(width: EltWidth),
9628 bitPosition: IsLittleEndian ? BitPos : VTSize - EltWidth - BitPos);
9629 BitPos += EltWidth;
9630 }
9631
9632 for (unsigned J = 0; J < 16; ++J) {
9633 APInt ExtractValue = ConstValue.extractBits(numBits: 8, bitPosition: J * 8);
9634 if (ExtractValue != 0x00 && ExtractValue != 0xFF)
9635 return false;
9636 if (ExtractValue == 0xFF)
9637 BitMask.setBit(J);
9638 }
9639 return true;
9640}
9641
9642// If this is a case we can't handle, return null and let the default
9643// expansion code take care of it. If we CAN select this case, and if it
9644// selects to a single instruction, return Op. Otherwise, if we can codegen
9645// this case more efficiently than a constant pool load, lower it to the
9646// sequence of ops that should be used.
9647SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op,
9648 SelectionDAG &DAG) const {
9649 SDLoc dl(Op);
9650 BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(Val: Op.getNode());
9651 assert(BVN && "Expected a BuildVectorSDNode in LowerBUILD_VECTOR");
9652
9653 if (Subtarget.hasP10Vector()) {
9654 APInt BitMask(32, 0);
9655 // If the value of the vector is all zeros or all ones,
9656 // we do not convert it to MTVSRBMI.
9657 // The xxleqv instruction sets a vector with all ones.
9658 // The xxlxor instruction sets a vector with all zeros.
9659 if (isValidMtVsrBmi(BitMask, BVN&: *BVN, IsLittleEndian: Subtarget.isLittleEndian()) &&
9660 BitMask != 0 && BitMask != 0xffff) {
9661 SDValue SDConstant = DAG.getTargetConstant(Val: BitMask, DL: dl, VT: MVT::i32);
9662 MachineSDNode *MSDNode =
9663 DAG.getMachineNode(Opcode: PPC::MTVSRBMI, dl, VT: MVT::v16i8, Op1: SDConstant);
9664 SDValue SDV = SDValue(MSDNode, 0);
9665 EVT DVT = BVN->getValueType(ResNo: 0);
9666 EVT SVT = SDV.getValueType();
9667 if (SVT != DVT) {
9668 SDV = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: DVT, Operand: SDV);
9669 }
9670 return SDV;
9671 }
9672 // Recognize build vector patterns to emit VSX vector instructions
9673 // instead of loading value from memory.
9674 if (SDValue VecPat = combineBVLoadsSpecialValue(Operand: Op, DAG))
9675 return VecPat;
9676 }
9677 // Check if this is a splat of a constant value.
9678 APInt APSplatBits, APSplatUndef;
9679 unsigned SplatBitSize = 0;
9680 bool HasAnyUndefs;
9681 bool BVNIsConstantSplat =
9682 BVN->isConstantSplat(SplatValue&: APSplatBits, SplatUndef&: APSplatUndef, SplatBitSize,
9683 HasAnyUndefs, MinSplatBits: 0, isBigEndian: !Subtarget.isLittleEndian());
9684
9685 // If it is a splat of a double, check if we can shrink it to a 32 bit
9686 // non-denormal float which when converted back to double gives us the same
9687 // double. This is to exploit the XXSPLTIDP instruction.
9688 // If we lose precision, we use XXSPLTI32DX.
9689 if (BVNIsConstantSplat && (SplatBitSize == 64) &&
9690 Subtarget.hasPrefixInstrs() && Subtarget.hasP10Vector()) {
9691 // Check the type first to short-circuit so we don't modify APSplatBits if
9692 // this block isn't executed.
9693 if ((Op->getValueType(ResNo: 0) == MVT::v2f64) &&
9694 convertToNonDenormSingle(ArgAPInt&: APSplatBits)) {
9695 SDValue SplatNode = DAG.getNode(
9696 Opcode: PPCISD::XXSPLTI_SP_TO_DP, DL: dl, VT: MVT::v2f64,
9697 Operand: DAG.getTargetConstant(Val: APSplatBits.getZExtValue(), DL: dl, VT: MVT::i32));
9698 return DAG.getBitcast(VT: Op.getValueType(), V: SplatNode);
9699 } else {
9700 // We may lose precision, so we have to use XXSPLTI32DX.
9701
9702 uint32_t Hi = Hi_32(Value: APSplatBits.getZExtValue());
9703 uint32_t Lo = Lo_32(Value: APSplatBits.getZExtValue());
9704 SDValue SplatNode = DAG.getUNDEF(VT: MVT::v2i64);
9705
9706 if (!Hi || !Lo)
9707 // If either load is 0, then we should generate XXLXOR to set to 0.
9708 SplatNode = DAG.getTargetConstant(Val: 0, DL: dl, VT: MVT::v2i64);
9709
9710 if (Hi)
9711 SplatNode = DAG.getNode(
9712 Opcode: PPCISD::XXSPLTI32DX, DL: dl, VT: MVT::v2i64, N1: SplatNode,
9713 N2: DAG.getTargetConstant(Val: 0, DL: dl, VT: MVT::i32),
9714 N3: DAG.getTargetConstant(Val: Hi, DL: dl, VT: MVT::i32));
9715
9716 if (Lo)
9717 SplatNode =
9718 DAG.getNode(Opcode: PPCISD::XXSPLTI32DX, DL: dl, VT: MVT::v2i64, N1: SplatNode,
9719 N2: DAG.getTargetConstant(Val: 1, DL: dl, VT: MVT::i32),
9720 N3: DAG.getTargetConstant(Val: Lo, DL: dl, VT: MVT::i32));
9721
9722 return DAG.getBitcast(VT: Op.getValueType(), V: SplatNode);
9723 }
9724 }
9725
9726 if (SDValue V =
9727 LowerVecSplatSmallFP(Op, DAG, BVNIsConstantSplat, SplatBitSize))
9728 return V;
9729
9730 bool IsSplat64 = false;
9731 uint64_t SplatBits = 0;
9732 int32_t SextVal = 0;
9733 if (BVNIsConstantSplat && SplatBitSize <= 64) {
9734 SplatBits = APSplatBits.getZExtValue();
9735 if (SplatBitSize <= 32) {
9736 SextVal = SignExtend32(X: SplatBits, B: SplatBitSize);
9737 } else if (SplatBitSize == 64 && Subtarget.hasP8Altivec()) {
9738 int64_t Splat64Val = static_cast<int64_t>(SplatBits);
9739 bool P9Vector = Subtarget.hasP9Vector();
9740 int32_t Hi = P9Vector ? 127 : 15;
9741 int32_t Lo = P9Vector ? -128 : -16;
9742 IsSplat64 = Splat64Val >= Lo && Splat64Val <= Hi;
9743 SextVal = static_cast<int32_t>(SplatBits);
9744 }
9745 }
9746
9747 if (!BVNIsConstantSplat || (SplatBitSize > 32 && !IsSplat64)) {
9748 unsigned NewOpcode = PPCISD::LD_SPLAT;
9749
9750 // Handle load-and-splat patterns as we have instructions that will do this
9751 // in one go.
9752 if (DAG.isSplatValue(V: Op, AllowUndefs: true) &&
9753 isValidSplatLoad(Subtarget, Op, Opcode&: NewOpcode)) {
9754 const SDValue *InputLoad = &Op.getOperand(i: 0);
9755 LoadSDNode *LD = cast<LoadSDNode>(Val: *InputLoad);
9756
9757 // If the input load is an extending load, it will be an i32 -> i64
9758 // extending load and isValidSplatLoad() will update NewOpcode.
9759 unsigned MemorySize = LD->getMemoryVT().getScalarSizeInBits();
9760 unsigned ElementSize =
9761 MemorySize * ((NewOpcode == PPCISD::LD_SPLAT) ? 1 : 2);
9762
9763 assert(((ElementSize == 2 * MemorySize)
9764 ? (NewOpcode == PPCISD::ZEXT_LD_SPLAT ||
9765 NewOpcode == PPCISD::SEXT_LD_SPLAT)
9766 : (NewOpcode == PPCISD::LD_SPLAT)) &&
9767 "Unmatched element size and opcode!\n");
9768
9769 // Checking for a single use of this load, we have to check for vector
9770 // width (128 bits) / ElementSize uses (since each operand of the
9771 // BUILD_VECTOR is a separate use of the value.
9772 unsigned NumUsesOfInputLD = 128 / ElementSize;
9773 for (SDValue BVInOp : Op->ops())
9774 if (BVInOp.isUndef())
9775 NumUsesOfInputLD--;
9776
9777 // Exclude somes case where LD_SPLAT is worse than scalar_to_vector:
9778 // Below cases should also happen for "lfiwzx/lfiwax + LE target + index
9779 // 1" and "lxvrhx + BE target + index 7" and "lxvrbx + BE target + index
9780 // 15", but function IsValidSplatLoad() now will only return true when
9781 // the data at index 0 is not nullptr. So we will not get into trouble for
9782 // these cases.
9783 //
9784 // case 1 - lfiwzx/lfiwax
9785 // 1.1: load result is i32 and is sign/zero extend to i64;
9786 // 1.2: build a v2i64 vector type with above loaded value;
9787 // 1.3: the vector has only one value at index 0, others are all undef;
9788 // 1.4: on BE target, so that lfiwzx/lfiwax does not need any permute.
9789 if (NumUsesOfInputLD == 1 &&
9790 (Op->getValueType(ResNo: 0) == MVT::v2i64 && NewOpcode != PPCISD::LD_SPLAT &&
9791 !Subtarget.isLittleEndian() && Subtarget.hasVSX() &&
9792 Subtarget.hasLFIWAX()))
9793 return SDValue();
9794
9795 // case 2 - lxvr[hb]x
9796 // 2.1: load result is at most i16;
9797 // 2.2: build a vector with above loaded value;
9798 // 2.3: the vector has only one value at index 0, others are all undef;
9799 // 2.4: on LE target, so that lxvr[hb]x does not need any permute.
9800 if (NumUsesOfInputLD == 1 && Subtarget.isLittleEndian() &&
9801 Subtarget.isISA3_1() && ElementSize <= 16)
9802 return SDValue();
9803
9804 assert(NumUsesOfInputLD > 0 && "No uses of input LD of a build_vector?");
9805 if (InputLoad->getNode()->hasNUsesOfValue(NUses: NumUsesOfInputLD, Value: 0) &&
9806 Subtarget.hasVSX()) {
9807 SDValue Ops[] = {
9808 LD->getChain(), // Chain
9809 LD->getBasePtr(), // Ptr
9810 DAG.getValueType(Op.getValueType()) // VT
9811 };
9812 SDValue LdSplt = DAG.getMemIntrinsicNode(
9813 Opcode: NewOpcode, dl, VTList: DAG.getVTList(VT1: Op.getValueType(), VT2: MVT::Other), Ops,
9814 MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand());
9815 // Replace all uses of the output chain of the original load with the
9816 // output chain of the new load.
9817 DAG.ReplaceAllUsesOfValueWith(From: InputLoad->getValue(R: 1),
9818 To: LdSplt.getValue(R: 1));
9819 return LdSplt;
9820 }
9821 }
9822
9823 // In 64BIT mode BUILD_VECTOR nodes that are not constant splats of up to
9824 // 32-bits can be lowered to VSX instructions under certain conditions.
9825 // Without VSX, there is no pattern more efficient than expanding the node.
9826 if (Subtarget.hasVSX() && Subtarget.isPPC64() &&
9827 haveEfficientBuildVectorPattern(V: BVN, HasDirectMove: Subtarget.hasDirectMove(),
9828 HasP8Vector: Subtarget.hasP8Vector()))
9829 return Op;
9830 return SDValue();
9831 }
9832
9833 uint64_t SplatUndef = APSplatUndef.getZExtValue();
9834 unsigned SplatSize = SplatBitSize / 8;
9835
9836 // First, handle single instruction cases.
9837
9838 // All zeros?
9839 if (SplatBits == 0) {
9840 // Canonicalize all zero vectors to be v4i32.
9841 if (Op.getValueType() != MVT::v4i32 || HasAnyUndefs) {
9842 SDValue Z = DAG.getConstant(Val: 0, DL: dl, VT: MVT::v4i32);
9843 Op = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: Op.getValueType(), Operand: Z);
9844 }
9845 return Op;
9846 }
9847
9848 // We have XXSPLTIW for constant splats four bytes wide.
9849 // Given vector length is a multiple of 4, 2-byte splats can be replaced
9850 // with 4-byte splats. We replicate the SplatBits in case of 2-byte splat to
9851 // make a 4-byte splat element. For example: 2-byte splat of 0xABAB can be
9852 // turned into a 4-byte splat of 0xABABABAB.
9853 if (Subtarget.hasPrefixInstrs() && Subtarget.hasP10Vector() && SplatSize == 2)
9854 return getCanonicalConstSplat(Val: SplatBits | (SplatBits << 16), SplatSize: SplatSize * 2,
9855 VT: Op.getValueType(), DAG, dl);
9856
9857 if (Subtarget.hasPrefixInstrs() && Subtarget.hasP10Vector() && SplatSize == 4)
9858 return getCanonicalConstSplat(Val: SplatBits, SplatSize, VT: Op.getValueType(), DAG,
9859 dl);
9860
9861 // We have XXSPLTIB for constant splats one byte wide.
9862 if (Subtarget.hasP9Vector() && SplatSize == 1)
9863 return getCanonicalConstSplat(Val: SplatBits, SplatSize, VT: Op.getValueType(), DAG,
9864 dl);
9865
9866 // If the sign extended value is in the range [-16,15], use VSPLTI[bhw].
9867 // Use VSPLTIW/VUPKLSW for v2i64 in range [-16,15].
9868 if (SextVal >= -16 && SextVal <= 15) {
9869 // SplatSize may be 1, 2, 4, or 8. Use size 4 instead of 8 for the splat to
9870 // generate a splat word with extend for size 8.
9871 unsigned UseSize = SplatSize == 8 ? 4 : SplatSize;
9872 SDValue Res =
9873 getCanonicalConstSplat(Val: SextVal, SplatSize: UseSize, VT: Op.getValueType(), DAG, dl);
9874 if (SplatSize != 8)
9875 return Res;
9876 SDValue IntrinsicOp =
9877 BuildIntrinsicOp(IID: Intrinsic::ppc_altivec_vupklsw,
9878 Op: DAG.getBitcast(VT: MVT::v4i32, V: Res), DAG, dl, DestVT: MVT::v2i64);
9879 return DAG.getBitcast(VT: Op.getValueType(), V: IntrinsicOp);
9880 }
9881
9882 // Two instruction sequences.
9883
9884 if (Subtarget.hasP9Vector() && SextVal >= -128 && SextVal <= 127) {
9885 SDValue C = DAG.getConstant(Val: (unsigned char)SextVal, DL: dl, VT: MVT::i32);
9886 SmallVector<SDValue, 16> Ops(16, C);
9887 SDValue BV = DAG.getBuildVector(VT: MVT::v16i8, DL: dl, Ops);
9888 unsigned IID;
9889 EVT VT;
9890 switch (SplatSize) {
9891 default:
9892 llvm_unreachable("Unexpected type for vector constant.");
9893 case 2:
9894 IID = Intrinsic::ppc_altivec_vupklsb;
9895 VT = MVT::v8i16;
9896 break;
9897 case 4:
9898 IID = Intrinsic::ppc_altivec_vextsb2w;
9899 VT = MVT::v4i32;
9900 break;
9901 case 8:
9902 IID = Intrinsic::ppc_altivec_vextsb2d;
9903 VT = MVT::v2i64;
9904 break;
9905 }
9906 SDValue Extend = BuildIntrinsicOp(IID, Op: BV, DAG, dl, DestVT: VT);
9907 return DAG.getBitcast(VT: Op->getValueType(ResNo: 0), V: Extend);
9908 }
9909 assert(!IsSplat64 && "Unhandled 64-bit splat pattern");
9910
9911 // If this value is in the range [-32,30] and is even, use:
9912 // VSPLTI[bhw](val/2) + VSPLTI[bhw](val/2)
9913 // If this value is in the range [17,31] and is odd, use:
9914 // VSPLTI[bhw](val-16) - VSPLTI[bhw](-16)
9915 // If this value is in the range [-31,-17] and is odd, use:
9916 // VSPLTI[bhw](val+16) + VSPLTI[bhw](-16)
9917 // Note the last two are three-instruction sequences.
9918 if (SextVal >= -32 && SextVal <= 31) {
9919 // To avoid having these optimizations undone by constant folding,
9920 // we convert to a pseudo that will be expanded later into one of
9921 // the above forms.
9922 SDValue Elt = DAG.getSignedConstant(Val: SextVal, DL: dl, VT: MVT::i32);
9923 EVT VT = (SplatSize == 1 ? MVT::v16i8 :
9924 (SplatSize == 2 ? MVT::v8i16 : MVT::v4i32));
9925 SDValue EltSize = DAG.getConstant(Val: SplatSize, DL: dl, VT: MVT::i32);
9926 SDValue RetVal = DAG.getNode(Opcode: PPCISD::VADD_SPLAT, DL: dl, VT, N1: Elt, N2: EltSize);
9927 if (VT == Op.getValueType())
9928 return RetVal;
9929 else
9930 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: Op.getValueType(), Operand: RetVal);
9931 }
9932
9933 // If this is 0x8000_0000 x 4, turn into vspltisw + vslw. If it is
9934 // 0x7FFF_FFFF x 4, turn it into not(0x8000_0000). This is important
9935 // for fneg/fabs.
9936 if (SplatSize == 4 && SplatBits == (0x7FFFFFFF&~SplatUndef)) {
9937 // Make -1 and vspltisw -1:
9938 SDValue OnesV = getCanonicalConstSplat(Val: -1, SplatSize: 4, VT: MVT::v4i32, DAG, dl);
9939
9940 // Make the VSLW intrinsic, computing 0x8000_0000.
9941 SDValue Res = BuildIntrinsicOp(IID: Intrinsic::ppc_altivec_vslw, LHS: OnesV,
9942 RHS: OnesV, DAG, dl);
9943
9944 // xor by OnesV to invert it.
9945 Res = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: MVT::v4i32, N1: Res, N2: OnesV);
9946 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: Op.getValueType(), Operand: Res);
9947 }
9948
9949 // Check to see if this is a wide variety of vsplti*, binop self cases.
9950 static const signed char SplatCsts[] = {
9951 -1, 1, -2, 2, -3, 3, -4, 4, -5, 5, -6, 6, -7, 7,
9952 -8, 8, -9, 9, -10, 10, -11, 11, -12, 12, -13, 13, 14, -14, 15, -15, -16
9953 };
9954
9955 for (unsigned idx = 0; idx < std::size(SplatCsts); ++idx) {
9956 // Indirect through the SplatCsts array so that we favor 'vsplti -1' for
9957 // cases which are ambiguous (e.g. formation of 0x8000_0000). 'vsplti -1'
9958 int i = SplatCsts[idx];
9959
9960 // Figure out what shift amount will be used by altivec if shifted by i in
9961 // this splat size.
9962 unsigned TypeShiftAmt = i & (SplatBitSize-1);
9963
9964 // vsplti + shl self.
9965 if (SextVal == (int)((unsigned)i << TypeShiftAmt)) {
9966 SDValue Res = getCanonicalConstSplat(Val: i, SplatSize, VT: MVT::Other, DAG, dl);
9967 static const unsigned IIDs[] = { // Intrinsic to use for each size.
9968 Intrinsic::ppc_altivec_vslb, Intrinsic::ppc_altivec_vslh, 0,
9969 Intrinsic::ppc_altivec_vslw
9970 };
9971 Res = BuildIntrinsicOp(IID: IIDs[SplatSize-1], LHS: Res, RHS: Res, DAG, dl);
9972 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: Op.getValueType(), Operand: Res);
9973 }
9974
9975 // vsplti + srl self.
9976 if (SextVal == (int)((unsigned)i >> TypeShiftAmt)) {
9977 SDValue Res = getCanonicalConstSplat(Val: i, SplatSize, VT: MVT::Other, DAG, dl);
9978 static const unsigned IIDs[] = { // Intrinsic to use for each size.
9979 Intrinsic::ppc_altivec_vsrb, Intrinsic::ppc_altivec_vsrh, 0,
9980 Intrinsic::ppc_altivec_vsrw
9981 };
9982 Res = BuildIntrinsicOp(IID: IIDs[SplatSize-1], LHS: Res, RHS: Res, DAG, dl);
9983 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: Op.getValueType(), Operand: Res);
9984 }
9985
9986 // vsplti + rol self.
9987 if (SextVal == (int)(((unsigned)i << TypeShiftAmt) |
9988 ((unsigned)i >> (SplatBitSize-TypeShiftAmt)))) {
9989 SDValue Res = getCanonicalConstSplat(Val: i, SplatSize, VT: MVT::Other, DAG, dl);
9990 static const unsigned IIDs[] = { // Intrinsic to use for each size.
9991 Intrinsic::ppc_altivec_vrlb, Intrinsic::ppc_altivec_vrlh, 0,
9992 Intrinsic::ppc_altivec_vrlw
9993 };
9994 Res = BuildIntrinsicOp(IID: IIDs[SplatSize-1], LHS: Res, RHS: Res, DAG, dl);
9995 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: Op.getValueType(), Operand: Res);
9996 }
9997
9998 // t = vsplti c, result = vsldoi t, t, 1
9999 if (SextVal == (int)(((unsigned)i << 8) | (i < 0 ? 0xFF : 0))) {
10000 SDValue T = getCanonicalConstSplat(Val: i, SplatSize, VT: MVT::v16i8, DAG, dl);
10001 unsigned Amt = Subtarget.isLittleEndian() ? 15 : 1;
10002 return BuildVSLDOI(LHS: T, RHS: T, Amt, VT: Op.getValueType(), DAG, dl);
10003 }
10004 // t = vsplti c, result = vsldoi t, t, 2
10005 if (SextVal == (int)(((unsigned)i << 16) | (i < 0 ? 0xFFFF : 0))) {
10006 SDValue T = getCanonicalConstSplat(Val: i, SplatSize, VT: MVT::v16i8, DAG, dl);
10007 unsigned Amt = Subtarget.isLittleEndian() ? 14 : 2;
10008 return BuildVSLDOI(LHS: T, RHS: T, Amt, VT: Op.getValueType(), DAG, dl);
10009 }
10010 // t = vsplti c, result = vsldoi t, t, 3
10011 if (SextVal == (int)(((unsigned)i << 24) | (i < 0 ? 0xFFFFFF : 0))) {
10012 SDValue T = getCanonicalConstSplat(Val: i, SplatSize, VT: MVT::v16i8, DAG, dl);
10013 unsigned Amt = Subtarget.isLittleEndian() ? 13 : 3;
10014 return BuildVSLDOI(LHS: T, RHS: T, Amt, VT: Op.getValueType(), DAG, dl);
10015 }
10016 }
10017
10018 return SDValue();
10019}
10020
10021/// GeneratePerfectShuffle - Given an entry in the perfect-shuffle table, emit
10022/// the specified operations to build the shuffle.
10023static SDValue GeneratePerfectShuffle(unsigned PFEntry, SDValue LHS,
10024 SDValue RHS, SelectionDAG &DAG,
10025 const SDLoc &dl) {
10026 unsigned OpNum = (PFEntry >> 26) & 0x0F;
10027 unsigned LHSID = (PFEntry >> 13) & ((1 << 13)-1);
10028 unsigned RHSID = (PFEntry >> 0) & ((1 << 13)-1);
10029
10030 enum {
10031 OP_COPY = 0, // Copy, used for things like <u,u,u,3> to say it is <0,1,2,3>
10032 OP_VMRGHW,
10033 OP_VMRGLW,
10034 OP_VSPLTISW0,
10035 OP_VSPLTISW1,
10036 OP_VSPLTISW2,
10037 OP_VSPLTISW3,
10038 OP_VSLDOI4,
10039 OP_VSLDOI8,
10040 OP_VSLDOI12
10041 };
10042
10043 if (OpNum == OP_COPY) {
10044 if (LHSID == (1*9+2)*9+3) return LHS;
10045 assert(LHSID == ((4*9+5)*9+6)*9+7 && "Illegal OP_COPY!");
10046 return RHS;
10047 }
10048
10049 SDValue OpLHS, OpRHS;
10050 OpLHS = GeneratePerfectShuffle(PFEntry: PerfectShuffleTable[LHSID], LHS, RHS, DAG, dl);
10051 OpRHS = GeneratePerfectShuffle(PFEntry: PerfectShuffleTable[RHSID], LHS, RHS, DAG, dl);
10052
10053 int ShufIdxs[16];
10054 switch (OpNum) {
10055 default: llvm_unreachable("Unknown i32 permute!");
10056 case OP_VMRGHW:
10057 ShufIdxs[ 0] = 0; ShufIdxs[ 1] = 1; ShufIdxs[ 2] = 2; ShufIdxs[ 3] = 3;
10058 ShufIdxs[ 4] = 16; ShufIdxs[ 5] = 17; ShufIdxs[ 6] = 18; ShufIdxs[ 7] = 19;
10059 ShufIdxs[ 8] = 4; ShufIdxs[ 9] = 5; ShufIdxs[10] = 6; ShufIdxs[11] = 7;
10060 ShufIdxs[12] = 20; ShufIdxs[13] = 21; ShufIdxs[14] = 22; ShufIdxs[15] = 23;
10061 break;
10062 case OP_VMRGLW:
10063 ShufIdxs[ 0] = 8; ShufIdxs[ 1] = 9; ShufIdxs[ 2] = 10; ShufIdxs[ 3] = 11;
10064 ShufIdxs[ 4] = 24; ShufIdxs[ 5] = 25; ShufIdxs[ 6] = 26; ShufIdxs[ 7] = 27;
10065 ShufIdxs[ 8] = 12; ShufIdxs[ 9] = 13; ShufIdxs[10] = 14; ShufIdxs[11] = 15;
10066 ShufIdxs[12] = 28; ShufIdxs[13] = 29; ShufIdxs[14] = 30; ShufIdxs[15] = 31;
10067 break;
10068 case OP_VSPLTISW0:
10069 for (unsigned i = 0; i != 16; ++i)
10070 ShufIdxs[i] = (i&3)+0;
10071 break;
10072 case OP_VSPLTISW1:
10073 for (unsigned i = 0; i != 16; ++i)
10074 ShufIdxs[i] = (i&3)+4;
10075 break;
10076 case OP_VSPLTISW2:
10077 for (unsigned i = 0; i != 16; ++i)
10078 ShufIdxs[i] = (i&3)+8;
10079 break;
10080 case OP_VSPLTISW3:
10081 for (unsigned i = 0; i != 16; ++i)
10082 ShufIdxs[i] = (i&3)+12;
10083 break;
10084 case OP_VSLDOI4:
10085 return BuildVSLDOI(LHS: OpLHS, RHS: OpRHS, Amt: 4, VT: OpLHS.getValueType(), DAG, dl);
10086 case OP_VSLDOI8:
10087 return BuildVSLDOI(LHS: OpLHS, RHS: OpRHS, Amt: 8, VT: OpLHS.getValueType(), DAG, dl);
10088 case OP_VSLDOI12:
10089 return BuildVSLDOI(LHS: OpLHS, RHS: OpRHS, Amt: 12, VT: OpLHS.getValueType(), DAG, dl);
10090 }
10091 EVT VT = OpLHS.getValueType();
10092 OpLHS = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: OpLHS);
10093 OpRHS = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: OpRHS);
10094 SDValue T = DAG.getVectorShuffle(VT: MVT::v16i8, dl, N1: OpLHS, N2: OpRHS, Mask: ShufIdxs);
10095 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: T);
10096}
10097
10098/// lowerToVINSERTB - Return the SDValue if this VECTOR_SHUFFLE can be handled
10099/// by the VINSERTB instruction introduced in ISA 3.0, else just return default
10100/// SDValue.
10101SDValue PPCTargetLowering::lowerToVINSERTB(ShuffleVectorSDNode *N,
10102 SelectionDAG &DAG) const {
10103 const unsigned BytesInVector = 16;
10104 bool IsLE = Subtarget.isLittleEndian();
10105 SDLoc dl(N);
10106 SDValue V1 = N->getOperand(Num: 0);
10107 SDValue V2 = N->getOperand(Num: 1);
10108 unsigned ShiftElts = 0, InsertAtByte = 0;
10109 bool Swap = false;
10110
10111 // Shifts required to get the byte we want at element 7.
10112 unsigned LittleEndianShifts[] = {8, 7, 6, 5, 4, 3, 2, 1,
10113 0, 15, 14, 13, 12, 11, 10, 9};
10114 unsigned BigEndianShifts[] = {9, 10, 11, 12, 13, 14, 15, 0,
10115 1, 2, 3, 4, 5, 6, 7, 8};
10116
10117 ArrayRef<int> Mask = N->getMask();
10118 int OriginalOrder[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
10119
10120 // For each mask element, find out if we're just inserting something
10121 // from V2 into V1 or vice versa.
10122 // Possible permutations inserting an element from V2 into V1:
10123 // X, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
10124 // 0, X, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
10125 // ...
10126 // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, X
10127 // Inserting from V1 into V2 will be similar, except mask range will be
10128 // [16,31].
10129
10130 bool FoundCandidate = false;
10131 // If both vector operands for the shuffle are the same vector, the mask
10132 // will contain only elements from the first one and the second one will be
10133 // undef.
10134 unsigned VINSERTBSrcElem = IsLE ? 8 : 7;
10135 // Go through the mask of half-words to find an element that's being moved
10136 // from one vector to the other.
10137 for (unsigned i = 0; i < BytesInVector; ++i) {
10138 unsigned CurrentElement = Mask[i];
10139 // If 2nd operand is undefined, we should only look for element 7 in the
10140 // Mask.
10141 if (V2.isUndef() && CurrentElement != VINSERTBSrcElem)
10142 continue;
10143
10144 bool OtherElementsInOrder = true;
10145 // Examine the other elements in the Mask to see if they're in original
10146 // order.
10147 for (unsigned j = 0; j < BytesInVector; ++j) {
10148 if (j == i)
10149 continue;
10150 // If CurrentElement is from V1 [0,15], then we the rest of the Mask to be
10151 // from V2 [16,31] and vice versa. Unless the 2nd operand is undefined,
10152 // in which we always assume we're always picking from the 1st operand.
10153 int MaskOffset =
10154 (!V2.isUndef() && CurrentElement < BytesInVector) ? BytesInVector : 0;
10155 if (Mask[j] != OriginalOrder[j] + MaskOffset) {
10156 OtherElementsInOrder = false;
10157 break;
10158 }
10159 }
10160 // If other elements are in original order, we record the number of shifts
10161 // we need to get the element we want into element 7. Also record which byte
10162 // in the vector we should insert into.
10163 if (OtherElementsInOrder) {
10164 // If 2nd operand is undefined, we assume no shifts and no swapping.
10165 if (V2.isUndef()) {
10166 ShiftElts = 0;
10167 Swap = false;
10168 } else {
10169 // Only need the last 4-bits for shifts because operands will be swapped if CurrentElement is >= 2^4.
10170 ShiftElts = IsLE ? LittleEndianShifts[CurrentElement & 0xF]
10171 : BigEndianShifts[CurrentElement & 0xF];
10172 Swap = CurrentElement < BytesInVector;
10173 }
10174 InsertAtByte = IsLE ? BytesInVector - (i + 1) : i;
10175 FoundCandidate = true;
10176 break;
10177 }
10178 }
10179
10180 if (!FoundCandidate)
10181 return SDValue();
10182
10183 // Candidate found, construct the proper SDAG sequence with VINSERTB,
10184 // optionally with VECSHL if shift is required.
10185 if (Swap)
10186 std::swap(a&: V1, b&: V2);
10187 if (V2.isUndef())
10188 V2 = V1;
10189 if (ShiftElts) {
10190 SDValue Shl = DAG.getNode(Opcode: PPCISD::VECSHL, DL: dl, VT: MVT::v16i8, N1: V2, N2: V2,
10191 N3: DAG.getConstant(Val: ShiftElts, DL: dl, VT: MVT::i32));
10192 return DAG.getNode(Opcode: PPCISD::VECINSERT, DL: dl, VT: MVT::v16i8, N1: V1, N2: Shl,
10193 N3: DAG.getConstant(Val: InsertAtByte, DL: dl, VT: MVT::i32));
10194 }
10195 return DAG.getNode(Opcode: PPCISD::VECINSERT, DL: dl, VT: MVT::v16i8, N1: V1, N2: V2,
10196 N3: DAG.getConstant(Val: InsertAtByte, DL: dl, VT: MVT::i32));
10197}
10198
10199/// lowerToVINSERTH - Return the SDValue if this VECTOR_SHUFFLE can be handled
10200/// by the VINSERTH instruction introduced in ISA 3.0, else just return default
10201/// SDValue.
10202SDValue PPCTargetLowering::lowerToVINSERTH(ShuffleVectorSDNode *N,
10203 SelectionDAG &DAG) const {
10204 const unsigned NumHalfWords = 8;
10205 const unsigned BytesInVector = NumHalfWords * 2;
10206 // Check that the shuffle is on half-words.
10207 if (!isNByteElemShuffleMask(N, Width: 2, StepLen: 1))
10208 return SDValue();
10209
10210 bool IsLE = Subtarget.isLittleEndian();
10211 SDLoc dl(N);
10212 SDValue V1 = N->getOperand(Num: 0);
10213 SDValue V2 = N->getOperand(Num: 1);
10214 unsigned ShiftElts = 0, InsertAtByte = 0;
10215 bool Swap = false;
10216
10217 // Shifts required to get the half-word we want at element 3.
10218 unsigned LittleEndianShifts[] = {4, 3, 2, 1, 0, 7, 6, 5};
10219 unsigned BigEndianShifts[] = {5, 6, 7, 0, 1, 2, 3, 4};
10220
10221 uint32_t Mask = 0;
10222 uint32_t OriginalOrderLow = 0x1234567;
10223 uint32_t OriginalOrderHigh = 0x89ABCDEF;
10224 // Now we look at mask elements 0,2,4,6,8,10,12,14. Pack the mask into a
10225 // 32-bit space, only need 4-bit nibbles per element.
10226 for (unsigned i = 0; i < NumHalfWords; ++i) {
10227 unsigned MaskShift = (NumHalfWords - 1 - i) * 4;
10228 Mask |= ((uint32_t)(N->getMaskElt(Idx: i * 2) / 2) << MaskShift);
10229 }
10230
10231 // For each mask element, find out if we're just inserting something
10232 // from V2 into V1 or vice versa. Possible permutations inserting an element
10233 // from V2 into V1:
10234 // X, 1, 2, 3, 4, 5, 6, 7
10235 // 0, X, 2, 3, 4, 5, 6, 7
10236 // 0, 1, X, 3, 4, 5, 6, 7
10237 // 0, 1, 2, X, 4, 5, 6, 7
10238 // 0, 1, 2, 3, X, 5, 6, 7
10239 // 0, 1, 2, 3, 4, X, 6, 7
10240 // 0, 1, 2, 3, 4, 5, X, 7
10241 // 0, 1, 2, 3, 4, 5, 6, X
10242 // Inserting from V1 into V2 will be similar, except mask range will be [8,15].
10243
10244 bool FoundCandidate = false;
10245 // Go through the mask of half-words to find an element that's being moved
10246 // from one vector to the other.
10247 for (unsigned i = 0; i < NumHalfWords; ++i) {
10248 unsigned MaskShift = (NumHalfWords - 1 - i) * 4;
10249 uint32_t MaskOneElt = (Mask >> MaskShift) & 0xF;
10250 uint32_t MaskOtherElts = ~(0xF << MaskShift);
10251 uint32_t TargetOrder = 0x0;
10252
10253 // If both vector operands for the shuffle are the same vector, the mask
10254 // will contain only elements from the first one and the second one will be
10255 // undef.
10256 if (V2.isUndef()) {
10257 ShiftElts = 0;
10258 unsigned VINSERTHSrcElem = IsLE ? 4 : 3;
10259 TargetOrder = OriginalOrderLow;
10260 Swap = false;
10261 // Skip if not the correct element or mask of other elements don't equal
10262 // to our expected order.
10263 if (MaskOneElt == VINSERTHSrcElem &&
10264 (Mask & MaskOtherElts) == (TargetOrder & MaskOtherElts)) {
10265 InsertAtByte = IsLE ? BytesInVector - (i + 1) * 2 : i * 2;
10266 FoundCandidate = true;
10267 break;
10268 }
10269 } else { // If both operands are defined.
10270 // Target order is [8,15] if the current mask is between [0,7].
10271 TargetOrder =
10272 (MaskOneElt < NumHalfWords) ? OriginalOrderHigh : OriginalOrderLow;
10273 // Skip if mask of other elements don't equal our expected order.
10274 if ((Mask & MaskOtherElts) == (TargetOrder & MaskOtherElts)) {
10275 // We only need the last 3 bits for the number of shifts.
10276 ShiftElts = IsLE ? LittleEndianShifts[MaskOneElt & 0x7]
10277 : BigEndianShifts[MaskOneElt & 0x7];
10278 InsertAtByte = IsLE ? BytesInVector - (i + 1) * 2 : i * 2;
10279 Swap = MaskOneElt < NumHalfWords;
10280 FoundCandidate = true;
10281 break;
10282 }
10283 }
10284 }
10285
10286 if (!FoundCandidate)
10287 return SDValue();
10288
10289 // Candidate found, construct the proper SDAG sequence with VINSERTH,
10290 // optionally with VECSHL if shift is required.
10291 if (Swap)
10292 std::swap(a&: V1, b&: V2);
10293 if (V2.isUndef())
10294 V2 = V1;
10295 SDValue Conv1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v8i16, Operand: V1);
10296 if (ShiftElts) {
10297 // Double ShiftElts because we're left shifting on v16i8 type.
10298 SDValue Shl = DAG.getNode(Opcode: PPCISD::VECSHL, DL: dl, VT: MVT::v16i8, N1: V2, N2: V2,
10299 N3: DAG.getConstant(Val: 2 * ShiftElts, DL: dl, VT: MVT::i32));
10300 SDValue Conv2 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v8i16, Operand: Shl);
10301 SDValue Ins = DAG.getNode(Opcode: PPCISD::VECINSERT, DL: dl, VT: MVT::v8i16, N1: Conv1, N2: Conv2,
10302 N3: DAG.getConstant(Val: InsertAtByte, DL: dl, VT: MVT::i32));
10303 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: Ins);
10304 }
10305 SDValue Conv2 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v8i16, Operand: V2);
10306 SDValue Ins = DAG.getNode(Opcode: PPCISD::VECINSERT, DL: dl, VT: MVT::v8i16, N1: Conv1, N2: Conv2,
10307 N3: DAG.getConstant(Val: InsertAtByte, DL: dl, VT: MVT::i32));
10308 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: Ins);
10309}
10310
10311/// lowerToXXSPLTI32DX - Return the SDValue if this VECTOR_SHUFFLE can be
10312/// handled by the XXSPLTI32DX instruction introduced in ISA 3.1, otherwise
10313/// return the default SDValue.
10314SDValue PPCTargetLowering::lowerToXXSPLTI32DX(ShuffleVectorSDNode *SVN,
10315 SelectionDAG &DAG) const {
10316 // The LHS and RHS may be bitcasts to v16i8 as we canonicalize shuffles
10317 // to v16i8. Peek through the bitcasts to get the actual operands.
10318 SDValue LHS = peekThroughBitcasts(V: SVN->getOperand(Num: 0));
10319 SDValue RHS = peekThroughBitcasts(V: SVN->getOperand(Num: 1));
10320
10321 auto ShuffleMask = SVN->getMask();
10322 SDValue VecShuffle(SVN, 0);
10323 SDLoc DL(SVN);
10324
10325 // Check that we have a four byte shuffle.
10326 if (!isNByteElemShuffleMask(N: SVN, Width: 4, StepLen: 1))
10327 return SDValue();
10328
10329 // Canonicalize the RHS being a BUILD_VECTOR when lowering to xxsplti32dx.
10330 if (RHS->getOpcode() != ISD::BUILD_VECTOR) {
10331 std::swap(a&: LHS, b&: RHS);
10332 VecShuffle = peekThroughBitcasts(V: DAG.getCommutedVectorShuffle(SV: *SVN));
10333 ShuffleVectorSDNode *CommutedSV = dyn_cast<ShuffleVectorSDNode>(Val&: VecShuffle);
10334 if (!CommutedSV)
10335 return SDValue();
10336 ShuffleMask = CommutedSV->getMask();
10337 }
10338
10339 // Ensure that the RHS is a vector of constants.
10340 BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(Val: RHS.getNode());
10341 if (!BVN)
10342 return SDValue();
10343
10344 // Check if RHS is a splat of 4-bytes (or smaller).
10345 APInt APSplatValue, APSplatUndef;
10346 unsigned SplatBitSize;
10347 bool HasAnyUndefs;
10348 if (!BVN->isConstantSplat(SplatValue&: APSplatValue, SplatUndef&: APSplatUndef, SplatBitSize,
10349 HasAnyUndefs, MinSplatBits: 0, isBigEndian: !Subtarget.isLittleEndian()) ||
10350 SplatBitSize > 32)
10351 return SDValue();
10352
10353 // Check that the shuffle mask matches the semantics of XXSPLTI32DX.
10354 // The instruction splats a constant C into two words of the source vector
10355 // producing { C, Unchanged, C, Unchanged } or { Unchanged, C, Unchanged, C }.
10356 // Thus we check that the shuffle mask is the equivalent of
10357 // <0, [4-7], 2, [4-7]> or <[4-7], 1, [4-7], 3> respectively.
10358 // Note: the check above of isNByteElemShuffleMask() ensures that the bytes
10359 // within each word are consecutive, so we only need to check the first byte.
10360 SDValue Index;
10361 bool IsLE = Subtarget.isLittleEndian();
10362 if ((ShuffleMask[0] == 0 && ShuffleMask[8] == 8) &&
10363 (ShuffleMask[4] % 4 == 0 && ShuffleMask[12] % 4 == 0 &&
10364 ShuffleMask[4] > 15 && ShuffleMask[12] > 15))
10365 Index = DAG.getTargetConstant(Val: IsLE ? 0 : 1, DL, VT: MVT::i32);
10366 else if ((ShuffleMask[4] == 4 && ShuffleMask[12] == 12) &&
10367 (ShuffleMask[0] % 4 == 0 && ShuffleMask[8] % 4 == 0 &&
10368 ShuffleMask[0] > 15 && ShuffleMask[8] > 15))
10369 Index = DAG.getTargetConstant(Val: IsLE ? 1 : 0, DL, VT: MVT::i32);
10370 else
10371 return SDValue();
10372
10373 // If the splat is narrower than 32-bits, we need to get the 32-bit value
10374 // for XXSPLTI32DX.
10375 unsigned SplatVal = APSplatValue.getZExtValue();
10376 for (; SplatBitSize < 32; SplatBitSize <<= 1)
10377 SplatVal |= (SplatVal << SplatBitSize);
10378
10379 SDValue SplatNode = DAG.getNode(
10380 Opcode: PPCISD::XXSPLTI32DX, DL, VT: MVT::v2i64, N1: DAG.getBitcast(VT: MVT::v2i64, V: LHS),
10381 N2: Index, N3: DAG.getTargetConstant(Val: SplatVal, DL, VT: MVT::i32));
10382 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v16i8, Operand: SplatNode);
10383}
10384
10385/// LowerROTL - Custom lowering for ROTL(v1i128) to vector_shuffle(v16i8).
10386/// We lower ROTL(v1i128) to vector_shuffle(v16i8) only if shift amount is
10387/// a multiple of 8. Otherwise convert it to a scalar rotation(i128)
10388/// i.e (or (shl x, C1), (srl x, 128-C1)).
10389SDValue PPCTargetLowering::LowerROTL(SDValue Op, SelectionDAG &DAG) const {
10390 assert(Op.getOpcode() == ISD::ROTL && "Should only be called for ISD::ROTL");
10391 assert(Op.getValueType() == MVT::v1i128 &&
10392 "Only set v1i128 as custom, other type shouldn't reach here!");
10393 SDLoc dl(Op);
10394 SDValue N0 = peekThroughBitcasts(V: Op.getOperand(i: 0));
10395 SDValue N1 = peekThroughBitcasts(V: Op.getOperand(i: 1));
10396 unsigned SHLAmt = N1.getConstantOperandVal(i: 0);
10397 if (SHLAmt % 8 == 0) {
10398 std::array<int, 16> Mask;
10399 std::iota(first: Mask.begin(), last: Mask.end(), value: 0);
10400 std::rotate(first: Mask.begin(), middle: Mask.begin() + SHLAmt / 8, last: Mask.end());
10401 if (SDValue Shuffle =
10402 DAG.getVectorShuffle(VT: MVT::v16i8, dl, N1: DAG.getBitcast(VT: MVT::v16i8, V: N0),
10403 N2: DAG.getUNDEF(VT: MVT::v16i8), Mask))
10404 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v1i128, Operand: Shuffle);
10405 }
10406 SDValue ArgVal = DAG.getBitcast(VT: MVT::i128, V: N0);
10407 SDValue SHLOp = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: MVT::i128, N1: ArgVal,
10408 N2: DAG.getConstant(Val: SHLAmt, DL: dl, VT: MVT::i32));
10409 SDValue SRLOp = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i128, N1: ArgVal,
10410 N2: DAG.getConstant(Val: 128 - SHLAmt, DL: dl, VT: MVT::i32));
10411 SDValue OROp = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: MVT::i128, N1: SHLOp, N2: SRLOp);
10412 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v1i128, Operand: OROp);
10413}
10414
10415/// LowerVECTOR_SHUFFLE - Return the code we lower for VECTOR_SHUFFLE. If this
10416/// is a shuffle we can handle in a single instruction, return it. Otherwise,
10417/// return the code it can be lowered into. Worst case, it can always be
10418/// lowered into a vperm.
10419SDValue PPCTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
10420 SelectionDAG &DAG) const {
10421 SDLoc dl(Op);
10422 SDValue V1 = Op.getOperand(i: 0);
10423 SDValue V2 = Op.getOperand(i: 1);
10424 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Val&: Op);
10425
10426 // Any nodes that were combined in the target-independent combiner prior
10427 // to vector legalization will not be sent to the target combine. Try to
10428 // combine it here.
10429 if (SDValue NewShuffle = combineVectorShuffle(SVN: SVOp, DAG)) {
10430 if (!isa<ShuffleVectorSDNode>(Val: NewShuffle))
10431 return NewShuffle;
10432 Op = NewShuffle;
10433 SVOp = cast<ShuffleVectorSDNode>(Val&: Op);
10434 V1 = Op.getOperand(i: 0);
10435 V2 = Op.getOperand(i: 1);
10436 }
10437 EVT VT = Op.getValueType();
10438 bool isLittleEndian = Subtarget.isLittleEndian();
10439
10440 unsigned ShiftElts, InsertAtByte;
10441 bool Swap = false;
10442
10443 // If this is a load-and-splat, we can do that with a single instruction
10444 // in some cases. However if the load has multiple uses, we don't want to
10445 // combine it because that will just produce multiple loads.
10446 bool IsPermutedLoad = false;
10447 const SDValue *InputLoad = getNormalLoadInput(Op: V1, IsPermuted&: IsPermutedLoad);
10448 if (InputLoad && Subtarget.hasVSX() && V2.isUndef() &&
10449 (PPC::isSplatShuffleMask(N: SVOp, EltSize: 4) || PPC::isSplatShuffleMask(N: SVOp, EltSize: 8)) &&
10450 InputLoad->hasOneUse()) {
10451 bool IsFourByte = PPC::isSplatShuffleMask(N: SVOp, EltSize: 4);
10452 int SplatIdx =
10453 PPC::getSplatIdxForPPCMnemonics(N: SVOp, EltSize: IsFourByte ? 4 : 8, DAG);
10454
10455 // The splat index for permuted loads will be in the left half of the vector
10456 // which is strictly wider than the loaded value by 8 bytes. So we need to
10457 // adjust the splat index to point to the correct address in memory.
10458 if (IsPermutedLoad) {
10459 assert((isLittleEndian || IsFourByte) &&
10460 "Unexpected size for permuted load on big endian target");
10461 SplatIdx += IsFourByte ? 2 : 1;
10462 assert((SplatIdx < (IsFourByte ? 4 : 2)) &&
10463 "Splat of a value outside of the loaded memory");
10464 }
10465
10466 LoadSDNode *LD = cast<LoadSDNode>(Val: *InputLoad);
10467 // For 4-byte load-and-splat, we need Power9.
10468 if ((IsFourByte && Subtarget.hasP9Vector()) || !IsFourByte) {
10469 uint64_t Offset = 0;
10470 if (IsFourByte)
10471 Offset = isLittleEndian ? (3 - SplatIdx) * 4 : SplatIdx * 4;
10472 else
10473 Offset = isLittleEndian ? (1 - SplatIdx) * 8 : SplatIdx * 8;
10474
10475 // If the width of the load is the same as the width of the splat,
10476 // loading with an offset would load the wrong memory.
10477 if (LD->getValueType(ResNo: 0).getSizeInBits() == (IsFourByte ? 32 : 64))
10478 Offset = 0;
10479
10480 SDValue BasePtr = LD->getBasePtr();
10481 if (Offset != 0)
10482 BasePtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: getPointerTy(DL: DAG.getDataLayout()),
10483 N1: BasePtr, N2: DAG.getIntPtrConstant(Val: Offset, DL: dl));
10484 SDValue Ops[] = {
10485 LD->getChain(), // Chain
10486 BasePtr, // BasePtr
10487 DAG.getValueType(Op.getValueType()) // VT
10488 };
10489 SDVTList VTL =
10490 DAG.getVTList(VT1: IsFourByte ? MVT::v4i32 : MVT::v2i64, VT2: MVT::Other);
10491 SDValue LdSplt =
10492 DAG.getMemIntrinsicNode(Opcode: PPCISD::LD_SPLAT, dl, VTList: VTL,
10493 Ops, MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand());
10494 DAG.ReplaceAllUsesOfValueWith(From: InputLoad->getValue(R: 1), To: LdSplt.getValue(R: 1));
10495 if (LdSplt.getValueType() != SVOp->getValueType(ResNo: 0))
10496 LdSplt = DAG.getBitcast(VT: SVOp->getValueType(ResNo: 0), V: LdSplt);
10497 return LdSplt;
10498 }
10499 }
10500
10501 // All v2i64 and v2f64 shuffles are legal
10502 if (VT == MVT::v2i64 || VT == MVT::v2f64)
10503 return Op;
10504
10505 if (Subtarget.hasP9Vector() &&
10506 PPC::isXXINSERTWMask(N: SVOp, ShiftElts, InsertAtByte, Swap,
10507 IsLE: isLittleEndian)) {
10508 if (V2.isUndef())
10509 V2 = V1;
10510 else if (Swap)
10511 std::swap(a&: V1, b&: V2);
10512 SDValue Conv1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v4i32, Operand: V1);
10513 SDValue Conv2 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v4i32, Operand: V2);
10514 if (ShiftElts) {
10515 SDValue Shl = DAG.getNode(Opcode: PPCISD::VECSHL, DL: dl, VT: MVT::v4i32, N1: Conv2, N2: Conv2,
10516 N3: DAG.getConstant(Val: ShiftElts, DL: dl, VT: MVT::i32));
10517 SDValue Ins = DAG.getNode(Opcode: PPCISD::VECINSERT, DL: dl, VT: MVT::v4i32, N1: Conv1, N2: Shl,
10518 N3: DAG.getConstant(Val: InsertAtByte, DL: dl, VT: MVT::i32));
10519 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: Ins);
10520 }
10521 SDValue Ins = DAG.getNode(Opcode: PPCISD::VECINSERT, DL: dl, VT: MVT::v4i32, N1: Conv1, N2: Conv2,
10522 N3: DAG.getConstant(Val: InsertAtByte, DL: dl, VT: MVT::i32));
10523 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: Ins);
10524 }
10525
10526 if (Subtarget.hasPrefixInstrs() && Subtarget.hasP10Vector()) {
10527 SDValue SplatInsertNode;
10528 if ((SplatInsertNode = lowerToXXSPLTI32DX(SVN: SVOp, DAG)))
10529 return SplatInsertNode;
10530 }
10531
10532 if (Subtarget.hasP9Altivec()) {
10533 SDValue NewISDNode;
10534 if ((NewISDNode = lowerToVINSERTH(N: SVOp, DAG)))
10535 return NewISDNode;
10536
10537 if ((NewISDNode = lowerToVINSERTB(N: SVOp, DAG)))
10538 return NewISDNode;
10539 }
10540
10541 if (Subtarget.hasVSX() &&
10542 PPC::isXXSLDWIShuffleMask(N: SVOp, ShiftElts, Swap, IsLE: isLittleEndian)) {
10543 if (Swap)
10544 std::swap(a&: V1, b&: V2);
10545 SDValue Conv1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v4i32, Operand: V1);
10546 SDValue Conv2 =
10547 DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v4i32, Operand: V2.isUndef() ? V1 : V2);
10548
10549 SDValue Shl = DAG.getNode(Opcode: PPCISD::VECSHL, DL: dl, VT: MVT::v4i32, N1: Conv1, N2: Conv2,
10550 N3: DAG.getConstant(Val: ShiftElts, DL: dl, VT: MVT::i32));
10551 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: Shl);
10552 }
10553
10554 if (Subtarget.hasVSX() &&
10555 PPC::isXXPERMDIShuffleMask(N: SVOp, DM&: ShiftElts, Swap, IsLE: isLittleEndian)) {
10556 if (Swap)
10557 std::swap(a&: V1, b&: V2);
10558 SDValue Conv1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v2i64, Operand: V1);
10559 SDValue Conv2 =
10560 DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v2i64, Operand: V2.isUndef() ? V1 : V2);
10561
10562 SDValue PermDI = DAG.getNode(Opcode: PPCISD::XXPERMDI, DL: dl, VT: MVT::v2i64, N1: Conv1, N2: Conv2,
10563 N3: DAG.getConstant(Val: ShiftElts, DL: dl, VT: MVT::i32));
10564 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: PermDI);
10565 }
10566
10567 if (Subtarget.hasP9Vector()) {
10568 if (PPC::isXXBRHShuffleMask(N: SVOp)) {
10569 SDValue Conv = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v8i16, Operand: V1);
10570 SDValue ReveHWord = DAG.getNode(Opcode: ISD::BSWAP, DL: dl, VT: MVT::v8i16, Operand: Conv);
10571 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: ReveHWord);
10572 } else if (PPC::isXXBRWShuffleMask(N: SVOp)) {
10573 SDValue Conv = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v4i32, Operand: V1);
10574 SDValue ReveWord = DAG.getNode(Opcode: ISD::BSWAP, DL: dl, VT: MVT::v4i32, Operand: Conv);
10575 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: ReveWord);
10576 } else if (PPC::isXXBRDShuffleMask(N: SVOp)) {
10577 SDValue Conv = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v2i64, Operand: V1);
10578 SDValue ReveDWord = DAG.getNode(Opcode: ISD::BSWAP, DL: dl, VT: MVT::v2i64, Operand: Conv);
10579 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: ReveDWord);
10580 } else if (PPC::isXXBRQShuffleMask(N: SVOp)) {
10581 SDValue Conv = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v1i128, Operand: V1);
10582 SDValue ReveQWord = DAG.getNode(Opcode: ISD::BSWAP, DL: dl, VT: MVT::v1i128, Operand: Conv);
10583 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: ReveQWord);
10584 }
10585 }
10586
10587 if (Subtarget.hasVSX()) {
10588 if (V2.isUndef() && PPC::isSplatShuffleMask(N: SVOp, EltSize: 4)) {
10589 int SplatIdx = PPC::getSplatIdxForPPCMnemonics(N: SVOp, EltSize: 4, DAG);
10590
10591 SDValue Conv = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v4i32, Operand: V1);
10592 SDValue Splat = DAG.getNode(Opcode: PPCISD::XXSPLT, DL: dl, VT: MVT::v4i32, N1: Conv,
10593 N2: DAG.getConstant(Val: SplatIdx, DL: dl, VT: MVT::i32));
10594 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: Splat);
10595 }
10596
10597 // Left shifts of 8 bytes are actually swaps. Convert accordingly.
10598 if (V2.isUndef() && PPC::isVSLDOIShuffleMask(N: SVOp, ShuffleKind: 1, DAG) == 8) {
10599 SDValue Conv = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v2f64, Operand: V1);
10600 SDValue Swap = DAG.getNode(Opcode: PPCISD::SWAP_NO_CHAIN, DL: dl, VT: MVT::v2f64, Operand: Conv);
10601 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: Swap);
10602 }
10603 }
10604
10605 // Cases that are handled by instructions that take permute immediates
10606 // (such as vsplt*) should be left as VECTOR_SHUFFLE nodes so they can be
10607 // selected by the instruction selector.
10608 if (V2.isUndef()) {
10609 if (PPC::isSplatShuffleMask(N: SVOp, EltSize: 1) ||
10610 PPC::isSplatShuffleMask(N: SVOp, EltSize: 2) ||
10611 PPC::isSplatShuffleMask(N: SVOp, EltSize: 4) ||
10612 PPC::isVPKUWUMShuffleMask(N: SVOp, ShuffleKind: 1, DAG) ||
10613 PPC::isVPKUHUMShuffleMask(N: SVOp, ShuffleKind: 1, DAG) ||
10614 PPC::isVSLDOIShuffleMask(N: SVOp, ShuffleKind: 1, DAG) != -1 ||
10615 PPC::isVMRGLShuffleMask(N: SVOp, UnitSize: 1, ShuffleKind: 1, DAG) ||
10616 PPC::isVMRGLShuffleMask(N: SVOp, UnitSize: 2, ShuffleKind: 1, DAG) ||
10617 PPC::isVMRGLShuffleMask(N: SVOp, UnitSize: 4, ShuffleKind: 1, DAG) ||
10618 PPC::isVMRGHShuffleMask(N: SVOp, UnitSize: 1, ShuffleKind: 1, DAG) ||
10619 PPC::isVMRGHShuffleMask(N: SVOp, UnitSize: 2, ShuffleKind: 1, DAG) ||
10620 PPC::isVMRGHShuffleMask(N: SVOp, UnitSize: 4, ShuffleKind: 1, DAG) ||
10621 (Subtarget.hasP8Altivec() && (
10622 PPC::isVPKUDUMShuffleMask(N: SVOp, ShuffleKind: 1, DAG) ||
10623 PPC::isVMRGEOShuffleMask(N: SVOp, CheckEven: true, ShuffleKind: 1, DAG) ||
10624 PPC::isVMRGEOShuffleMask(N: SVOp, CheckEven: false, ShuffleKind: 1, DAG)))) {
10625 return Op;
10626 }
10627 }
10628
10629 // Altivec has a variety of "shuffle immediates" that take two vector inputs
10630 // and produce a fixed permutation. If any of these match, do not lower to
10631 // VPERM.
10632 unsigned int ShuffleKind = isLittleEndian ? 2 : 0;
10633 if (PPC::isVPKUWUMShuffleMask(N: SVOp, ShuffleKind, DAG) ||
10634 PPC::isVPKUHUMShuffleMask(N: SVOp, ShuffleKind, DAG) ||
10635 PPC::isVSLDOIShuffleMask(N: SVOp, ShuffleKind, DAG) != -1 ||
10636 PPC::isVMRGLShuffleMask(N: SVOp, UnitSize: 1, ShuffleKind, DAG) ||
10637 PPC::isVMRGLShuffleMask(N: SVOp, UnitSize: 2, ShuffleKind, DAG) ||
10638 PPC::isVMRGLShuffleMask(N: SVOp, UnitSize: 4, ShuffleKind, DAG) ||
10639 PPC::isVMRGHShuffleMask(N: SVOp, UnitSize: 1, ShuffleKind, DAG) ||
10640 PPC::isVMRGHShuffleMask(N: SVOp, UnitSize: 2, ShuffleKind, DAG) ||
10641 PPC::isVMRGHShuffleMask(N: SVOp, UnitSize: 4, ShuffleKind, DAG) ||
10642 (Subtarget.hasP8Altivec() && (
10643 PPC::isVPKUDUMShuffleMask(N: SVOp, ShuffleKind, DAG) ||
10644 PPC::isVMRGEOShuffleMask(N: SVOp, CheckEven: true, ShuffleKind, DAG) ||
10645 PPC::isVMRGEOShuffleMask(N: SVOp, CheckEven: false, ShuffleKind, DAG))))
10646 return Op;
10647
10648 // Check to see if this is a shuffle of 4-byte values. If so, we can use our
10649 // perfect shuffle table to emit an optimal matching sequence.
10650 ArrayRef<int> PermMask = SVOp->getMask();
10651
10652 if (!DisablePerfectShuffle && !isLittleEndian) {
10653 unsigned PFIndexes[4];
10654 bool isFourElementShuffle = true;
10655 for (unsigned i = 0; i != 4 && isFourElementShuffle;
10656 ++i) { // Element number
10657 unsigned EltNo = 8; // Start out undef.
10658 for (unsigned j = 0; j != 4; ++j) { // Intra-element byte.
10659 if (PermMask[i * 4 + j] < 0)
10660 continue; // Undef, ignore it.
10661
10662 unsigned ByteSource = PermMask[i * 4 + j];
10663 if ((ByteSource & 3) != j) {
10664 isFourElementShuffle = false;
10665 break;
10666 }
10667
10668 if (EltNo == 8) {
10669 EltNo = ByteSource / 4;
10670 } else if (EltNo != ByteSource / 4) {
10671 isFourElementShuffle = false;
10672 break;
10673 }
10674 }
10675 PFIndexes[i] = EltNo;
10676 }
10677
10678 // If this shuffle can be expressed as a shuffle of 4-byte elements, use the
10679 // perfect shuffle vector to determine if it is cost effective to do this as
10680 // discrete instructions, or whether we should use a vperm.
10681 // For now, we skip this for little endian until such time as we have a
10682 // little-endian perfect shuffle table.
10683 if (isFourElementShuffle) {
10684 // Compute the index in the perfect shuffle table.
10685 unsigned PFTableIndex = PFIndexes[0] * 9 * 9 * 9 + PFIndexes[1] * 9 * 9 +
10686 PFIndexes[2] * 9 + PFIndexes[3];
10687
10688 unsigned PFEntry = PerfectShuffleTable[PFTableIndex];
10689 unsigned Cost = (PFEntry >> 30);
10690
10691 // Determining when to avoid vperm is tricky. Many things affect the cost
10692 // of vperm, particularly how many times the perm mask needs to be
10693 // computed. For example, if the perm mask can be hoisted out of a loop or
10694 // is already used (perhaps because there are multiple permutes with the
10695 // same shuffle mask?) the vperm has a cost of 1. OTOH, hoisting the
10696 // permute mask out of the loop requires an extra register.
10697 //
10698 // As a compromise, we only emit discrete instructions if the shuffle can
10699 // be generated in 3 or fewer operations. When we have loop information
10700 // available, if this block is within a loop, we should avoid using vperm
10701 // for 3-operation perms and use a constant pool load instead.
10702 if (Cost < 3)
10703 return GeneratePerfectShuffle(PFEntry, LHS: V1, RHS: V2, DAG, dl);
10704 }
10705 }
10706
10707 // Lower this to a VPERM(V1, V2, V3) expression, where V3 is a constant
10708 // vector that will get spilled to the constant pool.
10709 if (V2.isUndef()) V2 = V1;
10710
10711 return LowerVPERM(Op, DAG, PermMask, VT, V1, V2);
10712}
10713
10714SDValue PPCTargetLowering::LowerVPERM(SDValue Op, SelectionDAG &DAG,
10715 ArrayRef<int> PermMask, EVT VT,
10716 SDValue V1, SDValue V2) const {
10717 unsigned Opcode = PPCISD::VPERM;
10718 EVT ValType = V1.getValueType();
10719 SDLoc dl(Op);
10720 bool NeedSwap = false;
10721 bool isLittleEndian = Subtarget.isLittleEndian();
10722 bool isPPC64 = Subtarget.isPPC64();
10723
10724 if (Subtarget.hasVSX() && Subtarget.hasP9Vector() &&
10725 (V1->hasOneUse() || V2->hasOneUse())) {
10726 LLVM_DEBUG(dbgs() << "At least one of two input vectors are dead - using "
10727 "XXPERM instead\n");
10728 Opcode = PPCISD::XXPERM;
10729
10730 // The second input to XXPERM is also an output so if the second input has
10731 // multiple uses then copying is necessary, as a result we want the
10732 // single-use operand to be used as the second input to prevent copying.
10733 if ((!isLittleEndian && !V2->hasOneUse() && V1->hasOneUse()) ||
10734 (isLittleEndian && !V1->hasOneUse() && V2->hasOneUse())) {
10735 std::swap(a&: V1, b&: V2);
10736 NeedSwap = !NeedSwap;
10737 }
10738 }
10739
10740 // The SHUFFLE_VECTOR mask is almost exactly what we want for vperm, except
10741 // that it is in input element units, not in bytes. Convert now.
10742
10743 // For little endian, the order of the input vectors is reversed, and
10744 // the permutation mask is complemented with respect to 31. This is
10745 // necessary to produce proper semantics with the big-endian-based vperm
10746 // instruction.
10747 EVT EltVT = V1.getValueType().getVectorElementType();
10748 unsigned BytesPerElement = EltVT.getSizeInBits() / 8;
10749
10750 bool V1HasXXSWAPD = V1->getOperand(Num: 0)->getOpcode() == PPCISD::XXSWAPD;
10751 bool V2HasXXSWAPD = V2->getOperand(Num: 0)->getOpcode() == PPCISD::XXSWAPD;
10752
10753 /*
10754 Vectors will be appended like so: [ V1 | v2 ]
10755 XXSWAPD on V1:
10756 [ A | B | C | D ] -> [ C | D | A | B ]
10757 0-3 4-7 8-11 12-15 0-3 4-7 8-11 12-15
10758 i.e. index of A, B += 8, and index of C, D -= 8.
10759 XXSWAPD on V2:
10760 [ E | F | G | H ] -> [ G | H | E | F ]
10761 16-19 20-23 24-27 28-31 16-19 20-23 24-27 28-31
10762 i.e. index of E, F += 8, index of G, H -= 8
10763 Swap V1 and V2:
10764 [ V1 | V2 ] -> [ V2 | V1 ]
10765 0-15 16-31 0-15 16-31
10766 i.e. index of V1 += 16, index of V2 -= 16
10767 */
10768
10769 SmallVector<SDValue, 16> ResultMask;
10770 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i) {
10771 unsigned SrcElt = PermMask[i] < 0 ? 0 : PermMask[i];
10772
10773 if (V1HasXXSWAPD) {
10774 if (SrcElt < 8)
10775 SrcElt += 8;
10776 else if (SrcElt < 16)
10777 SrcElt -= 8;
10778 }
10779 if (V2HasXXSWAPD) {
10780 if (SrcElt > 23)
10781 SrcElt -= 8;
10782 else if (SrcElt > 15)
10783 SrcElt += 8;
10784 }
10785 if (NeedSwap) {
10786 if (SrcElt < 16)
10787 SrcElt += 16;
10788 else
10789 SrcElt -= 16;
10790 }
10791 for (unsigned j = 0; j != BytesPerElement; ++j)
10792 if (isLittleEndian)
10793 ResultMask.push_back(
10794 Elt: DAG.getConstant(Val: 31 - (SrcElt * BytesPerElement + j), DL: dl, VT: MVT::i32));
10795 else
10796 ResultMask.push_back(
10797 Elt: DAG.getConstant(Val: SrcElt * BytesPerElement + j, DL: dl, VT: MVT::i32));
10798 }
10799
10800 if (V1HasXXSWAPD) {
10801 dl = SDLoc(V1->getOperand(Num: 0));
10802 V1 = V1->getOperand(Num: 0)->getOperand(Num: 1);
10803 }
10804 if (V2HasXXSWAPD) {
10805 dl = SDLoc(V2->getOperand(Num: 0));
10806 V2 = V2->getOperand(Num: 0)->getOperand(Num: 1);
10807 }
10808
10809 if (isPPC64 && (V1HasXXSWAPD || V2HasXXSWAPD)) {
10810 if (ValType != MVT::v2f64)
10811 V1 = DAG.getBitcast(VT: MVT::v2f64, V: V1);
10812 if (V2.getValueType() != MVT::v2f64)
10813 V2 = DAG.getBitcast(VT: MVT::v2f64, V: V2);
10814 }
10815
10816 ShufflesHandledWithVPERM++;
10817 SDValue VPermMask = DAG.getBuildVector(VT: MVT::v16i8, DL: dl, Ops: ResultMask);
10818 LLVM_DEBUG({
10819 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
10820 if (Opcode == PPCISD::XXPERM) {
10821 dbgs() << "Emitting a XXPERM for the following shuffle:\n";
10822 } else {
10823 dbgs() << "Emitting a VPERM for the following shuffle:\n";
10824 }
10825 SVOp->dump();
10826 dbgs() << "With the following permute control vector:\n";
10827 VPermMask.dump();
10828 });
10829
10830 if (Opcode == PPCISD::XXPERM)
10831 VPermMask = DAG.getBitcast(VT: MVT::v4i32, V: VPermMask);
10832
10833 // Only need to place items backwards in LE,
10834 // the mask was properly calculated.
10835 if (isLittleEndian)
10836 std::swap(a&: V1, b&: V2);
10837
10838 SDValue VPERMNode =
10839 DAG.getNode(Opcode, DL: dl, VT: V1.getValueType(), N1: V1, N2: V2, N3: VPermMask);
10840
10841 VPERMNode = DAG.getBitcast(VT: ValType, V: VPERMNode);
10842 return VPERMNode;
10843}
10844
10845/// getVectorCompareInfo - Given an intrinsic, return false if it is not a
10846/// vector comparison. If it is, return true and fill in Opc/isDot with
10847/// information about the intrinsic.
10848static bool getVectorCompareInfo(SDValue Intrin, int &CompareOpc,
10849 bool &isDot, const PPCSubtarget &Subtarget) {
10850 unsigned IntrinsicID = Intrin.getConstantOperandVal(i: 0);
10851 CompareOpc = -1;
10852 isDot = false;
10853 switch (IntrinsicID) {
10854 default:
10855 return false;
10856 // Comparison predicates.
10857 case Intrinsic::ppc_altivec_vcmpbfp_p:
10858 CompareOpc = 966;
10859 isDot = true;
10860 break;
10861 case Intrinsic::ppc_altivec_vcmpeqfp_p:
10862 CompareOpc = 198;
10863 isDot = true;
10864 break;
10865 case Intrinsic::ppc_altivec_vcmpequb_p:
10866 CompareOpc = 6;
10867 isDot = true;
10868 break;
10869 case Intrinsic::ppc_altivec_vcmpequh_p:
10870 CompareOpc = 70;
10871 isDot = true;
10872 break;
10873 case Intrinsic::ppc_altivec_vcmpequw_p:
10874 CompareOpc = 134;
10875 isDot = true;
10876 break;
10877 case Intrinsic::ppc_altivec_vcmpequd_p:
10878 if (Subtarget.hasVSX() || Subtarget.hasP8Altivec()) {
10879 CompareOpc = 199;
10880 isDot = true;
10881 } else
10882 return false;
10883 break;
10884 case Intrinsic::ppc_altivec_vcmpneb_p:
10885 case Intrinsic::ppc_altivec_vcmpneh_p:
10886 case Intrinsic::ppc_altivec_vcmpnew_p:
10887 case Intrinsic::ppc_altivec_vcmpnezb_p:
10888 case Intrinsic::ppc_altivec_vcmpnezh_p:
10889 case Intrinsic::ppc_altivec_vcmpnezw_p:
10890 if (Subtarget.hasP9Altivec()) {
10891 switch (IntrinsicID) {
10892 default:
10893 llvm_unreachable("Unknown comparison intrinsic.");
10894 case Intrinsic::ppc_altivec_vcmpneb_p:
10895 CompareOpc = 7;
10896 break;
10897 case Intrinsic::ppc_altivec_vcmpneh_p:
10898 CompareOpc = 71;
10899 break;
10900 case Intrinsic::ppc_altivec_vcmpnew_p:
10901 CompareOpc = 135;
10902 break;
10903 case Intrinsic::ppc_altivec_vcmpnezb_p:
10904 CompareOpc = 263;
10905 break;
10906 case Intrinsic::ppc_altivec_vcmpnezh_p:
10907 CompareOpc = 327;
10908 break;
10909 case Intrinsic::ppc_altivec_vcmpnezw_p:
10910 CompareOpc = 391;
10911 break;
10912 }
10913 isDot = true;
10914 } else
10915 return false;
10916 break;
10917 case Intrinsic::ppc_altivec_vcmpgefp_p:
10918 CompareOpc = 454;
10919 isDot = true;
10920 break;
10921 case Intrinsic::ppc_altivec_vcmpgtfp_p:
10922 CompareOpc = 710;
10923 isDot = true;
10924 break;
10925 case Intrinsic::ppc_altivec_vcmpgtsb_p:
10926 CompareOpc = 774;
10927 isDot = true;
10928 break;
10929 case Intrinsic::ppc_altivec_vcmpgtsh_p:
10930 CompareOpc = 838;
10931 isDot = true;
10932 break;
10933 case Intrinsic::ppc_altivec_vcmpgtsw_p:
10934 CompareOpc = 902;
10935 isDot = true;
10936 break;
10937 case Intrinsic::ppc_altivec_vcmpgtsd_p:
10938 if (Subtarget.hasVSX() || Subtarget.hasP8Altivec()) {
10939 CompareOpc = 967;
10940 isDot = true;
10941 } else
10942 return false;
10943 break;
10944 case Intrinsic::ppc_altivec_vcmpgtub_p:
10945 CompareOpc = 518;
10946 isDot = true;
10947 break;
10948 case Intrinsic::ppc_altivec_vcmpgtuh_p:
10949 CompareOpc = 582;
10950 isDot = true;
10951 break;
10952 case Intrinsic::ppc_altivec_vcmpgtuw_p:
10953 CompareOpc = 646;
10954 isDot = true;
10955 break;
10956 case Intrinsic::ppc_altivec_vcmpgtud_p:
10957 if (Subtarget.hasVSX() || Subtarget.hasP8Altivec()) {
10958 CompareOpc = 711;
10959 isDot = true;
10960 } else
10961 return false;
10962 break;
10963
10964 case Intrinsic::ppc_altivec_vcmpequq:
10965 case Intrinsic::ppc_altivec_vcmpgtsq:
10966 case Intrinsic::ppc_altivec_vcmpgtuq:
10967 if (!Subtarget.isISA3_1())
10968 return false;
10969 switch (IntrinsicID) {
10970 default:
10971 llvm_unreachable("Unknown comparison intrinsic.");
10972 case Intrinsic::ppc_altivec_vcmpequq:
10973 CompareOpc = 455;
10974 break;
10975 case Intrinsic::ppc_altivec_vcmpgtsq:
10976 CompareOpc = 903;
10977 break;
10978 case Intrinsic::ppc_altivec_vcmpgtuq:
10979 CompareOpc = 647;
10980 break;
10981 }
10982 break;
10983
10984 // VSX predicate comparisons use the same infrastructure
10985 case Intrinsic::ppc_vsx_xvcmpeqdp_p:
10986 case Intrinsic::ppc_vsx_xvcmpgedp_p:
10987 case Intrinsic::ppc_vsx_xvcmpgtdp_p:
10988 case Intrinsic::ppc_vsx_xvcmpeqsp_p:
10989 case Intrinsic::ppc_vsx_xvcmpgesp_p:
10990 case Intrinsic::ppc_vsx_xvcmpgtsp_p:
10991 if (Subtarget.hasVSX()) {
10992 switch (IntrinsicID) {
10993 case Intrinsic::ppc_vsx_xvcmpeqdp_p:
10994 CompareOpc = 99;
10995 break;
10996 case Intrinsic::ppc_vsx_xvcmpgedp_p:
10997 CompareOpc = 115;
10998 break;
10999 case Intrinsic::ppc_vsx_xvcmpgtdp_p:
11000 CompareOpc = 107;
11001 break;
11002 case Intrinsic::ppc_vsx_xvcmpeqsp_p:
11003 CompareOpc = 67;
11004 break;
11005 case Intrinsic::ppc_vsx_xvcmpgesp_p:
11006 CompareOpc = 83;
11007 break;
11008 case Intrinsic::ppc_vsx_xvcmpgtsp_p:
11009 CompareOpc = 75;
11010 break;
11011 }
11012 isDot = true;
11013 } else
11014 return false;
11015 break;
11016
11017 // Normal Comparisons.
11018 case Intrinsic::ppc_altivec_vcmpbfp:
11019 CompareOpc = 966;
11020 break;
11021 case Intrinsic::ppc_altivec_vcmpeqfp:
11022 CompareOpc = 198;
11023 break;
11024 case Intrinsic::ppc_altivec_vcmpequb:
11025 CompareOpc = 6;
11026 break;
11027 case Intrinsic::ppc_altivec_vcmpequh:
11028 CompareOpc = 70;
11029 break;
11030 case Intrinsic::ppc_altivec_vcmpequw:
11031 CompareOpc = 134;
11032 break;
11033 case Intrinsic::ppc_altivec_vcmpequd:
11034 if (Subtarget.hasP8Altivec())
11035 CompareOpc = 199;
11036 else
11037 return false;
11038 break;
11039 case Intrinsic::ppc_altivec_vcmpneb:
11040 case Intrinsic::ppc_altivec_vcmpneh:
11041 case Intrinsic::ppc_altivec_vcmpnew:
11042 case Intrinsic::ppc_altivec_vcmpnezb:
11043 case Intrinsic::ppc_altivec_vcmpnezh:
11044 case Intrinsic::ppc_altivec_vcmpnezw:
11045 if (Subtarget.hasP9Altivec())
11046 switch (IntrinsicID) {
11047 default:
11048 llvm_unreachable("Unknown comparison intrinsic.");
11049 case Intrinsic::ppc_altivec_vcmpneb:
11050 CompareOpc = 7;
11051 break;
11052 case Intrinsic::ppc_altivec_vcmpneh:
11053 CompareOpc = 71;
11054 break;
11055 case Intrinsic::ppc_altivec_vcmpnew:
11056 CompareOpc = 135;
11057 break;
11058 case Intrinsic::ppc_altivec_vcmpnezb:
11059 CompareOpc = 263;
11060 break;
11061 case Intrinsic::ppc_altivec_vcmpnezh:
11062 CompareOpc = 327;
11063 break;
11064 case Intrinsic::ppc_altivec_vcmpnezw:
11065 CompareOpc = 391;
11066 break;
11067 }
11068 else
11069 return false;
11070 break;
11071 case Intrinsic::ppc_altivec_vcmpgefp:
11072 CompareOpc = 454;
11073 break;
11074 case Intrinsic::ppc_altivec_vcmpgtfp:
11075 CompareOpc = 710;
11076 break;
11077 case Intrinsic::ppc_altivec_vcmpgtsb:
11078 CompareOpc = 774;
11079 break;
11080 case Intrinsic::ppc_altivec_vcmpgtsh:
11081 CompareOpc = 838;
11082 break;
11083 case Intrinsic::ppc_altivec_vcmpgtsw:
11084 CompareOpc = 902;
11085 break;
11086 case Intrinsic::ppc_altivec_vcmpgtsd:
11087 if (Subtarget.hasP8Altivec())
11088 CompareOpc = 967;
11089 else
11090 return false;
11091 break;
11092 case Intrinsic::ppc_altivec_vcmpgtub:
11093 CompareOpc = 518;
11094 break;
11095 case Intrinsic::ppc_altivec_vcmpgtuh:
11096 CompareOpc = 582;
11097 break;
11098 case Intrinsic::ppc_altivec_vcmpgtuw:
11099 CompareOpc = 646;
11100 break;
11101 case Intrinsic::ppc_altivec_vcmpgtud:
11102 if (Subtarget.hasP8Altivec())
11103 CompareOpc = 711;
11104 else
11105 return false;
11106 break;
11107 case Intrinsic::ppc_altivec_vcmpequq_p:
11108 case Intrinsic::ppc_altivec_vcmpgtsq_p:
11109 case Intrinsic::ppc_altivec_vcmpgtuq_p:
11110 if (!Subtarget.isISA3_1())
11111 return false;
11112 switch (IntrinsicID) {
11113 default:
11114 llvm_unreachable("Unknown comparison intrinsic.");
11115 case Intrinsic::ppc_altivec_vcmpequq_p:
11116 CompareOpc = 455;
11117 break;
11118 case Intrinsic::ppc_altivec_vcmpgtsq_p:
11119 CompareOpc = 903;
11120 break;
11121 case Intrinsic::ppc_altivec_vcmpgtuq_p:
11122 CompareOpc = 647;
11123 break;
11124 }
11125 isDot = true;
11126 break;
11127 }
11128 return true;
11129}
11130
11131/// LowerINTRINSIC_WO_CHAIN - If this is an intrinsic that we want to custom
11132/// lower, do it, otherwise return null.
11133SDValue PPCTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
11134 SelectionDAG &DAG) const {
11135 unsigned IntrinsicID = Op.getConstantOperandVal(i: 0);
11136
11137 SDLoc dl(Op);
11138 // Note: BCD instructions expect the immediate operand in vector form (v4i32),
11139 // but the builtin provides it as a scalar. To satisfy the instruction
11140 // encoding, we splat the scalar across all lanes using SPLAT_VECTOR.
11141 auto MapNodeWithSplatVector =
11142 [&](unsigned Opcode,
11143 std::initializer_list<SDValue> ExtraOps = {}) -> SDValue {
11144 SDValue SplatVal =
11145 DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: MVT::v4i32, Operand: Op.getOperand(i: 2));
11146
11147 SmallVector<SDValue, 4> Ops{SplatVal, Op.getOperand(i: 1)};
11148 Ops.append(in_start: ExtraOps.begin(), in_end: ExtraOps.end());
11149 return DAG.getNode(Opcode, DL: dl, VT: MVT::v16i8, Ops);
11150 };
11151
11152 switch (IntrinsicID) {
11153 case Intrinsic::thread_pointer:
11154 // Reads the thread pointer register, used for __builtin_thread_pointer.
11155 if (Subtarget.isPPC64())
11156 return DAG.getRegister(Reg: PPC::X13, VT: MVT::i64);
11157 return DAG.getRegister(Reg: PPC::R2, VT: MVT::i32);
11158
11159 case Intrinsic::ppc_rldimi: {
11160 assert(Subtarget.isPPC64() && "rldimi is only available in 64-bit!");
11161 SDValue Src = Op.getOperand(i: 1);
11162 APInt Mask = Op.getConstantOperandAPInt(i: 4);
11163 if (Mask.isZero())
11164 return Op.getOperand(i: 2);
11165 if (Mask.isAllOnes())
11166 return DAG.getNode(Opcode: ISD::ROTL, DL: dl, VT: MVT::i64, N1: Src, N2: Op.getOperand(i: 3));
11167 uint64_t SH = Op.getConstantOperandVal(i: 3);
11168 unsigned MB = 0, ME = 0;
11169 if (!isRunOfOnes64(Val: Mask.getZExtValue(), MB, ME))
11170 report_fatal_error(reason: "invalid rldimi mask!");
11171 // rldimi requires ME=63-SH, otherwise rotation is needed before rldimi.
11172 if (ME < 63 - SH) {
11173 Src = DAG.getNode(Opcode: ISD::ROTL, DL: dl, VT: MVT::i64, N1: Src,
11174 N2: DAG.getConstant(Val: ME + SH + 1, DL: dl, VT: MVT::i32));
11175 } else if (ME > 63 - SH) {
11176 Src = DAG.getNode(Opcode: ISD::ROTL, DL: dl, VT: MVT::i64, N1: Src,
11177 N2: DAG.getConstant(Val: ME + SH - 63, DL: dl, VT: MVT::i32));
11178 }
11179 return SDValue(
11180 DAG.getMachineNode(Opcode: PPC::RLDIMI, dl, VT: MVT::i64,
11181 Ops: {Op.getOperand(i: 2), Src,
11182 DAG.getTargetConstant(Val: 63 - ME, DL: dl, VT: MVT::i32),
11183 DAG.getTargetConstant(Val: MB, DL: dl, VT: MVT::i32)}),
11184 0);
11185 }
11186
11187 case Intrinsic::ppc_rlwimi: {
11188 APInt Mask = Op.getConstantOperandAPInt(i: 4);
11189 if (Mask.isZero())
11190 return Op.getOperand(i: 2);
11191 if (Mask.isAllOnes())
11192 return DAG.getNode(Opcode: ISD::ROTL, DL: dl, VT: MVT::i32, N1: Op.getOperand(i: 1),
11193 N2: Op.getOperand(i: 3));
11194 unsigned MB = 0, ME = 0;
11195 if (!isRunOfOnes(Val: Mask.getZExtValue(), MB, ME))
11196 report_fatal_error(reason: "invalid rlwimi mask!");
11197 return SDValue(DAG.getMachineNode(
11198 Opcode: PPC::RLWIMI, dl, VT: MVT::i32,
11199 Ops: {Op.getOperand(i: 2), Op.getOperand(i: 1), Op.getOperand(i: 3),
11200 DAG.getTargetConstant(Val: MB, DL: dl, VT: MVT::i32),
11201 DAG.getTargetConstant(Val: ME, DL: dl, VT: MVT::i32)}),
11202 0);
11203 }
11204
11205 case Intrinsic::ppc_bcdshift:
11206 return MapNodeWithSplatVector(PPCISD::BCDSHIFT, {Op.getOperand(i: 3)});
11207 case Intrinsic::ppc_bcdshiftround:
11208 return MapNodeWithSplatVector(PPCISD::BCDSHIFTROUND, {Op.getOperand(i: 3)});
11209 case Intrinsic::ppc_bcdtruncate:
11210 return MapNodeWithSplatVector(PPCISD::BCDTRUNC, {Op.getOperand(i: 3)});
11211 case Intrinsic::ppc_bcdunsignedtruncate:
11212 return MapNodeWithSplatVector(PPCISD::BCDUTRUNC);
11213 case Intrinsic::ppc_bcdunsignedshift:
11214 return MapNodeWithSplatVector(PPCISD::BCDUSHIFT);
11215
11216 case Intrinsic::ppc_rlwnm: {
11217 if (Op.getConstantOperandVal(i: 3) == 0)
11218 return DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32);
11219 unsigned MB = 0, ME = 0;
11220 if (!isRunOfOnes(Val: Op.getConstantOperandVal(i: 3), MB, ME))
11221 report_fatal_error(reason: "invalid rlwnm mask!");
11222 return SDValue(
11223 DAG.getMachineNode(Opcode: PPC::RLWNM, dl, VT: MVT::i32,
11224 Ops: {Op.getOperand(i: 1), Op.getOperand(i: 2),
11225 DAG.getTargetConstant(Val: MB, DL: dl, VT: MVT::i32),
11226 DAG.getTargetConstant(Val: ME, DL: dl, VT: MVT::i32)}),
11227 0);
11228 }
11229
11230 case Intrinsic::ppc_mma_disassemble_acc: {
11231 if (Subtarget.isISAFuture()) {
11232 EVT ReturnTypes[] = {MVT::v256i1, MVT::v256i1};
11233 SDValue WideVec =
11234 SDValue(DAG.getMachineNode(Opcode: PPC::DMXXEXTFDMR512, dl, ResultTys: ReturnTypes,
11235 Ops: Op.getOperand(i: 1)),
11236 0);
11237 SmallVector<SDValue, 4> RetOps;
11238 SDValue Value = SDValue(WideVec.getNode(), 0);
11239 SDValue Value2 = SDValue(WideVec.getNode(), 1);
11240
11241 SDValue Extract;
11242 Extract = DAG.getNode(
11243 Opcode: PPCISD::EXTRACT_VSX_REG, DL: dl, VT: MVT::v16i8,
11244 N1: Subtarget.isLittleEndian() ? Value2 : Value,
11245 N2: DAG.getConstant(Val: Subtarget.isLittleEndian() ? 1 : 0,
11246 DL: dl, VT: getPointerTy(DL: DAG.getDataLayout())));
11247 RetOps.push_back(Elt: Extract);
11248 Extract = DAG.getNode(
11249 Opcode: PPCISD::EXTRACT_VSX_REG, DL: dl, VT: MVT::v16i8,
11250 N1: Subtarget.isLittleEndian() ? Value2 : Value,
11251 N2: DAG.getConstant(Val: Subtarget.isLittleEndian() ? 0 : 1,
11252 DL: dl, VT: getPointerTy(DL: DAG.getDataLayout())));
11253 RetOps.push_back(Elt: Extract);
11254 Extract = DAG.getNode(
11255 Opcode: PPCISD::EXTRACT_VSX_REG, DL: dl, VT: MVT::v16i8,
11256 N1: Subtarget.isLittleEndian() ? Value : Value2,
11257 N2: DAG.getConstant(Val: Subtarget.isLittleEndian() ? 1 : 0,
11258 DL: dl, VT: getPointerTy(DL: DAG.getDataLayout())));
11259 RetOps.push_back(Elt: Extract);
11260 Extract = DAG.getNode(
11261 Opcode: PPCISD::EXTRACT_VSX_REG, DL: dl, VT: MVT::v16i8,
11262 N1: Subtarget.isLittleEndian() ? Value : Value2,
11263 N2: DAG.getConstant(Val: Subtarget.isLittleEndian() ? 0 : 1,
11264 DL: dl, VT: getPointerTy(DL: DAG.getDataLayout())));
11265 RetOps.push_back(Elt: Extract);
11266 return DAG.getMergeValues(Ops: RetOps, dl);
11267 }
11268 [[fallthrough]];
11269 }
11270 case Intrinsic::ppc_vsx_disassemble_pair: {
11271 int NumVecs = 2;
11272 SDValue WideVec = Op.getOperand(i: 1);
11273 if (IntrinsicID == Intrinsic::ppc_mma_disassemble_acc) {
11274 NumVecs = 4;
11275 WideVec = DAG.getNode(Opcode: PPCISD::XXMFACC, DL: dl, VT: MVT::v512i1, Operand: WideVec);
11276 }
11277 SmallVector<SDValue, 4> RetOps;
11278 for (int VecNo = 0; VecNo < NumVecs; VecNo++) {
11279 SDValue Extract = DAG.getNode(
11280 Opcode: PPCISD::EXTRACT_VSX_REG, DL: dl, VT: MVT::v16i8, N1: WideVec,
11281 N2: DAG.getConstant(Val: Subtarget.isLittleEndian() ? NumVecs - 1 - VecNo
11282 : VecNo,
11283 DL: dl, VT: getPointerTy(DL: DAG.getDataLayout())));
11284 RetOps.push_back(Elt: Extract);
11285 }
11286 return DAG.getMergeValues(Ops: RetOps, dl);
11287 }
11288
11289 case Intrinsic::ppc_build_dmr: {
11290 SmallVector<SDValue, 8> Pairs;
11291 SmallVector<SDValue, 8> Chains;
11292 for (int i = 1; i < 9; i += 2) {
11293 SDValue Hi = Op.getOperand(i);
11294 SDValue Lo = Op.getOperand(i: i + 1);
11295 if (Hi->getOpcode() == ISD::LOAD)
11296 Chains.push_back(Elt: Hi.getValue(R: 1));
11297 if (Lo->getOpcode() == ISD::LOAD)
11298 Chains.push_back(Elt: Lo.getValue(R: 1));
11299 Pairs.push_back(
11300 Elt: DAG.getNode(Opcode: PPCISD::PAIR_BUILD, DL: dl, VT: MVT::v256i1, Ops: {Hi, Lo}));
11301 }
11302 SDValue TF = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Chains);
11303 SDValue Value = DMFInsert1024(Pairs, dl: SDLoc(Op), DAG);
11304 return DAG.getMergeValues(Ops: {Value, TF}, dl);
11305 }
11306
11307 case Intrinsic::ppc_mma_dmxxextfdmr512: {
11308 assert(Subtarget.isISAFuture() && "dmxxextfdmr512 requires ISA Future");
11309 auto *Idx = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2));
11310 assert(Idx && (Idx->getSExtValue() == 0 || Idx->getSExtValue() == 1) &&
11311 "Specify P of 0 or 1 for lower or upper 512 bytes");
11312 unsigned HiLo = Idx->getSExtValue();
11313 unsigned Opcode;
11314 unsigned Subx;
11315 if (HiLo == 0) {
11316 Opcode = PPC::DMXXEXTFDMR512;
11317 Subx = PPC::sub_wacc_lo;
11318 } else {
11319 Opcode = PPC::DMXXEXTFDMR512_HI;
11320 Subx = PPC::sub_wacc_hi;
11321 }
11322 SDValue Subreg(
11323 DAG.getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG, dl, VT: MVT::v512i1,
11324 Op1: Op.getOperand(i: 1),
11325 Op2: DAG.getTargetConstant(Val: Subx, DL: dl, VT: MVT::i32)),
11326 0);
11327 EVT ReturnTypes[] = {MVT::v256i1, MVT::v256i1};
11328 return SDValue(DAG.getMachineNode(Opcode, dl, ResultTys: ReturnTypes, Ops: Subreg), 0);
11329 }
11330
11331 case Intrinsic::ppc_mma_dmxxextfdmr256: {
11332 assert(Subtarget.isISAFuture() && "dmxxextfdmr256 requires ISA Future");
11333 auto *Idx = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2));
11334 assert(Idx && (Idx->getSExtValue() >= 0 || Idx->getSExtValue() <= 3) &&
11335 "Specify a dmr row pair 0-3");
11336 unsigned IdxVal = Idx->getSExtValue();
11337 unsigned Subx;
11338 switch (IdxVal) {
11339 case 0:
11340 Subx = PPC::sub_dmrrowp0;
11341 break;
11342 case 1:
11343 Subx = PPC::sub_dmrrowp1;
11344 break;
11345 case 2:
11346 Subx = PPC::sub_wacc_hi_then_sub_dmrrowp0;
11347 break;
11348 case 3:
11349 Subx = PPC::sub_wacc_hi_then_sub_dmrrowp1;
11350 break;
11351 }
11352 SDValue Subreg(
11353 DAG.getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG, dl, VT: MVT::v256i1,
11354 Op1: Op.getOperand(i: 1),
11355 Op2: DAG.getTargetConstant(Val: Subx, DL: dl, VT: MVT::i32)),
11356 0);
11357 SDValue P = DAG.getTargetConstant(Val: IdxVal, DL: dl, VT: MVT::i32);
11358 return SDValue(
11359 DAG.getMachineNode(Opcode: PPC::DMXXEXTFDMR256, dl, VT: MVT::v256i1, Ops: {Subreg, P}),
11360 0);
11361 }
11362
11363 case Intrinsic::ppc_mma_dmxxinstdmr512: {
11364 assert(Subtarget.isISAFuture() && "dmxxinstdmr512 requires ISA Future");
11365 auto *Idx = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 4));
11366 assert(Idx && (Idx->getSExtValue() == 0 || Idx->getSExtValue() == 1) &&
11367 "Specify P of 0 or 1 for lower or upper 512 bytes");
11368 unsigned HiLo = Idx->getSExtValue();
11369 unsigned Opcode;
11370 unsigned Subx;
11371 if (HiLo == 0) {
11372 Opcode = PPCISD::INST512;
11373 Subx = PPC::sub_wacc_lo;
11374 } else {
11375 Opcode = PPCISD::INST512HI;
11376 Subx = PPC::sub_wacc_hi;
11377 }
11378 SDValue Wacc = DAG.getNode(Opcode, DL: dl, VT: MVT::v512i1, N1: Op.getOperand(i: 2),
11379 N2: Op.getOperand(i: 3));
11380 SDValue SubReg = DAG.getTargetConstant(Val: Subx, DL: dl, VT: MVT::i32);
11381 return SDValue(DAG.getMachineNode(Opcode: PPC::INSERT_SUBREG, dl, VT: MVT::v1024i1,
11382 Op1: Op.getOperand(i: 1), Op2: Wacc, Op3: SubReg),
11383 0);
11384 }
11385
11386 case Intrinsic::ppc_mma_dmxxinstdmr256: {
11387 assert(Subtarget.isISAFuture() && "dmxxinstdmr256 requires ISA Future");
11388 auto *Idx = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 3));
11389 assert(Idx && (Idx->getSExtValue() >= 0 || Idx->getSExtValue() <= 3) &&
11390 "Specify a dmr row pair 0-3");
11391 unsigned IdxVal = Idx->getSExtValue();
11392 unsigned Subx;
11393 switch (IdxVal) {
11394 case 0:
11395 Subx = PPC::sub_dmrrowp0;
11396 break;
11397 case 1:
11398 Subx = PPC::sub_dmrrowp1;
11399 break;
11400 case 2:
11401 Subx = PPC::sub_wacc_hi_then_sub_dmrrowp0;
11402 break;
11403 case 3:
11404 Subx = PPC::sub_wacc_hi_then_sub_dmrrowp1;
11405 break;
11406 }
11407 SDValue SubReg = DAG.getTargetConstant(Val: Subx, DL: dl, VT: MVT::i32);
11408 SDValue P = DAG.getTargetConstant(Val: IdxVal, DL: dl, VT: MVT::i32);
11409 SDValue DMRRowp =
11410 DAG.getNode(Opcode: PPCISD::INST256, DL: dl, VT: MVT::v256i1, N1: Op.getOperand(i: 2), N2: P);
11411 return SDValue(DAG.getMachineNode(Opcode: PPC::INSERT_SUBREG, dl, VT: MVT::v1024i1,
11412 Op1: Op.getOperand(i: 1), Op2: DMRRowp, Op3: SubReg),
11413 0);
11414 }
11415
11416 case Intrinsic::ppc_mma_xxmfacc:
11417 case Intrinsic::ppc_mma_xxmtacc: {
11418 // Allow pre-isa-future subtargets to lower as normal.
11419 if (!Subtarget.isISAFuture())
11420 return SDValue();
11421 // The intrinsics for xxmtacc and xxmfacc take one argument of
11422 // type v512i1, for future cpu the corresponding wacc instruction
11423 // dmxx[inst|extf]dmr512 is always generated for type v512i1, negating
11424 // the need to produce the xxm[t|f]acc.
11425 SDValue WideVec = Op.getOperand(i: 1);
11426 DAG.ReplaceAllUsesWith(From: Op, To: WideVec);
11427 return SDValue();
11428 }
11429
11430 case Intrinsic::ppc_unpack_longdouble: {
11431 auto *Idx = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2));
11432 assert(Idx && (Idx->getSExtValue() == 0 || Idx->getSExtValue() == 1) &&
11433 "Argument of long double unpack must be 0 or 1!");
11434 return DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL: dl, VT: MVT::f64, N1: Op.getOperand(i: 1),
11435 N2: DAG.getConstant(Val: !!(Idx->getSExtValue()), DL: dl,
11436 VT: Idx->getValueType(ResNo: 0)));
11437 }
11438
11439 case Intrinsic::ppc_compare_exp_lt:
11440 case Intrinsic::ppc_compare_exp_gt:
11441 case Intrinsic::ppc_compare_exp_eq:
11442 case Intrinsic::ppc_compare_exp_uo: {
11443 unsigned Pred;
11444 switch (IntrinsicID) {
11445 case Intrinsic::ppc_compare_exp_lt:
11446 Pred = PPC::PRED_LT;
11447 break;
11448 case Intrinsic::ppc_compare_exp_gt:
11449 Pred = PPC::PRED_GT;
11450 break;
11451 case Intrinsic::ppc_compare_exp_eq:
11452 Pred = PPC::PRED_EQ;
11453 break;
11454 case Intrinsic::ppc_compare_exp_uo:
11455 Pred = PPC::PRED_UN;
11456 break;
11457 }
11458 return SDValue(
11459 DAG.getMachineNode(
11460 Opcode: PPC::SELECT_CC_I4, dl, VT: MVT::i32,
11461 Ops: {SDValue(DAG.getMachineNode(Opcode: PPC::XSCMPEXPDP, dl, VT: MVT::i32,
11462 Op1: Op.getOperand(i: 1), Op2: Op.getOperand(i: 2)),
11463 0),
11464 DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32), DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32),
11465 DAG.getTargetConstant(Val: Pred, DL: dl, VT: MVT::i32)}),
11466 0);
11467 }
11468 case Intrinsic::ppc_test_data_class: {
11469 EVT OpVT = Op.getOperand(i: 1).getValueType();
11470 unsigned CmprOpc = OpVT == MVT::f128 ? PPC::XSTSTDCQP
11471 : (OpVT == MVT::f64 ? PPC::XSTSTDCDP
11472 : PPC::XSTSTDCSP);
11473 // Lower __builtin_ppc_test_data_class(value, mask) to XSTSTDC* instruction.
11474 // The XSTSTDC* instructions test if a floating-point value matches any of
11475 // the data classes specified in the mask, setting CR field bits
11476 // accordingly. We need to extract the EQ bit (bit 2) from the CR field and
11477 // convert it to an integer result (1 if match, 0 if no match).
11478 //
11479 // Note: Operands are swapped because XSTSTDC* expects (mask, value) but the
11480 // intrinsic provides (value, mask) as Op.getOperand(1) and
11481 // Op.getOperand(2).
11482 SDValue TestDataClass =
11483 SDValue(DAG.getMachineNode(Opcode: CmprOpc, dl, VT: MVT::i32,
11484 Ops: {Op.getOperand(i: 2), Op.getOperand(i: 1)}),
11485 0);
11486 if (Subtarget.isISA3_1()) {
11487 // ISA 3.1+: Use SETBC instruction to directly convert CR bit to integer.
11488 // This is more efficient than the SELECT_CC approach used in earlier
11489 // ISAs.
11490 SDValue SubRegIdx = DAG.getTargetConstant(Val: PPC::sub_eq, DL: dl, VT: MVT::i32);
11491 SDValue CRBit =
11492 SDValue(DAG.getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG, dl, VT: MVT::i1,
11493 Op1: TestDataClass, Op2: SubRegIdx),
11494 0);
11495
11496 return DAG.getNode(Opcode: PPCISD::SETBC, DL: dl, VT: MVT::i32, Operand: CRBit);
11497 }
11498
11499 // Pre-ISA 3.1: Use SELECT_CC to convert CR field to integer (1 or 0).
11500 return SDValue(
11501 DAG.getMachineNode(Opcode: PPC::SELECT_CC_I4, dl, VT: MVT::i32,
11502 Ops: {TestDataClass, DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32),
11503 DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32),
11504 DAG.getTargetConstant(Val: PPC::PRED_EQ, DL: dl, VT: MVT::i32)}),
11505 0);
11506 }
11507 case Intrinsic::ppc_fnmsub: {
11508 EVT VT = Op.getOperand(i: 1).getValueType();
11509 if (!Subtarget.hasVSX() || (!Subtarget.hasFloat128() && VT == MVT::f128))
11510 return DAG.getNode(
11511 Opcode: ISD::FNEG, DL: dl, VT,
11512 Operand: DAG.getNode(Opcode: ISD::FMA, DL: dl, VT, N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2),
11513 N3: DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT, Operand: Op.getOperand(i: 3))));
11514 return DAG.getNode(Opcode: PPCISD::FNMSUB, DL: dl, VT, N1: Op.getOperand(i: 1),
11515 N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
11516 }
11517 case Intrinsic::ppc_convert_f128_to_ppcf128:
11518 case Intrinsic::ppc_convert_ppcf128_to_f128: {
11519 RTLIB::Libcall LC = IntrinsicID == Intrinsic::ppc_convert_ppcf128_to_f128
11520 ? RTLIB::CONVERT_PPCF128_F128
11521 : RTLIB::CONVERT_F128_PPCF128;
11522 MakeLibCallOptions CallOptions;
11523 std::pair<SDValue, SDValue> Result =
11524 makeLibCall(DAG, LC, RetVT: Op.getValueType(), Ops: Op.getOperand(i: 1), CallOptions,
11525 dl, Chain: SDValue());
11526 return Result.first;
11527 }
11528 case Intrinsic::ppc_maxfe:
11529 case Intrinsic::ppc_maxfl:
11530 case Intrinsic::ppc_maxfs:
11531 case Intrinsic::ppc_minfe:
11532 case Intrinsic::ppc_minfl:
11533 case Intrinsic::ppc_minfs: {
11534 EVT VT = Op.getValueType();
11535 assert(
11536 all_of(Op->ops().drop_front(4),
11537 [VT](const SDUse &Use) { return Use.getValueType() == VT; }) &&
11538 "ppc_[max|min]f[e|l|s] must have uniform type arguments");
11539 (void)VT;
11540 ISD::CondCode CC = ISD::SETGT;
11541 if (IntrinsicID == Intrinsic::ppc_minfe ||
11542 IntrinsicID == Intrinsic::ppc_minfl ||
11543 IntrinsicID == Intrinsic::ppc_minfs)
11544 CC = ISD::SETLT;
11545 unsigned I = Op.getNumOperands() - 2, Cnt = I;
11546 SDValue Res = Op.getOperand(i: I);
11547 for (--I; Cnt != 0; --Cnt, I = (--I == 0 ? (Op.getNumOperands() - 1) : I)) {
11548 Res =
11549 DAG.getSelectCC(DL: dl, LHS: Res, RHS: Op.getOperand(i: I), True: Res, False: Op.getOperand(i: I), Cond: CC);
11550 }
11551 return Res;
11552 }
11553 }
11554
11555 // If this is a lowered altivec predicate compare, CompareOpc is set to the
11556 // opcode number of the comparison.
11557 int CompareOpc;
11558 bool isDot;
11559 if (!getVectorCompareInfo(Intrin: Op, CompareOpc, isDot, Subtarget))
11560 return SDValue(); // Don't custom lower most intrinsics.
11561
11562 // If this is a non-dot comparison, make the VCMP node and we are done.
11563 if (!isDot) {
11564 SDValue Tmp = DAG.getNode(Opcode: PPCISD::VCMP, DL: dl, VT: Op.getOperand(i: 2).getValueType(),
11565 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2),
11566 N3: DAG.getConstant(Val: CompareOpc, DL: dl, VT: MVT::i32));
11567 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: Op.getValueType(), Operand: Tmp);
11568 }
11569
11570 // Create the PPCISD altivec 'dot' comparison node.
11571 SDValue Ops[] = {
11572 Op.getOperand(i: 2), // LHS
11573 Op.getOperand(i: 3), // RHS
11574 DAG.getConstant(Val: CompareOpc, DL: dl, VT: MVT::i32)
11575 };
11576 EVT VTs[] = { Op.getOperand(i: 2).getValueType(), MVT::Glue };
11577 SDValue CompNode = DAG.getNode(Opcode: PPCISD::VCMP_rec, DL: dl, ResultTys: VTs, Ops);
11578
11579 // Unpack the result based on how the target uses it.
11580 unsigned BitNo; // Bit # of CR6.
11581 bool InvertBit; // Invert result?
11582 unsigned Bitx;
11583 unsigned SetOp;
11584 switch (Op.getConstantOperandVal(i: 1)) {
11585 default: // Can't happen, don't crash on invalid number though.
11586 case 0: // Return the value of the EQ bit of CR6.
11587 BitNo = 0;
11588 InvertBit = false;
11589 Bitx = PPC::sub_eq;
11590 SetOp = PPCISD::SETBC;
11591 break;
11592 case 1: // Return the inverted value of the EQ bit of CR6.
11593 BitNo = 0;
11594 InvertBit = true;
11595 Bitx = PPC::sub_eq;
11596 SetOp = PPCISD::SETBCR;
11597 break;
11598 case 2: // Return the value of the LT bit of CR6.
11599 BitNo = 2;
11600 InvertBit = false;
11601 Bitx = PPC::sub_lt;
11602 SetOp = PPCISD::SETBC;
11603 break;
11604 case 3: // Return the inverted value of the LT bit of CR6.
11605 BitNo = 2;
11606 InvertBit = true;
11607 Bitx = PPC::sub_lt;
11608 SetOp = PPCISD::SETBCR;
11609 break;
11610 }
11611
11612 SDValue GlueOp = CompNode.getValue(R: 1);
11613 if (Subtarget.isISA3_1()) {
11614 SDValue SubRegIdx = DAG.getTargetConstant(Val: Bitx, DL: dl, VT: MVT::i32);
11615 SDValue CR6Reg = DAG.getRegister(Reg: PPC::CR6, VT: MVT::i32);
11616 SDValue CRBit =
11617 SDValue(DAG.getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG, dl, VT: MVT::i1,
11618 Op1: CR6Reg, Op2: SubRegIdx, Op3: GlueOp),
11619 0);
11620 return DAG.getNode(Opcode: SetOp, DL: dl, VT: MVT::i32, Operand: CRBit);
11621 }
11622
11623 // Now that we have the comparison, emit a copy from the CR to a GPR.
11624 // This is flagged to the above dot comparison.
11625 SDValue Flags = DAG.getNode(Opcode: PPCISD::MFOCRF, DL: dl, VT: MVT::i32,
11626 N1: DAG.getRegister(Reg: PPC::CR6, VT: MVT::i32), N2: GlueOp);
11627
11628 // Shift the bit into the low position.
11629 Flags = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i32, N1: Flags,
11630 N2: DAG.getConstant(Val: 8 - (3 - BitNo), DL: dl, VT: MVT::i32));
11631 // Isolate the bit.
11632 Flags = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32, N1: Flags,
11633 N2: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
11634
11635 // If we are supposed to, toggle the bit.
11636 if (InvertBit)
11637 Flags = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: MVT::i32, N1: Flags,
11638 N2: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
11639 return Flags;
11640}
11641
11642SDValue PPCTargetLowering::LowerINTRINSIC_VOID(SDValue Op,
11643 SelectionDAG &DAG) const {
11644 // SelectionDAGBuilder::visitTargetIntrinsic may insert one extra chain to
11645 // the beginning of the argument list.
11646 int ArgStart = isa<ConstantSDNode>(Val: Op.getOperand(i: 0)) ? 0 : 1;
11647 SDLoc DL(Op);
11648 switch (Op.getConstantOperandVal(i: ArgStart)) {
11649 case Intrinsic::ppc_cfence: {
11650 assert(ArgStart == 1 && "llvm.ppc.cfence must carry a chain argument.");
11651 SDValue Val = Op.getOperand(i: ArgStart + 1);
11652 EVT Ty = Val.getValueType();
11653 if (Ty == MVT::i128) {
11654 // FIXME: Testing one of two paired registers is sufficient to guarantee
11655 // ordering?
11656 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i64, Operand: Val);
11657 }
11658 unsigned Opcode = Subtarget.isPPC64() ? PPC::CFENCE8 : PPC::CFENCE;
11659 return SDValue(
11660 DAG.getMachineNode(
11661 Opcode, dl: DL, VT: MVT::Other,
11662 Op1: DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: Subtarget.getScalarIntVT(), Operand: Val),
11663 Op2: Op.getOperand(i: 0)),
11664 0);
11665 }
11666 case Intrinsic::ppc_disassemble_dmr: {
11667 assert(ArgStart == 1 &&
11668 "llvm.ppc.disassemble.dmr must carry a chain argument.");
11669 return DAG.getStore(Chain: Op.getOperand(i: 0), dl: DL, Val: Op.getOperand(i: ArgStart + 2),
11670 Ptr: Op.getOperand(i: ArgStart + 1), PtrInfo: MachinePointerInfo());
11671 }
11672 default:
11673 break;
11674 }
11675 return SDValue();
11676}
11677
11678// Lower scalar BSWAP64 to xxbrd.
11679SDValue PPCTargetLowering::LowerBSWAP(SDValue Op, SelectionDAG &DAG) const {
11680 SDLoc dl(Op);
11681 if (!Subtarget.isPPC64())
11682 return Op;
11683
11684 if (Subtarget.hasP9Vector()) {
11685 // MTVSRDD
11686 Op = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: dl, VT: MVT::v2i64, N1: Op.getOperand(i: 0),
11687 N2: Op.getOperand(i: 0));
11688 // XXBRD
11689 Op = DAG.getNode(Opcode: ISD::BSWAP, DL: dl, VT: MVT::v2i64, Operand: Op);
11690 // MFVSRD
11691 int VectorIndex = 0;
11692 if (Subtarget.isLittleEndian())
11693 VectorIndex = 1;
11694 Op = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: MVT::i64, N1: Op,
11695 N2: DAG.getTargetConstant(Val: VectorIndex, DL: dl, VT: MVT::i32));
11696 return Op;
11697 }
11698
11699 // For Power8, use parallel rotate instructions for faster bswap64.
11700 SDValue Input = Op.getOperand(i: 0);
11701 // Helper to create rotate-and-insert operations (RLWIMI/RLDIMI).
11702 auto CreateRotateInsert =
11703 [&](unsigned Opcode, MVT VT, SDValue Dest, SDValue Src, unsigned RotAmt,
11704 unsigned MaskBegin,
11705 std::optional<unsigned> MaskEnd = std::nullopt) -> SDValue {
11706 SmallVector<SDValue, 5> Ops = {
11707 Dest, Src, DAG.getTargetConstant(Val: RotAmt, DL: dl, VT: MVT::i32),
11708 DAG.getTargetConstant(Val: MaskBegin, DL: dl, VT: MVT::i32)};
11709 if (MaskEnd.has_value())
11710 Ops.push_back(Elt: DAG.getTargetConstant(Val: *MaskEnd, DL: dl, VT: MVT::i32));
11711
11712 return SDValue(DAG.getMachineNode(Opcode, dl, VT, Ops), 0);
11713 };
11714
11715 // Helper to perform 32-bit byte swap using rotl(8) + 2x rlwimi.
11716 auto Swap32 = [&](SDValue Val32) -> SDValue {
11717 SDValue Rot = DAG.getNode(Opcode: ISD::ROTL, DL: dl, VT: MVT::i32, N1: Val32,
11718 N2: DAG.getConstant(Val: 8, DL: dl, VT: MVT::i32));
11719 // Insert bits [24:31] from Val32 into Rot at position [0:7].
11720 SDValue Swap =
11721 CreateRotateInsert(PPC::RLWIMI, MVT::i32, Rot, Val32, 24, 0, 7);
11722 // Insert bits [16:23] from Val32 into Swap at position [16:23].
11723 return CreateRotateInsert(PPC::RLWIMI, MVT::i32, Swap, Val32, 24, 16, 23);
11724 };
11725 // Extract and swap high and low 32-bit halves independently for parallelism.
11726 SDValue Hi32 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i32,
11727 Operand: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i64, N1: Input,
11728 N2: DAG.getConstant(Val: 32, DL: dl, VT: MVT::i64)));
11729 SDValue Lo32 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i32, Operand: Input);
11730
11731 // Combine swapped halves: rotate LoSwap left by 32 bits and insert into
11732 // HiSwap to swap their positions, completing the 64-bit byte reversal.
11733 SDValue HiSwap = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: MVT::i64, Operand: Swap32(Hi32));
11734 SDValue LoSwap = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: MVT::i64, Operand: Swap32(Lo32));
11735
11736 return CreateRotateInsert(PPC::RLDIMI, MVT::i64, HiSwap, LoSwap, 32, 0);
11737}
11738
11739// ATOMIC_CMP_SWAP for i8/i16 needs to zero-extend its input since it will be
11740// compared to a value that is atomically loaded (atomic loads zero-extend).
11741SDValue PPCTargetLowering::LowerATOMIC_CMP_SWAP(SDValue Op,
11742 SelectionDAG &DAG) const {
11743 assert(Op.getOpcode() == ISD::ATOMIC_CMP_SWAP &&
11744 "Expecting an atomic compare-and-swap here.");
11745 SDLoc dl(Op);
11746 auto *AtomicNode = cast<AtomicSDNode>(Val: Op.getNode());
11747 EVT MemVT = AtomicNode->getMemoryVT();
11748 if (MemVT.getSizeInBits() >= 32)
11749 return Op;
11750
11751 SDValue CmpOp = Op.getOperand(i: 2);
11752 // If this is already correctly zero-extended, leave it alone.
11753 auto HighBits = APInt::getHighBitsSet(numBits: 32, hiBitsSet: 32 - MemVT.getSizeInBits());
11754 if (DAG.MaskedValueIsZero(Op: CmpOp, Mask: HighBits))
11755 return Op;
11756
11757 // Clear the high bits of the compare operand.
11758 unsigned MaskVal = (1 << MemVT.getSizeInBits()) - 1;
11759 SDValue NewCmpOp =
11760 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32, N1: CmpOp,
11761 N2: DAG.getConstant(Val: MaskVal, DL: dl, VT: MVT::i32));
11762
11763 // Replace the existing compare operand with the properly zero-extended one.
11764 SmallVector<SDValue, 4> Ops;
11765 for (int i = 0, e = AtomicNode->getNumOperands(); i < e; i++)
11766 Ops.push_back(Elt: AtomicNode->getOperand(Num: i));
11767 Ops[2] = NewCmpOp;
11768 MachineMemOperand *MMO = AtomicNode->getMemOperand();
11769 SDVTList Tys = DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other);
11770 auto NodeTy =
11771 (MemVT == MVT::i8) ? PPCISD::ATOMIC_CMP_SWAP_8 : PPCISD::ATOMIC_CMP_SWAP_16;
11772 return DAG.getMemIntrinsicNode(Opcode: NodeTy, dl, VTList: Tys, Ops, MemVT, MMO);
11773}
11774
11775SDValue PPCTargetLowering::LowerATOMIC_LOAD_STORE(SDValue Op,
11776 SelectionDAG &DAG) const {
11777 AtomicSDNode *N = cast<AtomicSDNode>(Val: Op.getNode());
11778 EVT MemVT = N->getMemoryVT();
11779 assert(MemVT.getSimpleVT() == MVT::i128 &&
11780 "Expect quadword atomic operations");
11781 SDLoc dl(N);
11782 unsigned Opc = N->getOpcode();
11783 switch (Opc) {
11784 case ISD::ATOMIC_LOAD: {
11785 // Lower quadword atomic load to int_ppc_atomic_load_i128 which will be
11786 // lowered to ppc instructions by pattern matching instruction selector.
11787 SDVTList Tys = DAG.getVTList(VT1: MVT::i64, VT2: MVT::i64, VT3: MVT::Other);
11788 SmallVector<SDValue, 4> Ops{
11789 N->getOperand(Num: 0),
11790 DAG.getConstant(Val: Intrinsic::ppc_atomic_load_i128, DL: dl, VT: MVT::i32)};
11791 for (int I = 1, E = N->getNumOperands(); I < E; ++I)
11792 Ops.push_back(Elt: N->getOperand(Num: I));
11793 SDValue LoadedVal = DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_W_CHAIN, dl, VTList: Tys,
11794 Ops, MemVT, MMO: N->getMemOperand());
11795 SDValue ValLo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: MVT::i128, Operand: LoadedVal);
11796 SDValue ValHi =
11797 DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: MVT::i128, Operand: LoadedVal.getValue(R: 1));
11798 ValHi = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: MVT::i128, N1: ValHi,
11799 N2: DAG.getConstant(Val: 64, DL: dl, VT: MVT::i32));
11800 SDValue Val =
11801 DAG.getNode(Opcode: ISD::OR, DL: dl, ResultTys: {MVT::i128, MVT::Other}, Ops: {ValLo, ValHi});
11802 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: dl, ResultTys: {MVT::i128, MVT::Other},
11803 Ops: {Val, LoadedVal.getValue(R: 2)});
11804 }
11805 case ISD::ATOMIC_STORE: {
11806 // Lower quadword atomic store to int_ppc_atomic_store_i128 which will be
11807 // lowered to ppc instructions by pattern matching instruction selector.
11808 SDVTList Tys = DAG.getVTList(VT: MVT::Other);
11809 SmallVector<SDValue, 4> Ops{
11810 N->getOperand(Num: 0),
11811 DAG.getConstant(Val: Intrinsic::ppc_atomic_store_i128, DL: dl, VT: MVT::i32)};
11812 SDValue Val = N->getOperand(Num: 1);
11813 SDValue ValLo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i64, Operand: Val);
11814 SDValue ValHi = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i128, N1: Val,
11815 N2: DAG.getConstant(Val: 64, DL: dl, VT: MVT::i32));
11816 ValHi = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i64, Operand: ValHi);
11817 Ops.push_back(Elt: ValLo);
11818 Ops.push_back(Elt: ValHi);
11819 Ops.push_back(Elt: N->getOperand(Num: 2));
11820 return DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_VOID, dl, VTList: Tys, Ops, MemVT,
11821 MMO: N->getMemOperand());
11822 }
11823 default:
11824 llvm_unreachable("Unexpected atomic opcode");
11825 }
11826}
11827
11828static SDValue getDataClassTest(SDValue Op, FPClassTest Mask, const SDLoc &Dl,
11829 SelectionDAG &DAG,
11830 const PPCSubtarget &Subtarget) {
11831 assert(Mask <= fcAllFlags && "Invalid fp_class flags!");
11832
11833 enum DataClassMask {
11834 DC_NAN = 1 << 6,
11835 DC_NEG_INF = 1 << 4,
11836 DC_POS_INF = 1 << 5,
11837 DC_NEG_ZERO = 1 << 2,
11838 DC_POS_ZERO = 1 << 3,
11839 DC_NEG_SUBNORM = 1,
11840 DC_POS_SUBNORM = 1 << 1,
11841 };
11842
11843 EVT VT = Op.getValueType();
11844
11845 unsigned TestOp = VT == MVT::f128 ? PPC::XSTSTDCQP
11846 : VT == MVT::f64 ? PPC::XSTSTDCDP
11847 : PPC::XSTSTDCSP;
11848
11849 if (Mask == fcAllFlags)
11850 return DAG.getBoolConstant(V: true, DL: Dl, VT: MVT::i1, OpVT: VT);
11851 if (Mask == 0)
11852 return DAG.getBoolConstant(V: false, DL: Dl, VT: MVT::i1, OpVT: VT);
11853
11854 // When it's cheaper or necessary to test reverse flags.
11855 if ((Mask & fcNormal) == fcNormal || Mask == ~fcQNan || Mask == ~fcSNan) {
11856 SDValue Rev = getDataClassTest(Op, Mask: ~Mask, Dl, DAG, Subtarget);
11857 return DAG.getNOT(DL: Dl, Val: Rev, VT: MVT::i1);
11858 }
11859
11860 // Power doesn't support testing whether a value is 'normal'. Test the rest
11861 // first, and test if it's 'not not-normal' with expected sign.
11862 if (Mask & fcNormal) {
11863 SDValue Rev(DAG.getMachineNode(
11864 Opcode: TestOp, dl: Dl, VT: MVT::i32,
11865 Op1: DAG.getTargetConstant(Val: DC_NAN | DC_NEG_INF | DC_POS_INF |
11866 DC_NEG_ZERO | DC_POS_ZERO |
11867 DC_NEG_SUBNORM | DC_POS_SUBNORM,
11868 DL: Dl, VT: MVT::i32),
11869 Op2: Op),
11870 0);
11871 // Sign are stored in CR bit 0, result are in CR bit 2.
11872 SDValue Sign(
11873 DAG.getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG, dl: Dl, VT: MVT::i1, Op1: Rev,
11874 Op2: DAG.getTargetConstant(Val: PPC::sub_lt, DL: Dl, VT: MVT::i32)),
11875 0);
11876 SDValue Normal(DAG.getNOT(
11877 DL: Dl,
11878 Val: SDValue(DAG.getMachineNode(
11879 Opcode: TargetOpcode::EXTRACT_SUBREG, dl: Dl, VT: MVT::i1, Op1: Rev,
11880 Op2: DAG.getTargetConstant(Val: PPC::sub_eq, DL: Dl, VT: MVT::i32)),
11881 0),
11882 VT: MVT::i1));
11883 if (Mask & fcPosNormal)
11884 Sign = DAG.getNOT(DL: Dl, Val: Sign, VT: MVT::i1);
11885 SDValue Result = DAG.getNode(Opcode: ISD::AND, DL: Dl, VT: MVT::i1, N1: Sign, N2: Normal);
11886 if (Mask == fcPosNormal || Mask == fcNegNormal)
11887 return Result;
11888
11889 return DAG.getNode(
11890 Opcode: ISD::OR, DL: Dl, VT: MVT::i1,
11891 N1: getDataClassTest(Op, Mask: Mask & ~fcNormal, Dl, DAG, Subtarget), N2: Result);
11892 }
11893
11894 // The instruction doesn't differentiate between signaling or quiet NaN. Test
11895 // the rest first, and test if it 'is NaN and is signaling/quiet'.
11896 if ((Mask & fcNan) == fcQNan || (Mask & fcNan) == fcSNan) {
11897 bool IsQuiet = Mask & fcQNan;
11898 SDValue NanCheck = getDataClassTest(Op, Mask: fcNan, Dl, DAG, Subtarget);
11899
11900 // Quietness is determined by the first bit in fraction field.
11901 uint64_t QuietMask = 0;
11902 SDValue HighWord;
11903 if (VT == MVT::f128) {
11904 HighWord = DAG.getNode(
11905 Opcode: ISD::EXTRACT_VECTOR_ELT, DL: Dl, VT: MVT::i32, N1: DAG.getBitcast(VT: MVT::v4i32, V: Op),
11906 N2: DAG.getVectorIdxConstant(Val: Subtarget.isLittleEndian() ? 3 : 0, DL: Dl));
11907 QuietMask = 0x8000;
11908 } else if (VT == MVT::f64) {
11909 if (Subtarget.isPPC64()) {
11910 HighWord = DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL: Dl, VT: MVT::i32,
11911 N1: DAG.getBitcast(VT: MVT::i64, V: Op),
11912 N2: DAG.getConstant(Val: 1, DL: Dl, VT: MVT::i32));
11913 } else {
11914 SDValue Vec = DAG.getBitcast(
11915 VT: MVT::v4i32, V: DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: Dl, VT: MVT::v2f64, Operand: Op));
11916 HighWord = DAG.getNode(
11917 Opcode: ISD::EXTRACT_VECTOR_ELT, DL: Dl, VT: MVT::i32, N1: Vec,
11918 N2: DAG.getVectorIdxConstant(Val: Subtarget.isLittleEndian() ? 1 : 0, DL: Dl));
11919 }
11920 QuietMask = 0x80000;
11921 } else if (VT == MVT::f32) {
11922 HighWord = DAG.getBitcast(VT: MVT::i32, V: Op);
11923 QuietMask = 0x400000;
11924 }
11925 SDValue NanRes = DAG.getSetCC(
11926 DL: Dl, VT: MVT::i1,
11927 LHS: DAG.getNode(Opcode: ISD::AND, DL: Dl, VT: MVT::i32, N1: HighWord,
11928 N2: DAG.getConstant(Val: QuietMask, DL: Dl, VT: MVT::i32)),
11929 RHS: DAG.getConstant(Val: 0, DL: Dl, VT: MVT::i32), Cond: IsQuiet ? ISD::SETNE : ISD::SETEQ);
11930 NanRes = DAG.getNode(Opcode: ISD::AND, DL: Dl, VT: MVT::i1, N1: NanCheck, N2: NanRes);
11931 if (Mask == fcQNan || Mask == fcSNan)
11932 return NanRes;
11933
11934 return DAG.getNode(Opcode: ISD::OR, DL: Dl, VT: MVT::i1,
11935 N1: getDataClassTest(Op, Mask: Mask & ~fcNan, Dl, DAG, Subtarget),
11936 N2: NanRes);
11937 }
11938
11939 unsigned NativeMask = 0;
11940 if ((Mask & fcNan) == fcNan)
11941 NativeMask |= DC_NAN;
11942 if (Mask & fcNegInf)
11943 NativeMask |= DC_NEG_INF;
11944 if (Mask & fcPosInf)
11945 NativeMask |= DC_POS_INF;
11946 if (Mask & fcNegZero)
11947 NativeMask |= DC_NEG_ZERO;
11948 if (Mask & fcPosZero)
11949 NativeMask |= DC_POS_ZERO;
11950 if (Mask & fcNegSubnormal)
11951 NativeMask |= DC_NEG_SUBNORM;
11952 if (Mask & fcPosSubnormal)
11953 NativeMask |= DC_POS_SUBNORM;
11954 return SDValue(
11955 DAG.getMachineNode(
11956 Opcode: TargetOpcode::EXTRACT_SUBREG, dl: Dl, VT: MVT::i1,
11957 Op1: SDValue(DAG.getMachineNode(
11958 Opcode: TestOp, dl: Dl, VT: MVT::i32,
11959 Op1: DAG.getTargetConstant(Val: NativeMask, DL: Dl, VT: MVT::i32), Op2: Op),
11960 0),
11961 Op2: DAG.getTargetConstant(Val: PPC::sub_eq, DL: Dl, VT: MVT::i32)),
11962 0);
11963}
11964
11965SDValue PPCTargetLowering::LowerIS_FPCLASS(SDValue Op,
11966 SelectionDAG &DAG) const {
11967 assert(Subtarget.hasP9Vector() && "Test data class requires Power9");
11968 SDValue LHS = Op.getOperand(i: 0);
11969 uint64_t RHSC = Op.getConstantOperandVal(i: 1);
11970 SDLoc Dl(Op);
11971 FPClassTest Category = static_cast<FPClassTest>(RHSC);
11972 if (LHS.getValueType() == MVT::ppcf128) {
11973 // The higher part determines the value class.
11974 LHS = DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL: Dl, VT: MVT::f64, N1: LHS,
11975 N2: DAG.getConstant(Val: 1, DL: Dl, VT: MVT::i32));
11976 }
11977
11978 return getDataClassTest(Op: LHS, Mask: Category, Dl, DAG, Subtarget);
11979}
11980
11981// Adjust the length value for a load/store with length to account for the
11982// instructions requiring a left justified length, and for non-byte element
11983// types requiring scaling by element size.
11984static SDValue AdjustLength(SDValue Val, unsigned Bits, bool Left,
11985 SelectionDAG &DAG) {
11986 SDLoc dl(Val);
11987 EVT VT = Val->getValueType(ResNo: 0);
11988 unsigned LeftAdj = Left ? VT.getSizeInBits() - 8 : 0;
11989 unsigned TypeAdj = llvm::countr_zero<uint32_t>(Val: Bits / 8);
11990 SDValue SHLAmt = DAG.getConstant(Val: LeftAdj + TypeAdj, DL: dl, VT);
11991 return DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Val, N2: SHLAmt);
11992}
11993
11994SDValue PPCTargetLowering::LowerVP_LOAD(SDValue Op, SelectionDAG &DAG) const {
11995 auto VPLD = cast<VPLoadSDNode>(Val&: Op);
11996 bool Future = Subtarget.isISAFuture();
11997 SDLoc dl(Op);
11998 assert(ISD::isConstantSplatVectorAllOnes(Op->getOperand(3).getNode(), true) &&
11999 "Mask predication not supported");
12000 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
12001 SDValue Len = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: PtrVT, Operand: VPLD->getOperand(Num: 4));
12002 unsigned IID = Future ? Intrinsic::ppc_vsx_lxvrl : Intrinsic::ppc_vsx_lxvl;
12003 unsigned EltBits = Op->getValueType(ResNo: 0).getScalarType().getSizeInBits();
12004 Len = AdjustLength(Val: Len, Bits: EltBits, Left: !Future, DAG);
12005 SDValue Ops[] = {VPLD->getChain(), DAG.getConstant(Val: IID, DL: dl, VT: MVT::i32),
12006 VPLD->getOperand(Num: 1), Len};
12007 SDVTList Tys = DAG.getVTList(VT1: Op->getValueType(ResNo: 0), VT2: MVT::Other);
12008 SDValue VPL =
12009 DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_W_CHAIN, dl, VTList: Tys, Ops,
12010 MemVT: VPLD->getMemoryVT(), MMO: VPLD->getMemOperand());
12011 return VPL;
12012}
12013
12014SDValue PPCTargetLowering::LowerVP_STORE(SDValue Op, SelectionDAG &DAG) const {
12015 auto VPST = cast<VPStoreSDNode>(Val&: Op);
12016 assert(ISD::isConstantSplatVectorAllOnes(Op->getOperand(4).getNode(), true) &&
12017 "Mask predication not supported");
12018 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
12019 SDLoc dl(Op);
12020 SDValue Len = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: PtrVT, Operand: VPST->getOperand(Num: 5));
12021 unsigned EltBits =
12022 Op->getOperand(Num: 1).getValueType().getScalarType().getSizeInBits();
12023 bool Future = Subtarget.isISAFuture();
12024 unsigned IID = Future ? Intrinsic::ppc_vsx_stxvrl : Intrinsic::ppc_vsx_stxvl;
12025 Len = AdjustLength(Val: Len, Bits: EltBits, Left: !Future, DAG);
12026 SDValue Ops[] = {
12027 VPST->getChain(), DAG.getConstant(Val: IID, DL: dl, VT: MVT::i32),
12028 DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v4i32, Operand: VPST->getOperand(Num: 1)),
12029 VPST->getOperand(Num: 2), Len};
12030 SDVTList Tys = DAG.getVTList(VT: MVT::Other);
12031 SDValue VPS =
12032 DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_VOID, dl, VTList: Tys, Ops,
12033 MemVT: VPST->getMemoryVT(), MMO: VPST->getMemOperand());
12034 return VPS;
12035}
12036
12037SDValue PPCTargetLowering::LowerPartialReduce(SDValue Op,
12038 SelectionDAG &DAG) const {
12039 SDValue Acc = Op.getOperand(i: 0);
12040 SDValue Op1 = Op.getOperand(i: 1);
12041 SDValue Op2 = Op.getOperand(i: 2);
12042
12043 assert(Op.getOpcode() == ISD::PARTIAL_REDUCE_UMLA &&
12044 "Unexpected partial reduction");
12045
12046 if (Acc.getValueType() != MVT::v4i32)
12047 return SDValue();
12048 if (Op1.getValueType() != MVT::v16i32 || Op1.getOpcode() != ISD::SIGN_EXTEND)
12049 return SDValue();
12050 SDValue Op1Input = Op1.getOperand(i: 0);
12051 if (Op1Input.getValueType() != MVT::v16i8 || !llvm::isOneOrOneSplat(V: Op2))
12052 return SDValue();
12053
12054 SDLoc dl(Op);
12055 SDValue Ones = DAG.getConstant(Val: 1, DL: dl, VT: MVT::v16i8);
12056 return DAG.getNode(Opcode: ISD::PARTIAL_REDUCE_SUMLA, DL: dl, VT: MVT::v4i32, N1: Acc, N2: Op1Input,
12057 N3: Ones);
12058}
12059
12060SDValue PPCTargetLowering::LowerSCALAR_TO_VECTOR(SDValue Op,
12061 SelectionDAG &DAG) const {
12062 SDLoc dl(Op);
12063
12064 MachineFunction &MF = DAG.getMachineFunction();
12065 SDValue Op0 = Op.getOperand(i: 0);
12066 EVT ValVT = Op0.getValueType();
12067 unsigned EltSize = Op.getValueType().getScalarSizeInBits();
12068 if (isa<ConstantSDNode>(Val: Op0) && EltSize <= 32) {
12069 int64_t IntVal = Op.getConstantOperandVal(i: 0);
12070 if (IntVal >= -16 && IntVal <= 15)
12071 return getCanonicalConstSplat(Val: IntVal, SplatSize: EltSize / 8, VT: Op.getValueType(), DAG,
12072 dl);
12073 }
12074
12075 ReuseLoadInfo RLI;
12076 if (Subtarget.hasLFIWAX() && Subtarget.hasVSX() &&
12077 Op.getValueType() == MVT::v4i32 && Op0.getOpcode() == ISD::LOAD &&
12078 Op0.getValueType() == MVT::i32 && Op0.hasOneUse() &&
12079 canReuseLoadAddress(Op: Op0, MemVT: MVT::i32, RLI, DAG, ET: ISD::NON_EXTLOAD)) {
12080
12081 MachineMemOperand *MMO = MF.getMachineMemOperand(
12082 PtrInfo: RLI.MPI, F: MachineMemOperand::MOLoad, Size: 4, BaseAlignment: RLI.Alignment,
12083 Metadata: MMOMetadata(RLI.AAInfo, RLI.Ranges));
12084 SDValue Ops[] = {RLI.Chain, RLI.Ptr, DAG.getValueType(Op.getValueType())};
12085 SDValue Bits = DAG.getMemIntrinsicNode(
12086 Opcode: PPCISD::LD_SPLAT, dl, VTList: DAG.getVTList(VT1: MVT::v4i32, VT2: MVT::Other), Ops,
12087 MemVT: MVT::i32, MMO);
12088 if (RLI.ResChain)
12089 DAG.makeEquivalentMemoryOrdering(OldChain: RLI.ResChain, NewMemOpChain: Bits.getValue(R: 1));
12090 return Bits.getValue(R: 0);
12091 }
12092
12093 // Create a stack slot that is 16-byte aligned.
12094 MachineFrameInfo &MFI = MF.getFrameInfo();
12095 int FrameIdx = MFI.CreateStackObject(Size: 16, Alignment: Align(16), isSpillSlot: false);
12096 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
12097 SDValue FIdx = DAG.getFrameIndex(FI: FrameIdx, VT: PtrVT);
12098
12099 SDValue Val = Op0;
12100 // P10 hardware store forwarding requires that a single store contains all
12101 // the data for the load. P10 is able to merge a pair of adjacent stores. Try
12102 // to avoid load hit store on P10 when running binaries compiled for older
12103 // processors by generating two mergeable scalar stores to forward with the
12104 // vector load.
12105 if (!DisableP10StoreForward && Subtarget.isPPC64() &&
12106 !Subtarget.isLittleEndian() && ValVT.isInteger() &&
12107 ValVT.getSizeInBits() <= 64) {
12108 Val = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: MVT::i64, Operand: Val);
12109 EVT ShiftAmountTy = getShiftAmountTy(LHSTy: MVT::i64, DL: DAG.getDataLayout());
12110 SDValue ShiftBy = DAG.getConstant(
12111 Val: 64 - Op.getValueType().getScalarSizeInBits(), DL: dl, VT: ShiftAmountTy);
12112 Val = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: MVT::i64, N1: Val, N2: ShiftBy);
12113 SDValue Plus8 =
12114 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: FIdx, N2: DAG.getConstant(Val: 8, DL: dl, VT: PtrVT));
12115 SDValue Store2 =
12116 DAG.getStore(Chain: DAG.getEntryNode(), dl, Val, Ptr: Plus8, PtrInfo: MachinePointerInfo());
12117 SDValue Store = DAG.getStore(Chain: Store2, dl, Val, Ptr: FIdx, PtrInfo: MachinePointerInfo());
12118 return DAG.getLoad(VT: Op.getValueType(), dl, Chain: Store, Ptr: FIdx,
12119 PtrInfo: MachinePointerInfo());
12120 }
12121
12122 // Store the input value into Value#0 of the stack slot.
12123 SDValue Store =
12124 DAG.getStore(Chain: DAG.getEntryNode(), dl, Val, Ptr: FIdx, PtrInfo: MachinePointerInfo());
12125 // Load it out.
12126 return DAG.getLoad(VT: Op.getValueType(), dl, Chain: Store, Ptr: FIdx, PtrInfo: MachinePointerInfo());
12127}
12128
12129SDValue PPCTargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
12130 SelectionDAG &DAG) const {
12131 assert(Op.getOpcode() == ISD::INSERT_VECTOR_ELT &&
12132 "Should only be called for ISD::INSERT_VECTOR_ELT");
12133
12134 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2));
12135
12136 EVT VT = Op.getValueType();
12137 SDLoc dl(Op);
12138 SDValue V1 = Op.getOperand(i: 0);
12139 SDValue V2 = Op.getOperand(i: 1);
12140
12141 if (VT == MVT::v2f64 && C)
12142 return Op;
12143
12144 if (Subtarget.hasP9Vector()) {
12145 // A f32 load feeding into a v4f32 insert_vector_elt is handled in this way
12146 // because on P10, it allows this specific insert_vector_elt load pattern to
12147 // utilize the refactored load and store infrastructure in order to exploit
12148 // prefixed loads.
12149 // On targets with inexpensive direct moves (Power9 and up), a
12150 // (insert_vector_elt v4f32:$vec, (f32 load)) is always better as an integer
12151 // load since a single precision load will involve conversion to double
12152 // precision on the load followed by another conversion to single precision.
12153 if ((VT == MVT::v4f32) && (V2.getValueType() == MVT::f32) &&
12154 (isa<LoadSDNode>(Val: V2))) {
12155 SDValue BitcastVector = DAG.getBitcast(VT: MVT::v4i32, V: V1);
12156 SDValue BitcastLoad = DAG.getBitcast(VT: MVT::i32, V: V2);
12157 SDValue InsVecElt =
12158 DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: MVT::v4i32, N1: BitcastVector,
12159 N2: BitcastLoad, N3: Op.getOperand(i: 2));
12160 return DAG.getBitcast(VT: MVT::v4f32, V: InsVecElt);
12161 }
12162 }
12163
12164 if (Subtarget.isISA3_1()) {
12165 if ((VT == MVT::v2i64 || VT == MVT::v2f64) && !Subtarget.isPPC64())
12166 return SDValue();
12167 // On P10, we have legal lowering for constant and variable indices for
12168 // all vectors.
12169 if (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32 ||
12170 VT == MVT::v2i64 || VT == MVT::v4f32 || VT == MVT::v2f64)
12171 return Op;
12172 }
12173
12174 // Before P10, we have legal lowering for constant indices but not for
12175 // variable ones.
12176 if (!C)
12177 return SDValue();
12178
12179 // We can use MTVSRZ + VECINSERT for v8i16 and v16i8 types.
12180 if (VT == MVT::v8i16 || VT == MVT::v16i8) {
12181 SDValue Mtvsrz = DAG.getNode(Opcode: PPCISD::MTVSRZ, DL: dl, VT, Operand: V2);
12182 unsigned BytesInEachElement = VT.getVectorElementType().getSizeInBits() / 8;
12183 unsigned InsertAtElement = C->getZExtValue();
12184 unsigned InsertAtByte = InsertAtElement * BytesInEachElement;
12185 if (Subtarget.isLittleEndian()) {
12186 InsertAtByte = (16 - BytesInEachElement) - InsertAtByte;
12187 }
12188 return DAG.getNode(Opcode: PPCISD::VECINSERT, DL: dl, VT, N1: V1, N2: Mtvsrz,
12189 N3: DAG.getConstant(Val: InsertAtByte, DL: dl, VT: MVT::i32));
12190 }
12191 return Op;
12192}
12193
12194SDValue PPCTargetLowering::LowerDMFVectorLoad(SDValue Op,
12195 SelectionDAG &DAG) const {
12196 SDLoc dl(Op);
12197 LoadSDNode *LN = cast<LoadSDNode>(Val: Op.getNode());
12198 SDValue LoadChain = LN->getChain();
12199 SDValue BasePtr = LN->getBasePtr();
12200 EVT VT = Op.getValueType();
12201 bool IsV1024i1 = VT == MVT::v1024i1;
12202 bool IsV2048i1 = VT == MVT::v2048i1;
12203
12204 // The types v1024i1 and v2048i1 are used for Dense Math dmr registers and
12205 // Dense Math dmr pair registers, respectively.
12206 assert((IsV1024i1 || IsV2048i1) && "Unsupported type.");
12207 (void)IsV2048i1;
12208 assert((Subtarget.hasMMA() && Subtarget.isISAFuture()) &&
12209 "Dense Math support required.");
12210 assert(Subtarget.pairedVectorMemops() && "Vector pair support required.");
12211
12212 SmallVector<SDValue, 8> Loads;
12213 SmallVector<SDValue, 8> LoadChains;
12214
12215 SDValue IntrinID = DAG.getConstant(Val: Intrinsic::ppc_vsx_lxvp, DL: dl, VT: MVT::i32);
12216 SDValue LoadOps[] = {LoadChain, IntrinID, BasePtr};
12217 MachineMemOperand *MMO = LN->getMemOperand();
12218 unsigned NumVecs = VT.getSizeInBits() / 256;
12219 for (unsigned Idx = 0; Idx < NumVecs; ++Idx) {
12220 MachineMemOperand *NewMMO =
12221 DAG.getMachineFunction().getMachineMemOperand(MMO, Offset: Idx * 32, Size: 32);
12222 if (Idx > 0) {
12223 BasePtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: BasePtr.getValueType(), N1: BasePtr,
12224 N2: DAG.getConstant(Val: 32, DL: dl, VT: BasePtr.getValueType()));
12225 LoadOps[2] = BasePtr;
12226 }
12227 SDValue Ld = DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_W_CHAIN, dl,
12228 VTList: DAG.getVTList(VT1: MVT::v256i1, VT2: MVT::Other),
12229 Ops: LoadOps, MemVT: MVT::v256i1, MMO: NewMMO);
12230 LoadChains.push_back(Elt: Ld.getValue(R: 1));
12231 Loads.push_back(Elt: Ld);
12232 }
12233
12234 if (Subtarget.isLittleEndian()) {
12235 std::reverse(first: Loads.begin(), last: Loads.end());
12236 std::reverse(first: LoadChains.begin(), last: LoadChains.end());
12237 }
12238
12239 SDValue TF = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: LoadChains);
12240 SDValue Value = DMFInsert1024(Pairs: Loads, dl, DAG);
12241
12242 if (IsV1024i1) {
12243 return DAG.getMergeValues(Ops: {Value, TF}, dl);
12244 }
12245
12246 // Handle Loads for V2048i1 which represents a dmr pair.
12247 SmallVector<SDValue, 4> MoreLoads{Loads[4], Loads[5], Loads[6], Loads[7]};
12248 SDValue Dmr1Value = DMFInsert1024(Pairs: MoreLoads, dl, DAG);
12249
12250 SDValue Dmr0Sub = DAG.getTargetConstant(Val: PPC::sub_dmr0, DL: dl, VT: MVT::i32);
12251 SDValue Dmr1Sub = DAG.getTargetConstant(Val: PPC::sub_dmr1, DL: dl, VT: MVT::i32);
12252
12253 SDValue DmrPRC = DAG.getTargetConstant(Val: PPC::DMRpRCRegClassID, DL: dl, VT: MVT::i32);
12254 const SDValue DmrPOps[] = {DmrPRC, Value, Dmr0Sub, Dmr1Value, Dmr1Sub};
12255
12256 SDValue DmrPValue = SDValue(
12257 DAG.getMachineNode(Opcode: PPC::REG_SEQUENCE, dl, VT: MVT::v2048i1, Ops: DmrPOps), 0);
12258
12259 return DAG.getMergeValues(Ops: {DmrPValue, TF}, dl);
12260}
12261
12262SDValue PPCTargetLowering::DMFInsert1024(const SmallVectorImpl<SDValue> &Pairs,
12263 const SDLoc &dl,
12264 SelectionDAG &DAG) const {
12265 SDValue Lo =
12266 DAG.getNode(Opcode: PPCISD::INST512, DL: dl, VT: MVT::v512i1, N1: Pairs[0], N2: Pairs[1]);
12267 SDValue LoSub = DAG.getTargetConstant(Val: PPC::sub_wacc_lo, DL: dl, VT: MVT::i32);
12268 SDValue Hi =
12269 DAG.getNode(Opcode: PPCISD::INST512HI, DL: dl, VT: MVT::v512i1, N1: Pairs[2], N2: Pairs[3]);
12270 SDValue HiSub = DAG.getTargetConstant(Val: PPC::sub_wacc_hi, DL: dl, VT: MVT::i32);
12271 SDValue RC = DAG.getTargetConstant(Val: PPC::DMRRCRegClassID, DL: dl, VT: MVT::i32);
12272
12273 return SDValue(DAG.getMachineNode(Opcode: PPC::REG_SEQUENCE, dl, VT: MVT::v1024i1,
12274 Ops: {RC, Lo, LoSub, Hi, HiSub}),
12275 0);
12276}
12277
12278SDValue PPCTargetLowering::LowerVectorLoad(SDValue Op,
12279 SelectionDAG &DAG) const {
12280 SDLoc dl(Op);
12281 LoadSDNode *LN = cast<LoadSDNode>(Val: Op.getNode());
12282 SDValue LoadChain = LN->getChain();
12283 SDValue BasePtr = LN->getBasePtr();
12284 EVT VT = Op.getValueType();
12285
12286 if (VT == MVT::v1024i1 || VT == MVT::v2048i1)
12287 return LowerDMFVectorLoad(Op, DAG);
12288
12289 if (VT != MVT::v256i1 && VT != MVT::v512i1)
12290 return Op;
12291
12292 // Type v256i1 is used for pairs and v512i1 is used for accumulators.
12293 assert((VT != MVT::v512i1 || Subtarget.hasMMA()) &&
12294 "Type unsupported without MMA");
12295 assert((VT != MVT::v256i1 || Subtarget.pairedVectorMemops()) &&
12296 "Type unsupported without paired vector support");
12297
12298 // For v256i1 on ISA Future, let the load go through to instruction selection
12299 // where it will be matched to lxvp/plxvp by the instruction patterns.
12300 if (VT == MVT::v256i1 && Subtarget.isISAFuture())
12301 return Op;
12302
12303 // For other cases, create 2 or 4 v16i8 loads to load the pair or accumulator
12304 // value in 2 or 4 vsx registers.
12305 Align Alignment = LN->getAlign();
12306 SmallVector<SDValue, 4> Loads;
12307 SmallVector<SDValue, 4> LoadChains;
12308 unsigned NumVecs = VT.getSizeInBits() / 128;
12309 for (unsigned Idx = 0; Idx < NumVecs; ++Idx) {
12310 SDValue Load =
12311 DAG.getLoad(VT: MVT::v16i8, dl, Chain: LoadChain, Ptr: BasePtr,
12312 PtrInfo: LN->getPointerInfo().getWithOffset(O: Idx * 16),
12313 Alignment: commonAlignment(A: Alignment, Offset: Idx * 16),
12314 MMOFlags: LN->getMemOperand()->getFlags(), Metadata: LN->getAAInfo());
12315 BasePtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: BasePtr.getValueType(), N1: BasePtr,
12316 N2: DAG.getConstant(Val: 16, DL: dl, VT: BasePtr.getValueType()));
12317 Loads.push_back(Elt: Load);
12318 LoadChains.push_back(Elt: Load.getValue(R: 1));
12319 }
12320 if (Subtarget.isLittleEndian()) {
12321 std::reverse(first: Loads.begin(), last: Loads.end());
12322 std::reverse(first: LoadChains.begin(), last: LoadChains.end());
12323 }
12324 SDValue TF = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: LoadChains);
12325 SDValue Value =
12326 DAG.getNode(Opcode: VT == MVT::v512i1 ? PPCISD::ACC_BUILD : PPCISD::PAIR_BUILD,
12327 DL: dl, VT, Ops: Loads);
12328 SDValue RetOps[] = {Value, TF};
12329 return DAG.getMergeValues(Ops: RetOps, dl);
12330}
12331
12332SDValue PPCTargetLowering::LowerDMFVectorStore(SDValue Op,
12333 SelectionDAG &DAG) const {
12334
12335 SDLoc dl(Op);
12336 StoreSDNode *SN = cast<StoreSDNode>(Val: Op.getNode());
12337 SDValue StoreChain = SN->getChain();
12338 SDValue BasePtr = SN->getBasePtr();
12339 SmallVector<SDValue, 8> Values;
12340 SmallVector<SDValue, 8> Stores;
12341 EVT VT = SN->getValue().getValueType();
12342 bool IsV1024i1 = VT == MVT::v1024i1;
12343 bool IsV2048i1 = VT == MVT::v2048i1;
12344
12345 // The types v1024i1 and v2048i1 are used for Dense Math dmr registers and
12346 // Dense Math dmr pair registers, respectively.
12347 assert((IsV1024i1 || IsV2048i1) && "Unsupported type.");
12348 (void)IsV2048i1;
12349 assert((Subtarget.hasMMA() && Subtarget.isISAFuture()) &&
12350 "Dense Math support required.");
12351 assert(Subtarget.pairedVectorMemops() && "Vector pair support required.");
12352
12353 EVT ReturnTypes[] = {MVT::v256i1, MVT::v256i1};
12354 if (IsV1024i1) {
12355 SDValue Lo(DAG.getMachineNode(
12356 Opcode: TargetOpcode::EXTRACT_SUBREG, dl, VT: MVT::v512i1,
12357 Op1: Op.getOperand(i: 1),
12358 Op2: DAG.getTargetConstant(Val: PPC::sub_wacc_lo, DL: dl, VT: MVT::i32)),
12359 0);
12360 SDValue Hi(DAG.getMachineNode(
12361 Opcode: TargetOpcode::EXTRACT_SUBREG, dl, VT: MVT::v512i1,
12362 Op1: Op.getOperand(i: 1),
12363 Op2: DAG.getTargetConstant(Val: PPC::sub_wacc_hi, DL: dl, VT: MVT::i32)),
12364 0);
12365 MachineSDNode *ExtNode =
12366 DAG.getMachineNode(Opcode: PPC::DMXXEXTFDMR512, dl, ResultTys: ReturnTypes, Ops: Lo);
12367 Values.push_back(Elt: SDValue(ExtNode, 0));
12368 Values.push_back(Elt: SDValue(ExtNode, 1));
12369 ExtNode = DAG.getMachineNode(Opcode: PPC::DMXXEXTFDMR512_HI, dl, ResultTys: ReturnTypes, Ops: Hi);
12370 Values.push_back(Elt: SDValue(ExtNode, 0));
12371 Values.push_back(Elt: SDValue(ExtNode, 1));
12372 } else {
12373 // This corresponds to v2048i1 which represents a dmr pair.
12374 SDValue Dmr0(
12375 DAG.getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG, dl, VT: MVT::v1024i1,
12376 Op1: Op.getOperand(i: 1),
12377 Op2: DAG.getTargetConstant(Val: PPC::sub_dmr0, DL: dl, VT: MVT::i32)),
12378 0);
12379
12380 SDValue Dmr1(
12381 DAG.getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG, dl, VT: MVT::v1024i1,
12382 Op1: Op.getOperand(i: 1),
12383 Op2: DAG.getTargetConstant(Val: PPC::sub_dmr1, DL: dl, VT: MVT::i32)),
12384 0);
12385
12386 SDValue Dmr0Lo(DAG.getMachineNode(
12387 Opcode: TargetOpcode::EXTRACT_SUBREG, dl, VT: MVT::v512i1, Op1: Dmr0,
12388 Op2: DAG.getTargetConstant(Val: PPC::sub_wacc_lo, DL: dl, VT: MVT::i32)),
12389 0);
12390
12391 SDValue Dmr0Hi(DAG.getMachineNode(
12392 Opcode: TargetOpcode::EXTRACT_SUBREG, dl, VT: MVT::v512i1, Op1: Dmr0,
12393 Op2: DAG.getTargetConstant(Val: PPC::sub_wacc_hi, DL: dl, VT: MVT::i32)),
12394 0);
12395
12396 SDValue Dmr1Lo(DAG.getMachineNode(
12397 Opcode: TargetOpcode::EXTRACT_SUBREG, dl, VT: MVT::v512i1, Op1: Dmr1,
12398 Op2: DAG.getTargetConstant(Val: PPC::sub_wacc_lo, DL: dl, VT: MVT::i32)),
12399 0);
12400
12401 SDValue Dmr1Hi(DAG.getMachineNode(
12402 Opcode: TargetOpcode::EXTRACT_SUBREG, dl, VT: MVT::v512i1, Op1: Dmr1,
12403 Op2: DAG.getTargetConstant(Val: PPC::sub_wacc_hi, DL: dl, VT: MVT::i32)),
12404 0);
12405
12406 MachineSDNode *ExtNode =
12407 DAG.getMachineNode(Opcode: PPC::DMXXEXTFDMR512, dl, ResultTys: ReturnTypes, Ops: Dmr0Lo);
12408 Values.push_back(Elt: SDValue(ExtNode, 0));
12409 Values.push_back(Elt: SDValue(ExtNode, 1));
12410 ExtNode =
12411 DAG.getMachineNode(Opcode: PPC::DMXXEXTFDMR512_HI, dl, ResultTys: ReturnTypes, Ops: Dmr0Hi);
12412 Values.push_back(Elt: SDValue(ExtNode, 0));
12413 Values.push_back(Elt: SDValue(ExtNode, 1));
12414 ExtNode = DAG.getMachineNode(Opcode: PPC::DMXXEXTFDMR512, dl, ResultTys: ReturnTypes, Ops: Dmr1Lo);
12415 Values.push_back(Elt: SDValue(ExtNode, 0));
12416 Values.push_back(Elt: SDValue(ExtNode, 1));
12417 ExtNode =
12418 DAG.getMachineNode(Opcode: PPC::DMXXEXTFDMR512_HI, dl, ResultTys: ReturnTypes, Ops: Dmr1Hi);
12419 Values.push_back(Elt: SDValue(ExtNode, 0));
12420 Values.push_back(Elt: SDValue(ExtNode, 1));
12421 }
12422
12423 if (Subtarget.isLittleEndian())
12424 std::reverse(first: Values.begin(), last: Values.end());
12425
12426 SDVTList Tys = DAG.getVTList(VT: MVT::Other);
12427 SmallVector<SDValue, 4> Ops{
12428 StoreChain, DAG.getConstant(Val: Intrinsic::ppc_vsx_stxvp, DL: dl, VT: MVT::i32),
12429 Values[0], BasePtr};
12430 MachineMemOperand *MMO = SN->getMemOperand();
12431 unsigned NumVecs = VT.getSizeInBits() / 256;
12432 for (unsigned Idx = 0; Idx < NumVecs; ++Idx) {
12433 MachineMemOperand *NewMMO =
12434 DAG.getMachineFunction().getMachineMemOperand(MMO, Offset: Idx * 32, Size: 32);
12435 if (Idx > 0) {
12436 BasePtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: BasePtr.getValueType(), N1: BasePtr,
12437 N2: DAG.getConstant(Val: 32, DL: dl, VT: BasePtr.getValueType()));
12438 Ops[3] = BasePtr;
12439 }
12440 Ops[2] = Values[Idx];
12441 SDValue St = DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_VOID, dl, VTList: Tys, Ops,
12442 MemVT: MVT::v256i1, MMO: NewMMO);
12443 Stores.push_back(Elt: St);
12444 }
12445
12446 SDValue TF = DAG.getTokenFactor(DL: dl, Vals&: Stores);
12447 return TF;
12448}
12449
12450SDValue PPCTargetLowering::LowerVectorStore(SDValue Op,
12451 SelectionDAG &DAG) const {
12452 SDLoc dl(Op);
12453 StoreSDNode *SN = cast<StoreSDNode>(Val: Op.getNode());
12454 SDValue StoreChain = SN->getChain();
12455 SDValue BasePtr = SN->getBasePtr();
12456 SDValue Value = SN->getValue();
12457 SDValue Value2 = SN->getValue();
12458 EVT StoreVT = Value.getValueType();
12459
12460 if (StoreVT == MVT::v1024i1 || StoreVT == MVT::v2048i1)
12461 return LowerDMFVectorStore(Op, DAG);
12462
12463 if (StoreVT != MVT::v256i1 && StoreVT != MVT::v512i1)
12464 return Op;
12465
12466 // Type v256i1 is used for pairs and v512i1 is used for accumulators.
12467 assert((StoreVT != MVT::v512i1 || Subtarget.hasMMA()) &&
12468 "Type unsupported without MMA");
12469 assert((StoreVT != MVT::v256i1 || Subtarget.pairedVectorMemops()) &&
12470 "Type unsupported without paired vector support");
12471
12472 // For v256i1 on ISA Future, let the store go through to instruction selection
12473 // where it will be matched to stxvp/pstxvp by the instruction patterns.
12474 if (StoreVT == MVT::v256i1 && Subtarget.isISAFuture() &&
12475 !DisableAutoPairedVecSt)
12476 return Op;
12477
12478 // For other cases, create 2 or 4 v16i8 stores to store the pair or
12479 // accumulator underlying registers individually.
12480 Align Alignment = SN->getAlign();
12481 SmallVector<SDValue, 4> Stores;
12482 unsigned NumVecs = 2;
12483 if (StoreVT == MVT::v512i1) {
12484 if (Subtarget.isISAFuture()) {
12485 EVT ReturnTypes[] = {MVT::v256i1, MVT::v256i1};
12486 MachineSDNode *ExtNode = DAG.getMachineNode(
12487 Opcode: PPC::DMXXEXTFDMR512, dl, ResultTys: ReturnTypes, Ops: Op.getOperand(i: 1));
12488
12489 Value = SDValue(ExtNode, 0);
12490 Value2 = SDValue(ExtNode, 1);
12491 } else
12492 Value = DAG.getNode(Opcode: PPCISD::XXMFACC, DL: dl, VT: MVT::v512i1, Operand: Value);
12493 NumVecs = 4;
12494 }
12495 for (unsigned Idx = 0; Idx < NumVecs; ++Idx) {
12496 unsigned VecNum = Subtarget.isLittleEndian() ? NumVecs - 1 - Idx : Idx;
12497 SDValue Elt;
12498 if (Subtarget.isISAFuture()) {
12499 VecNum = Subtarget.isLittleEndian() ? 1 - (Idx % 2) : (Idx % 2);
12500 Elt = DAG.getNode(Opcode: PPCISD::EXTRACT_VSX_REG, DL: dl, VT: MVT::v16i8,
12501 N1: Idx > 1 ? Value2 : Value,
12502 N2: DAG.getConstant(Val: VecNum, DL: dl, VT: getPointerTy(DL: DAG.getDataLayout())));
12503 } else
12504 Elt = DAG.getNode(Opcode: PPCISD::EXTRACT_VSX_REG, DL: dl, VT: MVT::v16i8, N1: Value,
12505 N2: DAG.getConstant(Val: VecNum, DL: dl, VT: getPointerTy(DL: DAG.getDataLayout())));
12506
12507 SDValue Store =
12508 DAG.getStore(Chain: StoreChain, dl, Val: Elt, Ptr: BasePtr,
12509 PtrInfo: SN->getPointerInfo().getWithOffset(O: Idx * 16),
12510 Alignment: commonAlignment(A: Alignment, Offset: Idx * 16),
12511 MMOFlags: SN->getMemOperand()->getFlags(), Metadata: SN->getAAInfo());
12512 BasePtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: BasePtr.getValueType(), N1: BasePtr,
12513 N2: DAG.getConstant(Val: 16, DL: dl, VT: BasePtr.getValueType()));
12514 Stores.push_back(Elt: Store);
12515 }
12516 SDValue TF = DAG.getTokenFactor(DL: dl, Vals&: Stores);
12517 return TF;
12518}
12519
12520SDValue PPCTargetLowering::LowerMUL(SDValue Op, SelectionDAG &DAG) const {
12521 SDLoc dl(Op);
12522 if (Op.getValueType() == MVT::v4i32) {
12523 SDValue LHS = Op.getOperand(i: 0), RHS = Op.getOperand(i: 1);
12524
12525 SDValue Zero = getCanonicalConstSplat(Val: 0, SplatSize: 1, VT: MVT::v4i32, DAG, dl);
12526 // +16 as shift amt.
12527 SDValue Neg16 = getCanonicalConstSplat(Val: -16, SplatSize: 4, VT: MVT::v4i32, DAG, dl);
12528 SDValue RHSSwap = // = vrlw RHS, 16
12529 BuildIntrinsicOp(IID: Intrinsic::ppc_altivec_vrlw, LHS: RHS, RHS: Neg16, DAG, dl);
12530
12531 // Shrinkify inputs to v8i16.
12532 LHS = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v8i16, Operand: LHS);
12533 RHS = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v8i16, Operand: RHS);
12534 RHSSwap = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v8i16, Operand: RHSSwap);
12535
12536 // Low parts multiplied together, generating 32-bit results (we ignore the
12537 // top parts).
12538 SDValue LoProd = BuildIntrinsicOp(IID: Intrinsic::ppc_altivec_vmulouh,
12539 LHS, RHS, DAG, dl, DestVT: MVT::v4i32);
12540
12541 SDValue HiProd = BuildIntrinsicOp(IID: Intrinsic::ppc_altivec_vmsumuhm,
12542 Op0: LHS, Op1: RHSSwap, Op2: Zero, DAG, dl, DestVT: MVT::v4i32);
12543 // Shift the high parts up 16 bits.
12544 HiProd = BuildIntrinsicOp(IID: Intrinsic::ppc_altivec_vslw, LHS: HiProd,
12545 RHS: Neg16, DAG, dl);
12546 return DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::v4i32, N1: LoProd, N2: HiProd);
12547 } else if (Op.getValueType() == MVT::v16i8) {
12548 SDValue LHS = Op.getOperand(i: 0), RHS = Op.getOperand(i: 1);
12549 bool isLittleEndian = Subtarget.isLittleEndian();
12550
12551 // Multiply the even 8-bit parts, producing 16-bit sums.
12552 SDValue EvenParts = BuildIntrinsicOp(IID: Intrinsic::ppc_altivec_vmuleub,
12553 LHS, RHS, DAG, dl, DestVT: MVT::v8i16);
12554 EvenParts = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: EvenParts);
12555
12556 // Multiply the odd 8-bit parts, producing 16-bit sums.
12557 SDValue OddParts = BuildIntrinsicOp(IID: Intrinsic::ppc_altivec_vmuloub,
12558 LHS, RHS, DAG, dl, DestVT: MVT::v8i16);
12559 OddParts = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: OddParts);
12560
12561 // Merge the results together. Because vmuleub and vmuloub are
12562 // instructions with a big-endian bias, we must reverse the
12563 // element numbering and reverse the meaning of "odd" and "even"
12564 // when generating little endian code.
12565 int Ops[16];
12566 for (unsigned i = 0; i != 8; ++i) {
12567 if (isLittleEndian) {
12568 Ops[i*2 ] = 2*i;
12569 Ops[i*2+1] = 2*i+16;
12570 } else {
12571 Ops[i*2 ] = 2*i+1;
12572 Ops[i*2+1] = 2*i+1+16;
12573 }
12574 }
12575 if (isLittleEndian)
12576 return DAG.getVectorShuffle(VT: MVT::v16i8, dl, N1: OddParts, N2: EvenParts, Mask: Ops);
12577 else
12578 return DAG.getVectorShuffle(VT: MVT::v16i8, dl, N1: EvenParts, N2: OddParts, Mask: Ops);
12579 } else {
12580 llvm_unreachable("Unknown mul to lower!");
12581 }
12582}
12583
12584SDValue PPCTargetLowering::LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const {
12585 bool IsStrict = Op->isStrictFPOpcode();
12586 if (Op.getOperand(i: IsStrict ? 1 : 0).getValueType() == MVT::f128 &&
12587 !Subtarget.hasP9Vector())
12588 return SDValue();
12589
12590 return Op;
12591}
12592
12593// Custom lowering for fpext vf32 to v2f64
12594SDValue PPCTargetLowering::LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const {
12595
12596 assert(Op.getOpcode() == ISD::FP_EXTEND &&
12597 "Should only be called for ISD::FP_EXTEND");
12598
12599 // FIXME: handle extends from half precision float vectors on P9.
12600 // We only want to custom lower an extend from v2f32 to v2f64.
12601 if (Op.getValueType() != MVT::v2f64 ||
12602 Op.getOperand(i: 0).getValueType() != MVT::v2f32)
12603 return SDValue();
12604
12605 SDLoc dl(Op);
12606 SDValue Op0 = Op.getOperand(i: 0);
12607
12608 switch (Op0.getOpcode()) {
12609 default:
12610 return SDValue();
12611 case ISD::EXTRACT_SUBVECTOR: {
12612 assert(Op0.getNumOperands() == 2 &&
12613 isa<ConstantSDNode>(Op0->getOperand(1)) &&
12614 "Node should have 2 operands with second one being a constant!");
12615
12616 if (Op0.getOperand(i: 0).getValueType() != MVT::v4f32)
12617 return SDValue();
12618
12619 // Custom lower is only done for high or low doubleword.
12620 int Idx = Op0.getConstantOperandVal(i: 1);
12621 if (Idx % 2 != 0)
12622 return SDValue();
12623
12624 // Since input is v4f32, at this point Idx is either 0 or 2.
12625 // Shift to get the doubleword position we want.
12626 int DWord = Idx >> 1;
12627
12628 // High and low word positions are different on little endian.
12629 if (Subtarget.isLittleEndian())
12630 DWord ^= 0x1;
12631
12632 return DAG.getNode(Opcode: PPCISD::FP_EXTEND_HALF, DL: dl, VT: MVT::v2f64,
12633 N1: Op0.getOperand(i: 0), N2: DAG.getConstant(Val: DWord, DL: dl, VT: MVT::i32));
12634 }
12635 case ISD::FADD:
12636 case ISD::FMUL:
12637 case ISD::FSUB: {
12638 SDValue NewLoad[2];
12639 for (unsigned i = 0, ie = Op0.getNumOperands(); i != ie; ++i) {
12640 // Ensure both input are loads.
12641 SDValue LdOp = Op0.getOperand(i);
12642 if (LdOp.getOpcode() != ISD::LOAD)
12643 return SDValue();
12644 // Generate new load node.
12645 LoadSDNode *LD = cast<LoadSDNode>(Val&: LdOp);
12646 SDValue LoadOps[] = {LD->getChain(), LD->getBasePtr()};
12647 NewLoad[i] = DAG.getMemIntrinsicNode(
12648 Opcode: PPCISD::LD_VSX_LH, dl, VTList: DAG.getVTList(VT1: MVT::v4f32, VT2: MVT::Other), Ops: LoadOps,
12649 MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand());
12650 }
12651 SDValue NewOp =
12652 DAG.getNode(Opcode: Op0.getOpcode(), DL: SDLoc(Op0), VT: MVT::v4f32, N1: NewLoad[0],
12653 N2: NewLoad[1], Flags: Op0.getNode()->getFlags());
12654 return DAG.getNode(Opcode: PPCISD::FP_EXTEND_HALF, DL: dl, VT: MVT::v2f64, N1: NewOp,
12655 N2: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
12656 }
12657 case ISD::LOAD: {
12658 LoadSDNode *LD = cast<LoadSDNode>(Val&: Op0);
12659 SDValue LoadOps[] = {LD->getChain(), LD->getBasePtr()};
12660 SDValue NewLd = DAG.getMemIntrinsicNode(
12661 Opcode: PPCISD::LD_VSX_LH, dl, VTList: DAG.getVTList(VT1: MVT::v4f32, VT2: MVT::Other), Ops: LoadOps,
12662 MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand());
12663 return DAG.getNode(Opcode: PPCISD::FP_EXTEND_HALF, DL: dl, VT: MVT::v2f64, N1: NewLd,
12664 N2: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
12665 }
12666 }
12667 llvm_unreachable("ERROR:Should return for all cases within swtich.");
12668}
12669
12670static SDValue ConvertCarryValueToCarryFlag(EVT SumType, SDValue Value,
12671 SelectionDAG &DAG,
12672 const PPCSubtarget &STI) {
12673 SDLoc DL(Value);
12674 if (STI.useCRBits())
12675 Value = DAG.getNode(Opcode: ISD::SELECT, DL, VT: SumType, N1: Value,
12676 N2: DAG.getConstant(Val: 1, DL, VT: SumType),
12677 N3: DAG.getConstant(Val: 0, DL, VT: SumType));
12678 else
12679 Value = DAG.getZExtOrTrunc(Op: Value, DL, VT: SumType);
12680 SDValue Sum = DAG.getNode(Opcode: PPCISD::ADDC, DL, VTList: DAG.getVTList(VT1: SumType, VT2: MVT::i32),
12681 N1: Value, N2: DAG.getAllOnesConstant(DL, VT: SumType));
12682 return Sum.getValue(R: 1);
12683}
12684
12685static SDValue ConvertCarryFlagToCarryValue(EVT SumType, SDValue Flag,
12686 EVT CarryType, SelectionDAG &DAG,
12687 const PPCSubtarget &STI) {
12688 SDLoc DL(Flag);
12689 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: SumType);
12690 SDValue Carry = DAG.getNode(
12691 Opcode: PPCISD::ADDE, DL, VTList: DAG.getVTList(VT1: SumType, VT2: MVT::i32), N1: Zero, N2: Zero, N3: Flag);
12692 if (STI.useCRBits())
12693 return DAG.getSetCC(DL, VT: CarryType, LHS: Carry, RHS: Zero, Cond: ISD::SETNE);
12694 return DAG.getZExtOrTrunc(Op: Carry, DL, VT: CarryType);
12695}
12696
12697SDValue PPCTargetLowering::LowerADDSUBO(SDValue Op, SelectionDAG &DAG) const {
12698
12699 SDLoc DL(Op);
12700 SDNode *N = Op.getNode();
12701 EVT VT = N->getValueType(ResNo: 0);
12702 EVT CarryType = N->getValueType(ResNo: 1);
12703 unsigned Opc = N->getOpcode();
12704 bool IsAdd = Opc == ISD::UADDO;
12705 Opc = IsAdd ? PPCISD::ADDC : PPCISD::SUBC;
12706 SDValue Sum = DAG.getNode(Opcode: Opc, DL, VTList: DAG.getVTList(VT1: VT, VT2: MVT::i32),
12707 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1));
12708 SDValue Carry = ConvertCarryFlagToCarryValue(SumType: VT, Flag: Sum.getValue(R: 1), CarryType,
12709 DAG, STI: Subtarget);
12710 if (!IsAdd)
12711 Carry = DAG.getNode(Opcode: ISD::XOR, DL, VT: CarryType, N1: Carry,
12712 N2: DAG.getConstant(Val: 1UL, DL, VT: CarryType));
12713 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL, VTList: N->getVTList(), N1: Sum, N2: Carry);
12714}
12715
12716SDValue PPCTargetLowering::LowerADDSUBO_CARRY(SDValue Op,
12717 SelectionDAG &DAG) const {
12718 SDLoc DL(Op);
12719 SDNode *N = Op.getNode();
12720 unsigned Opc = N->getOpcode();
12721 EVT VT = N->getValueType(ResNo: 0);
12722 EVT CarryType = N->getValueType(ResNo: 1);
12723 SDValue CarryOp = N->getOperand(Num: 2);
12724 bool IsAdd = Opc == ISD::UADDO_CARRY;
12725 Opc = IsAdd ? PPCISD::ADDE : PPCISD::SUBE;
12726 if (!IsAdd)
12727 CarryOp = DAG.getNode(Opcode: ISD::XOR, DL, VT: CarryOp.getValueType(), N1: CarryOp,
12728 N2: DAG.getConstant(Val: 1UL, DL, VT: CarryOp.getValueType()));
12729 CarryOp = ConvertCarryValueToCarryFlag(SumType: VT, Value: CarryOp, DAG, STI: Subtarget);
12730 SDValue Sum = DAG.getNode(Opcode: Opc, DL, VTList: DAG.getVTList(VT1: VT, VT2: MVT::i32),
12731 N1: Op.getOperand(i: 0), N2: Op.getOperand(i: 1), N3: CarryOp);
12732 CarryOp = ConvertCarryFlagToCarryValue(SumType: VT, Flag: Sum.getValue(R: 1), CarryType, DAG,
12733 STI: Subtarget);
12734 if (!IsAdd)
12735 CarryOp = DAG.getNode(Opcode: ISD::XOR, DL, VT: CarryOp.getValueType(), N1: CarryOp,
12736 N2: DAG.getConstant(Val: 1UL, DL, VT: CarryOp.getValueType()));
12737 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL, VTList: N->getVTList(), N1: Sum, N2: CarryOp);
12738}
12739
12740SDValue PPCTargetLowering::LowerSSUBO(SDValue Op, SelectionDAG &DAG) const {
12741
12742 SDLoc dl(Op);
12743 SDValue LHS = Op.getOperand(i: 0);
12744 SDValue RHS = Op.getOperand(i: 1);
12745 EVT VT = Op.getNode()->getValueType(ResNo: 0);
12746
12747 SDValue Sub = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: LHS, N2: RHS);
12748
12749 SDValue Xor1 = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: RHS, N2: LHS);
12750 SDValue Xor2 = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: Sub, N2: LHS);
12751
12752 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Xor1, N2: Xor2);
12753
12754 SDValue Overflow =
12755 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: And,
12756 N2: DAG.getConstant(Val: VT.getSizeInBits() - 1, DL: dl, VT: MVT::i32));
12757
12758 SDValue OverflowTrunc =
12759 DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: Op.getNode()->getValueType(ResNo: 1), Operand: Overflow);
12760
12761 return DAG.getMergeValues(Ops: {Sub, OverflowTrunc}, dl);
12762}
12763
12764/// Implements signed add with overflow detection using the rule:
12765/// (x eqv y) & (sum xor x), where the overflow bit is extracted from the sign
12766SDValue PPCTargetLowering::LowerSADDO(SDValue Op, SelectionDAG &DAG) const {
12767
12768 SDLoc dl(Op);
12769 SDValue LHS = Op.getOperand(i: 0);
12770 SDValue RHS = Op.getOperand(i: 1);
12771 EVT VT = Op.getNode()->getValueType(ResNo: 0);
12772
12773 SDValue Sum = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: LHS, N2: RHS);
12774
12775 // Compute ~(x xor y)
12776 SDValue XorXY = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: LHS, N2: RHS);
12777 SDValue EqvXY = DAG.getNOT(DL: dl, Val: XorXY, VT);
12778 // Compute (s xor x)
12779 SDValue SumXorX = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: Sum, N2: LHS);
12780
12781 // overflow = (x eqv y) & (s xor x)
12782 SDValue OverflowInSign = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: EqvXY, N2: SumXorX);
12783
12784 // Shift sign bit down to LSB
12785 SDValue Overflow =
12786 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: OverflowInSign,
12787 N2: DAG.getConstant(Val: VT.getSizeInBits() - 1, DL: dl, VT: MVT::i32));
12788 // Truncate to the overflow type (i1)
12789 SDValue OverflowTrunc =
12790 DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: Op.getNode()->getValueType(ResNo: 1), Operand: Overflow);
12791
12792 return DAG.getMergeValues(Ops: {Sum, OverflowTrunc}, dl);
12793}
12794
12795/// Lower ABDU with negation pattern using branchless carry arithmetic.
12796/// Recognizes: abdu(a, sub(0, x)) and transforms to:
12797/// a - (0 - x) = a + x (mod 2^n)
12798/// Uses SUBC to compute result without branches.
12799SDValue PPCTargetLowering::LowerABDU(SDValue Op, SelectionDAG &DAG) const {
12800 SDLoc DL(Op);
12801 SDValue LHS = Op.getOperand(i: 0);
12802 SDValue RHS = Op.getOperand(i: 1);
12803 EVT OpVT = LHS.getValueType();
12804 EVT VT = Op.getValueType();
12805 bool IsNonNegative = DAG.SignBitIsZero(Op: LHS) && DAG.SignBitIsZero(Op: RHS);
12806
12807 // If the subtract doesn't overflow then just use abs(sub()).
12808 if (DAG.willNotOverflowSub(IsSigned: IsNonNegative, N0: LHS, N1: RHS))
12809 return DAG.getNode(Opcode: ISD::ABS, DL, VT,
12810 Operand: DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: LHS, N2: RHS));
12811
12812 if (DAG.willNotOverflowSub(IsSigned: IsNonNegative, N0: RHS, N1: LHS))
12813 return DAG.getNode(Opcode: ISD::ABS, DL, VT,
12814 Operand: DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: RHS, N2: LHS));
12815
12816 // General path: use SUBC (or ADDC when RHS is 0-X) to get
12817 // subtract-with-flags, then CMOV to select a-b or b-a. ADDC/SUBC produce the
12818 // flags we need.
12819 unsigned Opcode = PPCISD::SUBC;
12820
12821 // Check if RHS is a negation (0 - X). If so, we can use ADDC instead of SUBC:
12822 // a - (0 - x) = a + x (mod 2^n)
12823 // Same semantics as in LowerCMP; apply same safety checks.
12824 if (RHS.getOpcode() == ISD::SUB) {
12825 SDValue SubLHS = RHS.getOperand(i: 0);
12826 SDValue SubRHS = RHS.getOperand(i: 1);
12827
12828 if (isNullConstant(V: SubLHS) && DAG.isKnownNeverZero(Op: SubRHS)) {
12829 Opcode = PPCISD::ADDC;
12830 RHS = SubRHS;
12831 }
12832 }
12833
12834 // On PPC64, carry ops use the full 64-bit register. Operands are type-legal
12835 // i32 here; widen only for the carry path (fast abs(sub) stays at VT).
12836 if (Subtarget.isPPC64() && OpVT != MVT::i64) {
12837 LHS = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i64, Operand: LHS);
12838 RHS = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i64, Operand: RHS);
12839 OpVT = MVT::i64;
12840 }
12841
12842 // Calculate LHS - RHS and capture the carry (CA)
12843 SDVTList VTs = DAG.getVTList(VT1: OpVT, VT2: MVT::i32);
12844 SDValue Res = DAG.getNode(Opcode, DL, VTList: VTs, N1: LHS, N2: RHS);
12845 SDValue CA0 = Res.getValue(R: 1);
12846
12847 // t2 = A - B + CA0 using SUBE.
12848 SDValue ZeroOrNeg1 = DAG.getNode(Opcode: PPCISD::SUBE, DL, VTList: VTs, N1: Res, N2: Res, N3: CA0);
12849
12850 SDValue Xor = DAG.getNode(Opcode: ISD::XOR, DL, VT: OpVT, N1: Res, N2: ZeroOrNeg1);
12851
12852 Res = DAG.getNode(Opcode: ISD::SUB, DL, VT: OpVT, N1: Xor, N2: ZeroOrNeg1);
12853
12854 Res = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Res);
12855 return Res;
12856}
12857
12858// Lower unsigned 3-way compare producing -1/0/1.
12859SDValue PPCTargetLowering::LowerUCMP(SDValue Op, SelectionDAG &DAG) const {
12860 SDLoc DL(Op);
12861 SDValue A = DAG.getFreeze(V: Op.getOperand(i: 0));
12862 SDValue B = DAG.getFreeze(V: Op.getOperand(i: 1));
12863 EVT OpVT = A.getValueType();
12864 EVT ResVT = Op.getValueType();
12865
12866 // On PPC64, i32 carries are affected by the upper 32 bits of the registers.
12867 // We must zero-extend to i64 to ensure the carry reflects the 32-bit unsigned
12868 // comparison.
12869 if (Subtarget.isPPC64() && OpVT != MVT::i64) {
12870 A = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i64, Operand: A);
12871 B = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i64, Operand: B);
12872 OpVT = MVT::i64;
12873 }
12874
12875 // First compute diff = A - B.
12876 SDValue Diff = DAG.getNode(Opcode: ISD::SUB, DL, VT: OpVT, N1: A, N2: B);
12877
12878 // Generate B - A using SUBC to capture carry.
12879 SDVTList VTs = DAG.getVTList(VT1: OpVT, VT2: MVT::i32);
12880 SDValue SubC = DAG.getNode(Opcode: PPCISD::SUBC, DL, VTList: VTs, N1: B, N2: A);
12881 SDValue CA0 = SubC.getValue(R: 1);
12882
12883 // t2 = A - B + CA0 using SUBE.
12884 SDValue SubE1 = DAG.getNode(Opcode: PPCISD::SUBE, DL, VTList: VTs, N1: A, N2: B, N3: CA0);
12885 SDValue CA1 = SubE1.getValue(R: 1);
12886
12887 // res = diff - t2 + CA1 using SUBE (produces desired -1/0/1).
12888 SDValue ResPair = DAG.getNode(Opcode: PPCISD::SUBE, DL, VTList: VTs, N1: Diff, N2: SubE1, N3: CA1);
12889
12890 // Extract the first result and truncate to result type if needed.
12891 return DAG.getSExtOrTrunc(Op: ResPair.getValue(R: 0), DL, VT: ResVT);
12892}
12893
12894/// LowerOperation - Provide custom lowering hooks for some operations.
12895///
12896SDValue PPCTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
12897 switch (Op.getOpcode()) {
12898 default:
12899 llvm_unreachable("Wasn't expecting to be able to lower this!");
12900 case ISD::FPOW: return lowerPow(Op, DAG);
12901 case ISD::FSIN: return lowerSin(Op, DAG);
12902 case ISD::FCOS: return lowerCos(Op, DAG);
12903 case ISD::FLOG: return lowerLog(Op, DAG);
12904 case ISD::FLOG10: return lowerLog10(Op, DAG);
12905 case ISD::FEXP: return lowerExp(Op, DAG);
12906 case ISD::ConstantPool: return LowerConstantPool(Op, DAG);
12907 case ISD::BlockAddress: return LowerBlockAddress(Op, DAG);
12908 case ISD::GlobalAddress: return LowerGlobalAddress(Op, DAG);
12909 case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG);
12910 case ISD::JumpTable: return LowerJumpTable(Op, DAG);
12911 case ISD::STRICT_FSETCC:
12912 case ISD::STRICT_FSETCCS:
12913 case ISD::SETCC: return LowerSETCC(Op, DAG);
12914 case ISD::BR_CC: return LowerBR_CC(Op, DAG);
12915 case ISD::INIT_TRAMPOLINE: return LowerINIT_TRAMPOLINE(Op, DAG);
12916 case ISD::ADJUST_TRAMPOLINE: return LowerADJUST_TRAMPOLINE(Op, DAG);
12917 case ISD::SSUBO:
12918 return LowerSSUBO(Op, DAG);
12919 case ISD::SADDO:
12920 return LowerSADDO(Op, DAG);
12921
12922 case ISD::INLINEASM:
12923 case ISD::INLINEASM_BR: return LowerINLINEASM(Op, DAG);
12924 // Variable argument lowering.
12925 case ISD::VASTART: return LowerVASTART(Op, DAG);
12926 case ISD::VAARG: return LowerVAARG(Op, DAG);
12927 case ISD::VACOPY: return LowerVACOPY(Op, DAG);
12928
12929 case ISD::STACKRESTORE: return LowerSTACKRESTORE(Op, DAG);
12930 case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG);
12931 case ISD::GET_DYNAMIC_AREA_OFFSET:
12932 return LowerGET_DYNAMIC_AREA_OFFSET(Op, DAG);
12933
12934 // Exception handling lowering.
12935 case ISD::EH_DWARF_CFA: return LowerEH_DWARF_CFA(Op, DAG);
12936 case ISD::EH_SJLJ_SETJMP: return lowerEH_SJLJ_SETJMP(Op, DAG);
12937 case ISD::EH_SJLJ_LONGJMP: return lowerEH_SJLJ_LONGJMP(Op, DAG);
12938
12939 case ISD::LOAD: return LowerLOAD(Op, DAG);
12940 case ISD::STORE: return LowerSTORE(Op, DAG);
12941 case ISD::TRUNCATE: return LowerTRUNCATE(Op, DAG);
12942 case ISD::SELECT_CC: return LowerSELECT_CC(Op, DAG);
12943 case ISD::STRICT_FP_TO_UINT:
12944 case ISD::STRICT_FP_TO_SINT:
12945 case ISD::FP_TO_UINT:
12946 case ISD::FP_TO_SINT: return LowerFP_TO_INT(Op, DAG, dl: SDLoc(Op));
12947 case ISD::STRICT_UINT_TO_FP:
12948 case ISD::STRICT_SINT_TO_FP:
12949 case ISD::UINT_TO_FP:
12950 case ISD::SINT_TO_FP: return LowerINT_TO_FP(Op, DAG);
12951 case ISD::GET_ROUNDING: return LowerGET_ROUNDING(Op, DAG);
12952 case ISD::SET_ROUNDING:
12953 return LowerSET_ROUNDING(Op, DAG);
12954
12955 // Lower 64-bit shifts.
12956 case ISD::SHL_PARTS: return LowerSHL_PARTS(Op, DAG);
12957 case ISD::SRL_PARTS: return LowerSRL_PARTS(Op, DAG);
12958 case ISD::SRA_PARTS: return LowerSRA_PARTS(Op, DAG);
12959
12960 case ISD::FSHL: return LowerFunnelShift(Op, DAG);
12961 case ISD::FSHR: return LowerFunnelShift(Op, DAG);
12962
12963 // Vector-related lowering.
12964 case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG);
12965 case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG);
12966 case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG);
12967 case ISD::SCALAR_TO_VECTOR: return LowerSCALAR_TO_VECTOR(Op, DAG);
12968 case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG);
12969 case ISD::MUL: return LowerMUL(Op, DAG);
12970 case ISD::FP_EXTEND: return LowerFP_EXTEND(Op, DAG);
12971 case ISD::STRICT_FP_ROUND:
12972 case ISD::FP_ROUND:
12973 return LowerFP_ROUND(Op, DAG);
12974 case ISD::ROTL: return LowerROTL(Op, DAG);
12975
12976 // For counter-based loop handling.
12977 case ISD::INTRINSIC_W_CHAIN:
12978 return SDValue();
12979
12980 case ISD::BITCAST: return LowerBITCAST(Op, DAG);
12981
12982 // Frame & Return address.
12983 case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG);
12984 case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG);
12985
12986 case ISD::INTRINSIC_VOID:
12987 return LowerINTRINSIC_VOID(Op, DAG);
12988 case ISD::BSWAP:
12989 return LowerBSWAP(Op, DAG);
12990 case ISD::ATOMIC_CMP_SWAP:
12991 return LowerATOMIC_CMP_SWAP(Op, DAG);
12992 case ISD::ATOMIC_STORE:
12993 return LowerATOMIC_LOAD_STORE(Op, DAG);
12994 case ISD::IS_FPCLASS:
12995 return LowerIS_FPCLASS(Op, DAG);
12996 case ISD::UADDO:
12997 case ISD::USUBO:
12998 return LowerADDSUBO(Op, DAG);
12999 case ISD::UADDO_CARRY:
13000 case ISD::USUBO_CARRY:
13001 return LowerADDSUBO_CARRY(Op, DAG);
13002 case ISD::UCMP:
13003 return LowerUCMP(Op, DAG);
13004 case ISD::ABDU:
13005 return LowerABDU(Op, DAG);
13006 case ISD::STRICT_LRINT:
13007 case ISD::STRICT_LLRINT:
13008 case ISD::STRICT_LROUND:
13009 case ISD::STRICT_LLROUND:
13010 case ISD::STRICT_FNEARBYINT:
13011 if (Op->getFlags().hasNoFPExcept())
13012 return Op;
13013 return SDValue();
13014 case ISD::VP_LOAD:
13015 return LowerVP_LOAD(Op, DAG);
13016 case ISD::VP_STORE:
13017 return LowerVP_STORE(Op, DAG);
13018 case ISD::PARTIAL_REDUCE_UMLA:
13019 return LowerPartialReduce(Op, DAG);
13020 }
13021}
13022
13023void PPCTargetLowering::ReplaceNodeResults(SDNode *N,
13024 SmallVectorImpl<SDValue>&Results,
13025 SelectionDAG &DAG) const {
13026 SDLoc dl(N);
13027 switch (N->getOpcode()) {
13028 default:
13029 llvm_unreachable("Do not know how to custom type legalize this operation!");
13030 case ISD::ATOMIC_LOAD: {
13031 SDValue Res = LowerATOMIC_LOAD_STORE(Op: SDValue(N, 0), DAG);
13032 Results.push_back(Elt: Res);
13033 Results.push_back(Elt: Res.getValue(R: 1));
13034 break;
13035 }
13036 case ISD::READCYCLECOUNTER: {
13037 SDVTList VTs = DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32, VT3: MVT::Other);
13038 SDValue RTB = DAG.getNode(Opcode: PPCISD::READ_TIME_BASE, DL: dl, VTList: VTs, N: N->getOperand(Num: 0));
13039
13040 Results.push_back(
13041 Elt: DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: MVT::i64, N1: RTB, N2: RTB.getValue(R: 1)));
13042 Results.push_back(Elt: RTB.getValue(R: 2));
13043 break;
13044 }
13045 case ISD::INTRINSIC_W_CHAIN: {
13046 if (N->getConstantOperandVal(Num: 1) != Intrinsic::loop_decrement)
13047 break;
13048
13049 assert(N->getValueType(0) == MVT::i1 &&
13050 "Unexpected result type for CTR decrement intrinsic");
13051 EVT SVT = getSetCCResultType(DL: DAG.getDataLayout(), C&: *DAG.getContext(),
13052 VT: N->getValueType(ResNo: 0));
13053 SDVTList VTs = DAG.getVTList(VT1: SVT, VT2: MVT::Other);
13054 SDValue NewInt = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: VTs, N1: N->getOperand(Num: 0),
13055 N2: N->getOperand(Num: 1));
13056
13057 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i1, Operand: NewInt));
13058 Results.push_back(Elt: NewInt.getValue(R: 1));
13059 break;
13060 }
13061 case ISD::INTRINSIC_WO_CHAIN: {
13062 switch (N->getConstantOperandVal(Num: 0)) {
13063 case Intrinsic::ppc_pack_longdouble:
13064 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: MVT::ppcf128,
13065 N1: N->getOperand(Num: 2), N2: N->getOperand(Num: 1)));
13066 break;
13067 case Intrinsic::ppc_maxfe:
13068 case Intrinsic::ppc_minfe:
13069 case Intrinsic::ppc_fnmsub:
13070 case Intrinsic::ppc_convert_f128_to_ppcf128:
13071 Results.push_back(Elt: LowerINTRINSIC_WO_CHAIN(Op: SDValue(N, 0), DAG));
13072 break;
13073 }
13074 break;
13075 }
13076 case ISD::VAARG: {
13077 if (!Subtarget.isSVR4ABI() || Subtarget.isPPC64())
13078 return;
13079
13080 EVT VT = N->getValueType(ResNo: 0);
13081
13082 if (VT == MVT::i64) {
13083 SDValue NewNode = LowerVAARG(Op: SDValue(N, 1), DAG);
13084
13085 Results.push_back(Elt: NewNode);
13086 Results.push_back(Elt: NewNode.getValue(R: 1));
13087 }
13088 return;
13089 }
13090 case ISD::STRICT_FP_TO_SINT:
13091 case ISD::STRICT_FP_TO_UINT:
13092 case ISD::FP_TO_SINT:
13093 case ISD::FP_TO_UINT: {
13094 // LowerFP_TO_INT() can only handle f32 and f64.
13095 if (N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType() ==
13096 MVT::ppcf128)
13097 return;
13098 SDValue LoweredValue = LowerFP_TO_INT(Op: SDValue(N, 0), DAG, dl);
13099 Results.push_back(Elt: LoweredValue);
13100 if (N->isStrictFPOpcode())
13101 Results.push_back(Elt: LoweredValue.getValue(R: 1));
13102 return;
13103 }
13104 case ISD::TRUNCATE: {
13105 if (!N->getValueType(ResNo: 0).isVector())
13106 return;
13107 SDValue Lowered = LowerTRUNCATEVector(Op: SDValue(N, 0), DAG);
13108 if (Lowered)
13109 Results.push_back(Elt: Lowered);
13110 return;
13111 }
13112 case ISD::SCALAR_TO_VECTOR: {
13113 SDValue Lowered = LowerSCALAR_TO_VECTOR(Op: SDValue(N, 0), DAG);
13114 if (Lowered)
13115 Results.push_back(Elt: Lowered);
13116 return;
13117 }
13118 case ISD::FSHL:
13119 case ISD::FSHR:
13120 // Don't handle funnel shifts here.
13121 return;
13122 case ISD::BITCAST:
13123 // Don't handle bitcast here.
13124 return;
13125 case ISD::FP_EXTEND:
13126 SDValue Lowered = LowerFP_EXTEND(Op: SDValue(N, 0), DAG);
13127 if (Lowered)
13128 Results.push_back(Elt: Lowered);
13129 return;
13130 }
13131}
13132
13133//===----------------------------------------------------------------------===//
13134// Other Lowering Code
13135//===----------------------------------------------------------------------===//
13136
13137static CallInst *callIntrinsic(IRBuilderBase &Builder, Intrinsic::ID Id) {
13138 return Builder.CreateIntrinsicWithoutFolding(ID: Id, Args: {});
13139}
13140
13141Value *PPCTargetLowering::emitLoadLinked(IRBuilderBase &Builder, Type *ValueTy,
13142 Value *Addr,
13143 AtomicOrdering Ord) const {
13144 unsigned SZ = ValueTy->getPrimitiveSizeInBits();
13145
13146 assert((SZ == 8 || SZ == 16 || SZ == 32 || SZ == 64) &&
13147 "Only 8/16/32/64-bit atomic loads supported");
13148 Intrinsic::ID IntID;
13149 switch (SZ) {
13150 default:
13151 llvm_unreachable("Unexpected PrimitiveSize");
13152 case 8:
13153 IntID = Intrinsic::ppc_lbarx;
13154 assert(Subtarget.hasPartwordAtomics() && "No support partword atomics.");
13155 break;
13156 case 16:
13157 IntID = Intrinsic::ppc_lharx;
13158 assert(Subtarget.hasPartwordAtomics() && "No support partword atomics.");
13159 break;
13160 case 32:
13161 IntID = Intrinsic::ppc_lwarx;
13162 break;
13163 case 64:
13164 IntID = Intrinsic::ppc_ldarx;
13165 break;
13166 }
13167 Value *Call =
13168 Builder.CreateIntrinsic(ID: IntID, Args: Addr, /*FMFSource=*/nullptr, Name: "larx");
13169
13170 return Builder.CreateTruncOrBitCast(V: Call, DestTy: ValueTy);
13171}
13172
13173// Perform a store-conditional operation to Addr. Return the status of the
13174// store. This should be 0 if the store succeeded, non-zero otherwise.
13175Value *PPCTargetLowering::emitStoreConditional(IRBuilderBase &Builder,
13176 Value *Val, Value *Addr,
13177 AtomicOrdering Ord) const {
13178 Type *Ty = Val->getType();
13179 unsigned SZ = Ty->getPrimitiveSizeInBits();
13180
13181 assert((SZ == 8 || SZ == 16 || SZ == 32 || SZ == 64) &&
13182 "Only 8/16/32/64-bit atomic loads supported");
13183 Intrinsic::ID IntID;
13184 switch (SZ) {
13185 default:
13186 llvm_unreachable("Unexpected PrimitiveSize");
13187 case 8:
13188 IntID = Intrinsic::ppc_stbcx;
13189 assert(Subtarget.hasPartwordAtomics() && "No support partword atomics.");
13190 break;
13191 case 16:
13192 IntID = Intrinsic::ppc_sthcx;
13193 assert(Subtarget.hasPartwordAtomics() && "No support partword atomics.");
13194 break;
13195 case 32:
13196 IntID = Intrinsic::ppc_stwcx;
13197 break;
13198 case 64:
13199 IntID = Intrinsic::ppc_stdcx;
13200 break;
13201 }
13202
13203 if (SZ == 8 || SZ == 16)
13204 Val = Builder.CreateZExt(V: Val, DestTy: Builder.getInt32Ty());
13205
13206 Value *Call = Builder.CreateIntrinsic(ID: IntID, Args: {Addr, Val},
13207 /*FMFSource=*/nullptr, Name: "stcx");
13208 return Builder.CreateXor(LHS: Call, RHS: Builder.getInt32(C: 1));
13209}
13210
13211// The mappings for emitLeading/TrailingFence is taken from
13212// http://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html
13213Instruction *PPCTargetLowering::emitLeadingFence(IRBuilderBase &Builder,
13214 Instruction *Inst,
13215 AtomicOrdering Ord) const {
13216 if (Ord == AtomicOrdering::SequentiallyConsistent)
13217 return callIntrinsic(Builder, Id: Intrinsic::ppc_sync);
13218 if (isReleaseOrStronger(AO: Ord))
13219 return callIntrinsic(Builder, Id: Intrinsic::ppc_lwsync);
13220 return nullptr;
13221}
13222
13223Instruction *PPCTargetLowering::emitTrailingFence(IRBuilderBase &Builder,
13224 Instruction *Inst,
13225 AtomicOrdering Ord) const {
13226 if (Inst->hasAtomicLoad() && isAcquireOrStronger(AO: Ord)) {
13227 // See http://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html and
13228 // http://www.rdrop.com/users/paulmck/scalability/paper/N2745r.2011.03.04a.html
13229 // and http://www.cl.cam.ac.uk/~pes20/cppppc/ for justification.
13230 if (isa<LoadInst>(Val: Inst))
13231 return Builder.CreateIntrinsicWithoutFolding(ID: Intrinsic::ppc_cfence,
13232 OverloadTypes: {Inst->getType()}, Args: {Inst});
13233 // FIXME: Can use isync for rmw operation.
13234 return callIntrinsic(Builder, Id: Intrinsic::ppc_lwsync);
13235 }
13236 return nullptr;
13237}
13238
13239MachineBasicBlock *PPCTargetLowering::EmitAtomicBinary(MachineInstr &MI,
13240 MachineBasicBlock *BB,
13241 unsigned BinOpcode,
13242 unsigned CmpOpcode,
13243 unsigned CmpPred) const {
13244 // BinOpcode != 0: Handles atomic load with binary operator, e.g. NAND.
13245 // CmpOpcode != 0: Handles atomic load with MIN/MAX etc.
13246 // BinOpcode == 0 && CmpOpcode == 0: Handles ATOMIC_SWAP.
13247 const PPCInstrInfo *TII = Subtarget.getInstrInfo();
13248 unsigned AtomicSize = MI.getOperand(i: 3).getImm();
13249
13250 auto LoadMnemonic = PPC::LDARX;
13251 auto StoreMnemonic = PPC::STDCX;
13252 switch (AtomicSize) {
13253 default:
13254 llvm_unreachable("Unexpected size of atomic entity");
13255 case 1:
13256 LoadMnemonic = PPC::LBARX;
13257 StoreMnemonic = PPC::STBCX;
13258 assert(Subtarget.hasPartwordAtomics() && "Call this only with size >=4");
13259 break;
13260 case 2:
13261 LoadMnemonic = PPC::LHARX;
13262 StoreMnemonic = PPC::STHCX;
13263 assert(Subtarget.hasPartwordAtomics() && "Call this only with size >=4");
13264 break;
13265 case 4:
13266 LoadMnemonic = PPC::LWARX;
13267 StoreMnemonic = PPC::STWCX;
13268 break;
13269 case 8:
13270 LoadMnemonic = PPC::LDARX;
13271 StoreMnemonic = PPC::STDCX;
13272 break;
13273 }
13274
13275 const BasicBlock *LLVM_BB = BB->getBasicBlock();
13276 MachineFunction *F = BB->getParent();
13277 MachineFunction::iterator It = ++BB->getIterator();
13278
13279 if (CmpOpcode == PPC::CMPW && (AtomicSize == 1 || AtomicSize == 2))
13280 signExtendOperandIfUnknown(MI, BB, OpIdx: 4, /*IsByte=*/AtomicSize == 1, TII);
13281
13282 Register dest = MI.getOperand(i: 0).getReg();
13283 Register ptrA = MI.getOperand(i: 1).getReg();
13284 Register ptrB = MI.getOperand(i: 2).getReg();
13285 Register incr = MI.getOperand(i: 4).getReg();
13286 DebugLoc dl = MI.getDebugLoc();
13287
13288 MachineBasicBlock *loopMBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
13289 MachineBasicBlock *loop2MBB =
13290 CmpOpcode ? F->CreateMachineBasicBlock(BB: LLVM_BB) : nullptr;
13291 MachineBasicBlock *exitMBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
13292 F->insert(MBBI: It, MBB: loopMBB);
13293 if (CmpOpcode)
13294 F->insert(MBBI: It, MBB: loop2MBB);
13295 F->insert(MBBI: It, MBB: exitMBB);
13296 exitMBB->splice(Where: exitMBB->begin(), Other: BB,
13297 From: std::next(x: MachineBasicBlock::iterator(MI)), To: BB->end());
13298 exitMBB->transferSuccessorsAndUpdatePHIs(FromMBB: BB);
13299
13300 MachineRegisterInfo &RegInfo = F->getRegInfo();
13301 Register TmpReg = (!BinOpcode) ? incr :
13302 RegInfo.createVirtualRegister( RegClass: AtomicSize == 8 ? &PPC::G8RCRegClass
13303 : &PPC::GPRCRegClass);
13304
13305 // thisMBB:
13306 // ...
13307 // fallthrough --> loopMBB
13308 BB->addSuccessor(Succ: loopMBB);
13309
13310 // loopMBB:
13311 // l[wd]arx dest, ptr
13312 // add r0, dest, incr
13313 // st[wd]cx. r0, ptr
13314 // bne- loopMBB
13315 // fallthrough --> exitMBB
13316
13317 // For max/min...
13318 // loopMBB:
13319 // l[wd]arx dest, ptr
13320 // cmpl?[wd] dest, incr
13321 // bgt exitMBB
13322 // loop2MBB:
13323 // st[wd]cx. dest, ptr
13324 // bne- loopMBB
13325 // fallthrough --> exitMBB
13326
13327 BB = loopMBB;
13328 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: LoadMnemonic), DestReg: dest)
13329 .addReg(RegNo: ptrA).addReg(RegNo: ptrB);
13330 if (BinOpcode)
13331 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: BinOpcode), DestReg: TmpReg).addReg(RegNo: incr).addReg(RegNo: dest);
13332 if (CmpOpcode) {
13333 Register CrReg = RegInfo.createVirtualRegister(RegClass: &PPC::CRRCRegClass);
13334 // Signed comparisons of byte or halfword values must be sign-extended.
13335 if (CmpOpcode == PPC::CMPW && AtomicSize < 4) {
13336 Register ExtReg = RegInfo.createVirtualRegister(RegClass: &PPC::GPRCRegClass);
13337 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: AtomicSize == 1 ? PPC::EXTSB : PPC::EXTSH),
13338 DestReg: ExtReg).addReg(RegNo: dest);
13339 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: CmpOpcode), DestReg: CrReg).addReg(RegNo: ExtReg).addReg(RegNo: incr);
13340 } else
13341 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: CmpOpcode), DestReg: CrReg).addReg(RegNo: dest).addReg(RegNo: incr);
13342
13343 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::BCC))
13344 .addImm(Val: CmpPred)
13345 .addReg(RegNo: CrReg)
13346 .addMBB(MBB: exitMBB);
13347 BB->addSuccessor(Succ: loop2MBB);
13348 BB->addSuccessor(Succ: exitMBB);
13349 BB = loop2MBB;
13350 }
13351 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: StoreMnemonic))
13352 .addReg(RegNo: TmpReg).addReg(RegNo: ptrA).addReg(RegNo: ptrB);
13353 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::BCC))
13354 .addImm(Val: PPC::PRED_NE_MINUS)
13355 .addReg(RegNo: PPC::CR0)
13356 .addMBB(MBB: loopMBB);
13357 BB->addSuccessor(Succ: loopMBB);
13358 BB->addSuccessor(Succ: exitMBB);
13359
13360 // exitMBB:
13361 // ...
13362 BB = exitMBB;
13363 return BB;
13364}
13365
13366static bool isSignExtended(MachineInstr &MI, const PPCInstrInfo *TII) {
13367 switch(MI.getOpcode()) {
13368 default:
13369 return false;
13370 case PPC::COPY:
13371 return TII->isSignExtended(Reg: MI.getOperand(i: 1).getReg(),
13372 MRI: &MI.getMF()->getRegInfo());
13373 case PPC::LHA:
13374 case PPC::LHA8:
13375 case PPC::LHAU:
13376 case PPC::LHAU8:
13377 case PPC::LHAUX:
13378 case PPC::LHAUX8:
13379 case PPC::LHAX:
13380 case PPC::LHAX8:
13381 case PPC::LWA:
13382 case PPC::LWAUX:
13383 case PPC::LWAX:
13384 case PPC::LWAX_32:
13385 case PPC::LWA_32:
13386 case PPC::PLHA:
13387 case PPC::PLHA8:
13388 case PPC::PLHA8pc:
13389 case PPC::PLHApc:
13390 case PPC::PLWA:
13391 case PPC::PLWA8:
13392 case PPC::PLWA8pc:
13393 case PPC::PLWApc:
13394 case PPC::EXTSB:
13395 case PPC::EXTSB8:
13396 case PPC::EXTSB8_32_64:
13397 case PPC::EXTSB8_rec:
13398 case PPC::EXTSB_rec:
13399 case PPC::EXTSH:
13400 case PPC::EXTSH8:
13401 case PPC::EXTSH8_32_64:
13402 case PPC::EXTSH8_rec:
13403 case PPC::EXTSH_rec:
13404 case PPC::EXTSW:
13405 case PPC::EXTSWSLI:
13406 case PPC::EXTSWSLI_32_64:
13407 case PPC::EXTSWSLI_32_64_rec:
13408 case PPC::EXTSWSLI_rec:
13409 case PPC::EXTSW_32:
13410 case PPC::EXTSW_32_64:
13411 case PPC::EXTSW_32_64_rec:
13412 case PPC::EXTSW_rec:
13413 case PPC::SRAW:
13414 case PPC::SRAWI:
13415 case PPC::SRAWI_rec:
13416 case PPC::SRAW_rec:
13417 return true;
13418 }
13419 return false;
13420}
13421
13422// Sign extend operand OpIdx if the value is not known to be sign extended.
13423// Assumes the operand is a register. The flag IsByte controls which intruction
13424// is used for the sign extension.
13425static void signExtendOperandIfUnknown(MachineInstr &MI, MachineBasicBlock *BB,
13426 unsigned OpIdx, bool IsByte,
13427 const PPCInstrInfo *TII) {
13428 MachineFunction *F = MI.getMF();
13429 MachineRegisterInfo &RegInfo = F->getRegInfo();
13430 Register Reg = MI.getOperand(i: OpIdx).getReg();
13431 bool IsSignExtended =
13432 Reg.isVirtual() && isSignExtended(MI&: *RegInfo.getVRegDef(Reg), TII);
13433
13434 if (!IsSignExtended) {
13435 Register ValueReg = RegInfo.createVirtualRegister(RegClass: &PPC::GPRCRegClass);
13436 BuildMI(BB&: *BB, I&: MI, MIMD: MI.getDebugLoc(),
13437 MCID: TII->get(Opcode: IsByte ? PPC::EXTSB : PPC::EXTSH), DestReg: ValueReg)
13438 .addReg(RegNo: Reg);
13439 MI.getOperand(i: OpIdx).setReg(ValueReg);
13440 }
13441}
13442
13443MachineBasicBlock *PPCTargetLowering::EmitPartwordAtomicBinary(
13444 MachineInstr &MI, MachineBasicBlock *BB, unsigned BinOpcode,
13445 unsigned CmpOpcode, unsigned CmpPred) const {
13446 // BinOpcode != 0: Handles atomic load with binary operator, e.g. NAND.
13447 // CmpOpcode != 0: Handles atomic load with MIN/MAX etc.
13448 // BinOpcode == 0 && CmpOpcode == 0: Handles ATOMIC_SWAP.
13449 assert(!Subtarget.hasPartwordAtomics() &&
13450 "Assumes that part-word atomics are not available");
13451 const PPCInstrInfo *TII = Subtarget.getInstrInfo();
13452
13453 // If this is a signed comparison and the value being compared is not known
13454 // to be sign extended, sign extend it here.
13455 DebugLoc dl = MI.getDebugLoc();
13456 MachineFunction *F = BB->getParent();
13457 MachineRegisterInfo &RegInfo = F->getRegInfo();
13458 const bool is8bit = MI.getOperand(i: 3).getImm() == 1;
13459 if (CmpOpcode == PPC::CMPW)
13460 signExtendOperandIfUnknown(MI, BB, OpIdx: 4, IsByte: is8bit, TII);
13461 Register incr = MI.getOperand(i: 4).getReg();
13462
13463 // In 64 bit mode we have to use 64 bits for addresses, even though the
13464 // lwarx/stwcx are 32 bits. With the 32-bit atomics we can use address
13465 // registers without caring whether they're 32 or 64, but here we're
13466 // doing actual arithmetic on the addresses.
13467 bool is64bit = Subtarget.isPPC64();
13468 bool isLittleEndian = Subtarget.isLittleEndian();
13469 unsigned ZeroReg = is64bit ? PPC::ZERO8 : PPC::ZERO;
13470
13471 const BasicBlock *LLVM_BB = BB->getBasicBlock();
13472 MachineFunction::iterator It = ++BB->getIterator();
13473
13474 Register dest = MI.getOperand(i: 0).getReg();
13475 Register ptrA = MI.getOperand(i: 1).getReg();
13476 Register ptrB = MI.getOperand(i: 2).getReg();
13477
13478 MachineBasicBlock *loopMBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
13479 MachineBasicBlock *loop2MBB =
13480 CmpOpcode ? F->CreateMachineBasicBlock(BB: LLVM_BB) : nullptr;
13481 MachineBasicBlock *exitMBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
13482 F->insert(MBBI: It, MBB: loopMBB);
13483 if (CmpOpcode)
13484 F->insert(MBBI: It, MBB: loop2MBB);
13485 F->insert(MBBI: It, MBB: exitMBB);
13486 exitMBB->splice(Where: exitMBB->begin(), Other: BB,
13487 From: std::next(x: MachineBasicBlock::iterator(MI)), To: BB->end());
13488 exitMBB->transferSuccessorsAndUpdatePHIs(FromMBB: BB);
13489
13490 const TargetRegisterClass *RC =
13491 is64bit ? &PPC::G8RCRegClass : &PPC::GPRCRegClass;
13492 const TargetRegisterClass *GPRC = &PPC::GPRCRegClass;
13493
13494 Register PtrReg = RegInfo.createVirtualRegister(RegClass: RC);
13495 Register Shift1Reg = RegInfo.createVirtualRegister(RegClass: GPRC);
13496 Register ShiftReg =
13497 isLittleEndian ? Shift1Reg : RegInfo.createVirtualRegister(RegClass: GPRC);
13498 Register Incr2Reg = RegInfo.createVirtualRegister(RegClass: GPRC);
13499 Register MaskReg = RegInfo.createVirtualRegister(RegClass: GPRC);
13500 Register Mask2Reg = RegInfo.createVirtualRegister(RegClass: GPRC);
13501 Register Mask3Reg = RegInfo.createVirtualRegister(RegClass: GPRC);
13502 Register Tmp2Reg = RegInfo.createVirtualRegister(RegClass: GPRC);
13503 Register Tmp3Reg = RegInfo.createVirtualRegister(RegClass: GPRC);
13504 Register Tmp4Reg = RegInfo.createVirtualRegister(RegClass: GPRC);
13505 Register TmpDestReg = RegInfo.createVirtualRegister(RegClass: GPRC);
13506 Register SrwDestReg = RegInfo.createVirtualRegister(RegClass: GPRC);
13507 Register Ptr1Reg;
13508 Register TmpReg =
13509 (!BinOpcode) ? Incr2Reg : RegInfo.createVirtualRegister(RegClass: GPRC);
13510
13511 // thisMBB:
13512 // ...
13513 // fallthrough --> loopMBB
13514 BB->addSuccessor(Succ: loopMBB);
13515
13516 // The 4-byte load must be aligned, while a char or short may be
13517 // anywhere in the word. Hence all this nasty bookkeeping code.
13518 // add ptr1, ptrA, ptrB [copy if ptrA==0]
13519 // rlwinm shift1, ptr1, 3, 27, 28 [3, 27, 27]
13520 // xori shift, shift1, 24 [16]
13521 // rlwinm ptr, ptr1, 0, 0, 29
13522 // slw incr2, incr, shift
13523 // li mask2, 255 [li mask3, 0; ori mask2, mask3, 65535]
13524 // slw mask, mask2, shift
13525 // loopMBB:
13526 // lwarx tmpDest, ptr
13527 // add tmp, tmpDest, incr2
13528 // andc tmp2, tmpDest, mask
13529 // and tmp3, tmp, mask
13530 // or tmp4, tmp3, tmp2
13531 // stwcx. tmp4, ptr
13532 // bne- loopMBB
13533 // fallthrough --> exitMBB
13534 // srw SrwDest, tmpDest, shift
13535 // rlwinm SrwDest, SrwDest, 0, 24 [16], 31
13536 if (ptrA != ZeroReg) {
13537 Ptr1Reg = RegInfo.createVirtualRegister(RegClass: RC);
13538 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: is64bit ? PPC::ADD8 : PPC::ADD4), DestReg: Ptr1Reg)
13539 .addReg(RegNo: ptrA)
13540 .addReg(RegNo: ptrB);
13541 } else {
13542 Ptr1Reg = ptrB;
13543 }
13544 // We need use 32-bit subregister to avoid mismatch register class in 64-bit
13545 // mode.
13546 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::RLWINM), DestReg: Shift1Reg)
13547 .addReg(RegNo: Ptr1Reg, Flags: {}, SubReg: is64bit ? PPC::sub_32 : 0)
13548 .addImm(Val: 3)
13549 .addImm(Val: 27)
13550 .addImm(Val: is8bit ? 28 : 27);
13551 if (!isLittleEndian)
13552 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::XORI), DestReg: ShiftReg)
13553 .addReg(RegNo: Shift1Reg)
13554 .addImm(Val: is8bit ? 24 : 16);
13555 if (is64bit)
13556 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::RLDICR), DestReg: PtrReg)
13557 .addReg(RegNo: Ptr1Reg)
13558 .addImm(Val: 0)
13559 .addImm(Val: 61);
13560 else
13561 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::RLWINM), DestReg: PtrReg)
13562 .addReg(RegNo: Ptr1Reg)
13563 .addImm(Val: 0)
13564 .addImm(Val: 0)
13565 .addImm(Val: 29);
13566 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::SLW), DestReg: Incr2Reg).addReg(RegNo: incr).addReg(RegNo: ShiftReg);
13567 if (is8bit)
13568 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::LI), DestReg: Mask2Reg).addImm(Val: 255);
13569 else {
13570 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::LI), DestReg: Mask3Reg).addImm(Val: 0);
13571 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::ORI), DestReg: Mask2Reg)
13572 .addReg(RegNo: Mask3Reg)
13573 .addImm(Val: 65535);
13574 }
13575 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::SLW), DestReg: MaskReg)
13576 .addReg(RegNo: Mask2Reg)
13577 .addReg(RegNo: ShiftReg);
13578
13579 BB = loopMBB;
13580 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::LWARX), DestReg: TmpDestReg)
13581 .addReg(RegNo: ZeroReg)
13582 .addReg(RegNo: PtrReg);
13583 if (BinOpcode)
13584 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: BinOpcode), DestReg: TmpReg)
13585 .addReg(RegNo: Incr2Reg)
13586 .addReg(RegNo: TmpDestReg);
13587 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::ANDC), DestReg: Tmp2Reg)
13588 .addReg(RegNo: TmpDestReg)
13589 .addReg(RegNo: MaskReg);
13590 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::AND), DestReg: Tmp3Reg).addReg(RegNo: TmpReg).addReg(RegNo: MaskReg);
13591 if (CmpOpcode) {
13592 // For unsigned comparisons, we can directly compare the shifted values.
13593 // For signed comparisons we shift and sign extend.
13594 Register SReg = RegInfo.createVirtualRegister(RegClass: GPRC);
13595 Register CrReg = RegInfo.createVirtualRegister(RegClass: &PPC::CRRCRegClass);
13596 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::AND), DestReg: SReg)
13597 .addReg(RegNo: TmpDestReg)
13598 .addReg(RegNo: MaskReg);
13599 unsigned ValueReg = SReg;
13600 unsigned CmpReg = Incr2Reg;
13601 if (CmpOpcode == PPC::CMPW) {
13602 ValueReg = RegInfo.createVirtualRegister(RegClass: GPRC);
13603 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::SRW), DestReg: ValueReg)
13604 .addReg(RegNo: SReg)
13605 .addReg(RegNo: ShiftReg);
13606 Register ValueSReg = RegInfo.createVirtualRegister(RegClass: GPRC);
13607 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: is8bit ? PPC::EXTSB : PPC::EXTSH), DestReg: ValueSReg)
13608 .addReg(RegNo: ValueReg);
13609 ValueReg = ValueSReg;
13610 CmpReg = incr;
13611 }
13612 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: CmpOpcode), DestReg: CrReg).addReg(RegNo: ValueReg).addReg(RegNo: CmpReg);
13613 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::BCC))
13614 .addImm(Val: CmpPred)
13615 .addReg(RegNo: CrReg)
13616 .addMBB(MBB: exitMBB);
13617 BB->addSuccessor(Succ: loop2MBB);
13618 BB->addSuccessor(Succ: exitMBB);
13619 BB = loop2MBB;
13620 }
13621 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::OR), DestReg: Tmp4Reg).addReg(RegNo: Tmp3Reg).addReg(RegNo: Tmp2Reg);
13622 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::STWCX))
13623 .addReg(RegNo: Tmp4Reg)
13624 .addReg(RegNo: ZeroReg)
13625 .addReg(RegNo: PtrReg);
13626 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::BCC))
13627 .addImm(Val: PPC::PRED_NE_MINUS)
13628 .addReg(RegNo: PPC::CR0)
13629 .addMBB(MBB: loopMBB);
13630 BB->addSuccessor(Succ: loopMBB);
13631 BB->addSuccessor(Succ: exitMBB);
13632
13633 // exitMBB:
13634 // ...
13635 BB = exitMBB;
13636 // Since the shift amount is not a constant, we need to clear
13637 // the upper bits with a separate RLWINM.
13638 BuildMI(BB&: *BB, I: BB->begin(), MIMD: dl, MCID: TII->get(Opcode: PPC::RLWINM), DestReg: dest)
13639 .addReg(RegNo: SrwDestReg)
13640 .addImm(Val: 0)
13641 .addImm(Val: is8bit ? 24 : 16)
13642 .addImm(Val: 31);
13643 BuildMI(BB&: *BB, I: BB->begin(), MIMD: dl, MCID: TII->get(Opcode: PPC::SRW), DestReg: SrwDestReg)
13644 .addReg(RegNo: TmpDestReg)
13645 .addReg(RegNo: ShiftReg);
13646 return BB;
13647}
13648
13649llvm::MachineBasicBlock *
13650PPCTargetLowering::emitEHSjLjSetJmp(MachineInstr &MI,
13651 MachineBasicBlock *MBB) const {
13652 DebugLoc DL = MI.getDebugLoc();
13653 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
13654 const PPCRegisterInfo *TRI = Subtarget.getRegisterInfo();
13655
13656 MachineFunction *MF = MBB->getParent();
13657 MachineRegisterInfo &MRI = MF->getRegInfo();
13658
13659 const BasicBlock *BB = MBB->getBasicBlock();
13660 MachineFunction::iterator I = ++MBB->getIterator();
13661
13662 Register DstReg = MI.getOperand(i: 0).getReg();
13663 const TargetRegisterClass *RC = MRI.getRegClass(Reg: DstReg);
13664 assert(TRI->isTypeLegalForClass(*RC, MVT::i32) && "Invalid destination!");
13665 Register mainDstReg = MRI.createVirtualRegister(RegClass: RC);
13666 Register restoreDstReg = MRI.createVirtualRegister(RegClass: RC);
13667
13668 MVT PVT = getPointerTy(DL: MF->getDataLayout());
13669 assert((PVT == MVT::i64 || PVT == MVT::i32) &&
13670 "Invalid Pointer Size!");
13671 // For v = setjmp(buf), we generate
13672 //
13673 // thisMBB:
13674 // SjLjSetup mainMBB
13675 // bl mainMBB
13676 // v_restore = 1
13677 // b sinkMBB
13678 //
13679 // mainMBB:
13680 // buf[LabelOffset] = LR
13681 // v_main = 0
13682 //
13683 // sinkMBB:
13684 // v = phi(main, restore)
13685 //
13686
13687 MachineBasicBlock *thisMBB = MBB;
13688 MachineBasicBlock *mainMBB = MF->CreateMachineBasicBlock(BB);
13689 MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(BB);
13690 MF->insert(MBBI: I, MBB: mainMBB);
13691 MF->insert(MBBI: I, MBB: sinkMBB);
13692
13693 MachineInstrBuilder MIB;
13694
13695 // Transfer the remainder of BB and its successor edges to sinkMBB.
13696 sinkMBB->splice(Where: sinkMBB->begin(), Other: MBB,
13697 From: std::next(x: MachineBasicBlock::iterator(MI)), To: MBB->end());
13698 sinkMBB->transferSuccessorsAndUpdatePHIs(FromMBB: MBB);
13699
13700 // Note that the structure of the jmp_buf used here is not compatible
13701 // with that used by libc, and is not designed to be. Specifically, it
13702 // stores only those 'reserved' registers that LLVM does not otherwise
13703 // understand how to spill. Also, by convention, by the time this
13704 // intrinsic is called, Clang has already stored the frame address in the
13705 // first slot of the buffer and stack address in the third. Following the
13706 // X86 target code, we'll store the jump address in the second slot. We also
13707 // need to save the TOC pointer (R2) to handle jumps between shared
13708 // libraries, and that will be stored in the fourth slot. The thread
13709 // identifier (R13) is not affected.
13710
13711 // thisMBB:
13712 const int64_t LabelOffset = 1 * PVT.getStoreSize();
13713 const int64_t TOCOffset = 3 * PVT.getStoreSize();
13714 const int64_t BPOffset = 4 * PVT.getStoreSize();
13715
13716 // Prepare IP either in reg.
13717 const TargetRegisterClass *PtrRC = getRegClassFor(VT: PVT);
13718 Register LabelReg = MRI.createVirtualRegister(RegClass: PtrRC);
13719 Register BufReg = MI.getOperand(i: 1).getReg();
13720
13721 if (Subtarget.is64BitELFABI()) {
13722 setUsesTOCBasePtr(*MBB->getParent());
13723 MIB = BuildMI(BB&: *thisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::STD))
13724 .addReg(RegNo: PPC::X2)
13725 .addImm(Val: TOCOffset)
13726 .addReg(RegNo: BufReg)
13727 .cloneMemRefs(OtherMI: MI);
13728 }
13729
13730 // Naked functions never have a base pointer, and so we use r1. For all
13731 // other functions, this decision must be delayed until during PEI.
13732 unsigned BaseReg;
13733 if (MF->getFunction().hasFnAttribute(Kind: Attribute::Naked))
13734 BaseReg = Subtarget.isPPC64() ? PPC::X1 : PPC::R1;
13735 else
13736 BaseReg = Subtarget.isPPC64() ? PPC::BP8 : PPC::BP;
13737
13738 MIB = BuildMI(BB&: *thisMBB, I&: MI, MIMD: DL,
13739 MCID: TII->get(Opcode: Subtarget.isPPC64() ? PPC::STD : PPC::STW))
13740 .addReg(RegNo: BaseReg)
13741 .addImm(Val: BPOffset)
13742 .addReg(RegNo: BufReg)
13743 .cloneMemRefs(OtherMI: MI);
13744
13745 // Setup
13746 MIB = BuildMI(BB&: *thisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::BCLalways)).addMBB(MBB: mainMBB);
13747 MIB.addRegMask(Mask: TRI->getNoPreservedMask());
13748
13749 BuildMI(BB&: *thisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::LI), DestReg: restoreDstReg).addImm(Val: 1);
13750
13751 MIB = BuildMI(BB&: *thisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::EH_SjLj_Setup))
13752 .addMBB(MBB: mainMBB);
13753 MIB = BuildMI(BB&: *thisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::B)).addMBB(MBB: sinkMBB);
13754
13755 thisMBB->addSuccessor(Succ: mainMBB, Prob: BranchProbability::getZero());
13756 thisMBB->addSuccessor(Succ: sinkMBB, Prob: BranchProbability::getOne());
13757
13758 // mainMBB:
13759 // mainDstReg = 0
13760 MIB =
13761 BuildMI(BB: mainMBB, MIMD: DL,
13762 MCID: TII->get(Opcode: Subtarget.isPPC64() ? PPC::MFLR8 : PPC::MFLR), DestReg: LabelReg);
13763
13764 // Store IP
13765 if (Subtarget.isPPC64()) {
13766 MIB = BuildMI(BB: mainMBB, MIMD: DL, MCID: TII->get(Opcode: PPC::STD))
13767 .addReg(RegNo: LabelReg)
13768 .addImm(Val: LabelOffset)
13769 .addReg(RegNo: BufReg);
13770 } else {
13771 MIB = BuildMI(BB: mainMBB, MIMD: DL, MCID: TII->get(Opcode: PPC::STW))
13772 .addReg(RegNo: LabelReg)
13773 .addImm(Val: LabelOffset)
13774 .addReg(RegNo: BufReg);
13775 }
13776 MIB.cloneMemRefs(OtherMI: MI);
13777
13778 BuildMI(BB: mainMBB, MIMD: DL, MCID: TII->get(Opcode: PPC::LI), DestReg: mainDstReg).addImm(Val: 0);
13779 mainMBB->addSuccessor(Succ: sinkMBB);
13780
13781 // sinkMBB:
13782 BuildMI(BB&: *sinkMBB, I: sinkMBB->begin(), MIMD: DL,
13783 MCID: TII->get(Opcode: PPC::PHI), DestReg: DstReg)
13784 .addReg(RegNo: mainDstReg).addMBB(MBB: mainMBB)
13785 .addReg(RegNo: restoreDstReg).addMBB(MBB: thisMBB);
13786
13787 MI.eraseFromParent();
13788 return sinkMBB;
13789}
13790
13791MachineBasicBlock *
13792PPCTargetLowering::emitEHSjLjLongJmp(MachineInstr &MI,
13793 MachineBasicBlock *MBB) const {
13794 DebugLoc DL = MI.getDebugLoc();
13795 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
13796
13797 MachineFunction *MF = MBB->getParent();
13798 MachineRegisterInfo &MRI = MF->getRegInfo();
13799
13800 MVT PVT = getPointerTy(DL: MF->getDataLayout());
13801 assert((PVT == MVT::i64 || PVT == MVT::i32) &&
13802 "Invalid Pointer Size!");
13803
13804 const TargetRegisterClass *RC =
13805 (PVT == MVT::i64) ? &PPC::G8RCRegClass : &PPC::GPRCRegClass;
13806 Register Tmp = MRI.createVirtualRegister(RegClass: RC);
13807 // Since FP is only updated here but NOT referenced, it's treated as GPR.
13808 unsigned FP = (PVT == MVT::i64) ? PPC::X31 : PPC::R31;
13809 unsigned SP = (PVT == MVT::i64) ? PPC::X1 : PPC::R1;
13810 unsigned BP =
13811 (PVT == MVT::i64)
13812 ? PPC::X30
13813 : (Subtarget.isSVR4ABI() && isPositionIndependent() ? PPC::R29
13814 : PPC::R30);
13815
13816 MachineInstrBuilder MIB;
13817
13818 const int64_t LabelOffset = 1 * PVT.getStoreSize();
13819 const int64_t SPOffset = 2 * PVT.getStoreSize();
13820 const int64_t TOCOffset = 3 * PVT.getStoreSize();
13821 const int64_t BPOffset = 4 * PVT.getStoreSize();
13822
13823 Register BufReg = MI.getOperand(i: 0).getReg();
13824
13825 // Reload FP (the jumped-to function may not have had a
13826 // frame pointer, and if so, then its r31 will be restored
13827 // as necessary).
13828 if (PVT == MVT::i64) {
13829 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::LD), DestReg: FP)
13830 .addImm(Val: 0)
13831 .addReg(RegNo: BufReg);
13832 } else {
13833 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::LWZ), DestReg: FP)
13834 .addImm(Val: 0)
13835 .addReg(RegNo: BufReg);
13836 }
13837 MIB.cloneMemRefs(OtherMI: MI);
13838
13839 // Reload IP
13840 if (PVT == MVT::i64) {
13841 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::LD), DestReg: Tmp)
13842 .addImm(Val: LabelOffset)
13843 .addReg(RegNo: BufReg);
13844 } else {
13845 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::LWZ), DestReg: Tmp)
13846 .addImm(Val: LabelOffset)
13847 .addReg(RegNo: BufReg);
13848 }
13849 MIB.cloneMemRefs(OtherMI: MI);
13850
13851 // Reload SP
13852 if (PVT == MVT::i64) {
13853 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::LD), DestReg: SP)
13854 .addImm(Val: SPOffset)
13855 .addReg(RegNo: BufReg);
13856 } else {
13857 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::LWZ), DestReg: SP)
13858 .addImm(Val: SPOffset)
13859 .addReg(RegNo: BufReg);
13860 }
13861 MIB.cloneMemRefs(OtherMI: MI);
13862
13863 // Reload BP
13864 if (PVT == MVT::i64) {
13865 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::LD), DestReg: BP)
13866 .addImm(Val: BPOffset)
13867 .addReg(RegNo: BufReg);
13868 } else {
13869 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::LWZ), DestReg: BP)
13870 .addImm(Val: BPOffset)
13871 .addReg(RegNo: BufReg);
13872 }
13873 MIB.cloneMemRefs(OtherMI: MI);
13874
13875 // Reload TOC
13876 if (PVT == MVT::i64 && Subtarget.isSVR4ABI()) {
13877 setUsesTOCBasePtr(*MBB->getParent());
13878 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::LD), DestReg: PPC::X2)
13879 .addImm(Val: TOCOffset)
13880 .addReg(RegNo: BufReg)
13881 .cloneMemRefs(OtherMI: MI);
13882 }
13883
13884 // Jump
13885 BuildMI(BB&: *MBB, I&: MI, MIMD: DL,
13886 MCID: TII->get(Opcode: PVT == MVT::i64 ? PPC::MTCTR8 : PPC::MTCTR)).addReg(RegNo: Tmp);
13887 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PVT == MVT::i64 ? PPC::BCTR8 : PPC::BCTR));
13888
13889 MI.eraseFromParent();
13890 return MBB;
13891}
13892
13893bool PPCTargetLowering::hasInlineStackProbe(const MachineFunction &MF) const {
13894 // If the function specifically requests inline stack probes, emit them.
13895 if (MF.getFunction().hasFnAttribute(Kind: "probe-stack"))
13896 return MF.getFunction().getFnAttribute(Kind: "probe-stack").getValueAsString() ==
13897 "inline-asm";
13898 return false;
13899}
13900
13901unsigned PPCTargetLowering::getStackProbeSize(const MachineFunction &MF) const {
13902 const TargetFrameLowering *TFI = Subtarget.getFrameLowering();
13903 unsigned StackAlign = TFI->getStackAlignment();
13904 assert(StackAlign >= 1 && isPowerOf2_32(StackAlign) &&
13905 "Unexpected stack alignment");
13906 // The default stack probe size is 4096 if the function has no
13907 // stack-probe-size attribute.
13908 const Function &Fn = MF.getFunction();
13909 unsigned StackProbeSize =
13910 Fn.getFnAttributeAsParsedInteger(Kind: "stack-probe-size", Default: 4096);
13911 // Round down to the stack alignment.
13912 StackProbeSize &= ~(StackAlign - 1);
13913 return StackProbeSize ? StackProbeSize : StackAlign;
13914}
13915
13916// Lower dynamic stack allocation with probing. `emitProbedAlloca` is splitted
13917// into three phases. In the first phase, it uses pseudo instruction
13918// PREPARE_PROBED_ALLOCA to get the future result of actual FramePointer and
13919// FinalStackPtr. In the second phase, it generates a loop for probing blocks.
13920// At last, it uses pseudo instruction DYNAREAOFFSET to get the future result of
13921// MaxCallFrameSize so that it can calculate correct data area pointer.
13922MachineBasicBlock *
13923PPCTargetLowering::emitProbedAlloca(MachineInstr &MI,
13924 MachineBasicBlock *MBB) const {
13925 const bool isPPC64 = Subtarget.isPPC64();
13926 MachineFunction *MF = MBB->getParent();
13927 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
13928 DebugLoc DL = MI.getDebugLoc();
13929 const unsigned ProbeSize = getStackProbeSize(MF: *MF);
13930 const BasicBlock *ProbedBB = MBB->getBasicBlock();
13931 MachineRegisterInfo &MRI = MF->getRegInfo();
13932 // The CFG of probing stack looks as
13933 // +-----+
13934 // | MBB |
13935 // +--+--+
13936 // |
13937 // +----v----+
13938 // +--->+ TestMBB +---+
13939 // | +----+----+ |
13940 // | | |
13941 // | +-----v----+ |
13942 // +---+ BlockMBB | |
13943 // +----------+ |
13944 // |
13945 // +---------+ |
13946 // | TailMBB +<--+
13947 // +---------+
13948 // In MBB, calculate previous frame pointer and final stack pointer.
13949 // In TestMBB, test if sp is equal to final stack pointer, if so, jump to
13950 // TailMBB. In BlockMBB, update the sp atomically and jump back to TestMBB.
13951 // TailMBB is spliced via \p MI.
13952 MachineBasicBlock *TestMBB = MF->CreateMachineBasicBlock(BB: ProbedBB);
13953 MachineBasicBlock *TailMBB = MF->CreateMachineBasicBlock(BB: ProbedBB);
13954 MachineBasicBlock *BlockMBB = MF->CreateMachineBasicBlock(BB: ProbedBB);
13955
13956 MachineFunction::iterator MBBIter = ++MBB->getIterator();
13957 MF->insert(MBBI: MBBIter, MBB: TestMBB);
13958 MF->insert(MBBI: MBBIter, MBB: BlockMBB);
13959 MF->insert(MBBI: MBBIter, MBB: TailMBB);
13960
13961 const TargetRegisterClass *G8RC = &PPC::G8RCRegClass;
13962 const TargetRegisterClass *GPRC = &PPC::GPRCRegClass;
13963
13964 Register DstReg = MI.getOperand(i: 0).getReg();
13965 Register NegSizeReg = MI.getOperand(i: 1).getReg();
13966 Register SPReg = isPPC64 ? PPC::X1 : PPC::R1;
13967 Register FinalStackPtr = MRI.createVirtualRegister(RegClass: isPPC64 ? G8RC : GPRC);
13968 Register FramePointer = MRI.createVirtualRegister(RegClass: isPPC64 ? G8RC : GPRC);
13969 Register ActualNegSizeReg = MRI.createVirtualRegister(RegClass: isPPC64 ? G8RC : GPRC);
13970
13971 // Since value of NegSizeReg might be realigned in prologepilog, insert a
13972 // PREPARE_PROBED_ALLOCA pseudo instruction to get actual FramePointer and
13973 // NegSize.
13974 unsigned ProbeOpc;
13975 if (!MRI.hasOneNonDBGUse(RegNo: NegSizeReg))
13976 ProbeOpc =
13977 isPPC64 ? PPC::PREPARE_PROBED_ALLOCA_64 : PPC::PREPARE_PROBED_ALLOCA_32;
13978 else
13979 // By introducing PREPARE_PROBED_ALLOCA_NEGSIZE_OPT, ActualNegSizeReg
13980 // and NegSizeReg will be allocated in the same phyreg to avoid
13981 // redundant copy when NegSizeReg has only one use which is current MI and
13982 // will be replaced by PREPARE_PROBED_ALLOCA then.
13983 ProbeOpc = isPPC64 ? PPC::PREPARE_PROBED_ALLOCA_NEGSIZE_SAME_REG_64
13984 : PPC::PREPARE_PROBED_ALLOCA_NEGSIZE_SAME_REG_32;
13985 BuildMI(BB&: *MBB, I&: {MI}, MIMD: DL, MCID: TII->get(Opcode: ProbeOpc), DestReg: FramePointer)
13986 .addDef(RegNo: ActualNegSizeReg)
13987 .addReg(RegNo: NegSizeReg)
13988 .add(MO: MI.getOperand(i: 2))
13989 .add(MO: MI.getOperand(i: 3));
13990
13991 // Calculate final stack pointer, which equals to SP + ActualNegSize.
13992 BuildMI(BB&: *MBB, I&: {MI}, MIMD: DL, MCID: TII->get(Opcode: isPPC64 ? PPC::ADD8 : PPC::ADD4),
13993 DestReg: FinalStackPtr)
13994 .addReg(RegNo: SPReg)
13995 .addReg(RegNo: ActualNegSizeReg);
13996
13997 // Materialize a scratch register for update.
13998 int64_t NegProbeSize = -(int64_t)ProbeSize;
13999 assert(isInt<32>(NegProbeSize) && "Unhandled probe size!");
14000 Register ScratchReg = MRI.createVirtualRegister(RegClass: isPPC64 ? G8RC : GPRC);
14001 if (!isInt<16>(x: NegProbeSize)) {
14002 Register TempReg = MRI.createVirtualRegister(RegClass: isPPC64 ? G8RC : GPRC);
14003 BuildMI(BB&: *MBB, I&: {MI}, MIMD: DL, MCID: TII->get(Opcode: isPPC64 ? PPC::LIS8 : PPC::LIS), DestReg: TempReg)
14004 .addImm(Val: NegProbeSize >> 16);
14005 BuildMI(BB&: *MBB, I&: {MI}, MIMD: DL, MCID: TII->get(Opcode: isPPC64 ? PPC::ORI8 : PPC::ORI),
14006 DestReg: ScratchReg)
14007 .addReg(RegNo: TempReg)
14008 .addImm(Val: NegProbeSize & 0xFFFF);
14009 } else
14010 BuildMI(BB&: *MBB, I&: {MI}, MIMD: DL, MCID: TII->get(Opcode: isPPC64 ? PPC::LI8 : PPC::LI), DestReg: ScratchReg)
14011 .addImm(Val: NegProbeSize);
14012
14013 {
14014 // Probing leading residual part.
14015 Register Div = MRI.createVirtualRegister(RegClass: isPPC64 ? G8RC : GPRC);
14016 BuildMI(BB&: *MBB, I&: {MI}, MIMD: DL, MCID: TII->get(Opcode: isPPC64 ? PPC::DIVD : PPC::DIVW), DestReg: Div)
14017 .addReg(RegNo: ActualNegSizeReg)
14018 .addReg(RegNo: ScratchReg);
14019 Register Mul = MRI.createVirtualRegister(RegClass: isPPC64 ? G8RC : GPRC);
14020 BuildMI(BB&: *MBB, I&: {MI}, MIMD: DL, MCID: TII->get(Opcode: isPPC64 ? PPC::MULLD : PPC::MULLW), DestReg: Mul)
14021 .addReg(RegNo: Div)
14022 .addReg(RegNo: ScratchReg);
14023 Register NegMod = MRI.createVirtualRegister(RegClass: isPPC64 ? G8RC : GPRC);
14024 BuildMI(BB&: *MBB, I&: {MI}, MIMD: DL, MCID: TII->get(Opcode: isPPC64 ? PPC::SUBF8 : PPC::SUBF), DestReg: NegMod)
14025 .addReg(RegNo: Mul)
14026 .addReg(RegNo: ActualNegSizeReg);
14027 BuildMI(BB&: *MBB, I&: {MI}, MIMD: DL, MCID: TII->get(Opcode: isPPC64 ? PPC::STDUX : PPC::STWUX), DestReg: SPReg)
14028 .addReg(RegNo: FramePointer)
14029 .addReg(RegNo: SPReg)
14030 .addReg(RegNo: NegMod);
14031 }
14032
14033 {
14034 // Remaining part should be multiple of ProbeSize.
14035 Register CmpResult = MRI.createVirtualRegister(RegClass: &PPC::CRRCRegClass);
14036 BuildMI(BB: TestMBB, MIMD: DL, MCID: TII->get(Opcode: isPPC64 ? PPC::CMPD : PPC::CMPW), DestReg: CmpResult)
14037 .addReg(RegNo: SPReg)
14038 .addReg(RegNo: FinalStackPtr);
14039 BuildMI(BB: TestMBB, MIMD: DL, MCID: TII->get(Opcode: PPC::BCC))
14040 .addImm(Val: PPC::PRED_EQ)
14041 .addReg(RegNo: CmpResult)
14042 .addMBB(MBB: TailMBB);
14043 TestMBB->addSuccessor(Succ: BlockMBB);
14044 TestMBB->addSuccessor(Succ: TailMBB);
14045 }
14046
14047 {
14048 // Touch the block.
14049 // |P...|P...|P...
14050 BuildMI(BB: BlockMBB, MIMD: DL, MCID: TII->get(Opcode: isPPC64 ? PPC::STDUX : PPC::STWUX), DestReg: SPReg)
14051 .addReg(RegNo: FramePointer)
14052 .addReg(RegNo: SPReg)
14053 .addReg(RegNo: ScratchReg);
14054 BuildMI(BB: BlockMBB, MIMD: DL, MCID: TII->get(Opcode: PPC::B)).addMBB(MBB: TestMBB);
14055 BlockMBB->addSuccessor(Succ: TestMBB);
14056 }
14057
14058 // Calculation of MaxCallFrameSize is deferred to prologepilog, use
14059 // DYNAREAOFFSET pseudo instruction to get the future result.
14060 Register MaxCallFrameSizeReg =
14061 MRI.createVirtualRegister(RegClass: isPPC64 ? G8RC : GPRC);
14062 BuildMI(BB: TailMBB, MIMD: DL,
14063 MCID: TII->get(Opcode: isPPC64 ? PPC::DYNAREAOFFSET8 : PPC::DYNAREAOFFSET),
14064 DestReg: MaxCallFrameSizeReg)
14065 .add(MO: MI.getOperand(i: 2))
14066 .add(MO: MI.getOperand(i: 3));
14067 BuildMI(BB: TailMBB, MIMD: DL, MCID: TII->get(Opcode: isPPC64 ? PPC::ADD8 : PPC::ADD4), DestReg: DstReg)
14068 .addReg(RegNo: SPReg)
14069 .addReg(RegNo: MaxCallFrameSizeReg);
14070
14071 // Splice instructions after MI to TailMBB.
14072 TailMBB->splice(Where: TailMBB->end(), Other: MBB,
14073 From: std::next(x: MachineBasicBlock::iterator(MI)), To: MBB->end());
14074 TailMBB->transferSuccessorsAndUpdatePHIs(FromMBB: MBB);
14075 MBB->addSuccessor(Succ: TestMBB);
14076
14077 // Delete the pseudo instruction.
14078 MI.eraseFromParent();
14079
14080 ++NumDynamicAllocaProbed;
14081 return TailMBB;
14082}
14083
14084/// Check if the opcode is a SELECT or SELECT_CC variant.
14085/// @param Opcode The opcode to check
14086/// @param CheckOnlyCC If true, only return true for SELECT_CC variants;
14087/// if false, return true for both SELECT and SELECT_CC
14088static bool IsSelect(unsigned Opcode, bool CheckOnlyCC = false) {
14089 switch (Opcode) {
14090 // SELECT_CC variants - always return true
14091 case PPC::SELECT_CC_I4:
14092 case PPC::SELECT_CC_I8:
14093 case PPC::SELECT_CC_F4:
14094 case PPC::SELECT_CC_F8:
14095 case PPC::SELECT_CC_F16:
14096 case PPC::SELECT_CC_VRRC:
14097 case PPC::SELECT_CC_VSFRC:
14098 case PPC::SELECT_CC_VSSRC:
14099 case PPC::SELECT_CC_VSRC:
14100 case PPC::SELECT_CC_SPE4:
14101 case PPC::SELECT_CC_SPE:
14102 return true;
14103 // SELECT variants - only return true if CheckOnlyCC is false
14104 case PPC::SELECT_I4:
14105 case PPC::SELECT_I8:
14106 case PPC::SELECT_F4:
14107 case PPC::SELECT_F8:
14108 case PPC::SELECT_F16:
14109 case PPC::SELECT_SPE:
14110 case PPC::SELECT_SPE4:
14111 case PPC::SELECT_VRRC:
14112 case PPC::SELECT_VSFRC:
14113 case PPC::SELECT_VSSRC:
14114 case PPC::SELECT_VSRC:
14115 return !CheckOnlyCC; // true if checking all SELECTs, false if only CC
14116 default:
14117 return false;
14118 }
14119}
14120static bool IsSelectCC(unsigned Opcode) { return IsSelect(Opcode, CheckOnlyCC: true); }
14121
14122/// Emit SELECT instruction, using ISEL if available, otherwise use
14123/// branch-based control flow.
14124///
14125/// For targets with ISEL support (SELECT_CC_I4/I8, SELECT_I4/I8), this
14126/// generates a single ISEL instruction. Otherwise, it creates a
14127/// branch-based control flow pattern with PHI nodes.
14128static MachineBasicBlock *emitSelect(MachineInstr &MI, MachineBasicBlock *BB,
14129 const TargetInstrInfo *TII,
14130 const PPCSubtarget &Subtarget) {
14131 assert(IsSelect(MI.getOpcode()) && "Instruction must be a SELECT variant");
14132
14133 // Check if we can use ISEL for this SELECT
14134 if (Subtarget.hasISEL() &&
14135 (MI.getOpcode() == PPC::SELECT_CC_I4 ||
14136 MI.getOpcode() == PPC::SELECT_CC_I8 ||
14137 MI.getOpcode() == PPC::SELECT_I4 || MI.getOpcode() == PPC::SELECT_I8)) {
14138 SmallVector<MachineOperand, 2> Cond;
14139 if (MI.getOpcode() == PPC::SELECT_CC_I4 ||
14140 MI.getOpcode() == PPC::SELECT_CC_I8)
14141 Cond.push_back(Elt: MI.getOperand(i: 4));
14142 else
14143 Cond.push_back(Elt: MachineOperand::CreateImm(Val: PPC::PRED_BIT_SET));
14144 Cond.push_back(Elt: MI.getOperand(i: 1));
14145
14146 DebugLoc dl = MI.getDebugLoc();
14147 TII->insertSelect(MBB&: *BB, I: MI, DL: dl, DstReg: MI.getOperand(i: 0).getReg(), Cond,
14148 TrueReg: MI.getOperand(i: 2).getReg(), FalseReg: MI.getOperand(i: 3).getReg());
14149 MI.eraseFromParent();
14150 return BB;
14151 }
14152
14153 // Fall back to branch-based SELECT implementation
14154 MachineFunction *F = BB->getParent();
14155 const BasicBlock *LLVM_BB = BB->getBasicBlock();
14156 MachineFunction::iterator It = ++BB->getIterator();
14157 DebugLoc dl = MI.getDebugLoc();
14158
14159 MachineBasicBlock *thisMBB = BB;
14160 MachineBasicBlock *copy0MBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
14161 MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
14162 F->insert(MBBI: It, MBB: copy0MBB);
14163 F->insert(MBBI: It, MBB: sinkMBB);
14164
14165 if (isPhysRegUsedAfter(Reg: PPC::CARRY, MBI: MI.getIterator())) {
14166 copy0MBB->addLiveIn(PhysReg: PPC::CARRY);
14167 sinkMBB->addLiveIn(PhysReg: PPC::CARRY);
14168 }
14169
14170 // Set the call frame size on entry to the new basic blocks.
14171 unsigned CallFrameSize = TII->getCallFrameSizeAt(MI);
14172 copy0MBB->setCallFrameSize(CallFrameSize);
14173 sinkMBB->setCallFrameSize(CallFrameSize);
14174
14175 // Transfer the remainder of BB and its successor edges to sinkMBB.
14176 sinkMBB->splice(Where: sinkMBB->begin(), Other: BB,
14177 From: std::next(x: MachineBasicBlock::iterator(MI)), To: BB->end());
14178 sinkMBB->transferSuccessorsAndUpdatePHIs(FromMBB: BB);
14179
14180 // Add successors
14181 BB->addSuccessor(Succ: copy0MBB);
14182 BB->addSuccessor(Succ: sinkMBB);
14183
14184 // Build branch instruction
14185 if (IsSelectCC(Opcode: MI.getOpcode()))
14186 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::BCC))
14187 .addImm(Val: MI.getOperand(i: 4).getImm())
14188 .addReg(RegNo: MI.getOperand(i: 1).getReg())
14189 .addMBB(MBB: sinkMBB);
14190 else
14191 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::BC))
14192 .addReg(RegNo: MI.getOperand(i: 1).getReg())
14193 .addMBB(MBB: sinkMBB);
14194
14195 // copy0MBB: fallthrough to sinkMBB
14196 BB = copy0MBB;
14197 BB->addSuccessor(Succ: sinkMBB);
14198
14199 // sinkMBB: PHI instruction
14200 BB = sinkMBB;
14201 BuildMI(BB&: *BB, I: BB->begin(), MIMD: dl, MCID: TII->get(Opcode: PPC::PHI), DestReg: MI.getOperand(i: 0).getReg())
14202 .addReg(RegNo: MI.getOperand(i: 3).getReg())
14203 .addMBB(MBB: copy0MBB)
14204 .addReg(RegNo: MI.getOperand(i: 2).getReg())
14205 .addMBB(MBB: thisMBB);
14206 MI.eraseFromParent();
14207 return BB;
14208}
14209
14210/// Helper function to create basic blocks for atomic compare-and-swap.
14211/// Creates three basic blocks (loop1MBB, loop2MBB, exitMBB) and sets up
14212/// the control flow structure common to both hardware and software
14213/// implementations of atomic compare-and-swap operations.
14214static void createAtomicLoopBlocks(MachineFunction *F, MachineBasicBlock *BB,
14215 MachineBasicBlock *&loop1MBB,
14216 MachineBasicBlock *&loop2MBB,
14217 MachineBasicBlock *&exitMBB,
14218 MachineInstr &MI,
14219 MachineFunction::iterator It) {
14220 const BasicBlock *LLVM_BB = BB->getBasicBlock();
14221 loop1MBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
14222 loop2MBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
14223 exitMBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
14224 F->insert(MBBI: It, MBB: loop1MBB);
14225 F->insert(MBBI: It, MBB: loop2MBB);
14226 F->insert(MBBI: It, MBB: exitMBB);
14227 exitMBB->splice(Where: exitMBB->begin(), Other: BB,
14228 From: std::next(x: MachineBasicBlock::iterator(MI)), To: BB->end());
14229 exitMBB->transferSuccessorsAndUpdatePHIs(FromMBB: BB);
14230 BB->addSuccessor(Succ: loop1MBB);
14231}
14232
14233/// Emit hardware-supported atomic compare-and-swap for I32/I64 and I8/I16
14234/// with partword atomic support.
14235///
14236/// This uses native PowerPC atomic instructions (LBARX/LHARX/LWARX/LDARX for
14237/// load-and-reserve, STBCX/STHCX/STWCX/STDCX for store-conditional) to
14238/// implement atomic compare-and-swap at byte, halfword, word, or doubleword
14239/// granularity.
14240///
14241/// Control flow:
14242/// thisMBB -> loop1MBB -> loop2MBB -> exitMBB
14243/// | |
14244/// +------------+
14245///
14246/// loop1MBB:
14247/// - Load-and-reserve from memory
14248/// - Compare loaded value with expected old value
14249/// - Branch to exitMBB if not equal (CAS failed)
14250/// loop2MBB:
14251/// - Store-conditional new value to memory
14252/// - Branch back to loop1MBB if store failed (retry)
14253/// - Fall through to exitMBB on success
14254static MachineBasicBlock *
14255emitAtomicCmpSwapHardware(MachineInstr &MI, MachineBasicBlock *BB,
14256 const TargetInstrInfo *TII,
14257 const PPCSubtarget &Subtarget) {
14258 MachineFunction *F = BB->getParent();
14259 MachineFunction::iterator It = ++BB->getIterator();
14260
14261 bool is64bit = MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I64;
14262
14263 unsigned LoadMnemonic = PPC::LDARX;
14264 unsigned StoreMnemonic = PPC::STDCX;
14265 switch (MI.getOpcode()) {
14266 default:
14267 llvm_unreachable("Compare and swap of unknown size");
14268 case PPC::ATOMIC_CMP_SWAP_I8:
14269 LoadMnemonic = PPC::LBARX;
14270 StoreMnemonic = PPC::STBCX;
14271 assert(Subtarget.hasPartwordAtomics() && "No support partword atomics.");
14272 break;
14273 case PPC::ATOMIC_CMP_SWAP_I16:
14274 LoadMnemonic = PPC::LHARX;
14275 StoreMnemonic = PPC::STHCX;
14276 assert(Subtarget.hasPartwordAtomics() && "No support partword atomics.");
14277 break;
14278 case PPC::ATOMIC_CMP_SWAP_I32:
14279 LoadMnemonic = PPC::LWARX;
14280 StoreMnemonic = PPC::STWCX;
14281 break;
14282 case PPC::ATOMIC_CMP_SWAP_I64:
14283 LoadMnemonic = PPC::LDARX;
14284 StoreMnemonic = PPC::STDCX;
14285 break;
14286 }
14287
14288 MachineRegisterInfo &RegInfo = F->getRegInfo();
14289 Register dest = MI.getOperand(i: 0).getReg();
14290 Register ptrA = MI.getOperand(i: 1).getReg();
14291 Register ptrB = MI.getOperand(i: 2).getReg();
14292 Register oldval = MI.getOperand(i: 3).getReg();
14293 Register newval = MI.getOperand(i: 4).getReg();
14294 DebugLoc dl = MI.getDebugLoc();
14295
14296 MachineBasicBlock *loop1MBB, *loop2MBB, *exitMBB;
14297 createAtomicLoopBlocks(F, BB, loop1MBB, loop2MBB, exitMBB, MI, It);
14298
14299 Register CrReg = RegInfo.createVirtualRegister(RegClass: &PPC::CRRCRegClass);
14300
14301 // loop1MBB:
14302 // l[bhwd]arx dest, ptr
14303 // cmp[wd] dest, oldval
14304 // bne- exitBB
14305 BB = loop1MBB;
14306 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: LoadMnemonic), DestReg: dest).addReg(RegNo: ptrA).addReg(RegNo: ptrB);
14307 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: is64bit ? PPC::CMPD : PPC::CMPW), DestReg: CrReg)
14308 .addReg(RegNo: dest)
14309 .addReg(RegNo: oldval);
14310 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::BCC))
14311 .addImm(Val: PPC::PRED_NE_MINUS)
14312 .addReg(RegNo: CrReg)
14313 .addMBB(MBB: exitMBB);
14314 BB->addSuccessor(Succ: loop2MBB);
14315 BB->addSuccessor(Succ: exitMBB);
14316
14317 // loop2MBB:
14318 // st[bhwd]cx. newval, ptr
14319 // bne- loopMBB
14320 // b exitBB
14321 BB = loop2MBB;
14322 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: StoreMnemonic))
14323 .addReg(RegNo: newval)
14324 .addReg(RegNo: ptrA)
14325 .addReg(RegNo: ptrB);
14326 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::BCC))
14327 .addImm(Val: PPC::PRED_NE_MINUS)
14328 .addReg(RegNo: PPC::CR0)
14329 .addMBB(MBB: loop1MBB);
14330 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::B)).addMBB(MBB: exitMBB);
14331 BB->addSuccessor(Succ: loop1MBB);
14332 BB->addSuccessor(Succ: exitMBB);
14333
14334 return exitMBB;
14335}
14336
14337/// Emit software-emulated atomic compare-and-swap for I8/I16 without
14338/// hardware partword atomic support.
14339///
14340/// This emulates byte/halfword atomic operations using word (32-bit) atomic
14341/// instructions. Since PowerPC atomic instructions work at word granularity,
14342/// we must:
14343/// 1. Align the pointer to a word boundary
14344/// 2. Calculate the bit shift for the target byte/halfword within the word
14345/// 3. Create masks to isolate the target byte/halfword
14346/// 4. Shift old/new values into the correct bit position
14347/// 5. Use LWARX/STWCX on the full word
14348/// 6. Mask and merge to preserve other bytes in the word
14349/// 7. Extract and shift the result back
14350///
14351/// Control flow:
14352/// thisMBB -> loop1MBB -> loop2MBB -> exitMBB
14353/// | |
14354/// +------------+
14355///
14356/// loop1MBB:
14357/// - LWARX: Load-and-reserve full word
14358/// - Mask to extract target byte/halfword
14359/// - Compare with expected old value
14360/// - Branch to exitMBB if not equal (CAS failed)
14361/// loop2MBB:
14362/// - Merge new value with other bytes in the word
14363/// - STWCX: Store-conditional full word
14364/// - Branch back to loop1MBB if store failed (retry)
14365/// - Fall through to exitMBB on success
14366/// exitMBB:
14367/// - Extract and return the loaded value
14368static MachineBasicBlock *
14369emitAtomicCmpSwapSoftware(MachineInstr &MI, MachineBasicBlock *BB,
14370 const TargetInstrInfo *TII,
14371 const PPCSubtarget &Subtarget) {
14372 MachineFunction *F = BB->getParent();
14373 MachineFunction::iterator It = ++BB->getIterator();
14374
14375 bool is64bit = Subtarget.isPPC64();
14376 bool isLittleEndian = Subtarget.isLittleEndian();
14377 bool is8bit = MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I8;
14378
14379 Register dest = MI.getOperand(i: 0).getReg();
14380 Register ptrA = MI.getOperand(i: 1).getReg();
14381 Register ptrB = MI.getOperand(i: 2).getReg();
14382 Register oldval = MI.getOperand(i: 3).getReg();
14383 Register newval = MI.getOperand(i: 4).getReg();
14384 DebugLoc dl = MI.getDebugLoc();
14385
14386 MachineBasicBlock *loop1MBB, *loop2MBB, *exitMBB;
14387 createAtomicLoopBlocks(F, BB, loop1MBB, loop2MBB, exitMBB, MI, It);
14388
14389 MachineRegisterInfo &RegInfo = F->getRegInfo();
14390 const TargetRegisterClass *RC =
14391 is64bit ? &PPC::G8RCRegClass : &PPC::GPRCRegClass;
14392 const TargetRegisterClass *GPRC = &PPC::GPRCRegClass;
14393
14394 // Lambda to create virtual registers
14395 auto createVReg = [&](const TargetRegisterClass *RC) {
14396 return RegInfo.createVirtualRegister(RegClass: RC);
14397 };
14398
14399 Register PtrReg = createVReg(RC);
14400 Register Shift1Reg = createVReg(GPRC);
14401 Register ShiftReg = isLittleEndian ? Shift1Reg : createVReg(GPRC);
14402 Register NewVal2Reg = createVReg(GPRC);
14403 Register NewVal3Reg = createVReg(GPRC);
14404 Register OldVal2Reg = createVReg(GPRC);
14405 Register OldVal3Reg = createVReg(GPRC);
14406 Register MaskReg = createVReg(GPRC);
14407 Register Mask2Reg = createVReg(GPRC);
14408 Register Mask3Reg = createVReg(GPRC);
14409 Register Tmp2Reg = createVReg(GPRC);
14410 Register Tmp4Reg = createVReg(GPRC);
14411 Register TmpDestReg = createVReg(GPRC);
14412 Register TmpReg = createVReg(GPRC);
14413 Register ZeroReg = is64bit ? PPC::ZERO8 : PPC::ZERO;
14414 Register CrReg = createVReg(&PPC::CRRCRegClass);
14415
14416 // Compute aligned pointer and shift amount
14417 Register Ptr1Reg;
14418 if (ptrA != ZeroReg) {
14419 Ptr1Reg = createVReg(RC);
14420 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: is64bit ? PPC::ADD8 : PPC::ADD4), DestReg: Ptr1Reg)
14421 .addReg(RegNo: ptrA)
14422 .addReg(RegNo: ptrB);
14423 } else {
14424 Ptr1Reg = ptrB;
14425 }
14426
14427 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::RLWINM), DestReg: Shift1Reg)
14428 .addReg(RegNo: Ptr1Reg, Flags: {}, SubReg: is64bit ? PPC::sub_32 : 0)
14429 .addImm(Val: 3)
14430 .addImm(Val: 27)
14431 .addImm(Val: is8bit ? 28 : 27);
14432 if (!isLittleEndian)
14433 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::XORI), DestReg: ShiftReg)
14434 .addReg(RegNo: Shift1Reg)
14435 .addImm(Val: is8bit ? 24 : 16);
14436 if (is64bit)
14437 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::RLDICR), DestReg: PtrReg)
14438 .addReg(RegNo: Ptr1Reg)
14439 .addImm(Val: 0)
14440 .addImm(Val: 61);
14441 else
14442 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::RLWINM), DestReg: PtrReg)
14443 .addReg(RegNo: Ptr1Reg)
14444 .addImm(Val: 0)
14445 .addImm(Val: 0)
14446 .addImm(Val: 29);
14447
14448 // Prepare masked values
14449 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::SLW), DestReg: NewVal2Reg)
14450 .addReg(RegNo: newval)
14451 .addReg(RegNo: ShiftReg);
14452 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::SLW), DestReg: OldVal2Reg)
14453 .addReg(RegNo: oldval)
14454 .addReg(RegNo: ShiftReg);
14455 if (is8bit)
14456 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::LI), DestReg: Mask2Reg).addImm(Val: 255);
14457 else {
14458 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::LI), DestReg: Mask3Reg).addImm(Val: 0);
14459 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::ORI), DestReg: Mask2Reg)
14460 .addReg(RegNo: Mask3Reg)
14461 .addImm(Val: 65535);
14462 }
14463 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::SLW), DestReg: MaskReg)
14464 .addReg(RegNo: Mask2Reg)
14465 .addReg(RegNo: ShiftReg);
14466 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::AND), DestReg: NewVal3Reg)
14467 .addReg(RegNo: NewVal2Reg)
14468 .addReg(RegNo: MaskReg);
14469 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::AND), DestReg: OldVal3Reg)
14470 .addReg(RegNo: OldVal2Reg)
14471 .addReg(RegNo: MaskReg);
14472
14473 // loop1MBB:
14474 // lwarx tmpDest, ptr
14475 // and tmp, tmpDest, mask
14476 // cmpw tmp, oldval3
14477 // bne- exitBB
14478 BB = loop1MBB;
14479 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::LWARX), DestReg: TmpDestReg)
14480 .addReg(RegNo: ZeroReg)
14481 .addReg(RegNo: PtrReg);
14482 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::AND), DestReg: TmpReg)
14483 .addReg(RegNo: TmpDestReg)
14484 .addReg(RegNo: MaskReg);
14485 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::CMPW), DestReg: CrReg).addReg(RegNo: TmpReg).addReg(RegNo: OldVal3Reg);
14486 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::BCC))
14487 .addImm(Val: PPC::PRED_NE)
14488 .addReg(RegNo: CrReg)
14489 .addMBB(MBB: exitMBB);
14490 BB->addSuccessor(Succ: loop2MBB);
14491 BB->addSuccessor(Succ: exitMBB);
14492
14493 // loop2MBB:
14494 // andc tmp2, tmpDest, mask
14495 // or tmp4, tmp2, newval3
14496 // stwcx. tmp4, ptr
14497 // bne- loop1MBB
14498 // b exitBB
14499 BB = loop2MBB;
14500 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::ANDC), DestReg: Tmp2Reg)
14501 .addReg(RegNo: TmpDestReg)
14502 .addReg(RegNo: MaskReg);
14503 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::OR), DestReg: Tmp4Reg)
14504 .addReg(RegNo: Tmp2Reg)
14505 .addReg(RegNo: NewVal3Reg);
14506 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::STWCX))
14507 .addReg(RegNo: Tmp4Reg)
14508 .addReg(RegNo: ZeroReg)
14509 .addReg(RegNo: PtrReg);
14510 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::BCC))
14511 .addImm(Val: PPC::PRED_NE)
14512 .addReg(RegNo: PPC::CR0)
14513 .addMBB(MBB: loop1MBB);
14514 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::B)).addMBB(MBB: exitMBB);
14515 BB->addSuccessor(Succ: loop1MBB);
14516 BB->addSuccessor(Succ: exitMBB);
14517
14518 // exitMBB:
14519 // srw dest, tmpDest, shift
14520 BB = exitMBB;
14521 BuildMI(BB&: *BB, I: BB->begin(), MIMD: dl, MCID: TII->get(Opcode: PPC::SRW), DestReg: dest)
14522 .addReg(RegNo: TmpReg)
14523 .addReg(RegNo: ShiftReg);
14524
14525 return BB;
14526}
14527
14528MachineBasicBlock *
14529PPCTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
14530 MachineBasicBlock *BB) const {
14531 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
14532
14533 // To "insert" these instructions we actually have to insert their
14534 // control-flow patterns.
14535 const BasicBlock *LLVM_BB = BB->getBasicBlock();
14536 MachineFunction::iterator It = ++BB->getIterator();
14537
14538 MachineFunction *F = BB->getParent();
14539 MachineRegisterInfo &MRI = F->getRegInfo();
14540
14541 // Handle SELECT with ISEL support first (before generic SELECT handling)
14542 if (IsSelect(Opcode: MI.getOpcode()))
14543 return emitSelect(MI, BB, TII, Subtarget);
14544
14545 switch (MI.getOpcode()) {
14546 case TargetOpcode::STACKMAP:
14547 return emitPatchPoint(MI, MBB: BB);
14548 case TargetOpcode::PATCHPOINT:
14549 // Call lowering should have added an r2 operand to indicate a dependence
14550 // on the TOC base pointer value. It can't however, because there is no
14551 // way to mark the dependence as implicit there, and so the stackmap code
14552 // will confuse it with a regular operand. Instead, add the dependence
14553 // here.
14554 if (Subtarget.is64BitELFABI() && !Subtarget.isUsingPCRelativeCalls())
14555 MI.addOperand(Op: MachineOperand::CreateReg(Reg: PPC::X2, isDef: false, isImp: true));
14556 return emitPatchPoint(MI, MBB: BB);
14557
14558 case PPC::EH_SjLj_SetJmp32:
14559 case PPC::EH_SjLj_SetJmp64:
14560 return emitEHSjLjSetJmp(MI, MBB: BB);
14561
14562 case PPC::EH_SjLj_LongJmp32:
14563 case PPC::EH_SjLj_LongJmp64:
14564 return emitEHSjLjLongJmp(MI, MBB: BB);
14565
14566 case PPC::ReadTB: {
14567 // To read the 64-bit time-base register on a 32-bit target, we read the
14568 // two halves. Should the counter have wrapped while it was being read, we
14569 // need to try again.
14570 // ...
14571 // readLoop:
14572 // mfspr Rx,TBU # load from TBU
14573 // mfspr Ry,TB # load from TB
14574 // mfspr Rz,TBU # load from TBU
14575 // cmpw crX,Rx,Rz # check if 'old'='new'
14576 // bne readLoop # branch if they're not equal
14577 // ...
14578
14579 MachineBasicBlock *readMBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
14580 MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
14581 DebugLoc dl = MI.getDebugLoc();
14582 F->insert(MBBI: It, MBB: readMBB);
14583 F->insert(MBBI: It, MBB: sinkMBB);
14584
14585 // Transfer the remainder of BB and its successor edges to sinkMBB.
14586 sinkMBB->splice(Where: sinkMBB->begin(), Other: BB,
14587 From: std::next(x: MachineBasicBlock::iterator(MI)), To: BB->end());
14588 sinkMBB->transferSuccessorsAndUpdatePHIs(FromMBB: BB);
14589
14590 BB->addSuccessor(Succ: readMBB);
14591 BB = readMBB;
14592
14593 MachineRegisterInfo &RegInfo = F->getRegInfo();
14594 Register ReadAgainReg = RegInfo.createVirtualRegister(RegClass: &PPC::GPRCRegClass);
14595 Register LoReg = MI.getOperand(i: 0).getReg();
14596 Register HiReg = MI.getOperand(i: 1).getReg();
14597
14598 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::MFSPR), DestReg: HiReg).addImm(Val: 269);
14599 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::MFSPR), DestReg: LoReg).addImm(Val: 268);
14600 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::MFSPR), DestReg: ReadAgainReg).addImm(Val: 269);
14601
14602 Register CmpReg = RegInfo.createVirtualRegister(RegClass: &PPC::CRRCRegClass);
14603
14604 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::CMPW), DestReg: CmpReg)
14605 .addReg(RegNo: HiReg)
14606 .addReg(RegNo: ReadAgainReg);
14607 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::BCC))
14608 .addImm(Val: PPC::PRED_NE)
14609 .addReg(RegNo: CmpReg)
14610 .addMBB(MBB: readMBB);
14611
14612 BB->addSuccessor(Succ: readMBB);
14613 BB->addSuccessor(Succ: sinkMBB);
14614 break;
14615 }
14616 case PPC::ATOMIC_LOAD_ADD_NOWP:
14617 BB = EmitPartwordAtomicBinary(MI, BB, BinOpcode: PPC::ADD4);
14618 break;
14619 case PPC::ATOMIC_LOAD_ADD:
14620 BB = EmitAtomicBinary(MI, BB, BinOpcode: PPC::ADD4);
14621 break;
14622 case PPC::ATOMIC_LOAD_ADD_I64:
14623 BB = EmitAtomicBinary(MI, BB, BinOpcode: PPC::ADD8);
14624 break;
14625 case PPC::ATOMIC_LOAD_AND_NOWP:
14626 BB = EmitPartwordAtomicBinary(MI, BB, BinOpcode: PPC::AND);
14627 break;
14628 case PPC::ATOMIC_LOAD_AND:
14629 BB = EmitAtomicBinary(MI, BB, BinOpcode: PPC::AND);
14630 break;
14631 case PPC::ATOMIC_LOAD_AND_I64:
14632 BB = EmitAtomicBinary(MI, BB, BinOpcode: PPC::AND8);
14633 break;
14634 case PPC::ATOMIC_LOAD_OR_NOWP:
14635 BB = EmitPartwordAtomicBinary(MI, BB, BinOpcode: PPC::OR);
14636 break;
14637 case PPC::ATOMIC_LOAD_OR:
14638 BB = EmitAtomicBinary(MI, BB, BinOpcode: PPC::OR);
14639 break;
14640 case PPC::ATOMIC_LOAD_OR_I64:
14641 BB = EmitAtomicBinary(MI, BB, BinOpcode: PPC::OR8);
14642 break;
14643 case PPC::ATOMIC_LOAD_XOR_NOWP:
14644 BB = EmitPartwordAtomicBinary(MI, BB, BinOpcode: PPC::XOR);
14645 break;
14646 case PPC::ATOMIC_LOAD_XOR:
14647 BB = EmitAtomicBinary(MI, BB, BinOpcode: PPC::XOR);
14648 break;
14649 case PPC::ATOMIC_LOAD_XOR_I64:
14650 BB = EmitAtomicBinary(MI, BB, BinOpcode: PPC::XOR8);
14651 break;
14652 case PPC::ATOMIC_LOAD_NAND_NOWP:
14653 BB = EmitPartwordAtomicBinary(MI, BB, BinOpcode: PPC::NAND);
14654 break;
14655 case PPC::ATOMIC_LOAD_NAND:
14656 BB = EmitAtomicBinary(MI, BB, BinOpcode: PPC::NAND);
14657 break;
14658 case PPC::ATOMIC_LOAD_NAND_I64:
14659 BB = EmitAtomicBinary(MI, BB, BinOpcode: PPC::NAND8);
14660 break;
14661 case PPC::ATOMIC_LOAD_SUB_NOWP:
14662 BB = EmitPartwordAtomicBinary(MI, BB, BinOpcode: PPC::SUBF);
14663 break;
14664 case PPC::ATOMIC_LOAD_SUB:
14665 BB = EmitAtomicBinary(MI, BB, BinOpcode: PPC::SUBF);
14666 break;
14667 case PPC::ATOMIC_LOAD_SUB_I64:
14668 BB = EmitAtomicBinary(MI, BB, BinOpcode: PPC::SUBF8);
14669 break;
14670 case PPC::ATOMIC_LOAD_MIN_NOWP:
14671 BB = EmitPartwordAtomicBinary(MI, BB, BinOpcode: 0, CmpOpcode: PPC::CMPW, CmpPred: PPC::PRED_LT);
14672 break;
14673 case PPC::ATOMIC_LOAD_MIN:
14674 BB = EmitAtomicBinary(MI, BB, BinOpcode: 0, CmpOpcode: PPC::CMPW, CmpPred: PPC::PRED_LT);
14675 break;
14676 case PPC::ATOMIC_LOAD_MIN_I64:
14677 BB = EmitAtomicBinary(MI, BB, BinOpcode: 0, CmpOpcode: PPC::CMPD, CmpPred: PPC::PRED_LT);
14678 break;
14679 case PPC::ATOMIC_LOAD_MAX_NOWP:
14680 BB = EmitPartwordAtomicBinary(MI, BB, BinOpcode: 0, CmpOpcode: PPC::CMPW, CmpPred: PPC::PRED_GT);
14681 break;
14682 case PPC::ATOMIC_LOAD_MAX:
14683 BB = EmitAtomicBinary(MI, BB, BinOpcode: 0, CmpOpcode: PPC::CMPW, CmpPred: PPC::PRED_GT);
14684 break;
14685 case PPC::ATOMIC_LOAD_MAX_I64:
14686 BB = EmitAtomicBinary(MI, BB, BinOpcode: 0, CmpOpcode: PPC::CMPD, CmpPred: PPC::PRED_GT);
14687 break;
14688 case PPC::ATOMIC_LOAD_UMIN_NOWP:
14689 BB = EmitPartwordAtomicBinary(MI, BB, BinOpcode: 0, CmpOpcode: PPC::CMPLW, CmpPred: PPC::PRED_LT);
14690 break;
14691 case PPC::ATOMIC_LOAD_UMIN:
14692 BB = EmitAtomicBinary(MI, BB, BinOpcode: 0, CmpOpcode: PPC::CMPLW, CmpPred: PPC::PRED_LT);
14693 break;
14694 case PPC::ATOMIC_LOAD_UMIN_I64:
14695 BB = EmitAtomicBinary(MI, BB, BinOpcode: 0, CmpOpcode: PPC::CMPLD, CmpPred: PPC::PRED_LT);
14696 break;
14697 case PPC::ATOMIC_LOAD_UMAX_NOWP:
14698 BB = EmitPartwordAtomicBinary(MI, BB, BinOpcode: 0, CmpOpcode: PPC::CMPLW, CmpPred: PPC::PRED_GT);
14699 break;
14700 case PPC::ATOMIC_LOAD_UMAX:
14701 BB = EmitAtomicBinary(MI, BB, BinOpcode: 0, CmpOpcode: PPC::CMPLW, CmpPred: PPC::PRED_GT);
14702 break;
14703 case PPC::ATOMIC_LOAD_UMAX_I64:
14704 BB = EmitAtomicBinary(MI, BB, BinOpcode: 0, CmpOpcode: PPC::CMPLD, CmpPred: PPC::PRED_GT);
14705 break;
14706 case PPC::ATOMIC_SWAP_NOWP:
14707 BB = EmitPartwordAtomicBinary(MI, BB, BinOpcode: 0);
14708 break;
14709 case PPC::ATOMIC_SWAP:
14710 case PPC::ATOMIC_SWAP_I64:
14711 BB = EmitAtomicBinary(MI, BB, BinOpcode: 0);
14712 break;
14713 case PPC::ATOMIC_CMP_SWAP_I32:
14714 case PPC::ATOMIC_CMP_SWAP_I64:
14715 case PPC::ATOMIC_CMP_SWAP_I8:
14716 case PPC::ATOMIC_CMP_SWAP_I16: {
14717 // Use hardware-supported atomic operations if available
14718 bool useHardware = MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I32 ||
14719 MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I64 ||
14720 (Subtarget.hasPartwordAtomics() &&
14721 (MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I8 ||
14722 MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I16));
14723
14724 if (useHardware)
14725 BB = emitAtomicCmpSwapHardware(MI, BB, TII, Subtarget);
14726 else
14727 BB = emitAtomicCmpSwapSoftware(MI, BB, TII, Subtarget);
14728 break;
14729 }
14730 case PPC::FADDrtz: {
14731 // This pseudo performs an FADD with rounding mode temporarily forced
14732 // to round-to-zero. We emit this via custom inserter since the FPSCR
14733 // is not modeled at the SelectionDAG level.
14734 Register Dest = MI.getOperand(i: 0).getReg();
14735 Register Src1 = MI.getOperand(i: 1).getReg();
14736 Register Src2 = MI.getOperand(i: 2).getReg();
14737 DebugLoc dl = MI.getDebugLoc();
14738
14739 MachineRegisterInfo &RegInfo = F->getRegInfo();
14740 Register MFFSReg = RegInfo.createVirtualRegister(RegClass: &PPC::F8RCRegClass);
14741
14742 // Save FPSCR value.
14743 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: PPC::MFFS), DestReg: MFFSReg);
14744
14745 // Set rounding mode to round-to-zero.
14746 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: PPC::MTFSB1))
14747 .addImm(Val: 31)
14748 .addReg(RegNo: PPC::RM, Flags: RegState::ImplicitDefine);
14749
14750 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: PPC::MTFSB0))
14751 .addImm(Val: 30)
14752 .addReg(RegNo: PPC::RM, Flags: RegState::ImplicitDefine);
14753
14754 // Perform addition.
14755 auto MIB = BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: PPC::FADD), DestReg: Dest)
14756 .addReg(RegNo: Src1)
14757 .addReg(RegNo: Src2);
14758 if (MI.getFlag(Flag: MachineInstr::NoFPExcept))
14759 MIB.setMIFlag(MachineInstr::NoFPExcept);
14760
14761 // Restore FPSCR value.
14762 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: PPC::MTFSFb)).addImm(Val: 1).addReg(RegNo: MFFSReg);
14763 break;
14764 }
14765 case PPC::ANDI_rec_1_EQ_BIT:
14766 case PPC::ANDI_rec_1_GT_BIT:
14767 case PPC::ANDI_rec_1_EQ_BIT8:
14768 case PPC::ANDI_rec_1_GT_BIT8: {
14769 unsigned Opcode = (MI.getOpcode() == PPC::ANDI_rec_1_EQ_BIT8 ||
14770 MI.getOpcode() == PPC::ANDI_rec_1_GT_BIT8)
14771 ? PPC::ANDI8_rec
14772 : PPC::ANDI_rec;
14773 bool IsEQ = (MI.getOpcode() == PPC::ANDI_rec_1_EQ_BIT ||
14774 MI.getOpcode() == PPC::ANDI_rec_1_EQ_BIT8);
14775
14776 MachineRegisterInfo &RegInfo = F->getRegInfo();
14777 Register Dest = RegInfo.createVirtualRegister(
14778 RegClass: Opcode == PPC::ANDI_rec ? &PPC::GPRCRegClass : &PPC::G8RCRegClass);
14779
14780 DebugLoc Dl = MI.getDebugLoc();
14781 BuildMI(BB&: *BB, I&: MI, MIMD: Dl, MCID: TII->get(Opcode), DestReg: Dest)
14782 .addReg(RegNo: MI.getOperand(i: 1).getReg())
14783 .addImm(Val: 1);
14784 BuildMI(BB&: *BB, I&: MI, MIMD: Dl, MCID: TII->get(Opcode: TargetOpcode::COPY),
14785 DestReg: MI.getOperand(i: 0).getReg())
14786 .addReg(RegNo: IsEQ ? PPC::CR0EQ : PPC::CR0GT);
14787 break;
14788 }
14789 case PPC::TCHECK_RET: {
14790 DebugLoc Dl = MI.getDebugLoc();
14791 MachineRegisterInfo &RegInfo = F->getRegInfo();
14792 Register CRReg = RegInfo.createVirtualRegister(RegClass: &PPC::CRRCRegClass);
14793 BuildMI(BB&: *BB, I&: MI, MIMD: Dl, MCID: TII->get(Opcode: PPC::TCHECK), DestReg: CRReg);
14794 BuildMI(BB&: *BB, I&: MI, MIMD: Dl, MCID: TII->get(Opcode: TargetOpcode::COPY),
14795 DestReg: MI.getOperand(i: 0).getReg())
14796 .addReg(RegNo: CRReg);
14797 break;
14798 }
14799 case PPC::TBEGIN_RET: {
14800 DebugLoc Dl = MI.getDebugLoc();
14801 unsigned Imm = MI.getOperand(i: 1).getImm();
14802 BuildMI(BB&: *BB, I&: MI, MIMD: Dl, MCID: TII->get(Opcode: PPC::TBEGIN)).addImm(Val: Imm);
14803 BuildMI(BB&: *BB, I&: MI, MIMD: Dl, MCID: TII->get(Opcode: TargetOpcode::COPY),
14804 DestReg: MI.getOperand(i: 0).getReg())
14805 .addReg(RegNo: PPC::CR0EQ);
14806 break;
14807 }
14808 case PPC::SETRNDi: {
14809 DebugLoc dl = MI.getDebugLoc();
14810 Register OldFPSCRReg = MI.getOperand(i: 0).getReg();
14811
14812 // Save FPSCR value.
14813 if (MRI.use_empty(RegNo: OldFPSCRReg))
14814 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: TargetOpcode::IMPLICIT_DEF), DestReg: OldFPSCRReg);
14815 else
14816 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: PPC::MFFS), DestReg: OldFPSCRReg);
14817
14818 // The floating point rounding mode is in the bits 62:63 of FPCSR, and has
14819 // the following settings:
14820 // 00 Round to nearest
14821 // 01 Round to 0
14822 // 10 Round to +inf
14823 // 11 Round to -inf
14824
14825 // When the operand is immediate, using the two least significant bits of
14826 // the immediate to set the bits 62:63 of FPSCR.
14827 unsigned Mode = MI.getOperand(i: 1).getImm();
14828 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: (Mode & 1) ? PPC::MTFSB1 : PPC::MTFSB0))
14829 .addImm(Val: 31)
14830 .addReg(RegNo: PPC::RM, Flags: RegState::ImplicitDefine);
14831
14832 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: (Mode & 2) ? PPC::MTFSB1 : PPC::MTFSB0))
14833 .addImm(Val: 30)
14834 .addReg(RegNo: PPC::RM, Flags: RegState::ImplicitDefine);
14835 break;
14836 }
14837 case PPC::SETRND: {
14838 DebugLoc dl = MI.getDebugLoc();
14839
14840 // Copy register from F8RCRegClass::SrcReg to G8RCRegClass::DestReg
14841 // or copy register from G8RCRegClass::SrcReg to F8RCRegClass::DestReg.
14842 // If the target doesn't have DirectMove, we should use stack to do the
14843 // conversion, because the target doesn't have the instructions like mtvsrd
14844 // or mfvsrd to do this conversion directly.
14845 auto copyRegFromG8RCOrF8RC = [&] (unsigned DestReg, unsigned SrcReg) {
14846 if (Subtarget.hasDirectMove()) {
14847 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg)
14848 .addReg(RegNo: SrcReg);
14849 } else {
14850 // Use stack to do the register copy.
14851 unsigned StoreOp = PPC::STD, LoadOp = PPC::LFD;
14852 MachineRegisterInfo &RegInfo = F->getRegInfo();
14853 const TargetRegisterClass *RC = RegInfo.getRegClass(Reg: SrcReg);
14854 if (RC == &PPC::F8RCRegClass) {
14855 // Copy register from F8RCRegClass to G8RCRegclass.
14856 assert((RegInfo.getRegClass(DestReg) == &PPC::G8RCRegClass) &&
14857 "Unsupported RegClass.");
14858
14859 StoreOp = PPC::STFD;
14860 LoadOp = PPC::LD;
14861 } else {
14862 // Copy register from G8RCRegClass to F8RCRegclass.
14863 assert((RegInfo.getRegClass(SrcReg) == &PPC::G8RCRegClass) &&
14864 (RegInfo.getRegClass(DestReg) == &PPC::F8RCRegClass) &&
14865 "Unsupported RegClass.");
14866 }
14867
14868 MachineFrameInfo &MFI = F->getFrameInfo();
14869 int FrameIdx = MFI.CreateStackObject(Size: 8, Alignment: Align(8), isSpillSlot: false);
14870
14871 MachineMemOperand *MMOStore = F->getMachineMemOperand(
14872 PtrInfo: MachinePointerInfo::getFixedStack(MF&: *F, FI: FrameIdx, Offset: 0),
14873 F: MachineMemOperand::MOStore, Size: MFI.getObjectSize(ObjectIdx: FrameIdx),
14874 BaseAlignment: MFI.getObjectAlign(ObjectIdx: FrameIdx));
14875
14876 // Store the SrcReg into the stack.
14877 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: StoreOp))
14878 .addReg(RegNo: SrcReg)
14879 .addImm(Val: 0)
14880 .addFrameIndex(Idx: FrameIdx)
14881 .addMemOperand(MMO: MMOStore);
14882
14883 MachineMemOperand *MMOLoad = F->getMachineMemOperand(
14884 PtrInfo: MachinePointerInfo::getFixedStack(MF&: *F, FI: FrameIdx, Offset: 0),
14885 F: MachineMemOperand::MOLoad, Size: MFI.getObjectSize(ObjectIdx: FrameIdx),
14886 BaseAlignment: MFI.getObjectAlign(ObjectIdx: FrameIdx));
14887
14888 // Load from the stack where SrcReg is stored, and save to DestReg,
14889 // so we have done the RegClass conversion from RegClass::SrcReg to
14890 // RegClass::DestReg.
14891 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: LoadOp), DestReg)
14892 .addImm(Val: 0)
14893 .addFrameIndex(Idx: FrameIdx)
14894 .addMemOperand(MMO: MMOLoad);
14895 }
14896 };
14897
14898 Register OldFPSCRReg = MI.getOperand(i: 0).getReg();
14899
14900 // Save FPSCR value.
14901 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: PPC::MFFS), DestReg: OldFPSCRReg);
14902
14903 // When the operand is gprc register, use two least significant bits of the
14904 // register and mtfsf instruction to set the bits 62:63 of FPSCR.
14905 //
14906 // copy OldFPSCRTmpReg, OldFPSCRReg
14907 // (INSERT_SUBREG ExtSrcReg, (IMPLICIT_DEF ImDefReg), SrcOp, 1)
14908 // rldimi NewFPSCRTmpReg, ExtSrcReg, OldFPSCRReg, 0, 62
14909 // copy NewFPSCRReg, NewFPSCRTmpReg
14910 // mtfsf 255, NewFPSCRReg
14911 MachineOperand SrcOp = MI.getOperand(i: 1);
14912 MachineRegisterInfo &RegInfo = F->getRegInfo();
14913 Register OldFPSCRTmpReg = RegInfo.createVirtualRegister(RegClass: &PPC::G8RCRegClass);
14914
14915 copyRegFromG8RCOrF8RC(OldFPSCRTmpReg, OldFPSCRReg);
14916
14917 Register ImDefReg = RegInfo.createVirtualRegister(RegClass: &PPC::G8RCRegClass);
14918 Register ExtSrcReg = RegInfo.createVirtualRegister(RegClass: &PPC::G8RCRegClass);
14919
14920 // The first operand of INSERT_SUBREG should be a register which has
14921 // subregisters, we only care about its RegClass, so we should use an
14922 // IMPLICIT_DEF register.
14923 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: TargetOpcode::IMPLICIT_DEF), DestReg: ImDefReg);
14924 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: PPC::INSERT_SUBREG), DestReg: ExtSrcReg)
14925 .addReg(RegNo: ImDefReg)
14926 .add(MO: SrcOp)
14927 .addImm(Val: PPC::sub_32);
14928
14929 Register NewFPSCRTmpReg = RegInfo.createVirtualRegister(RegClass: &PPC::G8RCRegClass);
14930 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: PPC::RLDIMI), DestReg: NewFPSCRTmpReg)
14931 .addReg(RegNo: OldFPSCRTmpReg)
14932 .addReg(RegNo: ExtSrcReg)
14933 .addImm(Val: 0)
14934 .addImm(Val: 62);
14935
14936 Register NewFPSCRReg = RegInfo.createVirtualRegister(RegClass: &PPC::F8RCRegClass);
14937 copyRegFromG8RCOrF8RC(NewFPSCRReg, NewFPSCRTmpReg);
14938
14939 // The mask 255 means that put the 32:63 bits of NewFPSCRReg to the 32:63
14940 // bits of FPSCR.
14941 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: PPC::MTFSF))
14942 .addImm(Val: 255)
14943 .addReg(RegNo: NewFPSCRReg)
14944 .addImm(Val: 0)
14945 .addImm(Val: 0);
14946 break;
14947 }
14948 case PPC::SETFLM: {
14949 DebugLoc Dl = MI.getDebugLoc();
14950
14951 // Result of setflm is previous FPSCR content, so we need to save it first.
14952 Register OldFPSCRReg = MI.getOperand(i: 0).getReg();
14953 if (MRI.use_empty(RegNo: OldFPSCRReg))
14954 BuildMI(BB&: *BB, I&: MI, MIMD: Dl, MCID: TII->get(Opcode: TargetOpcode::IMPLICIT_DEF), DestReg: OldFPSCRReg);
14955 else
14956 BuildMI(BB&: *BB, I&: MI, MIMD: Dl, MCID: TII->get(Opcode: PPC::MFFS), DestReg: OldFPSCRReg);
14957
14958 // Put bits in 32:63 to FPSCR.
14959 Register NewFPSCRReg = MI.getOperand(i: 1).getReg();
14960 BuildMI(BB&: *BB, I&: MI, MIMD: Dl, MCID: TII->get(Opcode: PPC::MTFSF))
14961 .addImm(Val: 255)
14962 .addReg(RegNo: NewFPSCRReg)
14963 .addImm(Val: 0)
14964 .addImm(Val: 0);
14965 break;
14966 }
14967 case PPC::PROBED_ALLOCA_32:
14968 case PPC::PROBED_ALLOCA_64:
14969 return emitProbedAlloca(MI, MBB: BB);
14970
14971 case PPC::SPLIT_QUADWORD: {
14972 DebugLoc DL = MI.getDebugLoc();
14973 Register Src = MI.getOperand(i: 2).getReg();
14974 Register Lo = MI.getOperand(i: 0).getReg();
14975 Register Hi = MI.getOperand(i: 1).getReg();
14976 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::COPY))
14977 .addDef(RegNo: Lo)
14978 .addUse(RegNo: Src, Flags: {}, SubReg: PPC::sub_gp8_x1);
14979 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::COPY))
14980 .addDef(RegNo: Hi)
14981 .addUse(RegNo: Src, Flags: {}, SubReg: PPC::sub_gp8_x0);
14982 break;
14983 }
14984 case PPC::LQX_PSEUDO:
14985 case PPC::STQX_PSEUDO: {
14986 DebugLoc DL = MI.getDebugLoc();
14987 // Ptr is used as the ptr_rc_no_r0 part
14988 // of LQ/STQ's memory operand and adding result of RA and RB,
14989 // so it has to be g8rc_and_g8rc_nox0.
14990 Register Ptr =
14991 F->getRegInfo().createVirtualRegister(RegClass: &PPC::G8RC_and_G8RC_NOX0RegClass);
14992 Register Val = MI.getOperand(i: 0).getReg();
14993 Register RA = MI.getOperand(i: 1).getReg();
14994 Register RB = MI.getOperand(i: 2).getReg();
14995 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::ADD8), DestReg: Ptr).addReg(RegNo: RA).addReg(RegNo: RB);
14996 BuildMI(BB&: *BB, I&: MI, MIMD: DL,
14997 MCID: MI.getOpcode() == PPC::LQX_PSEUDO ? TII->get(Opcode: PPC::LQ)
14998 : TII->get(Opcode: PPC::STQ))
14999 .addReg(RegNo: Val, Flags: getDefRegState(B: MI.getOpcode() == PPC::LQX_PSEUDO))
15000 .addImm(Val: 0)
15001 .addReg(RegNo: Ptr);
15002 break;
15003 }
15004 default:
15005 llvm_unreachable("Unexpected instr type to insert");
15006 }
15007
15008 MI.eraseFromParent(); // The pseudo instruction is gone now.
15009 return BB;
15010}
15011
15012//===----------------------------------------------------------------------===//
15013// Target Optimization Hooks
15014//===----------------------------------------------------------------------===//
15015
15016static int getEstimateRefinementSteps(EVT VT, const PPCSubtarget &Subtarget) {
15017 // For the estimates, convergence is quadratic, so we essentially double the
15018 // number of digits correct after every iteration. For both FRE and FRSQRTE,
15019 // the minimum architected relative accuracy is 2^-5. When hasRecipPrec(),
15020 // this is 2^-14. IEEE float has 23 digits and double has 52 digits.
15021 int RefinementSteps = Subtarget.hasRecipPrec() ? 1 : 3;
15022 if (VT.getScalarType() == MVT::f64)
15023 RefinementSteps++;
15024 return RefinementSteps;
15025}
15026
15027SDValue PPCTargetLowering::getSqrtInputTest(SDValue Op, SelectionDAG &DAG,
15028 const DenormalMode &Mode,
15029 SDNodeFlags Flags) const {
15030 // We only have VSX Vector Test for software Square Root.
15031 EVT VT = Op.getValueType();
15032 if (!isTypeLegal(VT: MVT::i1) ||
15033 (VT != MVT::f64 &&
15034 ((VT != MVT::v2f64 && VT != MVT::v4f32) || !Subtarget.hasVSX())))
15035 return TargetLowering::getSqrtInputTest(Operand: Op, DAG, Mode, Flags);
15036
15037 SDLoc DL(Op);
15038 // The output register of FTSQRT is CR field.
15039 SDValue FTSQRT = DAG.getNode(Opcode: PPCISD::FTSQRT, DL, VT: MVT::i32, Operand: Op, Flags);
15040 // ftsqrt BF,FRB
15041 // Let e_b be the unbiased exponent of the double-precision
15042 // floating-point operand in register FRB.
15043 // fe_flag is set to 1 if either of the following conditions occurs.
15044 // - The double-precision floating-point operand in register FRB is a zero,
15045 // a NaN, or an infinity, or a negative value.
15046 // - e_b is less than or equal to -970.
15047 // Otherwise fe_flag is set to 0.
15048 // Both VSX and non-VSX versions would set EQ bit in the CR if the number is
15049 // not eligible for iteration. (zero/negative/infinity/nan or unbiased
15050 // exponent is less than -970)
15051 SDValue SRIdxVal = DAG.getTargetConstant(Val: PPC::sub_eq, DL, VT: MVT::i32);
15052 return SDValue(DAG.getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG, dl: DL, VT: MVT::i1,
15053 Op1: FTSQRT, Op2: SRIdxVal),
15054 0);
15055}
15056
15057SDValue
15058PPCTargetLowering::getSqrtResultForDenormInput(SDValue Op,
15059 SelectionDAG &DAG) const {
15060 // We only have VSX Vector Square Root.
15061 EVT VT = Op.getValueType();
15062 if (VT != MVT::f64 &&
15063 ((VT != MVT::v2f64 && VT != MVT::v4f32) || !Subtarget.hasVSX()))
15064 return TargetLowering::getSqrtResultForDenormInput(Operand: Op, DAG);
15065
15066 return DAG.getNode(Opcode: PPCISD::FSQRT, DL: SDLoc(Op), VT, Operand: Op);
15067}
15068
15069SDValue PPCTargetLowering::getSqrtEstimate(SDValue Operand, SelectionDAG &DAG,
15070 int Enabled, int &RefinementSteps,
15071 bool &UseOneConstNR,
15072 bool Reciprocal) const {
15073 EVT VT = Operand.getValueType();
15074 if ((VT == MVT::f32 && Subtarget.hasFRSQRTES()) ||
15075 (VT == MVT::f64 && Subtarget.hasFRSQRTE()) ||
15076 (VT == MVT::v4f32 && Subtarget.hasAltivec()) ||
15077 (VT == MVT::v2f64 && Subtarget.hasVSX())) {
15078 if (RefinementSteps == ReciprocalEstimate::Unspecified)
15079 RefinementSteps = getEstimateRefinementSteps(VT, Subtarget);
15080
15081 // The Newton-Raphson computation with a single constant does not provide
15082 // enough accuracy on some CPUs.
15083 UseOneConstNR = !Subtarget.needsTwoConstNR();
15084 return DAG.getNode(Opcode: PPCISD::FRSQRTE, DL: SDLoc(Operand), VT, Operand);
15085 }
15086 return SDValue();
15087}
15088
15089SDValue PPCTargetLowering::getRecipEstimate(SDValue Operand, SelectionDAG &DAG,
15090 int Enabled,
15091 int &RefinementSteps) const {
15092 EVT VT = Operand.getValueType();
15093 if ((VT == MVT::f32 && Subtarget.hasFRES()) ||
15094 (VT == MVT::f64 && Subtarget.hasFRE()) ||
15095 (VT == MVT::v4f32 && Subtarget.hasAltivec()) ||
15096 (VT == MVT::v2f64 && Subtarget.hasVSX())) {
15097 if (RefinementSteps == ReciprocalEstimate::Unspecified)
15098 RefinementSteps = getEstimateRefinementSteps(VT, Subtarget);
15099 return DAG.getNode(Opcode: PPCISD::FRE, DL: SDLoc(Operand), VT, Operand);
15100 }
15101 return SDValue();
15102}
15103
15104unsigned PPCTargetLowering::combineRepeatedFPDivisors() const {
15105 // Note: This functionality is used only when arcp is enabled, and
15106 // on cores with reciprocal estimates (which are used when arcp is
15107 // enabled for division), this functionality is redundant with the default
15108 // combiner logic (once the division -> reciprocal/multiply transformation
15109 // has taken place). As a result, this matters more for older cores than for
15110 // newer ones.
15111
15112 // Combine multiple FDIVs with the same divisor into multiple FMULs by the
15113 // reciprocal if there are two or more FDIVs (for embedded cores with only
15114 // one FP pipeline) for three or more FDIVs (for generic OOO cores).
15115 switch (Subtarget.getCPUDirective()) {
15116 default:
15117 return 3;
15118 case PPC::DIR_440:
15119 case PPC::DIR_A2:
15120 case PPC::DIR_E500:
15121 case PPC::DIR_E500mc:
15122 case PPC::DIR_E5500:
15123 return 2;
15124 }
15125}
15126
15127// isConsecutiveLSLoc needs to work even if all adds have not yet been
15128// collapsed, and so we need to look through chains of them.
15129static void getBaseWithConstantOffset(SDValue Loc, SDValue &Base,
15130 int64_t& Offset, SelectionDAG &DAG) {
15131 if (DAG.isBaseWithConstantOffset(Op: Loc)) {
15132 Base = Loc.getOperand(i: 0);
15133 Offset += cast<ConstantSDNode>(Val: Loc.getOperand(i: 1))->getSExtValue();
15134
15135 // The base might itself be a base plus an offset, and if so, accumulate
15136 // that as well.
15137 getBaseWithConstantOffset(Loc: Loc.getOperand(i: 0), Base, Offset, DAG);
15138 }
15139}
15140
15141static bool isConsecutiveLSLoc(SDValue Loc, EVT VT, LSBaseSDNode *Base,
15142 unsigned Bytes, int Dist,
15143 SelectionDAG &DAG) {
15144 if (VT.getSizeInBits() / 8 != Bytes)
15145 return false;
15146
15147 SDValue BaseLoc = Base->getBasePtr();
15148 if (Loc.getOpcode() == ISD::FrameIndex) {
15149 if (BaseLoc.getOpcode() != ISD::FrameIndex)
15150 return false;
15151 const MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
15152 int FI = cast<FrameIndexSDNode>(Val&: Loc)->getIndex();
15153 int BFI = cast<FrameIndexSDNode>(Val&: BaseLoc)->getIndex();
15154 int FS = MFI.getObjectSize(ObjectIdx: FI);
15155 int BFS = MFI.getObjectSize(ObjectIdx: BFI);
15156 if (FS != BFS || FS != (int)Bytes) return false;
15157 return MFI.getObjectOffset(ObjectIdx: FI) == (MFI.getObjectOffset(ObjectIdx: BFI) + Dist*Bytes);
15158 }
15159
15160 SDValue Base1 = Loc, Base2 = BaseLoc;
15161 int64_t Offset1 = 0, Offset2 = 0;
15162 getBaseWithConstantOffset(Loc, Base&: Base1, Offset&: Offset1, DAG);
15163 getBaseWithConstantOffset(Loc: BaseLoc, Base&: Base2, Offset&: Offset2, DAG);
15164 if (Base1 == Base2 && Offset1 == (Offset2 + Dist * Bytes))
15165 return true;
15166
15167 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
15168 const GlobalValue *GV1 = nullptr;
15169 const GlobalValue *GV2 = nullptr;
15170 Offset1 = 0;
15171 Offset2 = 0;
15172 bool isGA1 = TLI.isGAPlusOffset(N: Loc.getNode(), GA&: GV1, Offset&: Offset1);
15173 bool isGA2 = TLI.isGAPlusOffset(N: BaseLoc.getNode(), GA&: GV2, Offset&: Offset2);
15174 if (isGA1 && isGA2 && GV1 == GV2)
15175 return Offset1 == (Offset2 + Dist*Bytes);
15176 return false;
15177}
15178
15179// Like SelectionDAG::isConsecutiveLoad, but also works for stores, and does
15180// not enforce equality of the chain operands.
15181static bool isConsecutiveLS(SDNode *N, LSBaseSDNode *Base,
15182 unsigned Bytes, int Dist,
15183 SelectionDAG &DAG) {
15184 if (LSBaseSDNode *LS = dyn_cast<LSBaseSDNode>(Val: N)) {
15185 EVT VT = LS->getMemoryVT();
15186 SDValue Loc = LS->getBasePtr();
15187 return isConsecutiveLSLoc(Loc, VT, Base, Bytes, Dist, DAG);
15188 }
15189
15190 if (N->getOpcode() == ISD::INTRINSIC_W_CHAIN) {
15191 EVT VT;
15192 switch (N->getConstantOperandVal(Num: 1)) {
15193 default: return false;
15194 case Intrinsic::ppc_altivec_lvx:
15195 case Intrinsic::ppc_altivec_lvxl:
15196 case Intrinsic::ppc_vsx_lxvw4x:
15197 case Intrinsic::ppc_vsx_lxvw4x_be:
15198 VT = MVT::v4i32;
15199 break;
15200 case Intrinsic::ppc_vsx_lxvd2x:
15201 case Intrinsic::ppc_vsx_lxvd2x_be:
15202 VT = MVT::v2f64;
15203 break;
15204 case Intrinsic::ppc_altivec_lvebx:
15205 VT = MVT::i8;
15206 break;
15207 case Intrinsic::ppc_altivec_lvehx:
15208 VT = MVT::i16;
15209 break;
15210 case Intrinsic::ppc_altivec_lvewx:
15211 VT = MVT::i32;
15212 break;
15213 }
15214
15215 return isConsecutiveLSLoc(Loc: N->getOperand(Num: 2), VT, Base, Bytes, Dist, DAG);
15216 }
15217
15218 if (N->getOpcode() == ISD::INTRINSIC_VOID) {
15219 EVT VT;
15220 switch (N->getConstantOperandVal(Num: 1)) {
15221 default: return false;
15222 case Intrinsic::ppc_altivec_stvx:
15223 case Intrinsic::ppc_altivec_stvxl:
15224 case Intrinsic::ppc_vsx_stxvw4x:
15225 VT = MVT::v4i32;
15226 break;
15227 case Intrinsic::ppc_vsx_stxvd2x:
15228 VT = MVT::v2f64;
15229 break;
15230 case Intrinsic::ppc_vsx_stxvw4x_be:
15231 VT = MVT::v4i32;
15232 break;
15233 case Intrinsic::ppc_vsx_stxvd2x_be:
15234 VT = MVT::v2f64;
15235 break;
15236 case Intrinsic::ppc_altivec_stvebx:
15237 VT = MVT::i8;
15238 break;
15239 case Intrinsic::ppc_altivec_stvehx:
15240 VT = MVT::i16;
15241 break;
15242 case Intrinsic::ppc_altivec_stvewx:
15243 VT = MVT::i32;
15244 break;
15245 }
15246
15247 return isConsecutiveLSLoc(Loc: N->getOperand(Num: 3), VT, Base, Bytes, Dist, DAG);
15248 }
15249
15250 return false;
15251}
15252
15253// Return true is there is a nearyby consecutive load to the one provided
15254// (regardless of alignment). We search up and down the chain, looking though
15255// token factors and other loads (but nothing else). As a result, a true result
15256// indicates that it is safe to create a new consecutive load adjacent to the
15257// load provided.
15258static bool findConsecutiveLoad(LoadSDNode *LD, SelectionDAG &DAG) {
15259 SDValue Chain = LD->getChain();
15260 EVT VT = LD->getMemoryVT();
15261
15262 SmallPtrSet<SDNode *, 16> LoadRoots;
15263 SmallVector<SDNode *, 8> Queue(1, Chain.getNode());
15264 SmallPtrSet<SDNode *, 16> Visited;
15265
15266 // First, search up the chain, branching to follow all token-factor operands.
15267 // If we find a consecutive load, then we're done, otherwise, record all
15268 // nodes just above the top-level loads and token factors.
15269 while (!Queue.empty()) {
15270 SDNode *ChainNext = Queue.pop_back_val();
15271 if (!Visited.insert(Ptr: ChainNext).second)
15272 continue;
15273
15274 if (MemSDNode *ChainLD = dyn_cast<MemSDNode>(Val: ChainNext)) {
15275 if (isConsecutiveLS(N: ChainLD, Base: LD, Bytes: VT.getStoreSize(), Dist: 1, DAG))
15276 return true;
15277
15278 if (!Visited.count(Ptr: ChainLD->getChain().getNode()))
15279 Queue.push_back(Elt: ChainLD->getChain().getNode());
15280 } else if (ChainNext->getOpcode() == ISD::TokenFactor) {
15281 for (const SDUse &O : ChainNext->ops())
15282 if (!Visited.count(Ptr: O.getNode()))
15283 Queue.push_back(Elt: O.getNode());
15284 } else
15285 LoadRoots.insert(Ptr: ChainNext);
15286 }
15287
15288 // Second, search down the chain, starting from the top-level nodes recorded
15289 // in the first phase. These top-level nodes are the nodes just above all
15290 // loads and token factors. Starting with their uses, recursively look though
15291 // all loads (just the chain uses) and token factors to find a consecutive
15292 // load.
15293 Visited.clear();
15294 Queue.clear();
15295
15296 for (SDNode *I : LoadRoots) {
15297 Queue.push_back(Elt: I);
15298
15299 while (!Queue.empty()) {
15300 SDNode *LoadRoot = Queue.pop_back_val();
15301 if (!Visited.insert(Ptr: LoadRoot).second)
15302 continue;
15303
15304 if (MemSDNode *ChainLD = dyn_cast<MemSDNode>(Val: LoadRoot))
15305 if (isConsecutiveLS(N: ChainLD, Base: LD, Bytes: VT.getStoreSize(), Dist: 1, DAG))
15306 return true;
15307
15308 for (SDNode *U : LoadRoot->users())
15309 if (((isa<MemSDNode>(Val: U) &&
15310 cast<MemSDNode>(Val: U)->getChain().getNode() == LoadRoot) ||
15311 U->getOpcode() == ISD::TokenFactor) &&
15312 !Visited.count(Ptr: U))
15313 Queue.push_back(Elt: U);
15314 }
15315 }
15316
15317 return false;
15318}
15319
15320/// This function is called when we have proved that a SETCC node can be replaced
15321/// by subtraction (and other supporting instructions) so that the result of
15322/// comparison is kept in a GPR instead of CR. This function is purely for
15323/// codegen purposes and has some flags to guide the codegen process.
15324static SDValue generateEquivalentSub(SDNode *N, int Size, bool Complement,
15325 bool Swap, SDLoc &DL, SelectionDAG &DAG) {
15326 assert(N->getOpcode() == ISD::SETCC && "ISD::SETCC Expected.");
15327
15328 // Zero extend the operands to the largest legal integer. Originally, they
15329 // must be of a strictly smaller size.
15330 auto Op0 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i64, N1: N->getOperand(Num: 0),
15331 N2: DAG.getConstant(Val: Size, DL, VT: MVT::i32));
15332 auto Op1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i64, N1: N->getOperand(Num: 1),
15333 N2: DAG.getConstant(Val: Size, DL, VT: MVT::i32));
15334
15335 // Swap if needed. Depends on the condition code.
15336 if (Swap)
15337 std::swap(a&: Op0, b&: Op1);
15338
15339 // Subtract extended integers.
15340 auto SubNode = DAG.getNode(Opcode: ISD::SUB, DL, VT: MVT::i64, N1: Op0, N2: Op1);
15341
15342 // Move the sign bit to the least significant position and zero out the rest.
15343 // Now the least significant bit carries the result of original comparison.
15344 auto Shifted = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i64, N1: SubNode,
15345 N2: DAG.getConstant(Val: Size - 1, DL, VT: MVT::i32));
15346 auto Final = Shifted;
15347
15348 // Complement the result if needed. Based on the condition code.
15349 if (Complement)
15350 Final = DAG.getNode(Opcode: ISD::XOR, DL, VT: MVT::i64, N1: Shifted,
15351 N2: DAG.getConstant(Val: 1, DL, VT: MVT::i64));
15352
15353 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i1, Operand: Final);
15354}
15355
15356SDValue PPCTargetLowering::ConvertSETCCToSubtract(SDNode *N,
15357 DAGCombinerInfo &DCI) const {
15358 assert(N->getOpcode() == ISD::SETCC && "ISD::SETCC Expected.");
15359
15360 SelectionDAG &DAG = DCI.DAG;
15361 SDLoc DL(N);
15362
15363 // Size of integers being compared has a critical role in the following
15364 // analysis, so we prefer to do this when all types are legal.
15365 if (!DCI.isAfterLegalizeDAG())
15366 return SDValue();
15367
15368 // If all users of SETCC extend its value to a legal integer type
15369 // then we replace SETCC with a subtraction
15370 for (const SDNode *U : N->users())
15371 if (U->getOpcode() != ISD::ZERO_EXTEND)
15372 return SDValue();
15373
15374 ISD::CondCode CC = cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get();
15375 auto OpSize = N->getOperand(Num: 0).getValueSizeInBits();
15376
15377 unsigned Size = DAG.getDataLayout().getLargestLegalIntTypeSizeInBits();
15378
15379 if (OpSize < Size) {
15380 switch (CC) {
15381 default: break;
15382 case ISD::SETULT:
15383 return generateEquivalentSub(N, Size, Complement: false, Swap: false, DL, DAG);
15384 case ISD::SETULE:
15385 return generateEquivalentSub(N, Size, Complement: true, Swap: true, DL, DAG);
15386 case ISD::SETUGT:
15387 return generateEquivalentSub(N, Size, Complement: false, Swap: true, DL, DAG);
15388 case ISD::SETUGE:
15389 return generateEquivalentSub(N, Size, Complement: true, Swap: false, DL, DAG);
15390 }
15391 }
15392
15393 return SDValue();
15394}
15395
15396SDValue PPCTargetLowering::DAGCombineTruncBoolExt(SDNode *N,
15397 DAGCombinerInfo &DCI) const {
15398 SelectionDAG &DAG = DCI.DAG;
15399 SDLoc dl(N);
15400
15401 assert(Subtarget.useCRBits() && "Expecting to be tracking CR bits");
15402 // If we're tracking CR bits, we need to be careful that we don't have:
15403 // trunc(binary-ops(zext(x), zext(y)))
15404 // or
15405 // trunc(binary-ops(binary-ops(zext(x), zext(y)), ...)
15406 // such that we're unnecessarily moving things into GPRs when it would be
15407 // better to keep them in CR bits.
15408
15409 // Note that trunc here can be an actual i1 trunc, or can be the effective
15410 // truncation that comes from a setcc or select_cc.
15411 if (N->getOpcode() == ISD::TRUNCATE &&
15412 N->getValueType(ResNo: 0) != MVT::i1)
15413 return SDValue();
15414
15415 if (N->getOperand(Num: 0).getValueType() != MVT::i32 &&
15416 N->getOperand(Num: 0).getValueType() != MVT::i64)
15417 return SDValue();
15418
15419 if (N->getOpcode() == ISD::SETCC ||
15420 N->getOpcode() == ISD::SELECT_CC) {
15421 // If we're looking at a comparison, then we need to make sure that the
15422 // high bits (all except for the first) don't matter the result.
15423 ISD::CondCode CC =
15424 cast<CondCodeSDNode>(Val: N->getOperand(
15425 Num: N->getOpcode() == ISD::SETCC ? 2 : 4))->get();
15426 unsigned OpBits = N->getOperand(Num: 0).getValueSizeInBits();
15427
15428 if (ISD::isSignedIntSetCC(Code: CC)) {
15429 if (DAG.ComputeNumSignBits(Op: N->getOperand(Num: 0)) != OpBits ||
15430 DAG.ComputeNumSignBits(Op: N->getOperand(Num: 1)) != OpBits)
15431 return SDValue();
15432 } else if (ISD::isUnsignedIntSetCC(Code: CC)) {
15433 if (!DAG.MaskedValueIsZero(Op: N->getOperand(Num: 0),
15434 Mask: APInt::getHighBitsSet(numBits: OpBits, hiBitsSet: OpBits-1)) ||
15435 !DAG.MaskedValueIsZero(Op: N->getOperand(Num: 1),
15436 Mask: APInt::getHighBitsSet(numBits: OpBits, hiBitsSet: OpBits-1)))
15437 return (N->getOpcode() == ISD::SETCC ? ConvertSETCCToSubtract(N, DCI)
15438 : SDValue());
15439 } else {
15440 // This is neither a signed nor an unsigned comparison, just make sure
15441 // that the high bits are equal.
15442 KnownBits Op1Known = DAG.computeKnownBits(Op: N->getOperand(Num: 0));
15443 KnownBits Op2Known = DAG.computeKnownBits(Op: N->getOperand(Num: 1));
15444
15445 // We don't really care about what is known about the first bit (if
15446 // anything), so pretend that it is known zero for both to ensure they can
15447 // be compared as constants.
15448 Op1Known.Zero.setBit(0); Op1Known.One.clearBit(BitPosition: 0);
15449 Op2Known.Zero.setBit(0); Op2Known.One.clearBit(BitPosition: 0);
15450
15451 if (!Op1Known.isConstant() || !Op2Known.isConstant() ||
15452 Op1Known.getConstant() != Op2Known.getConstant())
15453 return SDValue();
15454 }
15455 }
15456
15457 // We now know that the higher-order bits are irrelevant, we just need to
15458 // make sure that all of the intermediate operations are bit operations, and
15459 // all inputs are extensions.
15460 if (N->getOperand(Num: 0).getOpcode() != ISD::AND &&
15461 N->getOperand(Num: 0).getOpcode() != ISD::OR &&
15462 N->getOperand(Num: 0).getOpcode() != ISD::XOR &&
15463 N->getOperand(Num: 0).getOpcode() != ISD::SELECT &&
15464 N->getOperand(Num: 0).getOpcode() != ISD::SELECT_CC &&
15465 N->getOperand(Num: 0).getOpcode() != ISD::TRUNCATE &&
15466 N->getOperand(Num: 0).getOpcode() != ISD::SIGN_EXTEND &&
15467 N->getOperand(Num: 0).getOpcode() != ISD::ZERO_EXTEND &&
15468 N->getOperand(Num: 0).getOpcode() != ISD::ANY_EXTEND)
15469 return SDValue();
15470
15471 if ((N->getOpcode() == ISD::SETCC || N->getOpcode() == ISD::SELECT_CC) &&
15472 N->getOperand(Num: 1).getOpcode() != ISD::AND &&
15473 N->getOperand(Num: 1).getOpcode() != ISD::OR &&
15474 N->getOperand(Num: 1).getOpcode() != ISD::XOR &&
15475 N->getOperand(Num: 1).getOpcode() != ISD::SELECT &&
15476 N->getOperand(Num: 1).getOpcode() != ISD::SELECT_CC &&
15477 N->getOperand(Num: 1).getOpcode() != ISD::TRUNCATE &&
15478 N->getOperand(Num: 1).getOpcode() != ISD::SIGN_EXTEND &&
15479 N->getOperand(Num: 1).getOpcode() != ISD::ZERO_EXTEND &&
15480 N->getOperand(Num: 1).getOpcode() != ISD::ANY_EXTEND)
15481 return SDValue();
15482
15483 SmallVector<SDValue, 4> Inputs;
15484 SmallVector<SDValue, 8> BinOps, PromOps;
15485 SmallPtrSet<SDNode *, 16> Visited;
15486
15487 for (unsigned i = 0; i < 2; ++i) {
15488 if (((N->getOperand(Num: i).getOpcode() == ISD::SIGN_EXTEND ||
15489 N->getOperand(Num: i).getOpcode() == ISD::ZERO_EXTEND ||
15490 N->getOperand(Num: i).getOpcode() == ISD::ANY_EXTEND) &&
15491 N->getOperand(Num: i).getOperand(i: 0).getValueType() == MVT::i1) ||
15492 isa<ConstantSDNode>(Val: N->getOperand(Num: i)))
15493 Inputs.push_back(Elt: N->getOperand(Num: i));
15494 else
15495 BinOps.push_back(Elt: N->getOperand(Num: i));
15496
15497 if (N->getOpcode() == ISD::TRUNCATE)
15498 break;
15499 }
15500
15501 // Visit all inputs, collect all binary operations (and, or, xor and
15502 // select) that are all fed by extensions.
15503 while (!BinOps.empty()) {
15504 SDValue BinOp = BinOps.pop_back_val();
15505
15506 if (!Visited.insert(Ptr: BinOp.getNode()).second)
15507 continue;
15508
15509 PromOps.push_back(Elt: BinOp);
15510
15511 for (unsigned i = 0, ie = BinOp.getNumOperands(); i != ie; ++i) {
15512 // The condition of the select is not promoted.
15513 if (BinOp.getOpcode() == ISD::SELECT && i == 0)
15514 continue;
15515 if (BinOp.getOpcode() == ISD::SELECT_CC && i != 2 && i != 3)
15516 continue;
15517
15518 if (((BinOp.getOperand(i).getOpcode() == ISD::SIGN_EXTEND ||
15519 BinOp.getOperand(i).getOpcode() == ISD::ZERO_EXTEND ||
15520 BinOp.getOperand(i).getOpcode() == ISD::ANY_EXTEND) &&
15521 BinOp.getOperand(i).getOperand(i: 0).getValueType() == MVT::i1) ||
15522 isa<ConstantSDNode>(Val: BinOp.getOperand(i))) {
15523 Inputs.push_back(Elt: BinOp.getOperand(i));
15524 } else if (BinOp.getOperand(i).getOpcode() == ISD::AND ||
15525 BinOp.getOperand(i).getOpcode() == ISD::OR ||
15526 BinOp.getOperand(i).getOpcode() == ISD::XOR ||
15527 BinOp.getOperand(i).getOpcode() == ISD::SELECT ||
15528 BinOp.getOperand(i).getOpcode() == ISD::SELECT_CC ||
15529 BinOp.getOperand(i).getOpcode() == ISD::TRUNCATE ||
15530 BinOp.getOperand(i).getOpcode() == ISD::SIGN_EXTEND ||
15531 BinOp.getOperand(i).getOpcode() == ISD::ZERO_EXTEND ||
15532 BinOp.getOperand(i).getOpcode() == ISD::ANY_EXTEND) {
15533 BinOps.push_back(Elt: BinOp.getOperand(i));
15534 } else {
15535 // We have an input that is not an extension or another binary
15536 // operation; we'll abort this transformation.
15537 return SDValue();
15538 }
15539 }
15540 }
15541
15542 // Make sure that this is a self-contained cluster of operations (which
15543 // is not quite the same thing as saying that everything has only one
15544 // use).
15545 for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) {
15546 if (isa<ConstantSDNode>(Val: Inputs[i]))
15547 continue;
15548
15549 for (const SDNode *User : Inputs[i].getNode()->users()) {
15550 if (User != N && !Visited.count(Ptr: User))
15551 return SDValue();
15552
15553 // Make sure that we're not going to promote the non-output-value
15554 // operand(s) or SELECT or SELECT_CC.
15555 // FIXME: Although we could sometimes handle this, and it does occur in
15556 // practice that one of the condition inputs to the select is also one of
15557 // the outputs, we currently can't deal with this.
15558 if (User->getOpcode() == ISD::SELECT) {
15559 if (User->getOperand(Num: 0) == Inputs[i])
15560 return SDValue();
15561 } else if (User->getOpcode() == ISD::SELECT_CC) {
15562 if (User->getOperand(Num: 0) == Inputs[i] ||
15563 User->getOperand(Num: 1) == Inputs[i])
15564 return SDValue();
15565 }
15566 }
15567 }
15568
15569 for (unsigned i = 0, ie = PromOps.size(); i != ie; ++i) {
15570 for (const SDNode *User : PromOps[i].getNode()->users()) {
15571 if (User != N && !Visited.count(Ptr: User))
15572 return SDValue();
15573
15574 // Make sure that we're not going to promote the non-output-value
15575 // operand(s) or SELECT or SELECT_CC.
15576 // FIXME: Although we could sometimes handle this, and it does occur in
15577 // practice that one of the condition inputs to the select is also one of
15578 // the outputs, we currently can't deal with this.
15579 if (User->getOpcode() == ISD::SELECT) {
15580 if (User->getOperand(Num: 0) == PromOps[i])
15581 return SDValue();
15582 } else if (User->getOpcode() == ISD::SELECT_CC) {
15583 if (User->getOperand(Num: 0) == PromOps[i] ||
15584 User->getOperand(Num: 1) == PromOps[i])
15585 return SDValue();
15586 }
15587 }
15588 }
15589
15590 // Replace all inputs with the extension operand.
15591 for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) {
15592 // Constants may have users outside the cluster of to-be-promoted nodes,
15593 // and so we need to replace those as we do the promotions.
15594 if (isa<ConstantSDNode>(Val: Inputs[i]))
15595 continue;
15596 else
15597 DAG.ReplaceAllUsesOfValueWith(From: Inputs[i], To: Inputs[i].getOperand(i: 0));
15598 }
15599
15600 std::list<HandleSDNode> PromOpHandles;
15601 for (auto &PromOp : PromOps)
15602 PromOpHandles.emplace_back(args&: PromOp);
15603
15604 // Replace all operations (these are all the same, but have a different
15605 // (i1) return type). DAG.getNode will validate that the types of
15606 // a binary operator match, so go through the list in reverse so that
15607 // we've likely promoted both operands first. Any intermediate truncations or
15608 // extensions disappear.
15609 while (!PromOpHandles.empty()) {
15610 SDValue PromOp = PromOpHandles.back().getValue();
15611 PromOpHandles.pop_back();
15612
15613 if (PromOp.getOpcode() == ISD::TRUNCATE ||
15614 PromOp.getOpcode() == ISD::SIGN_EXTEND ||
15615 PromOp.getOpcode() == ISD::ZERO_EXTEND ||
15616 PromOp.getOpcode() == ISD::ANY_EXTEND) {
15617 if (!isa<ConstantSDNode>(Val: PromOp.getOperand(i: 0)) &&
15618 PromOp.getOperand(i: 0).getValueType() != MVT::i1) {
15619 // The operand is not yet ready (see comment below).
15620 PromOpHandles.emplace_front(args&: PromOp);
15621 continue;
15622 }
15623
15624 SDValue RepValue = PromOp.getOperand(i: 0);
15625 if (isa<ConstantSDNode>(Val: RepValue))
15626 RepValue = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i1, Operand: RepValue);
15627
15628 DAG.ReplaceAllUsesOfValueWith(From: PromOp, To: RepValue);
15629 continue;
15630 }
15631
15632 unsigned C;
15633 switch (PromOp.getOpcode()) {
15634 default: C = 0; break;
15635 case ISD::SELECT: C = 1; break;
15636 case ISD::SELECT_CC: C = 2; break;
15637 }
15638
15639 if ((!isa<ConstantSDNode>(Val: PromOp.getOperand(i: C)) &&
15640 PromOp.getOperand(i: C).getValueType() != MVT::i1) ||
15641 (!isa<ConstantSDNode>(Val: PromOp.getOperand(i: C+1)) &&
15642 PromOp.getOperand(i: C+1).getValueType() != MVT::i1)) {
15643 // The to-be-promoted operands of this node have not yet been
15644 // promoted (this should be rare because we're going through the
15645 // list backward, but if one of the operands has several users in
15646 // this cluster of to-be-promoted nodes, it is possible).
15647 PromOpHandles.emplace_front(args&: PromOp);
15648 continue;
15649 }
15650
15651 SmallVector<SDValue, 3> Ops(PromOp.getNode()->ops());
15652
15653 // If there are any constant inputs, make sure they're replaced now.
15654 for (unsigned i = 0; i < 2; ++i)
15655 if (isa<ConstantSDNode>(Val: Ops[C+i]))
15656 Ops[C+i] = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i1, Operand: Ops[C+i]);
15657
15658 DAG.ReplaceAllUsesOfValueWith(From: PromOp,
15659 To: DAG.getNode(Opcode: PromOp.getOpcode(), DL: dl, VT: MVT::i1, Ops));
15660 }
15661
15662 // Now we're left with the initial truncation itself.
15663 if (N->getOpcode() == ISD::TRUNCATE)
15664 return N->getOperand(Num: 0);
15665
15666 // Otherwise, this is a comparison. The operands to be compared have just
15667 // changed type (to i1), but everything else is the same.
15668 return SDValue(N, 0);
15669}
15670
15671SDValue PPCTargetLowering::DAGCombineExtBoolTrunc(SDNode *N,
15672 DAGCombinerInfo &DCI) const {
15673 SelectionDAG &DAG = DCI.DAG;
15674 SDLoc dl(N);
15675
15676 // If we're tracking CR bits, we need to be careful that we don't have:
15677 // zext(binary-ops(trunc(x), trunc(y)))
15678 // or
15679 // zext(binary-ops(binary-ops(trunc(x), trunc(y)), ...)
15680 // such that we're unnecessarily moving things into CR bits that can more
15681 // efficiently stay in GPRs. Note that if we're not certain that the high
15682 // bits are set as required by the final extension, we still may need to do
15683 // some masking to get the proper behavior.
15684
15685 // This same functionality is important on PPC64 when dealing with
15686 // 32-to-64-bit extensions; these occur often when 32-bit values are used as
15687 // the return values of functions. Because it is so similar, it is handled
15688 // here as well.
15689
15690 if (N->getValueType(ResNo: 0) != MVT::i32 &&
15691 N->getValueType(ResNo: 0) != MVT::i64)
15692 return SDValue();
15693
15694 if (!((N->getOperand(Num: 0).getValueType() == MVT::i1 && Subtarget.useCRBits()) ||
15695 (N->getOperand(Num: 0).getValueType() == MVT::i32 && Subtarget.isPPC64())))
15696 return SDValue();
15697
15698 if (N->getOperand(Num: 0).getOpcode() != ISD::AND &&
15699 N->getOperand(Num: 0).getOpcode() != ISD::OR &&
15700 N->getOperand(Num: 0).getOpcode() != ISD::XOR &&
15701 N->getOperand(Num: 0).getOpcode() != ISD::SELECT &&
15702 N->getOperand(Num: 0).getOpcode() != ISD::SELECT_CC)
15703 return SDValue();
15704
15705 SmallVector<SDValue, 4> Inputs;
15706 SmallVector<SDValue, 8> BinOps(1, N->getOperand(Num: 0)), PromOps;
15707 SmallPtrSet<SDNode *, 16> Visited;
15708
15709 // Visit all inputs, collect all binary operations (and, or, xor and
15710 // select) that are all fed by truncations.
15711 while (!BinOps.empty()) {
15712 SDValue BinOp = BinOps.pop_back_val();
15713
15714 if (!Visited.insert(Ptr: BinOp.getNode()).second)
15715 continue;
15716
15717 PromOps.push_back(Elt: BinOp);
15718
15719 for (unsigned i = 0, ie = BinOp.getNumOperands(); i != ie; ++i) {
15720 // The condition of the select is not promoted.
15721 if (BinOp.getOpcode() == ISD::SELECT && i == 0)
15722 continue;
15723 if (BinOp.getOpcode() == ISD::SELECT_CC && i != 2 && i != 3)
15724 continue;
15725
15726 if (BinOp.getOperand(i).getOpcode() == ISD::TRUNCATE ||
15727 isa<ConstantSDNode>(Val: BinOp.getOperand(i))) {
15728 Inputs.push_back(Elt: BinOp.getOperand(i));
15729 } else if (BinOp.getOperand(i).getOpcode() == ISD::AND ||
15730 BinOp.getOperand(i).getOpcode() == ISD::OR ||
15731 BinOp.getOperand(i).getOpcode() == ISD::XOR ||
15732 BinOp.getOperand(i).getOpcode() == ISD::SELECT ||
15733 BinOp.getOperand(i).getOpcode() == ISD::SELECT_CC) {
15734 BinOps.push_back(Elt: BinOp.getOperand(i));
15735 } else {
15736 // We have an input that is not a truncation or another binary
15737 // operation; we'll abort this transformation.
15738 return SDValue();
15739 }
15740 }
15741 }
15742
15743 // The operands of a select that must be truncated when the select is
15744 // promoted because the operand is actually part of the to-be-promoted set.
15745 DenseMap<SDNode *, EVT> SelectTruncOp[2];
15746
15747 // Make sure that this is a self-contained cluster of operations (which
15748 // is not quite the same thing as saying that everything has only one
15749 // use).
15750 for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) {
15751 if (isa<ConstantSDNode>(Val: Inputs[i]))
15752 continue;
15753
15754 for (SDNode *User : Inputs[i].getNode()->users()) {
15755 if (User != N && !Visited.count(Ptr: User))
15756 return SDValue();
15757
15758 // If we're going to promote the non-output-value operand(s) or SELECT or
15759 // SELECT_CC, record them for truncation.
15760 if (User->getOpcode() == ISD::SELECT) {
15761 if (User->getOperand(Num: 0) == Inputs[i])
15762 SelectTruncOp[0].insert(KV: std::make_pair(x&: User,
15763 y: User->getOperand(Num: 0).getValueType()));
15764 } else if (User->getOpcode() == ISD::SELECT_CC) {
15765 if (User->getOperand(Num: 0) == Inputs[i])
15766 SelectTruncOp[0].insert(KV: std::make_pair(x&: User,
15767 y: User->getOperand(Num: 0).getValueType()));
15768 if (User->getOperand(Num: 1) == Inputs[i])
15769 SelectTruncOp[1].insert(KV: std::make_pair(x&: User,
15770 y: User->getOperand(Num: 1).getValueType()));
15771 }
15772 }
15773 }
15774
15775 for (unsigned i = 0, ie = PromOps.size(); i != ie; ++i) {
15776 for (SDNode *User : PromOps[i].getNode()->users()) {
15777 if (User != N && !Visited.count(Ptr: User))
15778 return SDValue();
15779
15780 // If we're going to promote the non-output-value operand(s) or SELECT or
15781 // SELECT_CC, record them for truncation.
15782 if (User->getOpcode() == ISD::SELECT) {
15783 if (User->getOperand(Num: 0) == PromOps[i])
15784 SelectTruncOp[0].insert(KV: std::make_pair(x&: User,
15785 y: User->getOperand(Num: 0).getValueType()));
15786 } else if (User->getOpcode() == ISD::SELECT_CC) {
15787 if (User->getOperand(Num: 0) == PromOps[i])
15788 SelectTruncOp[0].insert(KV: std::make_pair(x&: User,
15789 y: User->getOperand(Num: 0).getValueType()));
15790 if (User->getOperand(Num: 1) == PromOps[i])
15791 SelectTruncOp[1].insert(KV: std::make_pair(x&: User,
15792 y: User->getOperand(Num: 1).getValueType()));
15793 }
15794 }
15795 }
15796
15797 unsigned PromBits = N->getOperand(Num: 0).getValueSizeInBits();
15798 bool ReallyNeedsExt = false;
15799 if (N->getOpcode() != ISD::ANY_EXTEND) {
15800 // If all of the inputs are not already sign/zero extended, then
15801 // we'll still need to do that at the end.
15802 for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) {
15803 if (isa<ConstantSDNode>(Val: Inputs[i]))
15804 continue;
15805
15806 unsigned OpBits =
15807 Inputs[i].getOperand(i: 0).getValueSizeInBits();
15808 assert(PromBits < OpBits && "Truncation not to a smaller bit count?");
15809
15810 if ((N->getOpcode() == ISD::ZERO_EXTEND &&
15811 !DAG.MaskedValueIsZero(Op: Inputs[i].getOperand(i: 0),
15812 Mask: APInt::getHighBitsSet(numBits: OpBits,
15813 hiBitsSet: OpBits-PromBits))) ||
15814 (N->getOpcode() == ISD::SIGN_EXTEND &&
15815 DAG.ComputeNumSignBits(Op: Inputs[i].getOperand(i: 0)) <
15816 (OpBits-(PromBits-1)))) {
15817 ReallyNeedsExt = true;
15818 break;
15819 }
15820 }
15821 }
15822
15823 // Convert PromOps to handles before doing any RAUW operations, as these
15824 // may CSE with existing nodes, deleting the originals.
15825 std::list<HandleSDNode> PromOpHandles;
15826 for (auto &PromOp : PromOps)
15827 PromOpHandles.emplace_back(args&: PromOp);
15828
15829 // Replace all inputs, either with the truncation operand, or a
15830 // truncation or extension to the final output type.
15831 for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) {
15832 // Constant inputs need to be replaced with the to-be-promoted nodes that
15833 // use them because they might have users outside of the cluster of
15834 // promoted nodes.
15835 if (isa<ConstantSDNode>(Val: Inputs[i]))
15836 continue;
15837
15838 SDValue InSrc = Inputs[i].getOperand(i: 0);
15839 if (Inputs[i].getValueType() == N->getValueType(ResNo: 0))
15840 DAG.ReplaceAllUsesOfValueWith(From: Inputs[i], To: InSrc);
15841 else if (N->getOpcode() == ISD::SIGN_EXTEND)
15842 DAG.ReplaceAllUsesOfValueWith(From: Inputs[i],
15843 To: DAG.getSExtOrTrunc(Op: InSrc, DL: dl, VT: N->getValueType(ResNo: 0)));
15844 else if (N->getOpcode() == ISD::ZERO_EXTEND)
15845 DAG.ReplaceAllUsesOfValueWith(From: Inputs[i],
15846 To: DAG.getZExtOrTrunc(Op: InSrc, DL: dl, VT: N->getValueType(ResNo: 0)));
15847 else
15848 DAG.ReplaceAllUsesOfValueWith(From: Inputs[i],
15849 To: DAG.getAnyExtOrTrunc(Op: InSrc, DL: dl, VT: N->getValueType(ResNo: 0)));
15850 }
15851
15852 // Replace all operations (these are all the same, but have a different
15853 // (promoted) return type). DAG.getNode will validate that the types of
15854 // a binary operator match, so go through the list in reverse so that
15855 // we've likely promoted both operands first.
15856 while (!PromOpHandles.empty()) {
15857 SDValue PromOp = PromOpHandles.back().getValue();
15858 PromOpHandles.pop_back();
15859
15860 unsigned C;
15861 switch (PromOp.getOpcode()) {
15862 default: C = 0; break;
15863 case ISD::SELECT: C = 1; break;
15864 case ISD::SELECT_CC: C = 2; break;
15865 }
15866
15867 if ((!isa<ConstantSDNode>(Val: PromOp.getOperand(i: C)) &&
15868 PromOp.getOperand(i: C).getValueType() != N->getValueType(ResNo: 0)) ||
15869 (!isa<ConstantSDNode>(Val: PromOp.getOperand(i: C+1)) &&
15870 PromOp.getOperand(i: C+1).getValueType() != N->getValueType(ResNo: 0))) {
15871 // The to-be-promoted operands of this node have not yet been
15872 // promoted (this should be rare because we're going through the
15873 // list backward, but if one of the operands has several users in
15874 // this cluster of to-be-promoted nodes, it is possible).
15875 PromOpHandles.emplace_front(args&: PromOp);
15876 continue;
15877 }
15878
15879 // For SELECT and SELECT_CC nodes, we do a similar check for any
15880 // to-be-promoted comparison inputs.
15881 if (PromOp.getOpcode() == ISD::SELECT ||
15882 PromOp.getOpcode() == ISD::SELECT_CC) {
15883 if ((SelectTruncOp[0].count(Val: PromOp.getNode()) &&
15884 PromOp.getOperand(i: 0).getValueType() != N->getValueType(ResNo: 0)) ||
15885 (SelectTruncOp[1].count(Val: PromOp.getNode()) &&
15886 PromOp.getOperand(i: 1).getValueType() != N->getValueType(ResNo: 0))) {
15887 PromOpHandles.emplace_front(args&: PromOp);
15888 continue;
15889 }
15890 }
15891
15892 SmallVector<SDValue, 3> Ops(PromOp.getNode()->ops());
15893
15894 // If this node has constant inputs, then they'll need to be promoted here.
15895 for (unsigned i = 0; i < 2; ++i) {
15896 if (!isa<ConstantSDNode>(Val: Ops[C+i]))
15897 continue;
15898 if (Ops[C+i].getValueType() == N->getValueType(ResNo: 0))
15899 continue;
15900
15901 if (N->getOpcode() == ISD::SIGN_EXTEND)
15902 Ops[C+i] = DAG.getSExtOrTrunc(Op: Ops[C+i], DL: dl, VT: N->getValueType(ResNo: 0));
15903 else if (N->getOpcode() == ISD::ZERO_EXTEND)
15904 Ops[C+i] = DAG.getZExtOrTrunc(Op: Ops[C+i], DL: dl, VT: N->getValueType(ResNo: 0));
15905 else
15906 Ops[C+i] = DAG.getAnyExtOrTrunc(Op: Ops[C+i], DL: dl, VT: N->getValueType(ResNo: 0));
15907 }
15908
15909 // If we've promoted the comparison inputs of a SELECT or SELECT_CC,
15910 // truncate them again to the original value type.
15911 if (PromOp.getOpcode() == ISD::SELECT ||
15912 PromOp.getOpcode() == ISD::SELECT_CC) {
15913 auto SI0 = SelectTruncOp[0].find(Val: PromOp.getNode());
15914 if (SI0 != SelectTruncOp[0].end())
15915 Ops[0] = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: SI0->second, Operand: Ops[0]);
15916 auto SI1 = SelectTruncOp[1].find(Val: PromOp.getNode());
15917 if (SI1 != SelectTruncOp[1].end())
15918 Ops[1] = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: SI1->second, Operand: Ops[1]);
15919 }
15920
15921 DAG.ReplaceAllUsesOfValueWith(From: PromOp,
15922 To: DAG.getNode(Opcode: PromOp.getOpcode(), DL: dl, VT: N->getValueType(ResNo: 0), Ops));
15923 }
15924
15925 // Now we're left with the initial extension itself.
15926 if (!ReallyNeedsExt)
15927 return N->getOperand(Num: 0);
15928
15929 // To zero extend, just mask off everything except for the first bit (in the
15930 // i1 case).
15931 if (N->getOpcode() == ISD::ZERO_EXTEND)
15932 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 0),
15933 N2: DAG.getConstant(Val: APInt::getLowBitsSet(
15934 numBits: N->getValueSizeInBits(ResNo: 0), loBitsSet: PromBits),
15935 DL: dl, VT: N->getValueType(ResNo: 0)));
15936
15937 assert(N->getOpcode() == ISD::SIGN_EXTEND &&
15938 "Invalid extension type");
15939 EVT ShiftAmountTy = getShiftAmountTy(LHSTy: N->getValueType(ResNo: 0), DL: DAG.getDataLayout());
15940 SDValue ShiftCst =
15941 DAG.getConstant(Val: N->getValueSizeInBits(ResNo: 0) - PromBits, DL: dl, VT: ShiftAmountTy);
15942 return DAG.getNode(
15943 Opcode: ISD::SRA, DL: dl, VT: N->getValueType(ResNo: 0),
15944 N1: DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 0), N2: ShiftCst),
15945 N2: ShiftCst);
15946}
15947
15948// The function check a i128 load can convert to 16i8 load for Vcmpequb.
15949static bool canConvertToVcmpequb(SDValue &LHS, SDValue &RHS, bool IsPPC64) {
15950
15951 auto isValidForConvert = [IsPPC64](SDValue &Operand) {
15952 if (!Operand.hasOneUse())
15953 return false;
15954
15955 if (Operand.getValueType() != MVT::i128)
15956 return false;
15957
15958 if (Operand.getOpcode() == ISD::Constant) {
15959 auto *C = cast<ConstantSDNode>(Val&: Operand);
15960 const APInt &Val = C->getAPIntValue();
15961 // On PPC64, comparing an i128 value loaded from memory against a
15962 // constant smaller than 2^16 is usually better left to scalar lowering.
15963 // In that case, the compare can be lowered using xori (since xori has a
15964 // 16-bit immediate field), which is cheaper than materializing a vector
15965 // constant and using vcmpequb.
15966 if (IsPPC64 && Val.ult(RHS: 1ULL << 16))
15967 return false;
15968 return true;
15969 }
15970
15971 auto *LoadNode = dyn_cast<LoadSDNode>(Val&: Operand);
15972 if (!LoadNode)
15973 return false;
15974
15975 // If memory operation is volatile, do not perform any
15976 // optimization or transformation. Volatile operations must be preserved
15977 // as written to ensure correct program behavior, so we return an empty
15978 // SDValue to indicate no action.
15979
15980 if (LoadNode->isVolatile())
15981 return false;
15982
15983 // Only combine loads if both use the unindexed addressing mode.
15984 // PowerPC AltiVec/VMX does not support vector loads or stores with
15985 // pre/post-increment addressing. Indexed modes may imply implicit
15986 // pointer updates, which are not compatible with AltiVec vector
15987 // instructions.
15988 if (LoadNode->getAddressingMode() != ISD::UNINDEXED)
15989 return false;
15990
15991 // Only combine loads if both are non-extending loads
15992 // (ISD::NON_EXTLOAD). Extending loads (such as ISD::ZEXTLOAD or
15993 // ISD::SEXTLOAD) perform zero or sign extension, which may change the
15994 // loaded value's semantics and are not compatible with vector loads.
15995 if (LoadNode->getExtensionType() != ISD::NON_EXTLOAD)
15996 return false;
15997
15998 return true;
15999 };
16000
16001 return (isValidForConvert(LHS) && isValidForConvert(RHS));
16002}
16003
16004SDValue convertTwoLoadsAndCmpToVCMPEQUB(SelectionDAG &DAG, SDNode *N,
16005 const SDLoc &DL) {
16006
16007 assert(N->getOpcode() == ISD::SETCC && "Should be called with a SETCC node");
16008
16009 ISD::CondCode CC = cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get();
16010 assert((CC == ISD::SETNE || CC == ISD::SETEQ) &&
16011 "CC mus be ISD::SETNE or ISD::SETEQ");
16012
16013 auto getV16i8Load = [&](const SDValue &Operand) {
16014 if (Operand.getOpcode() == ISD::Constant)
16015 return DAG.getBitcast(VT: MVT::v16i8, V: Operand);
16016
16017 assert(Operand.getOpcode() == ISD::LOAD && "Must be LoadSDNode here.");
16018
16019 auto *LoadNode = cast<LoadSDNode>(Val: Operand);
16020 // Create a new MachineMemOperand without range metadata.
16021 // Range metadata is only valid for integer scalar types, not vectors.
16022 // The original i128 load may have range metadata, but when we convert
16023 // to v16i8, that metadata is no longer semantically valid.
16024 MachineMemOperand *MMO = LoadNode->getMemOperand();
16025 MachineFunction &MF = DAG.getMachineFunction();
16026 MachineMemOperand *NewMMO = MF.getMachineMemOperand(
16027 PtrInfo: MMO->getPointerInfo(), F: MMO->getFlags(), Size: MMO->getSize(), BaseAlignment: MMO->getAlign(),
16028 Metadata: MMO->getAAInfo(), SSID: MMO->getSyncScopeID(), Ordering: MMO->getSuccessOrdering(),
16029 FailureOrdering: MMO->getFailureOrdering());
16030 SDValue NewLoad = DAG.getLoad(VT: MVT::v16i8, dl: DL, Chain: LoadNode->getChain(),
16031 Ptr: LoadNode->getBasePtr(), MMO: NewMMO);
16032 DAG.ReplaceAllUsesOfValueWith(From: SDValue(LoadNode, 1), To: NewLoad.getValue(R: 1));
16033 return NewLoad;
16034 };
16035
16036 // Following code transforms the DAG
16037 // t0: ch,glue = EntryToken
16038 // t2: i64,ch = CopyFromReg t0, Register:i64 %0
16039 // t3: i128,ch = load<(load (s128) from %ir.a, align 1)> t0, t2,
16040 // undef:i64
16041 // t4: i64,ch = CopyFromReg t0, Register:i64 %1
16042 // t5: i128,ch =
16043 // load<(load (s128) from %ir.b, align 1)> t0, t4, undef:i64 t6: i1 =
16044 // setcc t3, t5, setne:ch
16045 //
16046 // ---->
16047 //
16048 // t0: ch,glue = EntryToken
16049 // t2: i64,ch = CopyFromReg t0, Register:i64 %0
16050 // t3: v16i8,ch = load<(load (s128) from %ir.a, align 1)> t0, t2,
16051 // undef:i64
16052 // t4: i64,ch = CopyFromReg t0, Register:i64 %1
16053 // t5: v16i8,ch =
16054 // load<(load (s128) from %ir.b, align 1)> t0, t4, undef:i64
16055 // t6: i32 =
16056 // llvm.ppc.altivec.vcmpequb.p TargetConstant:i32<10505>,
16057 // Constant:i32<2>, t3, t5
16058 // t7: i1 = setcc t6, Constant:i32<0>, seteq:ch
16059
16060 // Or transforms the DAG
16061 // t5: i128,ch = load<(load (s128) from %ir.X, align 1)> t0, t2, undef:i64
16062 // t8: i1 =
16063 // setcc Constant:i128<237684487579686500932345921536>, t5, setne:ch
16064 //
16065 // --->
16066 //
16067 // t5: v16i8,ch = load<(load (s128) from %ir.X, align 1)> t0, t2, undef:i64
16068 // t6: v16i8 = bitcast Constant:i128<237684487579686500932345921536>
16069 // t7: i32 =
16070 // llvm.ppc.altivec.vcmpequb.p Constant:i32<10962>, Constant:i32<2>, t5, t2
16071
16072 SDValue LHSVec = getV16i8Load(N->getOperand(Num: 0));
16073 SDValue RHSVec = getV16i8Load(N->getOperand(Num: 1));
16074
16075 SDValue IntrID =
16076 DAG.getConstant(Val: Intrinsic::ppc_altivec_vcmpequb_p, DL, VT: MVT::i32);
16077 SDValue CRSel = DAG.getConstant(Val: 2, DL, VT: MVT::i32); // which CR6 predicate field
16078 SDValue PredResult = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: MVT::i32,
16079 N1: IntrID, N2: CRSel, N3: LHSVec, N4: RHSVec);
16080 // ppc_altivec_vcmpequb_p returns 1 when two vectors are the same,
16081 // so we need to invert the CC opcode.
16082 return DAG.getSetCC(DL, VT: N->getValueType(ResNo: 0), LHS: PredResult,
16083 RHS: DAG.getConstant(Val: 0, DL, VT: MVT::i32),
16084 Cond: CC == ISD::SETNE ? ISD::SETEQ : ISD::SETNE);
16085}
16086
16087// Detect whether there is a pattern like (setcc (and X, 1), 0, eq).
16088// If it is , return true; otherwise return false.
16089static bool canConvertSETCCToXori(SDNode *N) {
16090 assert(N->getOpcode() == ISD::SETCC && "Should be SETCC SDNode here.");
16091
16092 ISD::CondCode CC = cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get();
16093 if (CC != ISD::SETEQ)
16094 return false;
16095
16096 SDValue LHS = N->getOperand(Num: 0);
16097 SDValue RHS = N->getOperand(Num: 1);
16098
16099 // Check the `SDValue &V` is from `and` with `1`.
16100 auto IsAndWithOne = [](SDValue &V) {
16101 if (V.getOpcode() == ISD::AND) {
16102 for (const SDValue &Op : V->ops())
16103 if (auto *C = dyn_cast<ConstantSDNode>(Val: Op))
16104 if (C->isOne())
16105 return true;
16106 }
16107 return false;
16108 };
16109
16110 // Check whether the SETCC compare with zero.
16111 auto IsCompareWithZero = [](SDValue &V) {
16112 if (auto *C = dyn_cast<ConstantSDNode>(Val&: V))
16113 if (C->isZero())
16114 return true;
16115 return false;
16116 };
16117
16118 return (IsAndWithOne(LHS) && IsCompareWithZero(RHS)) ||
16119 (IsAndWithOne(RHS) && IsCompareWithZero(LHS));
16120}
16121
16122// You must check whether the `SDNode* N` can be converted to Xori using
16123// the function `static bool canConvertSETCCToXori(SDNode *N)`
16124// before calling the function; otherwise, it may produce incorrect results.
16125static SDValue ConvertSETCCToXori(SDNode *N, SelectionDAG &DAG) {
16126
16127 assert(N->getOpcode() == ISD::SETCC && "Should be SETCC SDNode here.");
16128 SDValue LHS = N->getOperand(Num: 0);
16129 SDValue RHS = N->getOperand(Num: 1);
16130 SDLoc DL(N);
16131
16132 [[maybe_unused]] ISD::CondCode CC =
16133 cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get();
16134 assert((CC == ISD::SETEQ) && "CC must be ISD::SETEQ.");
16135 // Rewrite it as XORI (and X, 1), 1.
16136 auto MakeXor1 = [&](SDValue V) {
16137 EVT VT = V.getValueType();
16138 SDValue One = DAG.getConstant(Val: 1, DL, VT);
16139 SDValue Xor = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: V, N2: One);
16140 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i1, Operand: Xor);
16141 };
16142
16143 if (LHS.getOpcode() == ISD::AND && RHS.getOpcode() != ISD::AND)
16144 return MakeXor1(LHS);
16145
16146 if (RHS.getOpcode() == ISD::AND && LHS.getOpcode() != ISD::AND)
16147 return MakeXor1(RHS);
16148
16149 llvm_unreachable("Should not reach here.");
16150}
16151
16152// Match `sext(setcc X, 0, eq)` and turn it into an ADDIC/SUBFE sequence.
16153//
16154// This generates code for:
16155// X == 0 ? -1 : 0
16156//
16157// On pre-ISA 3.1 targets, this is better than the longer CNTLZW/SRWI/NEG
16158// sequence. This is useful for cases like:
16159// uint8_t f(uint8_t x) { return (x == 0) ? -1 : 0; }
16160//
16161// ISA 3.1+ is skipped because those targets can use SETBC.
16162
16163SDValue PPCTargetLowering::combineSignExtendSetCC(SDNode *N,
16164 DAGCombinerInfo &DCI) const {
16165 if (Subtarget.isISA3_1())
16166 return SDValue();
16167
16168 EVT VT = N->getValueType(ResNo: 0);
16169 if (VT != MVT::i32 && (VT != MVT::i64 || !Subtarget.isPPC64()))
16170 return SDValue();
16171
16172 SDValue N0 = N->getOperand(Num: 0);
16173 if (N0.getOpcode() != ISD::SETCC)
16174 return SDValue();
16175
16176 ISD::CondCode CC = cast<CondCodeSDNode>(Val: N0.getOperand(i: 2))->get();
16177 SDValue LHS = N0.getOperand(i: 0);
16178 SDValue RHS = N0.getOperand(i: 1);
16179
16180 // Not match: sext (setcc x, 0, eq) or sext (setcc 0, x, eq)
16181 if (CC != ISD::SETEQ || (!isNullConstant(V: LHS) && !isNullConstant(V: RHS)))
16182 return SDValue();
16183
16184 SDLoc dl(N);
16185 SelectionDAG &DAG = DCI.DAG;
16186 SDValue X = isNullConstant(V: LHS) ? RHS : LHS;
16187 EVT XVT = X.getValueType(); // The type of x in the setcc x, 0, eq.
16188
16189 // The type that ADDC/SUBE operate on. Reject larger types and zero-extend
16190 // smaller ones.
16191 MVT OpVT = Subtarget.isPPC64() ? MVT::i64 : MVT::i32;
16192 if (XVT.bitsGT(VT: OpVT))
16193 return SDValue();
16194
16195 if (XVT.bitsLT(VT: OpVT))
16196 X = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: OpVT, Operand: X);
16197
16198 // Generate: SUBFE(ADDC(X, -1)).
16199 SDValue MinusOne = DAG.getAllOnesConstant(DL: dl, VT: OpVT);
16200 SDValue Addc =
16201 DAG.getNode(Opcode: PPCISD::ADDC, DL: dl, VTList: DAG.getVTList(VT1: OpVT, VT2: MVT::i32), N1: X, N2: MinusOne);
16202 SDValue Carry = Addc.getValue(R: 1);
16203 SDValue Sube = DAG.getNode(Opcode: PPCISD::SUBE, DL: dl, VTList: DAG.getVTList(VT1: OpVT, VT2: MVT::i32),
16204 N1: Addc, N2: Addc, N3: Carry);
16205
16206 // Truncate back to i32 if we used i64 operations.
16207 if (OpVT == MVT::i64 && VT == MVT::i32)
16208 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: Sube);
16209
16210 return Sube;
16211}
16212
16213SDValue PPCTargetLowering::combineSetCC(SDNode *N,
16214 DAGCombinerInfo &DCI) const {
16215 assert(N->getOpcode() == ISD::SETCC &&
16216 "Should be called with a SETCC node");
16217
16218 // Check if the pattern (setcc (and X, 1), 0, eq) is present.
16219 // If it is, rewrite it as XORI (and X, 1), 1.
16220 if (canConvertSETCCToXori(N))
16221 return ConvertSETCCToXori(N, DAG&: DCI.DAG);
16222
16223 ISD::CondCode CC = cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get();
16224 if (CC == ISD::SETNE || CC == ISD::SETEQ) {
16225 SDValue LHS = N->getOperand(Num: 0);
16226 SDValue RHS = N->getOperand(Num: 1);
16227
16228 // If there is a '0 - y' pattern, canonicalize the pattern to the RHS.
16229 if (LHS.getOpcode() == ISD::SUB && isNullConstant(V: LHS.getOperand(i: 0)) &&
16230 LHS.hasOneUse())
16231 std::swap(a&: LHS, b&: RHS);
16232
16233 // x == 0-y --> x+y == 0
16234 // x != 0-y --> x+y != 0
16235 if (RHS.getOpcode() == ISD::SUB && isNullConstant(V: RHS.getOperand(i: 0)) &&
16236 RHS.hasOneUse()) {
16237 SDLoc DL(N);
16238 SelectionDAG &DAG = DCI.DAG;
16239 EVT VT = N->getValueType(ResNo: 0);
16240 EVT OpVT = LHS.getValueType();
16241 SDValue Add = DAG.getNode(Opcode: ISD::ADD, DL, VT: OpVT, N1: LHS, N2: RHS.getOperand(i: 1));
16242 return DAG.getSetCC(DL, VT, LHS: Add, RHS: DAG.getConstant(Val: 0, DL, VT: OpVT), Cond: CC);
16243 }
16244
16245 // Optimization: Fold i128 equality/inequality compares of two loads into a
16246 // vectorized compare using vcmpequb.p when Altivec is available.
16247 //
16248 // Rationale:
16249 // A scalar i128 SETCC (eq/ne) normally lowers to multiple scalar ops.
16250 // On VSX-capable subtargets, we can instead reinterpret the i128 loads
16251 // as v16i8 vectors and use the Altive vcmpequb.p instruction to
16252 // perform a full 128-bit equality check in a single vector compare.
16253 //
16254 // Example Result:
16255 // This transformation replaces memcmp(a, b, 16) with two vector loads
16256 // and one vector compare instruction.
16257
16258 if (Subtarget.hasAltivec() &&
16259 canConvertToVcmpequb(LHS, RHS, IsPPC64: Subtarget.isPPC64()))
16260 return convertTwoLoadsAndCmpToVCMPEQUB(DAG&: DCI.DAG, N, DL: SDLoc(N));
16261 }
16262
16263 return DAGCombineTruncBoolExt(N, DCI);
16264}
16265
16266// Is this an extending load from an f32 to an f64?
16267static bool isFPExtLoad(SDValue Op) {
16268 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Val: Op.getNode()))
16269 return LD->getExtensionType() == ISD::EXTLOAD &&
16270 Op.getValueType() == MVT::f64;
16271 return false;
16272}
16273
16274/// Reduces the number of fp-to-int conversion when building a vector.
16275///
16276/// If this vector is built out of floating to integer conversions,
16277/// transform it to a vector built out of floating point values followed by a
16278/// single floating to integer conversion of the vector.
16279/// Namely (build_vector (fptosi $A), (fptosi $B), ...)
16280/// becomes (fptosi (build_vector ($A, $B, ...)))
16281SDValue PPCTargetLowering::
16282combineElementTruncationToVectorTruncation(SDNode *N,
16283 DAGCombinerInfo &DCI) const {
16284 assert(N->getOpcode() == ISD::BUILD_VECTOR &&
16285 "Should be called with a BUILD_VECTOR node");
16286
16287 SelectionDAG &DAG = DCI.DAG;
16288 SDLoc dl(N);
16289
16290 SDValue FirstInput = N->getOperand(Num: 0);
16291 assert(FirstInput.getOpcode() == PPCISD::MFVSR &&
16292 "The input operand must be an fp-to-int conversion.");
16293
16294 // This combine happens after legalization so the fp_to_[su]i nodes are
16295 // already converted to PPCSISD nodes.
16296 unsigned FirstConversion = FirstInput.getOperand(i: 0).getOpcode();
16297 if (FirstConversion == PPCISD::FCTIDZ ||
16298 FirstConversion == PPCISD::FCTIDUZ ||
16299 FirstConversion == PPCISD::FCTIWZ ||
16300 FirstConversion == PPCISD::FCTIWUZ) {
16301 bool IsSplat = true;
16302 bool Is32Bit = FirstConversion == PPCISD::FCTIWZ ||
16303 FirstConversion == PPCISD::FCTIWUZ;
16304 EVT SrcVT = FirstInput.getOperand(i: 0).getValueType();
16305 SmallVector<SDValue, 4> Ops;
16306 EVT TargetVT = N->getValueType(ResNo: 0);
16307 for (int i = 0, e = N->getNumOperands(); i < e; ++i) {
16308 SDValue NextOp = N->getOperand(Num: i);
16309 if (NextOp.getOpcode() != PPCISD::MFVSR)
16310 return SDValue();
16311 unsigned NextConversion = NextOp.getOperand(i: 0).getOpcode();
16312 if (NextConversion != FirstConversion)
16313 return SDValue();
16314 // If we are converting to 32-bit integers, we need to add an FP_ROUND.
16315 // This is not valid if the input was originally double precision. It is
16316 // also not profitable to do unless this is an extending load in which
16317 // case doing this combine will allow us to combine consecutive loads.
16318 if (Is32Bit && !isFPExtLoad(Op: NextOp.getOperand(i: 0).getOperand(i: 0)))
16319 return SDValue();
16320 if (N->getOperand(Num: i) != FirstInput)
16321 IsSplat = false;
16322 }
16323
16324 // If this is a splat, we leave it as-is since there will be only a single
16325 // fp-to-int conversion followed by a splat of the integer. This is better
16326 // for 32-bit and smaller ints and neutral for 64-bit ints.
16327 if (IsSplat)
16328 return SDValue();
16329
16330 // Now that we know we have the right type of node, get its operands
16331 for (int i = 0, e = N->getNumOperands(); i < e; ++i) {
16332 SDValue In = N->getOperand(Num: i).getOperand(i: 0);
16333 if (Is32Bit) {
16334 // For 32-bit values, we need to add an FP_ROUND node (if we made it
16335 // here, we know that all inputs are extending loads so this is safe).
16336 if (In.isUndef())
16337 Ops.push_back(Elt: DAG.getUNDEF(VT: SrcVT));
16338 else {
16339 SDValue Trunc =
16340 DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: MVT::f32, N1: In.getOperand(i: 0),
16341 N2: DAG.getIntPtrConstant(Val: 1, DL: dl, /*isTarget=*/true));
16342 Ops.push_back(Elt: Trunc);
16343 }
16344 } else
16345 Ops.push_back(Elt: In.isUndef() ? DAG.getUNDEF(VT: SrcVT) : In.getOperand(i: 0));
16346 }
16347
16348 unsigned Opcode;
16349 if (FirstConversion == PPCISD::FCTIDZ ||
16350 FirstConversion == PPCISD::FCTIWZ)
16351 Opcode = ISD::FP_TO_SINT;
16352 else
16353 Opcode = ISD::FP_TO_UINT;
16354
16355 EVT NewVT = TargetVT == MVT::v2i64 ? MVT::v2f64 : MVT::v4f32;
16356 SDValue BV = DAG.getBuildVector(VT: NewVT, DL: dl, Ops);
16357 return DAG.getNode(Opcode, DL: dl, VT: TargetVT, Operand: BV);
16358 }
16359 return SDValue();
16360}
16361
16362// LXVKQ instruction load VSX vector with a special quadword value
16363// based on an immediate value. This helper method returns the details of the
16364// match as a tuple of {LXVKQ unsigned IMM Value, right_shift_amount}
16365// to help generate the LXVKQ instruction and the subsequent shift instruction
16366// required to match the original build vector pattern.
16367
16368// LXVKQPattern: {LXVKQ unsigned IMM Value, right_shift_amount}
16369using LXVKQPattern = std::tuple<uint32_t, uint8_t>;
16370
16371static std::optional<LXVKQPattern> getPatternInfo(const APInt &FullVal) {
16372
16373 // LXVKQ instruction loads the Quadword value:
16374 // 0x8000_0000_0000_0000_0000_0000_0000_0000 when imm = 0b10000
16375 static const APInt BasePattern = APInt(128, 0x8000000000000000ULL) << 64;
16376 static const uint32_t Uim = 16;
16377
16378 // Check for direct LXVKQ match (no shift needed)
16379 if (FullVal == BasePattern)
16380 return std::make_tuple(args: Uim, args: uint8_t{0});
16381
16382 // Check if FullValue is 1 (the result of the base pattern >> 127)
16383 if (FullVal == APInt(128, 1))
16384 return std::make_tuple(args: Uim, args: uint8_t{127});
16385
16386 return std::nullopt;
16387}
16388
16389/// Combine vector loads to a single load (using lxvkq) or splat with shift of a
16390/// constant (xxspltib + vsrq) by recognising patterns in the Build Vector.
16391/// LXVKQ instruction load VSX vector with a special quadword value based on an
16392/// immediate value. if UIM=0b10000 then LXVKQ loads VSR[32×TX+T] with value
16393/// 0x8000_0000_0000_0000_0000_0000_0000_0000.
16394/// This can be used to inline the build vector constants that have the
16395/// following patterns:
16396///
16397/// 0x8000_0000_0000_0000_0000_0000_0000_0000 (MSB set pattern)
16398/// 0x0000_0000_0000_0000_0000_0000_0000_0001 (LSB set pattern)
16399/// MSB pattern can directly loaded using LXVKQ while LSB is loaded using a
16400/// combination of splatting and right shift instructions.
16401
16402SDValue PPCTargetLowering::combineBVLoadsSpecialValue(SDValue Op,
16403 SelectionDAG &DAG) const {
16404
16405 assert((Op.getNode() && Op.getOpcode() == ISD::BUILD_VECTOR) &&
16406 "Expected a BuildVectorSDNode in combineBVLoadsSpecialValue");
16407
16408 // This transformation is only supported if we are loading either a byte,
16409 // halfword, word, or doubleword.
16410 EVT VT = Op.getValueType();
16411 if (!(VT == MVT::v8i16 || VT == MVT::v16i8 || VT == MVT::v4i32 ||
16412 VT == MVT::v2i64))
16413 return SDValue();
16414
16415 LLVM_DEBUG(llvm::dbgs() << "\ncombineBVLoadsSpecialValue: Build vector ("
16416 << VT.getEVTString() << "): ";
16417 Op->dump());
16418
16419 unsigned NumElems = VT.getVectorNumElements();
16420 unsigned ElemBits = VT.getScalarSizeInBits();
16421
16422 bool IsLittleEndian = DAG.getDataLayout().isLittleEndian();
16423
16424 // Check for Non-constant operand in the build vector.
16425 for (const SDValue &Operand : Op.getNode()->op_values()) {
16426 if (!isa<ConstantSDNode>(Val: Operand))
16427 return SDValue();
16428 }
16429
16430 // Assemble build vector operands as a 128-bit register value
16431 // We need to reconstruct what the 128-bit register pattern would be
16432 // that produces this vector when interpreted with the current endianness
16433 APInt FullVal = APInt::getZero(numBits: 128);
16434
16435 for (unsigned Index = 0; Index < NumElems; ++Index) {
16436 auto *C = cast<ConstantSDNode>(Val: Op.getOperand(i: Index));
16437
16438 // Get element value as raw bits (zero-extended)
16439 uint64_t ElemValue = C->getZExtValue();
16440
16441 // Mask to element size to ensure we only get the relevant bits
16442 if (ElemBits < 64)
16443 ElemValue &= ((1ULL << ElemBits) - 1);
16444
16445 // Calculate bit position for this element in the 128-bit register
16446 unsigned BitPos =
16447 (IsLittleEndian) ? (Index * ElemBits) : (128 - (Index + 1) * ElemBits);
16448
16449 // Create APInt for the element value and shift it to correct position
16450 APInt ElemAPInt(128, ElemValue);
16451 ElemAPInt <<= BitPos;
16452
16453 // Place the element value at the correct bit position
16454 FullVal |= ElemAPInt;
16455 }
16456
16457 if (FullVal.isZero() || FullVal.isAllOnes())
16458 return SDValue();
16459
16460 if (auto UIMOpt = getPatternInfo(FullVal)) {
16461 const auto &[Uim, ShiftAmount] = *UIMOpt;
16462 SDLoc Dl(Op);
16463
16464 // Generate LXVKQ instruction if the shift amount is zero.
16465 if (ShiftAmount == 0) {
16466 SDValue UimVal = DAG.getTargetConstant(Val: Uim, DL: Dl, VT: MVT::i32);
16467 SDValue LxvkqInstr =
16468 SDValue(DAG.getMachineNode(Opcode: PPC::LXVKQ, dl: Dl, VT, Op1: UimVal), 0);
16469 LLVM_DEBUG(llvm::dbgs()
16470 << "combineBVLoadsSpecialValue: Instruction Emitted ";
16471 LxvkqInstr.dump());
16472 return LxvkqInstr;
16473 }
16474
16475 assert(ShiftAmount == 127 && "Unexpected lxvkq shift amount value");
16476
16477 // The right shifted pattern can be constructed using a combination of
16478 // XXSPLTIB and VSRQ instruction. VSRQ uses the shift amount from the lower
16479 // 7 bits of byte 15. This can be specified using XXSPLTIB with immediate
16480 // value 255.
16481 SDValue ShiftAmountVec =
16482 SDValue(DAG.getMachineNode(Opcode: PPC::XXSPLTIB, dl: Dl, VT: MVT::v4i32,
16483 Op1: DAG.getTargetConstant(Val: 255, DL: Dl, VT: MVT::i32)),
16484 0);
16485 // Generate appropriate right shift instruction
16486 SDValue ShiftVec = SDValue(
16487 DAG.getMachineNode(Opcode: PPC::VSRQ, dl: Dl, VT, Op1: ShiftAmountVec, Op2: ShiftAmountVec),
16488 0);
16489 LLVM_DEBUG(llvm::dbgs()
16490 << "\n combineBVLoadsSpecialValue: Instruction Emitted ";
16491 ShiftVec.dump());
16492 return ShiftVec;
16493 }
16494 // No patterns matched for build vectors.
16495 return SDValue();
16496}
16497
16498/// Reduce the number of loads when building a vector.
16499///
16500/// Building a vector out of multiple loads can be converted to a load
16501/// of the vector type if the loads are consecutive. If the loads are
16502/// consecutive but in descending order, a shuffle is added at the end
16503/// to reorder the vector.
16504static SDValue combineBVOfConsecutiveLoads(SDNode *N, SelectionDAG &DAG) {
16505 assert(N->getOpcode() == ISD::BUILD_VECTOR &&
16506 "Should be called with a BUILD_VECTOR node");
16507
16508 SDLoc dl(N);
16509
16510 // Return early for non byte-sized type, as they can't be consecutive.
16511 if (!N->getValueType(ResNo: 0).getVectorElementType().isByteSized())
16512 return SDValue();
16513
16514 bool InputsAreConsecutiveLoads = true;
16515 bool InputsAreReverseConsecutive = true;
16516 unsigned ElemSize = N->getValueType(ResNo: 0).getScalarType().getStoreSize();
16517 SDValue FirstInput = N->getOperand(Num: 0);
16518 bool IsRoundOfExtLoad = false;
16519 LoadSDNode *FirstLoad = nullptr;
16520
16521 if (FirstInput.getOpcode() == ISD::FP_ROUND &&
16522 FirstInput.getOperand(i: 0).getOpcode() == ISD::LOAD) {
16523 FirstLoad = cast<LoadSDNode>(Val: FirstInput.getOperand(i: 0));
16524 IsRoundOfExtLoad = FirstLoad->getExtensionType() == ISD::EXTLOAD;
16525 }
16526 // Not a build vector of (possibly fp_rounded) loads.
16527 if ((!IsRoundOfExtLoad && FirstInput.getOpcode() != ISD::LOAD) ||
16528 N->getNumOperands() == 1)
16529 return SDValue();
16530
16531 if (!IsRoundOfExtLoad)
16532 FirstLoad = cast<LoadSDNode>(Val&: FirstInput);
16533
16534 SmallVector<LoadSDNode *, 4> InputLoads;
16535 InputLoads.push_back(Elt: FirstLoad);
16536 for (int i = 1, e = N->getNumOperands(); i < e; ++i) {
16537 // If any inputs are fp_round(extload), they all must be.
16538 if (IsRoundOfExtLoad && N->getOperand(Num: i).getOpcode() != ISD::FP_ROUND)
16539 return SDValue();
16540
16541 SDValue NextInput = IsRoundOfExtLoad ? N->getOperand(Num: i).getOperand(i: 0) :
16542 N->getOperand(Num: i);
16543 if (NextInput.getOpcode() != ISD::LOAD)
16544 return SDValue();
16545
16546 SDValue PreviousInput =
16547 IsRoundOfExtLoad ? N->getOperand(Num: i-1).getOperand(i: 0) : N->getOperand(Num: i-1);
16548 LoadSDNode *LD1 = cast<LoadSDNode>(Val&: PreviousInput);
16549 LoadSDNode *LD2 = cast<LoadSDNode>(Val&: NextInput);
16550
16551 // If any inputs are fp_round(extload), they all must be.
16552 if (IsRoundOfExtLoad && LD2->getExtensionType() != ISD::EXTLOAD)
16553 return SDValue();
16554
16555 // We only care about regular loads. The PPC-specific load intrinsics
16556 // will not lead to a merge opportunity.
16557 if (!DAG.areNonVolatileConsecutiveLoads(LD: LD2, Base: LD1, Bytes: ElemSize, Dist: 1))
16558 InputsAreConsecutiveLoads = false;
16559 if (!DAG.areNonVolatileConsecutiveLoads(LD: LD1, Base: LD2, Bytes: ElemSize, Dist: 1))
16560 InputsAreReverseConsecutive = false;
16561
16562 // Exit early if the loads are neither consecutive nor reverse consecutive.
16563 if (!InputsAreConsecutiveLoads && !InputsAreReverseConsecutive)
16564 return SDValue();
16565 InputLoads.push_back(Elt: LD2);
16566 }
16567
16568 assert(!(InputsAreConsecutiveLoads && InputsAreReverseConsecutive) &&
16569 "The loads cannot be both consecutive and reverse consecutive.");
16570
16571 SDValue WideLoad;
16572 SDValue ReturnSDVal;
16573 if (InputsAreConsecutiveLoads) {
16574 assert(FirstLoad && "Input needs to be a LoadSDNode.");
16575 WideLoad = DAG.getLoad(VT: N->getValueType(ResNo: 0), dl, Chain: FirstLoad->getChain(),
16576 Ptr: FirstLoad->getBasePtr(), PtrInfo: FirstLoad->getPointerInfo(),
16577 Alignment: FirstLoad->getAlign());
16578 ReturnSDVal = WideLoad;
16579 } else if (InputsAreReverseConsecutive) {
16580 LoadSDNode *LastLoad = InputLoads.back();
16581 assert(LastLoad && "Input needs to be a LoadSDNode.");
16582 WideLoad = DAG.getLoad(VT: N->getValueType(ResNo: 0), dl, Chain: LastLoad->getChain(),
16583 Ptr: LastLoad->getBasePtr(), PtrInfo: LastLoad->getPointerInfo(),
16584 Alignment: LastLoad->getAlign());
16585 SmallVector<int, 16> Ops;
16586 for (int i = N->getNumOperands() - 1; i >= 0; i--)
16587 Ops.push_back(Elt: i);
16588
16589 ReturnSDVal = DAG.getVectorShuffle(VT: N->getValueType(ResNo: 0), dl, N1: WideLoad,
16590 N2: DAG.getUNDEF(VT: N->getValueType(ResNo: 0)), Mask: Ops);
16591 } else
16592 return SDValue();
16593
16594 for (auto *LD : InputLoads)
16595 DAG.makeEquivalentMemoryOrdering(OldLoad: LD, NewMemOp: WideLoad);
16596 return ReturnSDVal;
16597}
16598
16599// This function adds the required vector_shuffle needed to get
16600// the elements of the vector extract in the correct position
16601// as specified by the CorrectElems encoding.
16602static SDValue addShuffleForVecExtend(SDNode *N, SelectionDAG &DAG,
16603 SDValue Input, uint64_t Elems,
16604 uint64_t CorrectElems) {
16605 SDLoc dl(N);
16606
16607 unsigned NumElems = Input.getValueType().getVectorNumElements();
16608 SmallVector<int, 16> ShuffleMask(NumElems, -1);
16609
16610 // Knowing the element indices being extracted from the original
16611 // vector and the order in which they're being inserted, just put
16612 // them at element indices required for the instruction.
16613 for (unsigned i = 0; i < N->getNumOperands(); i++) {
16614 if (DAG.getDataLayout().isLittleEndian())
16615 ShuffleMask[CorrectElems & 0xF] = Elems & 0xF;
16616 else
16617 ShuffleMask[(CorrectElems & 0xF0) >> 4] = (Elems & 0xF0) >> 4;
16618 CorrectElems = CorrectElems >> 8;
16619 Elems = Elems >> 8;
16620 }
16621
16622 SDValue Shuffle =
16623 DAG.getVectorShuffle(VT: Input.getValueType(), dl, N1: Input,
16624 N2: DAG.getUNDEF(VT: Input.getValueType()), Mask: ShuffleMask);
16625
16626 EVT VT = N->getValueType(ResNo: 0);
16627 SDValue Conv = DAG.getBitcast(VT, V: Shuffle);
16628
16629 EVT ExtVT = EVT::getVectorVT(Context&: *DAG.getContext(),
16630 VT: Input.getValueType().getVectorElementType(),
16631 NumElements: VT.getVectorNumElements());
16632 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl, VT, N1: Conv,
16633 N2: DAG.getValueType(ExtVT));
16634}
16635
16636// Look for build vector patterns where input operands come from sign
16637// extended vector_extract elements of specific indices. If the correct indices
16638// aren't used, add a vector shuffle to fix up the indices and create
16639// SIGN_EXTEND_INREG node which selects the vector sign extend instructions
16640// during instruction selection.
16641static SDValue combineBVOfVecSExt(SDNode *N, SelectionDAG &DAG) {
16642 // This array encodes the indices that the vector sign extend instructions
16643 // extract from when extending from one type to another for both BE and LE.
16644 // The right nibble of each byte corresponds to the LE incides.
16645 // and the left nibble of each byte corresponds to the BE incides.
16646 // For example: 0x3074B8FC byte->word
16647 // For LE: the allowed indices are: 0x0,0x4,0x8,0xC
16648 // For BE: the allowed indices are: 0x3,0x7,0xB,0xF
16649 // For example: 0x000070F8 byte->double word
16650 // For LE: the allowed indices are: 0x0,0x8
16651 // For BE: the allowed indices are: 0x7,0xF
16652 uint64_t TargetElems[] = {
16653 0x3074B8FC, // b->w
16654 0x000070F8, // b->d
16655 0x10325476, // h->w
16656 0x00003074, // h->d
16657 0x00001032, // w->d
16658 };
16659
16660 uint64_t Elems = 0;
16661 int Index;
16662 SDValue Input;
16663
16664 auto isSExtOfVecExtract = [&](SDValue Op) -> bool {
16665 if (!Op)
16666 return false;
16667 if (Op.getOpcode() != ISD::SIGN_EXTEND &&
16668 Op.getOpcode() != ISD::SIGN_EXTEND_INREG)
16669 return false;
16670
16671 // A SIGN_EXTEND_INREG might be fed by an ANY_EXTEND to produce a value
16672 // of the right width.
16673 SDValue Extract = Op.getOperand(i: 0);
16674 if (Extract.getOpcode() == ISD::ANY_EXTEND)
16675 Extract = Extract.getOperand(i: 0);
16676 if (Extract.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
16677 return false;
16678
16679 ConstantSDNode *ExtOp = dyn_cast<ConstantSDNode>(Val: Extract.getOperand(i: 1));
16680 if (!ExtOp)
16681 return false;
16682
16683 Index = ExtOp->getZExtValue();
16684 if (Input && Input != Extract.getOperand(i: 0))
16685 return false;
16686
16687 if (!Input)
16688 Input = Extract.getOperand(i: 0);
16689
16690 Elems = Elems << 8;
16691 Index = DAG.getDataLayout().isLittleEndian() ? Index : Index << 4;
16692 Elems |= Index;
16693
16694 return true;
16695 };
16696
16697 // If the build vector operands aren't sign extended vector extracts,
16698 // of the same input vector, then return.
16699 for (unsigned i = 0; i < N->getNumOperands(); i++) {
16700 if (!isSExtOfVecExtract(N->getOperand(Num: i))) {
16701 return SDValue();
16702 }
16703 }
16704
16705 // If the vector extract indices are not correct, add the appropriate
16706 // vector_shuffle.
16707 int TgtElemArrayIdx;
16708 int InputSize = Input.getValueType().getScalarSizeInBits();
16709 int OutputSize = N->getValueType(ResNo: 0).getScalarSizeInBits();
16710 if (InputSize + OutputSize == 40)
16711 TgtElemArrayIdx = 0;
16712 else if (InputSize + OutputSize == 72)
16713 TgtElemArrayIdx = 1;
16714 else if (InputSize + OutputSize == 48)
16715 TgtElemArrayIdx = 2;
16716 else if (InputSize + OutputSize == 80)
16717 TgtElemArrayIdx = 3;
16718 else if (InputSize + OutputSize == 96)
16719 TgtElemArrayIdx = 4;
16720 else
16721 return SDValue();
16722
16723 uint64_t CorrectElems = TargetElems[TgtElemArrayIdx];
16724 CorrectElems = DAG.getDataLayout().isLittleEndian()
16725 ? CorrectElems & 0x0F0F0F0F0F0F0F0F
16726 : CorrectElems & 0xF0F0F0F0F0F0F0F0;
16727 if (Elems != CorrectElems) {
16728 return addShuffleForVecExtend(N, DAG, Input, Elems, CorrectElems);
16729 }
16730
16731 // Regular lowering will catch cases where a shuffle is not needed.
16732 return SDValue();
16733}
16734
16735// Look for the pattern of a load from a narrow width to i128, feeding
16736// into a BUILD_VECTOR of v1i128. Replace this sequence with a PPCISD node
16737// (LXVRZX). This node represents a zero extending load that will be matched
16738// to the Load VSX Vector Rightmost instructions.
16739static SDValue combineBVZEXTLOAD(SDNode *N, SelectionDAG &DAG) {
16740 SDLoc DL(N);
16741
16742 // This combine is only eligible for a BUILD_VECTOR of v1i128.
16743 if (N->getValueType(ResNo: 0) != MVT::v1i128)
16744 return SDValue();
16745
16746 SDValue Operand = N->getOperand(Num: 0);
16747 // Proceed with the transformation if the operand to the BUILD_VECTOR
16748 // is a load instruction.
16749 if (Operand.getOpcode() != ISD::LOAD)
16750 return SDValue();
16751
16752 auto *LD = cast<LoadSDNode>(Val&: Operand);
16753 EVT MemoryType = LD->getMemoryVT();
16754
16755 // This transformation is only valid if the we are loading either a byte,
16756 // halfword, word, or doubleword.
16757 bool ValidLDType = MemoryType == MVT::i8 || MemoryType == MVT::i16 ||
16758 MemoryType == MVT::i32 || MemoryType == MVT::i64;
16759
16760 // Ensure that the load from the narrow width is being zero extended to i128.
16761 if (!ValidLDType ||
16762 (LD->getExtensionType() != ISD::ZEXTLOAD &&
16763 LD->getExtensionType() != ISD::EXTLOAD))
16764 return SDValue();
16765
16766 SDValue LoadOps[] = {
16767 LD->getChain(), LD->getBasePtr(),
16768 DAG.getIntPtrConstant(Val: MemoryType.getScalarSizeInBits(), DL)};
16769
16770 return DAG.getMemIntrinsicNode(Opcode: PPCISD::LXVRZX, dl: DL,
16771 VTList: DAG.getVTList(VT1: MVT::v1i128, VT2: MVT::Other),
16772 Ops: LoadOps, MemVT: MemoryType, MMO: LD->getMemOperand());
16773}
16774
16775SDValue PPCTargetLowering::DAGCombineBuildVector(SDNode *N,
16776 DAGCombinerInfo &DCI) const {
16777 assert(N->getOpcode() == ISD::BUILD_VECTOR &&
16778 "Should be called with a BUILD_VECTOR node");
16779
16780 SelectionDAG &DAG = DCI.DAG;
16781 SDLoc dl(N);
16782
16783 if (!Subtarget.hasVSX())
16784 return SDValue();
16785
16786 // The target independent DAG combiner will leave a build_vector of
16787 // float-to-int conversions intact. We can generate MUCH better code for
16788 // a float-to-int conversion of a vector of floats.
16789 SDValue FirstInput = N->getOperand(Num: 0);
16790 if (FirstInput.getOpcode() == PPCISD::MFVSR) {
16791 SDValue Reduced = combineElementTruncationToVectorTruncation(N, DCI);
16792 if (Reduced)
16793 return Reduced;
16794 }
16795
16796 // If we're building a vector out of consecutive loads, just load that
16797 // vector type.
16798 SDValue Reduced = combineBVOfConsecutiveLoads(N, DAG);
16799 if (Reduced)
16800 return Reduced;
16801
16802 // If we're building a vector out of extended elements from another vector
16803 // we have P9 vector integer extend instructions. The code assumes legal
16804 // input types (i.e. it can't handle things like v4i16) so do not run before
16805 // legalization.
16806 if (Subtarget.hasP9Altivec() && !DCI.isBeforeLegalize()) {
16807 Reduced = combineBVOfVecSExt(N, DAG);
16808 if (Reduced)
16809 return Reduced;
16810 }
16811
16812 // On Power10, the Load VSX Vector Rightmost instructions can be utilized
16813 // if this is a BUILD_VECTOR of v1i128, and if the operand to the BUILD_VECTOR
16814 // is a load from <valid narrow width> to i128.
16815 if (Subtarget.isISA3_1()) {
16816 SDValue BVOfZLoad = combineBVZEXTLOAD(N, DAG);
16817 if (BVOfZLoad)
16818 return BVOfZLoad;
16819 }
16820
16821 if (N->getValueType(ResNo: 0) != MVT::v2f64)
16822 return SDValue();
16823
16824 // Looking for:
16825 // (build_vector ([su]int_to_fp (extractelt 0)), [su]int_to_fp (extractelt 1))
16826 if (FirstInput.getOpcode() != ISD::SINT_TO_FP &&
16827 FirstInput.getOpcode() != ISD::UINT_TO_FP)
16828 return SDValue();
16829 if (N->getOperand(Num: 1).getOpcode() != ISD::SINT_TO_FP &&
16830 N->getOperand(Num: 1).getOpcode() != ISD::UINT_TO_FP)
16831 return SDValue();
16832 if (FirstInput.getOpcode() != N->getOperand(Num: 1).getOpcode())
16833 return SDValue();
16834
16835 SDValue Ext1 = FirstInput.getOperand(i: 0);
16836 SDValue Ext2 = N->getOperand(Num: 1).getOperand(i: 0);
16837 if(Ext1.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
16838 Ext2.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
16839 return SDValue();
16840
16841 ConstantSDNode *Ext1Op = dyn_cast<ConstantSDNode>(Val: Ext1.getOperand(i: 1));
16842 ConstantSDNode *Ext2Op = dyn_cast<ConstantSDNode>(Val: Ext2.getOperand(i: 1));
16843 if (!Ext1Op || !Ext2Op)
16844 return SDValue();
16845 if (Ext1.getOperand(i: 0).getValueType() != MVT::v4i32 ||
16846 Ext1.getOperand(i: 0) != Ext2.getOperand(i: 0))
16847 return SDValue();
16848
16849 int FirstElem = Ext1Op->getZExtValue();
16850 int SecondElem = Ext2Op->getZExtValue();
16851 int SubvecIdx;
16852 if (FirstElem == 0 && SecondElem == 1)
16853 SubvecIdx = Subtarget.isLittleEndian() ? 1 : 0;
16854 else if (FirstElem == 2 && SecondElem == 3)
16855 SubvecIdx = Subtarget.isLittleEndian() ? 0 : 1;
16856 else
16857 return SDValue();
16858
16859 SDValue SrcVec = Ext1.getOperand(i: 0);
16860 auto NodeType = (N->getOperand(Num: 1).getOpcode() == ISD::SINT_TO_FP) ?
16861 PPCISD::SINT_VEC_TO_FP : PPCISD::UINT_VEC_TO_FP;
16862 return DAG.getNode(Opcode: NodeType, DL: dl, VT: MVT::v2f64,
16863 N1: SrcVec, N2: DAG.getIntPtrConstant(Val: SubvecIdx, DL: dl));
16864}
16865
16866SDValue PPCTargetLowering::combineFPToIntToFP(SDNode *N,
16867 DAGCombinerInfo &DCI) const {
16868 assert((N->getOpcode() == ISD::SINT_TO_FP ||
16869 N->getOpcode() == ISD::UINT_TO_FP) &&
16870 "Need an int -> FP conversion node here");
16871
16872 if (useSoftFloat() || !Subtarget.has64BitSupport())
16873 return SDValue();
16874
16875 SelectionDAG &DAG = DCI.DAG;
16876 SDLoc dl(N);
16877 SDValue Op(N, 0);
16878
16879 // Don't handle ppc_fp128 here or conversions that are out-of-range capable
16880 // from the hardware.
16881 if (Op.getValueType() != MVT::f32 && Op.getValueType() != MVT::f64)
16882 return SDValue();
16883 if (!Op.getOperand(i: 0).getValueType().isSimple())
16884 return SDValue();
16885 if (Op.getOperand(i: 0).getValueType().getSimpleVT() <= MVT(MVT::i1) ||
16886 Op.getOperand(i: 0).getValueType().getSimpleVT() > MVT(MVT::i64))
16887 return SDValue();
16888
16889 SDValue FirstOperand(Op.getOperand(i: 0));
16890 bool SubWordLoad = FirstOperand.getOpcode() == ISD::LOAD &&
16891 (FirstOperand.getValueType() == MVT::i8 ||
16892 FirstOperand.getValueType() == MVT::i16);
16893 if (Subtarget.hasP9Vector() && Subtarget.hasP9Altivec() && SubWordLoad) {
16894 bool Signed = N->getOpcode() == ISD::SINT_TO_FP;
16895 bool DstDouble = Op.getValueType() == MVT::f64;
16896 unsigned ConvOp = Signed ?
16897 (DstDouble ? PPCISD::FCFID : PPCISD::FCFIDS) :
16898 (DstDouble ? PPCISD::FCFIDU : PPCISD::FCFIDUS);
16899 SDValue WidthConst =
16900 DAG.getIntPtrConstant(Val: FirstOperand.getValueType() == MVT::i8 ? 1 : 2,
16901 DL: dl, isTarget: false);
16902 LoadSDNode *LDN = cast<LoadSDNode>(Val: FirstOperand.getNode());
16903 SDValue Ops[] = { LDN->getChain(), LDN->getBasePtr(), WidthConst };
16904 SDValue Ld = DAG.getMemIntrinsicNode(Opcode: PPCISD::LXSIZX, dl,
16905 VTList: DAG.getVTList(VT1: MVT::f64, VT2: MVT::Other),
16906 Ops, MemVT: MVT::i8, MMO: LDN->getMemOperand());
16907 DAG.makeEquivalentMemoryOrdering(OldLoad: LDN, NewMemOp: Ld);
16908
16909 // For signed conversion, we need to sign-extend the value in the VSR
16910 if (Signed) {
16911 SDValue ExtOps[] = { Ld, WidthConst };
16912 SDValue Ext = DAG.getNode(Opcode: PPCISD::VEXTS, DL: dl, VT: MVT::f64, Ops: ExtOps);
16913 return DAG.getNode(Opcode: ConvOp, DL: dl, VT: DstDouble ? MVT::f64 : MVT::f32, Operand: Ext);
16914 } else
16915 return DAG.getNode(Opcode: ConvOp, DL: dl, VT: DstDouble ? MVT::f64 : MVT::f32, Operand: Ld);
16916 }
16917
16918
16919 // For i32 intermediate values, unfortunately, the conversion functions
16920 // leave the upper 32 bits of the value are undefined. Within the set of
16921 // scalar instructions, we have no method for zero- or sign-extending the
16922 // value. Thus, we cannot handle i32 intermediate values here.
16923 if (Op.getOperand(i: 0).getValueType() == MVT::i32)
16924 return SDValue();
16925
16926 assert((Op.getOpcode() == ISD::SINT_TO_FP || Subtarget.hasFPCVT()) &&
16927 "UINT_TO_FP is supported only with FPCVT");
16928
16929 // If we have FCFIDS, then use it when converting to single-precision.
16930 // Otherwise, convert to double-precision and then round.
16931 unsigned FCFOp = (Subtarget.hasFPCVT() && Op.getValueType() == MVT::f32)
16932 ? (Op.getOpcode() == ISD::UINT_TO_FP ? PPCISD::FCFIDUS
16933 : PPCISD::FCFIDS)
16934 : (Op.getOpcode() == ISD::UINT_TO_FP ? PPCISD::FCFIDU
16935 : PPCISD::FCFID);
16936 MVT FCFTy = (Subtarget.hasFPCVT() && Op.getValueType() == MVT::f32)
16937 ? MVT::f32
16938 : MVT::f64;
16939
16940 // If we're converting from a float, to an int, and back to a float again,
16941 // then we don't need the store/load pair at all.
16942 if ((Op.getOperand(i: 0).getOpcode() == ISD::FP_TO_UINT &&
16943 Subtarget.hasFPCVT()) ||
16944 (Op.getOperand(i: 0).getOpcode() == ISD::FP_TO_SINT)) {
16945 SDValue Src = Op.getOperand(i: 0).getOperand(i: 0);
16946 if (Src.getValueType() == MVT::f32) {
16947 Src = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: Src);
16948 DCI.AddToWorklist(N: Src.getNode());
16949 } else if (Src.getValueType() != MVT::f64) {
16950 // Make sure that we don't pick up a ppc_fp128 source value.
16951 return SDValue();
16952 }
16953
16954 unsigned FCTOp =
16955 Op.getOperand(i: 0).getOpcode() == ISD::FP_TO_SINT ? PPCISD::FCTIDZ :
16956 PPCISD::FCTIDUZ;
16957
16958 SDValue Tmp = DAG.getNode(Opcode: FCTOp, DL: dl, VT: MVT::f64, Operand: Src);
16959 SDValue FP = DAG.getNode(Opcode: FCFOp, DL: dl, VT: FCFTy, Operand: Tmp);
16960
16961 if (Op.getValueType() == MVT::f32 && !Subtarget.hasFPCVT()) {
16962 FP = DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: MVT::f32, N1: FP,
16963 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
16964 DCI.AddToWorklist(N: FP.getNode());
16965 }
16966
16967 return FP;
16968 }
16969
16970 return SDValue();
16971}
16972
16973// expandVSXLoadForLE - Convert VSX loads (which may be intrinsics for
16974// builtins) into loads with swaps.
16975SDValue PPCTargetLowering::expandVSXLoadForLE(SDNode *N,
16976 DAGCombinerInfo &DCI) const {
16977 // Delay VSX load for LE combine until after LegalizeOps to prioritize other
16978 // load combines.
16979 if (DCI.isBeforeLegalizeOps())
16980 return SDValue();
16981
16982 SelectionDAG &DAG = DCI.DAG;
16983 SDLoc dl(N);
16984 SDValue Chain;
16985 SDValue Base;
16986 MachineMemOperand *MMO;
16987
16988 switch (N->getOpcode()) {
16989 default:
16990 llvm_unreachable("Unexpected opcode for little endian VSX load");
16991 case ISD::LOAD: {
16992 LoadSDNode *LD = cast<LoadSDNode>(Val: N);
16993 Chain = LD->getChain();
16994 Base = LD->getBasePtr();
16995 MMO = LD->getMemOperand();
16996 // If the MMO suggests this isn't a load of a full vector, leave
16997 // things alone. For a built-in, we have to make the change for
16998 // correctness, so if there is a size problem that will be a bug.
16999 if (!MMO->getSize().hasValue() || MMO->getSize().getValue() < 16)
17000 return SDValue();
17001 break;
17002 }
17003 case ISD::INTRINSIC_W_CHAIN: {
17004 MemIntrinsicSDNode *Intrin = cast<MemIntrinsicSDNode>(Val: N);
17005 Chain = Intrin->getChain();
17006 // Similarly to the store case below, Intrin->getBasePtr() doesn't get
17007 // us what we want. Get operand 2 instead.
17008 Base = Intrin->getOperand(Num: 2);
17009 MMO = Intrin->getMemOperand();
17010 break;
17011 }
17012 }
17013
17014 MVT VecTy = N->getValueType(ResNo: 0).getSimpleVT();
17015
17016 SDValue LoadOps[] = { Chain, Base };
17017 SDValue Load = DAG.getMemIntrinsicNode(Opcode: PPCISD::LXVD2X, dl,
17018 VTList: DAG.getVTList(VT1: MVT::v2f64, VT2: MVT::Other),
17019 Ops: LoadOps, MemVT: MVT::v2f64, MMO);
17020
17021 DCI.AddToWorklist(N: Load.getNode());
17022 Chain = Load.getValue(R: 1);
17023 SDValue Swap = DAG.getNode(
17024 Opcode: PPCISD::XXSWAPD, DL: dl, VTList: DAG.getVTList(VT1: MVT::v2f64, VT2: MVT::Other), N1: Chain, N2: Load);
17025 DCI.AddToWorklist(N: Swap.getNode());
17026
17027 // Add a bitcast if the resulting load type doesn't match v2f64.
17028 if (VecTy != MVT::v2f64) {
17029 SDValue N = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: VecTy, Operand: Swap);
17030 DCI.AddToWorklist(N: N.getNode());
17031 // Package {bitcast value, swap's chain} to match Load's shape.
17032 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: dl, VTList: DAG.getVTList(VT1: VecTy, VT2: MVT::Other),
17033 N1: N, N2: Swap.getValue(R: 1));
17034 }
17035
17036 return Swap;
17037}
17038
17039// expandVSXStoreForLE - Convert VSX stores (which may be intrinsics for
17040// builtins) into stores with swaps.
17041SDValue PPCTargetLowering::expandVSXStoreForLE(SDNode *N,
17042 DAGCombinerInfo &DCI) const {
17043 // Delay VSX store for LE combine until after LegalizeOps to prioritize other
17044 // store combines.
17045 if (DCI.isBeforeLegalizeOps())
17046 return SDValue();
17047
17048 SelectionDAG &DAG = DCI.DAG;
17049 SDLoc dl(N);
17050 SDValue Chain;
17051 SDValue Base;
17052 unsigned SrcOpnd;
17053 MachineMemOperand *MMO;
17054
17055 switch (N->getOpcode()) {
17056 default:
17057 llvm_unreachable("Unexpected opcode for little endian VSX store");
17058 case ISD::STORE: {
17059 StoreSDNode *ST = cast<StoreSDNode>(Val: N);
17060 Chain = ST->getChain();
17061 Base = ST->getBasePtr();
17062 MMO = ST->getMemOperand();
17063 SrcOpnd = 1;
17064 // If the MMO suggests this isn't a store of a full vector, leave
17065 // things alone. For a built-in, we have to make the change for
17066 // correctness, so if there is a size problem that will be a bug.
17067 if (!MMO->getSize().hasValue() || MMO->getSize().getValue() < 16)
17068 return SDValue();
17069 break;
17070 }
17071 case ISD::INTRINSIC_VOID: {
17072 MemIntrinsicSDNode *Intrin = cast<MemIntrinsicSDNode>(Val: N);
17073 Chain = Intrin->getChain();
17074 // Intrin->getBasePtr() oddly does not get what we want.
17075 Base = Intrin->getOperand(Num: 3);
17076 MMO = Intrin->getMemOperand();
17077 SrcOpnd = 2;
17078 break;
17079 }
17080 }
17081
17082 SDValue Src = N->getOperand(Num: SrcOpnd);
17083 MVT VecTy = Src.getValueType().getSimpleVT();
17084
17085 // All stores are done as v2f64 and possible bit cast.
17086 if (VecTy != MVT::v2f64) {
17087 Src = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v2f64, Operand: Src);
17088 DCI.AddToWorklist(N: Src.getNode());
17089 }
17090
17091 SDValue Swap = DAG.getNode(Opcode: PPCISD::XXSWAPD, DL: dl,
17092 VTList: DAG.getVTList(VT1: MVT::v2f64, VT2: MVT::Other), N1: Chain, N2: Src);
17093 DCI.AddToWorklist(N: Swap.getNode());
17094 Chain = Swap.getValue(R: 1);
17095 SDValue StoreOps[] = { Chain, Swap, Base };
17096 SDValue Store = DAG.getMemIntrinsicNode(Opcode: PPCISD::STXVD2X, dl,
17097 VTList: DAG.getVTList(VT: MVT::Other),
17098 Ops: StoreOps, MemVT: VecTy, MMO);
17099 DCI.AddToWorklist(N: Store.getNode());
17100 return Store;
17101}
17102
17103// Handle DAG combine for STORE (FP_TO_INT F).
17104SDValue PPCTargetLowering::combineStoreFPToInt(SDNode *N,
17105 DAGCombinerInfo &DCI) const {
17106 SelectionDAG &DAG = DCI.DAG;
17107 SDLoc dl(N);
17108 unsigned Opcode = N->getOperand(Num: 1).getOpcode();
17109 (void)Opcode;
17110 bool Strict = N->getOperand(Num: 1)->isStrictFPOpcode();
17111
17112 assert((Opcode == ISD::FP_TO_SINT || Opcode == ISD::FP_TO_UINT ||
17113 Opcode == ISD::STRICT_FP_TO_SINT || Opcode == ISD::STRICT_FP_TO_UINT)
17114 && "Not a FP_TO_INT Instruction!");
17115
17116 SDValue Val = N->getOperand(Num: 1).getOperand(i: Strict ? 1 : 0);
17117 EVT Op1VT = N->getOperand(Num: 1).getValueType();
17118 EVT ResVT = Val.getValueType();
17119
17120 if (!Subtarget.hasVSX() || !Subtarget.hasFPCVT() || !isTypeLegal(VT: ResVT))
17121 return SDValue();
17122
17123 // Only perform combine for conversion to i64/i32 or power9 i16/i8.
17124 bool ValidTypeForStoreFltAsInt =
17125 (Op1VT == MVT::i32 || (Op1VT == MVT::i64 && Subtarget.isPPC64()) ||
17126 (Subtarget.hasP9Vector() && (Op1VT == MVT::i16 || Op1VT == MVT::i8)));
17127
17128 // TODO: Lower conversion from f128 on all VSX targets
17129 if (ResVT == MVT::ppcf128 || (ResVT == MVT::f128 && !Subtarget.hasP9Vector()))
17130 return SDValue();
17131
17132 if ((Op1VT != MVT::i64 && !Subtarget.hasP8Vector()) ||
17133 cast<StoreSDNode>(Val: N)->isTruncatingStore() || !ValidTypeForStoreFltAsInt)
17134 return SDValue();
17135
17136 Val = convertFPToInt(Op: N->getOperand(Num: 1), DAG, Subtarget);
17137
17138 // Set number of bytes being converted.
17139 unsigned ByteSize = Op1VT.getScalarSizeInBits() / 8;
17140 SDValue Ops[] = {N->getOperand(Num: 0), Val, N->getOperand(Num: 2),
17141 DAG.getIntPtrConstant(Val: ByteSize, DL: dl, isTarget: false),
17142 DAG.getValueType(Op1VT)};
17143
17144 Val = DAG.getMemIntrinsicNode(Opcode: PPCISD::ST_VSR_SCAL_INT, dl,
17145 VTList: DAG.getVTList(VT: MVT::Other), Ops,
17146 MemVT: cast<StoreSDNode>(Val: N)->getMemoryVT(),
17147 MMO: cast<StoreSDNode>(Val: N)->getMemOperand());
17148
17149 return Val;
17150}
17151
17152static bool isAlternatingShuffMask(const ArrayRef<int> &Mask, int NumElts) {
17153 // Check that the source of the element keeps flipping
17154 // (i.e. Mask[i] < NumElts -> Mask[i+i] >= NumElts).
17155 bool PrevElemFromFirstVec = Mask[0] < NumElts;
17156 for (int i = 1, e = Mask.size(); i < e; i++) {
17157 if (PrevElemFromFirstVec && Mask[i] < NumElts)
17158 return false;
17159 if (!PrevElemFromFirstVec && Mask[i] >= NumElts)
17160 return false;
17161 PrevElemFromFirstVec = !PrevElemFromFirstVec;
17162 }
17163 return true;
17164}
17165
17166static bool isSplatBV(SDValue Op) {
17167 if (Op.getOpcode() != ISD::BUILD_VECTOR)
17168 return false;
17169 SDValue FirstOp;
17170
17171 // Find first non-undef input.
17172 for (int i = 0, e = Op.getNumOperands(); i < e; i++) {
17173 FirstOp = Op.getOperand(i);
17174 if (!FirstOp.isUndef())
17175 break;
17176 }
17177
17178 // All inputs are undef or the same as the first non-undef input.
17179 for (int i = 1, e = Op.getNumOperands(); i < e; i++)
17180 if (Op.getOperand(i) != FirstOp && !Op.getOperand(i).isUndef())
17181 return false;
17182 return true;
17183}
17184
17185static SDValue isScalarToVec(SDValue Op) {
17186 if (Op.getOpcode() == ISD::SCALAR_TO_VECTOR)
17187 return Op;
17188 if (Op.getOpcode() != ISD::BITCAST)
17189 return SDValue();
17190 Op = Op.getOperand(i: 0);
17191 if (Op.getOpcode() == ISD::SCALAR_TO_VECTOR)
17192 return Op;
17193 return SDValue();
17194}
17195
17196// Fix up the shuffle mask to account for the fact that the result of
17197// scalar_to_vector is not in lane zero. This just takes all values in
17198// the ranges specified by the min/max indices and adds the number of
17199// elements required to ensure each element comes from the respective
17200// position in the valid lane.
17201// On little endian, that's just the corresponding element in the other
17202// half of the vector. On big endian, it is in the same half but right
17203// justified rather than left justified in that half.
17204static void fixupShuffleMaskForPermutedSToV(
17205 SmallVectorImpl<int> &ShuffV, int LHSFirstElt, int LHSLastElt,
17206 int RHSFirstElt, int RHSLastElt, int HalfVec, unsigned LHSNumValidElts,
17207 unsigned RHSNumValidElts, const PPCSubtarget &Subtarget) {
17208 int LHSEltFixup =
17209 Subtarget.isLittleEndian() ? HalfVec : HalfVec - LHSNumValidElts;
17210 int RHSEltFixup =
17211 Subtarget.isLittleEndian() ? HalfVec : HalfVec - RHSNumValidElts;
17212 for (int I = 0, E = ShuffV.size(); I < E; ++I) {
17213 int Idx = ShuffV[I];
17214 if (Idx >= LHSFirstElt && Idx <= LHSLastElt)
17215 ShuffV[I] += LHSEltFixup;
17216 else if (Idx >= RHSFirstElt && Idx <= RHSLastElt)
17217 ShuffV[I] += RHSEltFixup;
17218 }
17219}
17220
17221// Replace a SCALAR_TO_VECTOR with a SCALAR_TO_VECTOR_PERMUTED except if
17222// the original is:
17223// (<n x Ty> (scalar_to_vector (Ty (extract_elt <n x Ty> %a, C))))
17224// In such a case, just change the shuffle mask to extract the element
17225// from the permuted index.
17226static SDValue getSToVPermuted(SDValue OrigSToV, SelectionDAG &DAG,
17227 const PPCSubtarget &Subtarget) {
17228 SDLoc dl(OrigSToV);
17229 EVT VT = OrigSToV.getValueType();
17230 assert(OrigSToV.getOpcode() == ISD::SCALAR_TO_VECTOR &&
17231 "Expecting a SCALAR_TO_VECTOR here");
17232 SDValue Input = OrigSToV.getOperand(i: 0);
17233
17234 if (Input.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
17235 ConstantSDNode *Idx = dyn_cast<ConstantSDNode>(Val: Input.getOperand(i: 1));
17236 SDValue OrigVector = Input.getOperand(i: 0);
17237
17238 // Can't handle non-const element indices or different vector types
17239 // for the input to the extract and the output of the scalar_to_vector.
17240 if (Idx && VT == OrigVector.getValueType()) {
17241 unsigned NumElts = VT.getVectorNumElements();
17242 assert(
17243 NumElts > 1 &&
17244 "Cannot produce a permuted scalar_to_vector for one element vector");
17245 SmallVector<int, 16> NewMask(NumElts, -1);
17246 unsigned ResultInElt = NumElts / 2;
17247 ResultInElt -= Subtarget.isLittleEndian() ? 0 : 1;
17248 NewMask[ResultInElt] = Idx->getZExtValue();
17249 return DAG.getVectorShuffle(VT, dl, N1: OrigVector, N2: OrigVector, Mask: NewMask);
17250 }
17251 }
17252 return DAG.getNode(Opcode: PPCISD::SCALAR_TO_VECTOR_PERMUTED, DL: dl, VT,
17253 Operand: OrigSToV.getOperand(i: 0));
17254}
17255
17256static bool isShuffleMaskInRange(const SmallVectorImpl<int> &ShuffV,
17257 int HalfVec, int LHSLastElementDefined,
17258 int RHSLastElementDefined) {
17259 for (int Index : ShuffV) {
17260 if (Index < 0) // Skip explicitly undefined mask indices.
17261 continue;
17262 // Handle first input vector of the vector_shuffle.
17263 if ((LHSLastElementDefined >= 0) && (Index < HalfVec) &&
17264 (Index > LHSLastElementDefined))
17265 return false;
17266 // Handle second input vector of the vector_shuffle.
17267 if ((RHSLastElementDefined >= 0) &&
17268 (Index > HalfVec + RHSLastElementDefined))
17269 return false;
17270 }
17271 return true;
17272}
17273
17274static SDValue generateSToVPermutedForVecShuffle(
17275 int ScalarSize, uint64_t ShuffleEltWidth, unsigned &NumValidElts,
17276 int FirstElt, int &LastElt, SDValue VecShuffOperand, SDValue SToVNode,
17277 SelectionDAG &DAG, const PPCSubtarget &Subtarget) {
17278 EVT VecShuffOperandType = VecShuffOperand.getValueType();
17279 // Set up the values for the shuffle vector fixup.
17280 NumValidElts = ScalarSize / VecShuffOperandType.getScalarSizeInBits();
17281 // The last element depends on if the input comes from the LHS or RHS.
17282 //
17283 // For example:
17284 // (shuff (s_to_v i32), (bitcast (s_to_v i64), v4i32), ...)
17285 //
17286 // For the LHS: The last element that comes from the LHS is actually 0, not 3
17287 // because elements 1 and higher of a scalar_to_vector are undefined.
17288 // For the RHS: The last element that comes from the RHS is actually 5, not 7
17289 // because elements 1 and higher of a scalar_to_vector are undefined.
17290 // It is also not 4 because the original scalar_to_vector is wider and
17291 // actually contains two i32 elements.
17292 LastElt = (uint64_t)ScalarSize > ShuffleEltWidth
17293 ? ScalarSize / ShuffleEltWidth - 1 + FirstElt
17294 : FirstElt;
17295 SDValue SToVPermuted = getSToVPermuted(OrigSToV: SToVNode, DAG, Subtarget);
17296 if (SToVPermuted.getValueType() != VecShuffOperandType)
17297 SToVPermuted = DAG.getBitcast(VT: VecShuffOperandType, V: SToVPermuted);
17298 return SToVPermuted;
17299}
17300
17301// On little endian subtargets, combine shuffles such as:
17302// vector_shuffle<16,1,17,3,18,5,19,7,20,9,21,11,22,13,23,15>, <zero>, %b
17303// into:
17304// vector_shuffle<16,0,17,1,18,2,19,3,20,4,21,5,22,6,23,7>, <zero>, %b
17305// because the latter can be matched to a single instruction merge.
17306// Furthermore, SCALAR_TO_VECTOR on little endian always involves a permute
17307// to put the value into element zero. Adjust the shuffle mask so that the
17308// vector can remain in permuted form (to prevent a swap prior to a shuffle).
17309// On big endian targets, this is still useful for SCALAR_TO_VECTOR
17310// nodes with elements smaller than doubleword because all the ways
17311// of getting scalar data into a vector register put the value in the
17312// rightmost element of the left half of the vector.
17313SDValue PPCTargetLowering::combineVectorShuffle(ShuffleVectorSDNode *SVN,
17314 SelectionDAG &DAG) const {
17315 SDValue LHS = SVN->getOperand(Num: 0);
17316 SDValue RHS = SVN->getOperand(Num: 1);
17317 auto Mask = SVN->getMask();
17318 int NumElts = LHS.getValueType().getVectorNumElements();
17319 SDValue Res(SVN, 0);
17320 SDLoc dl(SVN);
17321 bool IsLittleEndian = Subtarget.isLittleEndian();
17322
17323 // On big endian targets this is only useful for subtargets with direct moves.
17324 // On little endian targets it would be useful for all subtargets with VSX.
17325 // However adding special handling for LE subtargets without direct moves
17326 // would be wasted effort since the minimum arch for LE is ISA 2.07 (Power8)
17327 // which includes direct moves.
17328 if (!Subtarget.hasDirectMove())
17329 return Res;
17330
17331 // If this is not a shuffle of a shuffle and the first element comes from
17332 // the second vector, canonicalize to the commuted form. This will make it
17333 // more likely to match one of the single instruction patterns.
17334 if (Mask[0] >= NumElts && LHS.getOpcode() != ISD::VECTOR_SHUFFLE &&
17335 RHS.getOpcode() != ISD::VECTOR_SHUFFLE) {
17336 std::swap(a&: LHS, b&: RHS);
17337 Res = DAG.getCommutedVectorShuffle(SV: *SVN);
17338
17339 if (!isa<ShuffleVectorSDNode>(Val: Res))
17340 return Res;
17341
17342 Mask = cast<ShuffleVectorSDNode>(Val&: Res)->getMask();
17343 }
17344
17345 // Adjust the shuffle mask if either input vector comes from a
17346 // SCALAR_TO_VECTOR and keep the respective input vector in permuted
17347 // form (to prevent the need for a swap).
17348 SmallVector<int, 16> ShuffV(Mask);
17349 SDValue SToVLHS = isScalarToVec(Op: LHS);
17350 SDValue SToVRHS = isScalarToVec(Op: RHS);
17351 if (SToVLHS || SToVRHS) {
17352 EVT VT = SVN->getValueType(ResNo: 0);
17353 uint64_t ShuffleEltWidth = VT.getVectorElementType().getSizeInBits();
17354 int ShuffleNumElts = ShuffV.size();
17355 int HalfVec = ShuffleNumElts / 2;
17356 // The width of the "valid lane" (i.e. the lane that contains the value that
17357 // is vectorized) needs to be expressed in terms of the number of elements
17358 // of the shuffle. It is thereby the ratio of the values before and after
17359 // any bitcast, which will be set later on if the LHS or RHS are
17360 // SCALAR_TO_VECTOR nodes.
17361 unsigned LHSNumValidElts = HalfVec;
17362 unsigned RHSNumValidElts = HalfVec;
17363
17364 // Initially assume that neither input is permuted. These will be adjusted
17365 // accordingly if either input is. Note, that -1 means that all elements
17366 // are undefined.
17367 int LHSFirstElt = 0;
17368 int RHSFirstElt = ShuffleNumElts;
17369 int LHSLastElt = -1;
17370 int RHSLastElt = -1;
17371
17372 // Get the permuted scalar to vector nodes for the source(s) that come from
17373 // ISD::SCALAR_TO_VECTOR.
17374 // On big endian systems, this only makes sense for element sizes smaller
17375 // than 64 bits since for 64-bit elements, all instructions already put
17376 // the value into element zero. Since scalar size of LHS and RHS may differ
17377 // after isScalarToVec, this should be checked using their own sizes.
17378 int LHSScalarSize = 0;
17379 int RHSScalarSize = 0;
17380 if (SToVLHS) {
17381 LHSScalarSize = SToVLHS.getValueType().getScalarSizeInBits();
17382 if (!IsLittleEndian && LHSScalarSize >= 64)
17383 return Res;
17384 }
17385 if (SToVRHS) {
17386 RHSScalarSize = SToVRHS.getValueType().getScalarSizeInBits();
17387 if (!IsLittleEndian && RHSScalarSize >= 64)
17388 return Res;
17389 }
17390 if (LHSScalarSize != 0)
17391 LHS = generateSToVPermutedForVecShuffle(
17392 ScalarSize: LHSScalarSize, ShuffleEltWidth, NumValidElts&: LHSNumValidElts, FirstElt: LHSFirstElt,
17393 LastElt&: LHSLastElt, VecShuffOperand: LHS, SToVNode: SToVLHS, DAG, Subtarget);
17394 if (RHSScalarSize != 0)
17395 RHS = generateSToVPermutedForVecShuffle(
17396 ScalarSize: RHSScalarSize, ShuffleEltWidth, NumValidElts&: RHSNumValidElts, FirstElt: RHSFirstElt,
17397 LastElt&: RHSLastElt, VecShuffOperand: RHS, SToVNode: SToVRHS, DAG, Subtarget);
17398
17399 if (!isShuffleMaskInRange(ShuffV, HalfVec, LHSLastElementDefined: LHSLastElt, RHSLastElementDefined: RHSLastElt))
17400 return Res;
17401
17402 // Fix up the shuffle mask to reflect where the desired element actually is.
17403 // The minimum and maximum indices that correspond to element zero for both
17404 // the LHS and RHS are computed and will control which shuffle mask entries
17405 // are to be changed. For example, if the RHS is permuted, any shuffle mask
17406 // entries in the range [RHSFirstElt,RHSLastElt] will be adjusted.
17407 fixupShuffleMaskForPermutedSToV(
17408 ShuffV, LHSFirstElt, LHSLastElt, RHSFirstElt, RHSLastElt, HalfVec,
17409 LHSNumValidElts, RHSNumValidElts, Subtarget);
17410 Res = DAG.getVectorShuffle(VT: SVN->getValueType(ResNo: 0), dl, N1: LHS, N2: RHS, Mask: ShuffV);
17411
17412 // We may have simplified away the shuffle. We won't be able to do anything
17413 // further with it here.
17414 if (!isa<ShuffleVectorSDNode>(Val: Res))
17415 return Res;
17416 Mask = cast<ShuffleVectorSDNode>(Val&: Res)->getMask();
17417 }
17418
17419 SDValue TheSplat = IsLittleEndian ? RHS : LHS;
17420 // The common case after we commuted the shuffle is that the RHS is a splat
17421 // and we have elements coming in from the splat at indices that are not
17422 // conducive to using a merge.
17423 // Example:
17424 // vector_shuffle<0,17,1,19,2,21,3,23,4,25,5,27,6,29,7,31> t1, <zero>
17425 if (!isSplatBV(Op: TheSplat))
17426 return Res;
17427
17428 // We are looking for a mask such that all even elements are from
17429 // one vector and all odd elements from the other.
17430 if (!isAlternatingShuffMask(Mask, NumElts))
17431 return Res;
17432
17433 // Adjust the mask so we are pulling in the same index from the splat
17434 // as the index from the interesting vector in consecutive elements.
17435 if (IsLittleEndian) {
17436 // Example (even elements from first vector):
17437 // vector_shuffle<0,16,1,17,2,18,3,19,4,20,5,21,6,22,7,23> t1, <zero>
17438 if (Mask[0] < NumElts)
17439 for (int i = 1, e = Mask.size(); i < e; i += 2) {
17440 if (ShuffV[i] < 0)
17441 continue;
17442 // If element from non-splat is undef, pick first element from splat.
17443 ShuffV[i] = (ShuffV[i - 1] >= 0 ? ShuffV[i - 1] : 0) + NumElts;
17444 }
17445 // Example (odd elements from first vector):
17446 // vector_shuffle<16,0,17,1,18,2,19,3,20,4,21,5,22,6,23,7> t1, <zero>
17447 else
17448 for (int i = 0, e = Mask.size(); i < e; i += 2) {
17449 if (ShuffV[i] < 0)
17450 continue;
17451 // If element from non-splat is undef, pick first element from splat.
17452 ShuffV[i] = (ShuffV[i + 1] >= 0 ? ShuffV[i + 1] : 0) + NumElts;
17453 }
17454 } else {
17455 // Example (even elements from first vector):
17456 // vector_shuffle<0,16,1,17,2,18,3,19,4,20,5,21,6,22,7,23> <zero>, t1
17457 if (Mask[0] < NumElts)
17458 for (int i = 0, e = Mask.size(); i < e; i += 2) {
17459 if (ShuffV[i] < 0)
17460 continue;
17461 // If element from non-splat is undef, pick first element from splat.
17462 ShuffV[i] = ShuffV[i + 1] >= 0 ? ShuffV[i + 1] - NumElts : 0;
17463 }
17464 // Example (odd elements from first vector):
17465 // vector_shuffle<16,0,17,1,18,2,19,3,20,4,21,5,22,6,23,7> <zero>, t1
17466 else
17467 for (int i = 1, e = Mask.size(); i < e; i += 2) {
17468 if (ShuffV[i] < 0)
17469 continue;
17470 // If element from non-splat is undef, pick first element from splat.
17471 ShuffV[i] = ShuffV[i - 1] >= 0 ? ShuffV[i - 1] - NumElts : 0;
17472 }
17473 }
17474
17475 // If the RHS has undefs, we need to remove them since we may have created
17476 // a shuffle that adds those instead of the splat value.
17477 SDValue SplatVal =
17478 cast<BuildVectorSDNode>(Val: TheSplat.getNode())->getSplatValue();
17479 TheSplat = DAG.getSplatBuildVector(VT: TheSplat.getValueType(), DL: dl, Op: SplatVal);
17480
17481 if (IsLittleEndian)
17482 RHS = TheSplat;
17483 else
17484 LHS = TheSplat;
17485 return DAG.getVectorShuffle(VT: SVN->getValueType(ResNo: 0), dl, N1: LHS, N2: RHS, Mask: ShuffV);
17486}
17487
17488SDValue PPCTargetLowering::combineVReverseMemOP(ShuffleVectorSDNode *SVN,
17489 LSBaseSDNode *LSBase,
17490 DAGCombinerInfo &DCI) const {
17491 assert((ISD::isNormalLoad(LSBase) || ISD::isNormalStore(LSBase)) &&
17492 "Not a reverse memop pattern!");
17493
17494 auto IsElementReverse = [](const ShuffleVectorSDNode *SVN) -> bool {
17495 auto Mask = SVN->getMask();
17496 int i = 0;
17497 auto I = Mask.rbegin();
17498 auto E = Mask.rend();
17499
17500 for (; I != E; ++I) {
17501 if (*I != i)
17502 return false;
17503 i++;
17504 }
17505 return true;
17506 };
17507
17508 SelectionDAG &DAG = DCI.DAG;
17509 EVT VT = SVN->getValueType(ResNo: 0);
17510
17511 if (!isTypeLegal(VT) || !Subtarget.isLittleEndian() || !Subtarget.hasVSX())
17512 return SDValue();
17513
17514 // Before P9, we have PPCVSXSwapRemoval pass to hack the element order.
17515 // See comment in PPCVSXSwapRemoval.cpp.
17516 // It is conflict with PPCVSXSwapRemoval opt. So we don't do it.
17517 if (!Subtarget.hasP9Vector())
17518 return SDValue();
17519
17520 if(!IsElementReverse(SVN))
17521 return SDValue();
17522
17523 if (LSBase->getOpcode() == ISD::LOAD) {
17524 // If the load return value 0 has more than one user except the
17525 // shufflevector instruction, it is not profitable to replace the
17526 // shufflevector with a reverse load.
17527 for (SDUse &Use : LSBase->uses())
17528 if (Use.getResNo() == 0 &&
17529 Use.getUser()->getOpcode() != ISD::VECTOR_SHUFFLE)
17530 return SDValue();
17531
17532 SDLoc dl(LSBase);
17533 SDValue LoadOps[] = {LSBase->getChain(), LSBase->getBasePtr()};
17534 return DAG.getMemIntrinsicNode(
17535 Opcode: PPCISD::LOAD_VEC_BE, dl, VTList: DAG.getVTList(VT1: VT, VT2: MVT::Other), Ops: LoadOps,
17536 MemVT: LSBase->getMemoryVT(), MMO: LSBase->getMemOperand());
17537 }
17538
17539 if (LSBase->getOpcode() == ISD::STORE) {
17540 // If there are other uses of the shuffle, the swap cannot be avoided.
17541 // Forcing the use of an X-Form (since swapped stores only have
17542 // X-Forms) without removing the swap is unprofitable.
17543 if (!SVN->hasOneUse())
17544 return SDValue();
17545
17546 SDLoc dl(LSBase);
17547 SDValue StoreOps[] = {LSBase->getChain(), SVN->getOperand(Num: 0),
17548 LSBase->getBasePtr()};
17549 return DAG.getMemIntrinsicNode(
17550 Opcode: PPCISD::STORE_VEC_BE, dl, VTList: DAG.getVTList(VT: MVT::Other), Ops: StoreOps,
17551 MemVT: LSBase->getMemoryVT(), MMO: LSBase->getMemOperand());
17552 }
17553
17554 llvm_unreachable("Expected a load or store node here");
17555}
17556
17557static bool isStoreConditional(SDValue Intrin, unsigned &StoreWidth) {
17558 unsigned IntrinsicID = Intrin.getConstantOperandVal(i: 1);
17559 if (IntrinsicID == Intrinsic::ppc_stdcx)
17560 StoreWidth = 8;
17561 else if (IntrinsicID == Intrinsic::ppc_stwcx)
17562 StoreWidth = 4;
17563 else if (IntrinsicID == Intrinsic::ppc_sthcx)
17564 StoreWidth = 2;
17565 else if (IntrinsicID == Intrinsic::ppc_stbcx)
17566 StoreWidth = 1;
17567 else
17568 return false;
17569 return true;
17570}
17571
17572static SDValue DAGCombineAddc(SDNode *N,
17573 llvm::PPCTargetLowering::DAGCombinerInfo &DCI) {
17574 if (N->getOpcode() == PPCISD::ADDC && N->hasAnyUseOfValue(Value: 1)) {
17575 // (ADDC (ADDE 0, 0, C), -1) -> C
17576 SDValue LHS = N->getOperand(Num: 0);
17577 SDValue RHS = N->getOperand(Num: 1);
17578 if (LHS->getOpcode() == PPCISD::ADDE &&
17579 isNullConstant(V: LHS->getOperand(Num: 0)) &&
17580 isNullConstant(V: LHS->getOperand(Num: 1)) && isAllOnesConstant(V: RHS)) {
17581 return DCI.CombineTo(N, Res0: SDValue(N, 0), Res1: LHS->getOperand(Num: 2));
17582 }
17583 }
17584 return SDValue();
17585}
17586
17587static SDValue DAGCombineSube(SDNode *N,
17588 llvm::PPCTargetLowering::DAGCombinerInfo &DCI) {
17589 if (N->getOpcode() == PPCISD::SUBE) {
17590 SDValue LHS = N->getOperand(Num: 0);
17591 SDValue RHS = N->getOperand(Num: 1);
17592 SDValue Carry = N->getOperand(Num: 2);
17593
17594 // SUBE(ADDC(ADDE(0, 0, C), -1), same, C) -> ADDC(ADDE(0, 0, C), -1)
17595 if (LHS == RHS && LHS.getOpcode() == PPCISD::ADDC) {
17596 SDValue AddcLHS = LHS.getOperand(i: 0);
17597 SDValue AddcRHS = LHS.getOperand(i: 1);
17598 if (AddcLHS.getOpcode() == PPCISD::ADDE &&
17599 isNullConstant(V: AddcLHS.getOperand(i: 0)) &&
17600 isNullConstant(V: AddcLHS.getOperand(i: 1)) && isAllOnesConstant(V: AddcRHS) &&
17601 Carry == AddcLHS.getOperand(i: 2)) {
17602 return LHS;
17603 }
17604 }
17605 }
17606 return SDValue();
17607}
17608
17609/// Optimize the bitfloor(X) pattern for PowerPC.
17610/// Transforms: select_cc X, 0, 0, (srl MinSignedValue, (ctlz X)), seteq
17611/// Into: srl MinSignedValue, (ctlz X)
17612///
17613/// This is safe on PowerPC because the srw instruction returns 0 when the
17614/// shift amount is == bitwidth, which matches the behavior we need for X=0.
17615static SDValue combineSELECT_CCBitFloor(SDNode *N, SelectionDAG &DAG) {
17616 if (N->getOpcode() != ISD::SELECT_CC)
17617 return SDValue();
17618
17619 // SELECT_CC operands: LHS, RHS, TrueVal, FalseVal, CC
17620 SDValue CmpLHS = N->getOperand(Num: 0);
17621 SDValue CmpRHS = N->getOperand(Num: 1);
17622 SDValue TrueVal = N->getOperand(Num: 2);
17623 SDValue FalseVal = N->getOperand(Num: 3);
17624 ISD::CondCode CC = cast<CondCodeSDNode>(Val: N->getOperand(Num: 4))->get();
17625
17626 // Check if condition is (X == 0)
17627 if (CC != ISD::SETEQ || !isNullConstant(V: CmpRHS))
17628 return SDValue();
17629
17630 // Check if TrueVal is constant 0
17631 if (!isNullConstant(V: TrueVal))
17632 return SDValue();
17633
17634 // This combine is replacing a select_cc with a PPC srl, not an srl with a
17635 // PPC srl. If the original srl had multiple uses it would just remain in the
17636 // code. This is at most a performance consideration.
17637 if (FalseVal.getOpcode() != ISD::SRL || !FalseVal.hasOneUse())
17638 return SDValue();
17639
17640 SDValue ShiftVal = FalseVal.getOperand(i: 0);
17641 SDValue ShiftAmt = FalseVal.getOperand(i: 1);
17642
17643 // Check if ShiftVal is MinSignedValue
17644 auto *ShiftConst = dyn_cast<ConstantSDNode>(Val&: ShiftVal);
17645 if (!ShiftConst || !ShiftConst->getAPIntValue().isMinSignedValue())
17646 return SDValue();
17647
17648 SDValue CtlzArg;
17649 // Check if ShiftAmt is (ctlz CmpLHS) or (truncate (ctlz ...))
17650 if (ShiftAmt.getOpcode() != ISD::CTLZ) {
17651 // Look through truncate if present (for i64 ctlz truncated to i32 shift
17652 // amount)
17653 if (ShiftAmt.getOpcode() != ISD::TRUNCATE)
17654 return SDValue();
17655
17656 // Verify the truncate target type is appropriate for shift amount (i32, not
17657 // i1 or other)
17658 if (ShiftAmt.getValueType() != MVT::i32)
17659 return SDValue();
17660
17661 SDValue CtlzNode = ShiftAmt.getOperand(i: 0);
17662
17663 if (CtlzNode.getOpcode() != ISD::CTLZ)
17664 return SDValue();
17665
17666 CtlzArg = CtlzNode.getOperand(i: 0);
17667 } else {
17668 CtlzArg = ShiftAmt.getOperand(i: 0);
17669 }
17670
17671 // Check if ctlz operates on the same value as the comparison
17672 if (CtlzArg != CmpLHS)
17673 return SDValue();
17674
17675 // Using PPCISD::SRL to ensure well-defined behavior.
17676 // On PowerPC, PPCISD::SRL guarantees that shift by bitwidth returns 0,
17677 // which is exactly what we need for the bitfloor(0) case.
17678 SDLoc DL(N);
17679 SDValue PPCSrl =
17680 DAG.getNode(Opcode: PPCISD::SRL, DL, VT: FalseVal.getValueType(), N1: ShiftVal, N2: ShiftAmt);
17681 return PPCSrl;
17682}
17683
17684// Optimize zero-extension of setcc when the compared value is known to be 0
17685// or 1.
17686//
17687// Pattern: zext(setcc(Value, 0, seteq/setne)) where Value is 0 or 1
17688// -> zext(xor(Value, 1)) for seteq
17689// -> zext(Value) for setne
17690//
17691// This optimization avoids the i32 -> i1 -> i32/i64 conversion sequence
17692// by keeping the value in its original i32 type throughout.
17693//
17694// Example:
17695// Before: zext(setcc(test_data_class(...), 0, seteq))
17696// // test_data_class returns 0 or 1 in i32
17697// // setcc converts i32 -> i1
17698// // zext converts i1 -> i64
17699// After: zext(xor(test_data_class(...), 1))
17700// // Stays in i32, then extends to i64
17701//
17702// This is beneficial because:
17703// 1. Eliminates the setcc instruction
17704// 2. Avoids i32 -> i1 truncation
17705// 3. Keeps computation in native integer width
17706
17707static SDValue combineZextSetccWithZero(SDNode *N, SelectionDAG &DAG) {
17708 // Check if this is a zero_extend
17709 if (N->getOpcode() != ISD::ZERO_EXTEND)
17710 return SDValue();
17711
17712 SDValue Src = N->getOperand(Num: 0);
17713
17714 // Check if the source is a setcc
17715 if (Src.getOpcode() != ISD::SETCC)
17716 return SDValue();
17717
17718 SDValue LHS = Src.getOperand(i: 0);
17719 SDValue RHS = Src.getOperand(i: 1);
17720 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Src.getOperand(i: 2))->get();
17721
17722 if (!isNullConstant(V: RHS) && !isNullConstant(V: LHS))
17723 return SDValue();
17724
17725 SDValue NonNullConstant = isNullConstant(V: RHS) ? LHS : RHS;
17726
17727 auto isZeroOrOne = [=](SDValue &V) {
17728 if (V.getOpcode() == ISD::INTRINSIC_WO_CHAIN &&
17729 V.getConstantOperandVal(i: 0) == Intrinsic::ppc_test_data_class)
17730 return true;
17731 return false;
17732 };
17733
17734 if (!isZeroOrOne(NonNullConstant))
17735 return SDValue();
17736
17737 // Check for pattern: zext(setcc (Value), 0, seteq)) or
17738 // zext(setcc (Value), 0, setne))
17739 if (CC == ISD::SETEQ || CC == ISD::SETNE) {
17740 // Replace with: zext(xor(Value, 1)) for seteq
17741 // or: zext(Value) for setne
17742 // This keeps the value in i32 instead of converting to i1
17743 SDLoc DL(N);
17744 EVT VType = N->getValueType(ResNo: 0);
17745 SDValue NewNonNullConstant = DAG.getZExtOrTrunc(Op: NonNullConstant, DL, VT: VType);
17746
17747 if (CC == ISD::SETNE)
17748 return NewNonNullConstant;
17749
17750 SDValue One = DAG.getConstant(Val: 1, DL, VT: VType);
17751 return DAG.getNode(Opcode: ISD::XOR, DL, VT: VType, N1: NewNonNullConstant, N2: One);
17752 }
17753
17754 return SDValue();
17755}
17756
17757// Combine XOR patterns with SELECT_CC_I4/I8, for Example:
17758// 1. XOR(SELECT_CC_I4(cond, 1, 0, cc), 1) -> SELECT_CC_I4(cond, 0, 1, cc)
17759// 2. XOR(ZEXT(SELECT_CC_I4(cond, 1, 0, cc)), 1) -> SELECT_CC_I4/I8(cond, 0,
17760// 1, cc))
17761// 3. XOR(ANYEXT(SELECT_CC_I4(cond, 1, 0, cc)), 1) -> SELECT_CC_I4/I8(cond,
17762// 0, 1, cc))
17763// 4. etc
17764static SDValue combineXorSelectCC(SDNode *N, SelectionDAG &DAG) {
17765 assert(N->getOpcode() == ISD::XOR && "Expected XOR node");
17766
17767 EVT XorVT = N->getValueType(ResNo: 0);
17768 if ((XorVT != MVT::i32 && XorVT != MVT::i64))
17769 return SDValue();
17770
17771 SDValue LHS = N->getOperand(Num: 0);
17772 SDValue RHS = N->getOperand(Num: 1);
17773
17774 // Check for XOR with constant 1
17775 ConstantSDNode *XorConst = dyn_cast<ConstantSDNode>(Val&: RHS);
17776 if (!XorConst || !XorConst->isOne()) {
17777 XorConst = dyn_cast<ConstantSDNode>(Val&: LHS);
17778 if (!XorConst || !XorConst->isOne())
17779 return SDValue();
17780 // Swap so LHS is the SELECT_CC_I4 (or extension) and RHS is the constant
17781 std::swap(a&: LHS, b&: RHS);
17782 }
17783
17784 // Check if LHS has only one use
17785 if (!LHS.hasOneUse())
17786 return SDValue();
17787
17788 // Handle extensions: ZEXT, ANYEXT
17789 SDValue SelectNode = LHS;
17790
17791 if (LHS.getOpcode() == ISD::ZERO_EXTEND ||
17792 LHS.getOpcode() == ISD::ANY_EXTEND) {
17793 SelectNode = LHS.getOperand(i: 0);
17794
17795 // Check if the extension input has only one use
17796 if (!SelectNode.hasOneUse())
17797 return SDValue();
17798 }
17799
17800 // Check if SelectNode is a MachineSDNode with SELECT_CC_I4/I8 opcode
17801 if (!SelectNode.isMachineOpcode())
17802 return SDValue();
17803
17804 unsigned MachineOpc = SelectNode.getMachineOpcode();
17805
17806 // Handle both SELECT_CC_I4 and SELECT_CC_I8
17807 if (MachineOpc != PPC::SELECT_CC_I4 && MachineOpc != PPC::SELECT_CC_I8)
17808 return SDValue();
17809
17810 // SELECT_CC_I4 operands: (cond, true_val, false_val, bropc)
17811 if (SelectNode.getNumOperands() != 4)
17812 return SDValue();
17813
17814 ConstantSDNode *ConstOp1 = dyn_cast<ConstantSDNode>(Val: SelectNode.getOperand(i: 1));
17815 ConstantSDNode *ConstOp2 = dyn_cast<ConstantSDNode>(Val: SelectNode.getOperand(i: 2));
17816
17817 if (!ConstOp1 || !ConstOp2)
17818 return SDValue();
17819
17820 // Only optimize if operands are {0, 1} or {1, 0}
17821 if (!((ConstOp1->isOne() && ConstOp2->isZero()) ||
17822 (ConstOp1->isZero() && ConstOp2->isOne())))
17823 return SDValue();
17824
17825 // Pattern matched! Create new SELECT_CC with swapped 0/1 operands to
17826 // eliminate XOR. If original was SELECT_CC(cond, 1, 0, pred), create
17827 // SELECT_CC(cond, 0, 1, pred). If original was SELECT_CC(cond, 0, 1, pred),
17828 // create SELECT_CC(cond, 1, 0, pred).
17829 SDLoc DL(N);
17830 MachineOpc = (XorVT == MVT::i32) ? PPC::SELECT_CC_I4 : PPC::SELECT_CC_I8;
17831
17832 bool ConstOp1IsOne = ConstOp1->isOne();
17833 return SDValue(
17834 DAG.getMachineNode(Opcode: MachineOpc, dl: DL, VT: XorVT,
17835 Ops: {SelectNode.getOperand(i: 0),
17836 DAG.getConstant(Val: ConstOp1IsOne ? 0 : 1, DL, VT: XorVT),
17837 DAG.getConstant(Val: ConstOp1IsOne ? 1 : 0, DL, VT: XorVT),
17838 SelectNode.getOperand(i: 3)}),
17839 0);
17840}
17841
17842SDValue PPCTargetLowering::PerformDAGCombine(SDNode *N,
17843 DAGCombinerInfo &DCI) const {
17844 SelectionDAG &DAG = DCI.DAG;
17845 SDLoc dl(N);
17846 switch (N->getOpcode()) {
17847 default: break;
17848 case ISD::ADD:
17849 return combineADD(N, DCI);
17850 case ISD::AND: {
17851 // We don't want (and (zext (shift...)), C) if C fits in the width of the
17852 // original input as that will prevent us from selecting optimal rotates.
17853 // This only matters if the input to the extend is i32 widened to i64.
17854 SDValue Op1 = N->getOperand(Num: 0);
17855 SDValue Op2 = N->getOperand(Num: 1);
17856 if ((Op1.getOpcode() != ISD::ZERO_EXTEND &&
17857 Op1.getOpcode() != ISD::ANY_EXTEND) ||
17858 !isa<ConstantSDNode>(Val: Op2) || N->getValueType(ResNo: 0) != MVT::i64 ||
17859 Op1.getOperand(i: 0).getValueType() != MVT::i32)
17860 break;
17861 SDValue NarrowOp = Op1.getOperand(i: 0);
17862 if (NarrowOp.getOpcode() != ISD::SHL && NarrowOp.getOpcode() != ISD::SRL &&
17863 NarrowOp.getOpcode() != ISD::ROTL && NarrowOp.getOpcode() != ISD::ROTR)
17864 break;
17865
17866 uint64_t Imm = Op2->getAsZExtVal();
17867 // Make sure that the constant is narrow enough to fit in the narrow type.
17868 if (!isUInt<32>(x: Imm))
17869 break;
17870 SDValue ConstOp = DAG.getConstant(Val: Imm, DL: dl, VT: MVT::i32);
17871 SDValue NarrowAnd = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32, N1: NarrowOp, N2: ConstOp);
17872 return DAG.getZExtOrTrunc(Op: NarrowAnd, DL: dl, VT: N->getValueType(ResNo: 0));
17873 }
17874 case ISD::XOR: {
17875 // Optimize XOR(ISEL(1,0,CR), 1) -> ISEL(0,1,CR)
17876 if (SDValue V = combineXorSelectCC(N, DAG))
17877 return V;
17878 break;
17879 }
17880 case ISD::SHL:
17881 return combineSHL(N, DCI);
17882 case ISD::SRA:
17883 return combineSRA(N, DCI);
17884 case ISD::SRL:
17885 return combineSRL(N, DCI);
17886 case ISD::MUL:
17887 return combineMUL(N, DCI);
17888 case ISD::FMA:
17889 case PPCISD::FNMSUB:
17890 return combineFMALike(N, DCI);
17891 case PPCISD::SHL:
17892 if (isNullConstant(V: N->getOperand(Num: 0))) // 0 << V -> 0.
17893 return N->getOperand(Num: 0);
17894 break;
17895 case PPCISD::SRL:
17896 if (isNullConstant(V: N->getOperand(Num: 0))) // 0 >>u V -> 0.
17897 return N->getOperand(Num: 0);
17898 break;
17899 case PPCISD::SRA:
17900 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 0))) {
17901 if (C->isZero() || // 0 >>s V -> 0.
17902 C->isAllOnes()) // -1 >>s V -> -1.
17903 return N->getOperand(Num: 0);
17904 }
17905 break;
17906 case ISD::SIGN_EXTEND:
17907 if (SDValue SECC = combineSignExtendSetCC(N, DCI))
17908 return SECC;
17909 [[fallthrough]];
17910 case ISD::ZERO_EXTEND:
17911 if (SDValue RetV = combineZextSetccWithZero(N, DAG&: DCI.DAG))
17912 return RetV;
17913 [[fallthrough]];
17914 case ISD::ANY_EXTEND:
17915 return DAGCombineExtBoolTrunc(N, DCI);
17916 case ISD::TRUNCATE:
17917 return combineTRUNCATE(N, DCI);
17918 case ISD::SETCC:
17919 if (SDValue CSCC = combineSetCC(N, DCI))
17920 return CSCC;
17921 [[fallthrough]];
17922 case ISD::SELECT_CC:
17923 if (SDValue V = combineSELECT_CCBitFloor(N, DAG))
17924 return V;
17925 return DAGCombineTruncBoolExt(N, DCI);
17926 case ISD::SINT_TO_FP:
17927 case ISD::UINT_TO_FP:
17928 return combineFPToIntToFP(N, DCI);
17929 case ISD::VECTOR_SHUFFLE:
17930 if (ISD::isNormalLoad(N: N->getOperand(Num: 0).getNode())) {
17931 LSBaseSDNode* LSBase = cast<LSBaseSDNode>(Val: N->getOperand(Num: 0));
17932 return combineVReverseMemOP(SVN: cast<ShuffleVectorSDNode>(Val: N), LSBase, DCI);
17933 }
17934 return combineVectorShuffle(SVN: cast<ShuffleVectorSDNode>(Val: N), DAG&: DCI.DAG);
17935 case ISD::STORE: {
17936
17937 EVT Op1VT = N->getOperand(Num: 1).getValueType();
17938 unsigned Opcode = N->getOperand(Num: 1).getOpcode();
17939
17940 if (Opcode == ISD::FP_TO_SINT || Opcode == ISD::FP_TO_UINT ||
17941 Opcode == ISD::STRICT_FP_TO_SINT || Opcode == ISD::STRICT_FP_TO_UINT) {
17942 SDValue Val = combineStoreFPToInt(N, DCI);
17943 if (Val)
17944 return Val;
17945 }
17946
17947 if (Opcode == ISD::VECTOR_SHUFFLE && ISD::isNormalStore(N)) {
17948 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val: N->getOperand(Num: 1));
17949 SDValue Val= combineVReverseMemOP(SVN, LSBase: cast<LSBaseSDNode>(Val: N), DCI);
17950 if (Val)
17951 return Val;
17952 }
17953
17954 // Turn STORE (BSWAP) -> sthbrx/stwbrx.
17955 if (cast<StoreSDNode>(Val: N)->isUnindexed() && Opcode == ISD::BSWAP &&
17956 N->getOperand(Num: 1).getNode()->hasOneUse() &&
17957 (Op1VT == MVT::i32 || Op1VT == MVT::i16 ||
17958 (Subtarget.hasLDBRX() && Subtarget.isPPC64() && Op1VT == MVT::i64))) {
17959
17960 // STBRX can only handle simple types and it makes no sense to store less
17961 // two bytes in byte-reversed order.
17962 EVT mVT = cast<StoreSDNode>(Val: N)->getMemoryVT();
17963 if (mVT.isExtended() || mVT.getSizeInBits() < 16)
17964 break;
17965
17966 SDValue BSwapOp = N->getOperand(Num: 1).getOperand(i: 0);
17967 // Do an any-extend to 32-bits if this is a half-word input.
17968 if (BSwapOp.getValueType() == MVT::i16)
17969 BSwapOp = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: MVT::i32, Operand: BSwapOp);
17970
17971 // If the type of BSWAP operand is wider than stored memory width
17972 // it need to be shifted to the right side before STBRX.
17973 if (Op1VT.bitsGT(VT: mVT)) {
17974 int Shift = Op1VT.getSizeInBits() - mVT.getSizeInBits();
17975 BSwapOp = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: Op1VT, N1: BSwapOp,
17976 N2: DAG.getConstant(Val: Shift, DL: dl, VT: MVT::i32));
17977 // Need to truncate if this is a bswap of i64 stored as i32/i16.
17978 if (Op1VT == MVT::i64)
17979 BSwapOp = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i32, Operand: BSwapOp);
17980 }
17981
17982 SDValue Ops[] = {
17983 N->getOperand(Num: 0), BSwapOp, N->getOperand(Num: 2), DAG.getValueType(mVT)
17984 };
17985 return
17986 DAG.getMemIntrinsicNode(Opcode: PPCISD::STBRX, dl, VTList: DAG.getVTList(VT: MVT::Other),
17987 Ops, MemVT: cast<StoreSDNode>(Val: N)->getMemoryVT(),
17988 MMO: cast<StoreSDNode>(Val: N)->getMemOperand());
17989 }
17990
17991 // STORE Constant:i32<0> -> STORE<trunc to i32> Constant:i64<0>
17992 // So it can increase the chance of CSE constant construction.
17993 if (Subtarget.isPPC64() && !DCI.isBeforeLegalize() &&
17994 isa<ConstantSDNode>(Val: N->getOperand(Num: 1)) && Op1VT == MVT::i32) {
17995 // Need to sign-extended to 64-bits to handle negative values.
17996 EVT MemVT = cast<StoreSDNode>(Val: N)->getMemoryVT();
17997 uint64_t Val64 = SignExtend64(X: N->getConstantOperandVal(Num: 1),
17998 B: MemVT.getSizeInBits());
17999 SDValue Const64 = DAG.getConstant(Val: Val64, DL: dl, VT: MVT::i64);
18000
18001 auto *ST = cast<StoreSDNode>(Val: N);
18002 SDValue NewST = DAG.getStore(Chain: ST->getChain(), dl, Val: Const64,
18003 Ptr: ST->getBasePtr(), Offset: ST->getOffset(), SVT: MemVT,
18004 MMO: ST->getMemOperand(), AM: ST->getAddressingMode(),
18005 /*IsTruncating=*/true);
18006 // Note we use CombineTo here to prevent DAGCombiner from visiting the
18007 // new store which will change the constant by removing non-demanded bits.
18008 return ST->isUnindexed()
18009 ? DCI.CombineTo(N, Res: NewST, /*AddTo=*/false)
18010 : DCI.CombineTo(N, Res0: NewST, Res1: NewST.getValue(R: 1), /*AddTo=*/false);
18011 }
18012
18013 // For little endian, VSX stores require generating xxswapd/lxvd2x.
18014 // Not needed on ISA 3.0 based CPUs since we have a non-permuting store.
18015 if (Op1VT.isSimple()) {
18016 MVT StoreVT = Op1VT.getSimpleVT();
18017 if (Subtarget.needsSwapsForVSXMemOps() &&
18018 (StoreVT == MVT::v2f64 || StoreVT == MVT::v2i64 ||
18019 StoreVT == MVT::v4f32 || StoreVT == MVT::v4i32))
18020 return expandVSXStoreForLE(N, DCI);
18021 }
18022 break;
18023 }
18024 case ISD::LOAD: {
18025 LoadSDNode *LD = cast<LoadSDNode>(Val: N);
18026 EVT VT = LD->getValueType(ResNo: 0);
18027
18028 // For little endian, VSX loads require generating lxvd2x/xxswapd.
18029 // Not needed on ISA 3.0 based CPUs since we have a non-permuting load.
18030 if (VT.isSimple()) {
18031 MVT LoadVT = VT.getSimpleVT();
18032 if (Subtarget.needsSwapsForVSXMemOps() &&
18033 (LoadVT == MVT::v2f64 || LoadVT == MVT::v2i64 ||
18034 LoadVT == MVT::v4f32 || LoadVT == MVT::v4i32))
18035 return expandVSXLoadForLE(N, DCI);
18036 }
18037
18038 // We sometimes end up with a 64-bit integer load, from which we extract
18039 // two single-precision floating-point numbers. This happens with
18040 // std::complex<float>, and other similar structures, because of the way we
18041 // canonicalize structure copies. However, if we lack direct moves,
18042 // then the final bitcasts from the extracted integer values to the
18043 // floating-point numbers turn into store/load pairs. Even with direct moves,
18044 // just loading the two floating-point numbers is likely better.
18045 auto ReplaceTwoFloatLoad = [&]() {
18046 if (VT != MVT::i64)
18047 return false;
18048
18049 if (LD->getExtensionType() != ISD::NON_EXTLOAD ||
18050 LD->isVolatile())
18051 return false;
18052
18053 // We're looking for a sequence like this:
18054 // t13: i64,ch = load<LD8[%ref.tmp]> t0, t6, undef:i64
18055 // t16: i64 = srl t13, Constant:i32<32>
18056 // t17: i32 = truncate t16
18057 // t18: f32 = bitcast t17
18058 // t19: i32 = truncate t13
18059 // t20: f32 = bitcast t19
18060
18061 if (!LD->hasNUsesOfValue(NUses: 2, Value: 0))
18062 return false;
18063
18064 auto UI = LD->user_begin();
18065 while (UI.getUse().getResNo() != 0) ++UI;
18066 SDNode *Trunc = *UI++;
18067 while (UI.getUse().getResNo() != 0) ++UI;
18068 SDNode *RightShift = *UI;
18069 if (Trunc->getOpcode() != ISD::TRUNCATE)
18070 std::swap(a&: Trunc, b&: RightShift);
18071
18072 if (Trunc->getOpcode() != ISD::TRUNCATE ||
18073 Trunc->getValueType(ResNo: 0) != MVT::i32 ||
18074 !Trunc->hasOneUse())
18075 return false;
18076 if (RightShift->getOpcode() != ISD::SRL ||
18077 !isa<ConstantSDNode>(Val: RightShift->getOperand(Num: 1)) ||
18078 RightShift->getConstantOperandVal(Num: 1) != 32 ||
18079 !RightShift->hasOneUse())
18080 return false;
18081
18082 SDNode *Trunc2 = *RightShift->user_begin();
18083 if (Trunc2->getOpcode() != ISD::TRUNCATE ||
18084 Trunc2->getValueType(ResNo: 0) != MVT::i32 ||
18085 !Trunc2->hasOneUse())
18086 return false;
18087
18088 SDNode *Bitcast = *Trunc->user_begin();
18089 SDNode *Bitcast2 = *Trunc2->user_begin();
18090
18091 if (Bitcast->getOpcode() != ISD::BITCAST ||
18092 Bitcast->getValueType(ResNo: 0) != MVT::f32)
18093 return false;
18094 if (Bitcast2->getOpcode() != ISD::BITCAST ||
18095 Bitcast2->getValueType(ResNo: 0) != MVT::f32)
18096 return false;
18097
18098 if (Subtarget.isLittleEndian())
18099 std::swap(a&: Bitcast, b&: Bitcast2);
18100
18101 // Bitcast has the second float (in memory-layout order) and Bitcast2
18102 // has the first one.
18103
18104 SDValue BasePtr = LD->getBasePtr();
18105 if (LD->isIndexed()) {
18106 assert(LD->getAddressingMode() == ISD::PRE_INC &&
18107 "Non-pre-inc AM on PPC?");
18108 BasePtr =
18109 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: BasePtr.getValueType(), N1: BasePtr,
18110 N2: LD->getOffset());
18111 }
18112
18113 auto MMOFlags =
18114 LD->getMemOperand()->getFlags() & ~MachineMemOperand::MOVolatile;
18115 SDValue FloatLoad = DAG.getLoad(VT: MVT::f32, dl, Chain: LD->getChain(), Ptr: BasePtr,
18116 PtrInfo: LD->getPointerInfo(), Alignment: LD->getAlign(),
18117 MMOFlags, Metadata: LD->getAAInfo());
18118 SDValue AddPtr =
18119 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: BasePtr.getValueType(),
18120 N1: BasePtr, N2: DAG.getIntPtrConstant(Val: 4, DL: dl));
18121 SDValue FloatLoad2 = DAG.getLoad(
18122 VT: MVT::f32, dl, Chain: SDValue(FloatLoad.getNode(), 1), Ptr: AddPtr,
18123 PtrInfo: LD->getPointerInfo().getWithOffset(O: 4),
18124 Alignment: commonAlignment(A: LD->getAlign(), Offset: 4), MMOFlags, Metadata: LD->getAAInfo());
18125
18126 if (LD->isIndexed()) {
18127 // Note that DAGCombine should re-form any pre-increment load(s) from
18128 // what is produced here if that makes sense.
18129 DAG.ReplaceAllUsesOfValueWith(From: SDValue(LD, 1), To: BasePtr);
18130 }
18131
18132 DCI.CombineTo(N: Bitcast2, Res: FloatLoad);
18133 DCI.CombineTo(N: Bitcast, Res: FloatLoad2);
18134
18135 DAG.ReplaceAllUsesOfValueWith(From: SDValue(LD, LD->isIndexed() ? 2 : 1),
18136 To: SDValue(FloatLoad2.getNode(), 1));
18137 return true;
18138 };
18139
18140 if (ReplaceTwoFloatLoad())
18141 return SDValue(N, 0);
18142
18143 EVT MemVT = LD->getMemoryVT();
18144 Type *Ty = MemVT.getTypeForEVT(Context&: *DAG.getContext());
18145 Align ABIAlignment = DAG.getDataLayout().getABITypeAlign(Ty);
18146 if (LD->isUnindexed() && VT.isVector() &&
18147 ((Subtarget.hasAltivec() && ISD::isNON_EXTLoad(N) &&
18148 // P8 and later hardware should just use LOAD.
18149 !Subtarget.hasP8Vector() &&
18150 (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32 ||
18151 VT == MVT::v4f32))) &&
18152 LD->getAlign() < ABIAlignment) {
18153 // This is a type-legal unaligned Altivec load.
18154 SDValue Chain = LD->getChain();
18155 SDValue Ptr = LD->getBasePtr();
18156 bool isLittleEndian = Subtarget.isLittleEndian();
18157
18158 // This implements the loading of unaligned vectors as described in
18159 // the venerable Apple Velocity Engine overview. Specifically:
18160 // https://developer.apple.com/hardwaredrivers/ve/alignment.html
18161 // https://developer.apple.com/hardwaredrivers/ve/code_optimization.html
18162 //
18163 // The general idea is to expand a sequence of one or more unaligned
18164 // loads into an alignment-based permutation-control instruction (lvsl
18165 // or lvsr), a series of regular vector loads (which always truncate
18166 // their input address to an aligned address), and a series of
18167 // permutations. The results of these permutations are the requested
18168 // loaded values. The trick is that the last "extra" load is not taken
18169 // from the address you might suspect (sizeof(vector) bytes after the
18170 // last requested load), but rather sizeof(vector) - 1 bytes after the
18171 // last requested vector. The point of this is to avoid a page fault if
18172 // the base address happened to be aligned. This works because if the
18173 // base address is aligned, then adding less than a full vector length
18174 // will cause the last vector in the sequence to be (re)loaded.
18175 // Otherwise, the next vector will be fetched as you might suspect was
18176 // necessary.
18177
18178 // We might be able to reuse the permutation generation from
18179 // a different base address offset from this one by an aligned amount.
18180 // The INTRINSIC_WO_CHAIN DAG combine will attempt to perform this
18181 // optimization later.
18182 Intrinsic::ID Intr, IntrLD, IntrPerm;
18183 MVT PermCntlTy, PermTy, LDTy;
18184 Intr = isLittleEndian ? Intrinsic::ppc_altivec_lvsr
18185 : Intrinsic::ppc_altivec_lvsl;
18186 IntrLD = Intrinsic::ppc_altivec_lvx;
18187 IntrPerm = Intrinsic::ppc_altivec_vperm;
18188 PermCntlTy = MVT::v16i8;
18189 PermTy = MVT::v4i32;
18190 LDTy = MVT::v4i32;
18191
18192 SDValue PermCntl = BuildIntrinsicOp(IID: Intr, Op: Ptr, DAG, dl, DestVT: PermCntlTy);
18193
18194 // Create the new MMO for the new base load. It is like the original MMO,
18195 // but represents an area in memory almost twice the vector size centered
18196 // on the original address. If the address is unaligned, we might start
18197 // reading up to (sizeof(vector)-1) bytes below the address of the
18198 // original unaligned load.
18199 MachineFunction &MF = DAG.getMachineFunction();
18200 MachineMemOperand *BaseMMO =
18201 MF.getMachineMemOperand(MMO: LD->getMemOperand(),
18202 Offset: -(int64_t)MemVT.getStoreSize()+1,
18203 Size: 2*MemVT.getStoreSize()-1);
18204
18205 // Create the new base load.
18206 SDValue LDXIntID =
18207 DAG.getTargetConstant(Val: IntrLD, DL: dl, VT: getPointerTy(DL: MF.getDataLayout()));
18208 SDValue BaseLoadOps[] = { Chain, LDXIntID, Ptr };
18209 SDValue BaseLoad =
18210 DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_W_CHAIN, dl,
18211 VTList: DAG.getVTList(VT1: PermTy, VT2: MVT::Other),
18212 Ops: BaseLoadOps, MemVT: LDTy, MMO: BaseMMO);
18213
18214 // Note that the value of IncOffset (which is provided to the next
18215 // load's pointer info offset value, and thus used to calculate the
18216 // alignment), and the value of IncValue (which is actually used to
18217 // increment the pointer value) are different! This is because we
18218 // require the next load to appear to be aligned, even though it
18219 // is actually offset from the base pointer by a lesser amount.
18220 int IncOffset = VT.getSizeInBits() / 8;
18221 int IncValue = IncOffset;
18222
18223 // Walk (both up and down) the chain looking for another load at the real
18224 // (aligned) offset (the alignment of the other load does not matter in
18225 // this case). If found, then do not use the offset reduction trick, as
18226 // that will prevent the loads from being later combined (as they would
18227 // otherwise be duplicates).
18228 if (!findConsecutiveLoad(LD, DAG))
18229 --IncValue;
18230
18231 SDValue Increment =
18232 DAG.getConstant(Val: IncValue, DL: dl, VT: getPointerTy(DL: MF.getDataLayout()));
18233 Ptr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: Ptr.getValueType(), N1: Ptr, N2: Increment);
18234
18235 MachineMemOperand *ExtraMMO =
18236 MF.getMachineMemOperand(MMO: LD->getMemOperand(),
18237 Offset: 1, Size: 2*MemVT.getStoreSize()-1);
18238 SDValue ExtraLoadOps[] = { Chain, LDXIntID, Ptr };
18239 SDValue ExtraLoad =
18240 DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_W_CHAIN, dl,
18241 VTList: DAG.getVTList(VT1: PermTy, VT2: MVT::Other),
18242 Ops: ExtraLoadOps, MemVT: LDTy, MMO: ExtraMMO);
18243
18244 SDValue TF = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other,
18245 N1: BaseLoad.getValue(R: 1), N2: ExtraLoad.getValue(R: 1));
18246
18247 // Because vperm has a big-endian bias, we must reverse the order
18248 // of the input vectors and complement the permute control vector
18249 // when generating little endian code. We have already handled the
18250 // latter by using lvsr instead of lvsl, so just reverse BaseLoad
18251 // and ExtraLoad here.
18252 SDValue Perm;
18253 if (isLittleEndian)
18254 Perm = BuildIntrinsicOp(IID: IntrPerm,
18255 Op0: ExtraLoad, Op1: BaseLoad, Op2: PermCntl, DAG, dl);
18256 else
18257 Perm = BuildIntrinsicOp(IID: IntrPerm,
18258 Op0: BaseLoad, Op1: ExtraLoad, Op2: PermCntl, DAG, dl);
18259
18260 if (VT != PermTy)
18261 Perm = Subtarget.hasAltivec()
18262 ? DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: Perm)
18263 : DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT, N1: Perm,
18264 N2: DAG.getTargetConstant(Val: 1, DL: dl, VT: MVT::i64));
18265 // second argument is 1 because this rounding
18266 // is always exact.
18267
18268 // The output of the permutation is our loaded result, the TokenFactor is
18269 // our new chain.
18270 DCI.CombineTo(N, Res0: Perm, Res1: TF);
18271 return SDValue(N, 0);
18272 }
18273 }
18274 break;
18275 case ISD::INTRINSIC_WO_CHAIN: {
18276 bool isLittleEndian = Subtarget.isLittleEndian();
18277 unsigned IID = N->getConstantOperandVal(Num: 0);
18278 Intrinsic::ID Intr = (isLittleEndian ? Intrinsic::ppc_altivec_lvsr
18279 : Intrinsic::ppc_altivec_lvsl);
18280 if (IID == Intr && N->getOperand(Num: 1)->getOpcode() == ISD::ADD) {
18281 SDValue Add = N->getOperand(Num: 1);
18282
18283 int Bits = 4 /* 16 byte alignment */;
18284
18285 if (DAG.MaskedValueIsZero(Op: Add->getOperand(Num: 1),
18286 Mask: APInt::getAllOnes(numBits: Bits /* alignment */)
18287 .zext(width: Add.getScalarValueSizeInBits()))) {
18288 SDNode *BasePtr = Add->getOperand(Num: 0).getNode();
18289 for (SDNode *U : BasePtr->users()) {
18290 if (U->getOpcode() == ISD::INTRINSIC_WO_CHAIN &&
18291 U->getConstantOperandVal(Num: 0) == IID) {
18292 // We've found another LVSL/LVSR, and this address is an aligned
18293 // multiple of that one. The results will be the same, so use the
18294 // one we've just found instead.
18295
18296 return SDValue(U, 0);
18297 }
18298 }
18299 }
18300
18301 if (isa<ConstantSDNode>(Val: Add->getOperand(Num: 1))) {
18302 SDNode *BasePtr = Add->getOperand(Num: 0).getNode();
18303 for (SDNode *U : BasePtr->users()) {
18304 if (U->getOpcode() == ISD::ADD &&
18305 isa<ConstantSDNode>(Val: U->getOperand(Num: 1)) &&
18306 (Add->getConstantOperandVal(Num: 1) - U->getConstantOperandVal(Num: 1)) %
18307 (1ULL << Bits) ==
18308 0) {
18309 SDNode *OtherAdd = U;
18310 for (SDNode *V : OtherAdd->users()) {
18311 if (V->getOpcode() == ISD::INTRINSIC_WO_CHAIN &&
18312 V->getConstantOperandVal(Num: 0) == IID) {
18313 return SDValue(V, 0);
18314 }
18315 }
18316 }
18317 }
18318 }
18319 }
18320
18321 // Combine vmaxsw/h/b(a, a's negation) to abs(a)
18322 // Expose the vabsduw/h/b opportunity for down stream
18323 if (!DCI.isAfterLegalizeDAG() && Subtarget.hasP9Altivec() &&
18324 (IID == Intrinsic::ppc_altivec_vmaxsw ||
18325 IID == Intrinsic::ppc_altivec_vmaxsh ||
18326 IID == Intrinsic::ppc_altivec_vmaxsb)) {
18327 SDValue V1 = N->getOperand(Num: 1);
18328 SDValue V2 = N->getOperand(Num: 2);
18329 if ((V1.getSimpleValueType() == MVT::v4i32 ||
18330 V1.getSimpleValueType() == MVT::v8i16 ||
18331 V1.getSimpleValueType() == MVT::v16i8) &&
18332 V1.getSimpleValueType() == V2.getSimpleValueType()) {
18333 // (0-a, a)
18334 if (V1.getOpcode() == ISD::SUB &&
18335 ISD::isBuildVectorAllZeros(N: V1.getOperand(i: 0).getNode()) &&
18336 V1.getOperand(i: 1) == V2) {
18337 return DAG.getNode(Opcode: ISD::ABS, DL: dl, VT: V2.getValueType(), Operand: V2);
18338 }
18339 // (a, 0-a)
18340 if (V2.getOpcode() == ISD::SUB &&
18341 ISD::isBuildVectorAllZeros(N: V2.getOperand(i: 0).getNode()) &&
18342 V2.getOperand(i: 1) == V1) {
18343 return DAG.getNode(Opcode: ISD::ABS, DL: dl, VT: V1.getValueType(), Operand: V1);
18344 }
18345 // (x-y, y-x)
18346 if (V1.getOpcode() == ISD::SUB && V2.getOpcode() == ISD::SUB &&
18347 V1.getOperand(i: 0) == V2.getOperand(i: 1) &&
18348 V1.getOperand(i: 1) == V2.getOperand(i: 0)) {
18349 return DAG.getNode(Opcode: ISD::ABS, DL: dl, VT: V1.getValueType(), Operand: V1);
18350 }
18351 }
18352 }
18353 }
18354
18355 break;
18356 case ISD::INTRINSIC_W_CHAIN:
18357 switch (N->getConstantOperandVal(Num: 1)) {
18358 default:
18359 break;
18360 case Intrinsic::ppc_altivec_vsum4sbs:
18361 case Intrinsic::ppc_altivec_vsum4shs:
18362 case Intrinsic::ppc_altivec_vsum4ubs: {
18363 // These sum-across intrinsics only have a chain due to the side effect
18364 // that they may set the SAT bit. If we know the SAT bit will not be set
18365 // for some inputs, we can replace any uses of their chain with the
18366 // input chain.
18367 if (BuildVectorSDNode *BVN =
18368 dyn_cast<BuildVectorSDNode>(Val: N->getOperand(Num: 3))) {
18369 APInt APSplatBits, APSplatUndef;
18370 unsigned SplatBitSize;
18371 bool HasAnyUndefs;
18372 bool BVNIsConstantSplat = BVN->isConstantSplat(
18373 SplatValue&: APSplatBits, SplatUndef&: APSplatUndef, SplatBitSize, HasAnyUndefs, MinSplatBits: 0,
18374 isBigEndian: !Subtarget.isLittleEndian());
18375 // If the constant splat vector is 0, the SAT bit will not be set.
18376 if (BVNIsConstantSplat && APSplatBits == 0)
18377 DAG.ReplaceAllUsesOfValueWith(From: SDValue(N, 1), To: N->getOperand(Num: 0));
18378 }
18379 return SDValue();
18380 }
18381 case Intrinsic::ppc_vsx_lxvw4x:
18382 case Intrinsic::ppc_vsx_lxvd2x:
18383 // For little endian, VSX loads require generating lxvd2x/xxswapd.
18384 // Not needed on ISA 3.0 based CPUs since we have a non-permuting load.
18385 if (Subtarget.needsSwapsForVSXMemOps())
18386 return expandVSXLoadForLE(N, DCI);
18387 break;
18388 }
18389 break;
18390 case ISD::INTRINSIC_VOID:
18391 // For little endian, VSX stores require generating xxswapd/stxvd2x.
18392 // Not needed on ISA 3.0 based CPUs since we have a non-permuting store.
18393 if (Subtarget.needsSwapsForVSXMemOps()) {
18394 switch (N->getConstantOperandVal(Num: 1)) {
18395 default:
18396 break;
18397 case Intrinsic::ppc_vsx_stxvw4x:
18398 case Intrinsic::ppc_vsx_stxvd2x:
18399 return expandVSXStoreForLE(N, DCI);
18400 }
18401 }
18402 break;
18403 case ISD::BSWAP: {
18404 // Turn BSWAP (LOAD) -> lhbrx/lwbrx.
18405 // For subtargets without LDBRX, we can still do better than the default
18406 // expansion even for 64-bit BSWAP (LOAD).
18407 bool Is64BitBswapOn64BitTgt =
18408 Subtarget.isPPC64() && N->getValueType(ResNo: 0) == MVT::i64;
18409 bool IsSingleUseNormalLd = ISD::isNormalLoad(N: N->getOperand(Num: 0).getNode()) &&
18410 N->getOperand(Num: 0).hasOneUse();
18411 if (IsSingleUseNormalLd &&
18412 (N->getValueType(ResNo: 0) == MVT::i32 || N->getValueType(ResNo: 0) == MVT::i16 ||
18413 (Subtarget.hasLDBRX() && Is64BitBswapOn64BitTgt))) {
18414 SDValue Load = N->getOperand(Num: 0);
18415 LoadSDNode *LD = cast<LoadSDNode>(Val&: Load);
18416 // Create the byte-swapping load.
18417 SDValue Ops[] = {
18418 LD->getChain(), // Chain
18419 LD->getBasePtr(), // Ptr
18420 DAG.getValueType(N->getValueType(ResNo: 0)) // VT
18421 };
18422 SDValue BSLoad =
18423 DAG.getMemIntrinsicNode(Opcode: PPCISD::LBRX, dl,
18424 VTList: DAG.getVTList(VT1: N->getValueType(ResNo: 0) == MVT::i64 ?
18425 MVT::i64 : MVT::i32, VT2: MVT::Other),
18426 Ops, MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand());
18427
18428 // If this is an i16 load, insert the truncate.
18429 SDValue ResVal = BSLoad;
18430 if (N->getValueType(ResNo: 0) == MVT::i16)
18431 ResVal = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i16, Operand: BSLoad);
18432
18433 // First, combine the bswap away. This makes the value produced by the
18434 // load dead.
18435 DCI.CombineTo(N, Res: ResVal);
18436
18437 // Next, combine the load away, we give it a bogus result value but a real
18438 // chain result. The result value is dead because the bswap is dead.
18439 DCI.CombineTo(N: Load.getNode(), Res0: ResVal, Res1: BSLoad.getValue(R: 1));
18440
18441 // Return N so it doesn't get rechecked!
18442 return SDValue(N, 0);
18443 }
18444 // Convert this to two 32-bit bswap loads and a BUILD_PAIR. Do this only
18445 // before legalization so that the BUILD_PAIR is handled correctly.
18446 if (!DCI.isBeforeLegalize() || !Is64BitBswapOn64BitTgt ||
18447 !IsSingleUseNormalLd)
18448 return SDValue();
18449 LoadSDNode *LD = cast<LoadSDNode>(Val: N->getOperand(Num: 0));
18450
18451 // Can't split volatile or atomic loads.
18452 if (!LD->isSimple())
18453 return SDValue();
18454 SDValue BasePtr = LD->getBasePtr();
18455 SDValue Lo = DAG.getLoad(VT: MVT::i32, dl, Chain: LD->getChain(), Ptr: BasePtr,
18456 PtrInfo: LD->getPointerInfo(), Alignment: LD->getAlign());
18457 Lo = DAG.getNode(Opcode: ISD::BSWAP, DL: dl, VT: MVT::i32, Operand: Lo);
18458 BasePtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: BasePtr.getValueType(), N1: BasePtr,
18459 N2: DAG.getIntPtrConstant(Val: 4, DL: dl));
18460 MachineMemOperand *NewMMO = DAG.getMachineFunction().getMachineMemOperand(
18461 MMO: LD->getMemOperand(), Offset: 4, Size: 4);
18462 SDValue Hi = DAG.getLoad(VT: MVT::i32, dl, Chain: LD->getChain(), Ptr: BasePtr, MMO: NewMMO);
18463 Hi = DAG.getNode(Opcode: ISD::BSWAP, DL: dl, VT: MVT::i32, Operand: Hi);
18464 SDValue Res;
18465 if (Subtarget.isLittleEndian())
18466 Res = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: MVT::i64, N1: Hi, N2: Lo);
18467 else
18468 Res = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: MVT::i64, N1: Lo, N2: Hi);
18469 SDValue TF =
18470 DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other,
18471 N1: Hi.getOperand(i: 0).getValue(R: 1), N2: Lo.getOperand(i: 0).getValue(R: 1));
18472 DAG.ReplaceAllUsesOfValueWith(From: SDValue(LD, 1), To: TF);
18473 return Res;
18474 }
18475 case PPCISD::VCMP:
18476 // If a VCMP_rec node already exists with exactly the same operands as this
18477 // node, use its result instead of this node (VCMP_rec computes both a CR6
18478 // and a normal output).
18479 //
18480 if (!N->getOperand(Num: 0).hasOneUse() &&
18481 !N->getOperand(Num: 1).hasOneUse() &&
18482 !N->getOperand(Num: 2).hasOneUse()) {
18483
18484 // Scan all of the users of the LHS, looking for VCMP_rec's that match.
18485 SDNode *VCMPrecNode = nullptr;
18486
18487 SDNode *LHSN = N->getOperand(Num: 0).getNode();
18488 for (SDNode *User : LHSN->users())
18489 if (User->getOpcode() == PPCISD::VCMP_rec &&
18490 User->getOperand(Num: 1) == N->getOperand(Num: 1) &&
18491 User->getOperand(Num: 2) == N->getOperand(Num: 2) &&
18492 User->getOperand(Num: 0) == N->getOperand(Num: 0)) {
18493 VCMPrecNode = User;
18494 break;
18495 }
18496
18497 // If there is no VCMP_rec node, or if the flag value has a single use,
18498 // don't transform this.
18499 if (!VCMPrecNode || VCMPrecNode->hasNUsesOfValue(NUses: 0, Value: 1))
18500 break;
18501
18502 // Look at the (necessarily single) use of the flag value. If it has a
18503 // chain, this transformation is more complex. Note that multiple things
18504 // could use the value result, which we should ignore.
18505 SDNode *FlagUser = nullptr;
18506 for (SDNode::use_iterator UI = VCMPrecNode->use_begin();
18507 FlagUser == nullptr; ++UI) {
18508 assert(UI != VCMPrecNode->use_end() && "Didn't find user!");
18509 SDNode *User = UI->getUser();
18510 for (unsigned i = 0, e = User->getNumOperands(); i != e; ++i) {
18511 if (User->getOperand(Num: i) == SDValue(VCMPrecNode, 1)) {
18512 FlagUser = User;
18513 break;
18514 }
18515 }
18516 }
18517
18518 // If the user is a MFOCRF instruction, we know this is safe.
18519 // Otherwise we give up for right now.
18520 if (FlagUser->getOpcode() == PPCISD::MFOCRF)
18521 return SDValue(VCMPrecNode, 0);
18522 }
18523 break;
18524 case ISD::BR_CC: {
18525 // If this is a branch on an altivec predicate comparison, lower this so
18526 // that we don't have to do a MFOCRF: instead, branch directly on CR6. This
18527 // lowering is done pre-legalize, because the legalizer lowers the predicate
18528 // compare down to code that is difficult to reassemble.
18529 // This code also handles branches that depend on the result of a store
18530 // conditional.
18531 ISD::CondCode CC = cast<CondCodeSDNode>(Val: N->getOperand(Num: 1))->get();
18532 SDValue LHS = N->getOperand(Num: 2), RHS = N->getOperand(Num: 3);
18533
18534 int CompareOpc;
18535 bool isDot;
18536
18537 if (!isa<ConstantSDNode>(Val: RHS) || (CC != ISD::SETEQ && CC != ISD::SETNE))
18538 break;
18539
18540 // Since we are doing this pre-legalize, the RHS can be a constant of
18541 // arbitrary bitwidth which may cause issues when trying to get the value
18542 // from the underlying APInt.
18543 auto RHSAPInt = RHS->getAsAPIntVal();
18544 if (!RHSAPInt.isIntN(N: 64))
18545 break;
18546
18547 unsigned Val = RHSAPInt.getZExtValue();
18548 auto isImpossibleCompare = [&]() {
18549 // If this is a comparison against something other than 0/1, then we know
18550 // that the condition is never/always true.
18551 if (Val != 0 && Val != 1) {
18552 if (CC == ISD::SETEQ) // Cond never true, remove branch.
18553 return N->getOperand(Num: 0);
18554 // Always !=, turn it into an unconditional branch.
18555 return DAG.getNode(Opcode: ISD::BR, DL: dl, VT: MVT::Other,
18556 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 4));
18557 }
18558 return SDValue();
18559 };
18560 // Combine branches fed by store conditional instructions (st[bhwd]cx).
18561 unsigned StoreWidth = 0;
18562 if (LHS.getOpcode() == ISD::INTRINSIC_W_CHAIN &&
18563 isStoreConditional(Intrin: LHS, StoreWidth)) {
18564 if (SDValue Impossible = isImpossibleCompare())
18565 return Impossible;
18566 PPC::Predicate CompOpc;
18567 // eq 0 => ne
18568 // ne 0 => eq
18569 // eq 1 => eq
18570 // ne 1 => ne
18571 if (Val == 0)
18572 CompOpc = CC == ISD::SETEQ ? PPC::PRED_NE : PPC::PRED_EQ;
18573 else
18574 CompOpc = CC == ISD::SETEQ ? PPC::PRED_EQ : PPC::PRED_NE;
18575
18576 SDValue Ops[] = {LHS.getOperand(i: 0), LHS.getOperand(i: 2), LHS.getOperand(i: 3),
18577 DAG.getConstant(Val: StoreWidth, DL: dl, VT: MVT::i32)};
18578 auto *MemNode = cast<MemSDNode>(Val&: LHS);
18579 SDValue ConstSt = DAG.getMemIntrinsicNode(
18580 Opcode: PPCISD::STORE_COND, dl,
18581 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other, VT3: MVT::Glue), Ops,
18582 MemVT: MemNode->getMemoryVT(), MMO: MemNode->getMemOperand());
18583
18584 SDValue InChain;
18585 // Unchain the branch from the original store conditional.
18586 if (N->getOperand(Num: 0) == LHS.getValue(R: 1))
18587 InChain = LHS.getOperand(i: 0);
18588 else if (N->getOperand(Num: 0).getOpcode() == ISD::TokenFactor) {
18589 SmallVector<SDValue, 4> InChains;
18590 SDValue InTF = N->getOperand(Num: 0);
18591 for (int i = 0, e = InTF.getNumOperands(); i < e; i++)
18592 if (InTF.getOperand(i) != LHS.getValue(R: 1))
18593 InChains.push_back(Elt: InTF.getOperand(i));
18594 InChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: InChains);
18595 }
18596
18597 return DAG.getNode(Opcode: PPCISD::COND_BRANCH, DL: dl, VT: MVT::Other, N1: InChain,
18598 N2: DAG.getConstant(Val: CompOpc, DL: dl, VT: MVT::i32),
18599 N3: DAG.getRegister(Reg: PPC::CR0, VT: MVT::i32), N4: N->getOperand(Num: 4),
18600 N5: ConstSt.getValue(R: 2));
18601 }
18602
18603 if (LHS.getOpcode() == ISD::INTRINSIC_WO_CHAIN &&
18604 getVectorCompareInfo(Intrin: LHS, CompareOpc, isDot, Subtarget)) {
18605 assert(isDot && "Can't compare against a vector result!");
18606
18607 if (SDValue Impossible = isImpossibleCompare())
18608 return Impossible;
18609
18610 bool BranchOnWhenPredTrue = (CC == ISD::SETEQ) ^ (Val == 0);
18611 // Create the PPCISD altivec 'dot' comparison node.
18612 SDValue Ops[] = {
18613 LHS.getOperand(i: 2), // LHS of compare
18614 LHS.getOperand(i: 3), // RHS of compare
18615 DAG.getConstant(Val: CompareOpc, DL: dl, VT: MVT::i32)
18616 };
18617 EVT VTs[] = { LHS.getOperand(i: 2).getValueType(), MVT::Glue };
18618 SDValue CompNode = DAG.getNode(Opcode: PPCISD::VCMP_rec, DL: dl, ResultTys: VTs, Ops);
18619
18620 // Unpack the result based on how the target uses it.
18621 PPC::Predicate CompOpc;
18622 switch (LHS.getConstantOperandVal(i: 1)) {
18623 default: // Can't happen, don't crash on invalid number though.
18624 case 0: // Branch on the value of the EQ bit of CR6.
18625 CompOpc = BranchOnWhenPredTrue ? PPC::PRED_EQ : PPC::PRED_NE;
18626 break;
18627 case 1: // Branch on the inverted value of the EQ bit of CR6.
18628 CompOpc = BranchOnWhenPredTrue ? PPC::PRED_NE : PPC::PRED_EQ;
18629 break;
18630 case 2: // Branch on the value of the LT bit of CR6.
18631 CompOpc = BranchOnWhenPredTrue ? PPC::PRED_LT : PPC::PRED_GE;
18632 break;
18633 case 3: // Branch on the inverted value of the LT bit of CR6.
18634 CompOpc = BranchOnWhenPredTrue ? PPC::PRED_GE : PPC::PRED_LT;
18635 break;
18636 }
18637
18638 return DAG.getNode(Opcode: PPCISD::COND_BRANCH, DL: dl, VT: MVT::Other, N1: N->getOperand(Num: 0),
18639 N2: DAG.getConstant(Val: CompOpc, DL: dl, VT: MVT::i32),
18640 N3: DAG.getRegister(Reg: PPC::CR6, VT: MVT::i32),
18641 N4: N->getOperand(Num: 4), N5: CompNode.getValue(R: 1));
18642 }
18643 break;
18644 }
18645 case ISD::BUILD_VECTOR:
18646 return DAGCombineBuildVector(N, DCI);
18647 case PPCISD::ADDC:
18648 return DAGCombineAddc(N, DCI);
18649 case PPCISD::SUBE:
18650 return DAGCombineSube(N, DCI);
18651
18652 case ISD::BITCAST:
18653 return DAGCombineBitcast(N, DCI);
18654 }
18655
18656 return SDValue();
18657}
18658
18659SDValue
18660PPCTargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
18661 SelectionDAG &DAG,
18662 SmallVectorImpl<SDNode *> &Created) const {
18663 // fold (sdiv X, pow2)
18664 EVT VT = N->getValueType(ResNo: 0);
18665 if (VT == MVT::i64 && !Subtarget.isPPC64())
18666 return SDValue();
18667 if ((VT != MVT::i32 && VT != MVT::i64) ||
18668 !(Divisor.isPowerOf2() || Divisor.isNegatedPowerOf2()))
18669 return SDValue();
18670
18671 SDLoc DL(N);
18672 SDValue N0 = N->getOperand(Num: 0);
18673
18674 bool IsNegPow2 = Divisor.isNegatedPowerOf2();
18675 unsigned Lg2 = (IsNegPow2 ? -Divisor : Divisor).countr_zero();
18676 SDValue ShiftAmt = DAG.getConstant(Val: Lg2, DL, VT);
18677
18678 SDValue Op = DAG.getNode(Opcode: PPCISD::SRA_ADDZE, DL, VT, N1: N0, N2: ShiftAmt);
18679 Created.push_back(Elt: Op.getNode());
18680
18681 if (IsNegPow2) {
18682 Op = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: DAG.getConstant(Val: 0, DL, VT), N2: Op);
18683 Created.push_back(Elt: Op.getNode());
18684 }
18685
18686 return Op;
18687}
18688
18689//===----------------------------------------------------------------------===//
18690// Inline Assembly Support
18691//===----------------------------------------------------------------------===//
18692
18693void PPCTargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
18694 KnownBits &Known,
18695 const APInt &DemandedElts,
18696 const SelectionDAG &DAG,
18697 unsigned Depth) const {
18698 Known.resetAll();
18699 switch (Op.getOpcode()) {
18700 default: break;
18701 case PPCISD::LBRX: {
18702 // lhbrx is known to have the top bits cleared out.
18703 if (cast<VTSDNode>(Val: Op.getOperand(i: 2))->getVT() == MVT::i16)
18704 Known.Zero = 0xFFFF0000;
18705 break;
18706 }
18707 case PPCISD::ADDE: {
18708 if (Op.getResNo() == 0) {
18709 // (0|1), _ = ADDE 0, 0, CARRY
18710 SDValue LHS = Op.getOperand(i: 0);
18711 SDValue RHS = Op.getOperand(i: 1);
18712 if (isNullConstant(V: LHS) && isNullConstant(V: RHS))
18713 Known.Zero = ~1ULL;
18714 }
18715 break;
18716 }
18717 case ISD::INTRINSIC_WO_CHAIN: {
18718 switch (Op.getConstantOperandVal(i: 0)) {
18719 default: break;
18720 case Intrinsic::ppc_altivec_vcmpbfp_p:
18721 case Intrinsic::ppc_altivec_vcmpeqfp_p:
18722 case Intrinsic::ppc_altivec_vcmpequb_p:
18723 case Intrinsic::ppc_altivec_vcmpequh_p:
18724 case Intrinsic::ppc_altivec_vcmpequw_p:
18725 case Intrinsic::ppc_altivec_vcmpequd_p:
18726 case Intrinsic::ppc_altivec_vcmpequq_p:
18727 case Intrinsic::ppc_altivec_vcmpgefp_p:
18728 case Intrinsic::ppc_altivec_vcmpgtfp_p:
18729 case Intrinsic::ppc_altivec_vcmpgtsb_p:
18730 case Intrinsic::ppc_altivec_vcmpgtsh_p:
18731 case Intrinsic::ppc_altivec_vcmpgtsw_p:
18732 case Intrinsic::ppc_altivec_vcmpgtsd_p:
18733 case Intrinsic::ppc_altivec_vcmpgtsq_p:
18734 case Intrinsic::ppc_altivec_vcmpgtub_p:
18735 case Intrinsic::ppc_altivec_vcmpgtuh_p:
18736 case Intrinsic::ppc_altivec_vcmpgtuw_p:
18737 case Intrinsic::ppc_altivec_vcmpgtud_p:
18738 case Intrinsic::ppc_altivec_vcmpgtuq_p:
18739 Known.Zero = ~1U; // All bits but the low one are known to be zero.
18740 break;
18741 }
18742 break;
18743 }
18744 case ISD::INTRINSIC_W_CHAIN: {
18745 switch (Op.getConstantOperandVal(i: 1)) {
18746 default:
18747 break;
18748 case Intrinsic::ppc_load2r:
18749 // Top bits are cleared for load2r (which is the same as lhbrx).
18750 Known.Zero = 0xFFFF0000;
18751 break;
18752 }
18753 break;
18754 }
18755 }
18756}
18757
18758Align PPCTargetLowering::getPrefLoopAlignment(MachineLoop *ML) const {
18759 switch (Subtarget.getCPUDirective()) {
18760 default: break;
18761 case PPC::DIR_970:
18762 case PPC::DIR_PWR4:
18763 case PPC::DIR_PWR5:
18764 case PPC::DIR_PWR5X:
18765 case PPC::DIR_PWR6:
18766 case PPC::DIR_PWR6X:
18767 case PPC::DIR_PWR7:
18768 case PPC::DIR_PWR8:
18769 case PPC::DIR_PWR9:
18770 case PPC::DIR_PWR10:
18771 case PPC::DIR_PWR11:
18772 case PPC::DIR_PWR_FUTURE: {
18773 if (!ML)
18774 break;
18775
18776 if (!DisableInnermostLoopAlign32) {
18777 // If the nested loop is an innermost loop, prefer to a 32-byte alignment,
18778 // so that we can decrease cache misses and branch-prediction misses.
18779 // Actual alignment of the loop will depend on the hotness check and other
18780 // logic in alignBlocks.
18781 if (ML->getLoopDepth() > 1 && ML->getSubLoops().empty())
18782 return Align(32);
18783 }
18784
18785 const PPCInstrInfo *TII = Subtarget.getInstrInfo();
18786
18787 // For small loops (between 5 and 8 instructions), align to a 32-byte
18788 // boundary so that the entire loop fits in one instruction-cache line.
18789 uint64_t LoopSize = 0;
18790 for (auto I = ML->block_begin(), IE = ML->block_end(); I != IE; ++I)
18791 for (const MachineInstr &J : **I) {
18792 LoopSize += TII->getInstSizeInBytes(MI: J);
18793 if (LoopSize > 32)
18794 break;
18795 }
18796
18797 if (LoopSize > 16 && LoopSize <= 32)
18798 return Align(32);
18799
18800 break;
18801 }
18802 }
18803
18804 return TargetLowering::getPrefLoopAlignment(ML);
18805}
18806
18807/// getConstraintType - Given a constraint, return the type of
18808/// constraint it is for this target.
18809PPCTargetLowering::ConstraintType
18810PPCTargetLowering::getConstraintType(StringRef Constraint) const {
18811 if (Constraint.size() == 1) {
18812 switch (Constraint[0]) {
18813 default: break;
18814 case 'b':
18815 case 'r':
18816 case 'f':
18817 case 'd':
18818 case 'v':
18819 case 'y':
18820 return C_RegisterClass;
18821 case 'Z':
18822 // FIXME: While Z does indicate a memory constraint, it specifically
18823 // indicates an r+r address (used in conjunction with the 'y' modifier
18824 // in the replacement string). Currently, we're forcing the base
18825 // register to be r0 in the asm printer (which is interpreted as zero)
18826 // and forming the complete address in the second register. This is
18827 // suboptimal.
18828 return C_Memory;
18829 }
18830 } else if (Constraint == "wc") { // individual CR bits.
18831 return C_RegisterClass;
18832 } else if (Constraint == "wa" || Constraint == "wd" ||
18833 Constraint == "wf" || Constraint == "ws" ||
18834 Constraint == "wi" || Constraint == "ww") {
18835 return C_RegisterClass; // VSX registers.
18836 }
18837 return TargetLowering::getConstraintType(Constraint);
18838}
18839
18840/// Examine constraint type and operand type and determine a weight value.
18841/// This object must already have been set up with the operand type
18842/// and the current alternative constraint selected.
18843TargetLowering::ConstraintWeight
18844PPCTargetLowering::getSingleConstraintMatchWeight(
18845 AsmOperandInfo &info, const char *constraint) const {
18846 ConstraintWeight weight = CW_Invalid;
18847 Value *CallOperandVal = info.CallOperandVal;
18848 // If we don't have a value, we can't do a match,
18849 // but allow it at the lowest weight.
18850 if (!CallOperandVal)
18851 return CW_Default;
18852 Type *type = CallOperandVal->getType();
18853
18854 // Look at the constraint type.
18855 if (StringRef(constraint) == "wc" && type->isIntegerTy(BitWidth: 1))
18856 return CW_Register; // an individual CR bit.
18857 else if ((StringRef(constraint) == "wa" ||
18858 StringRef(constraint) == "wd" ||
18859 StringRef(constraint) == "wf") &&
18860 type->isVectorTy())
18861 return CW_Register;
18862 else if (StringRef(constraint) == "wi" && type->isIntegerTy(BitWidth: 64))
18863 return CW_Register; // just hold 64-bit integers data.
18864 else if (StringRef(constraint) == "ws" && type->isDoubleTy())
18865 return CW_Register;
18866 else if (StringRef(constraint) == "ww" && type->isFloatTy())
18867 return CW_Register;
18868
18869 switch (*constraint) {
18870 default:
18871 weight = TargetLowering::getSingleConstraintMatchWeight(info, constraint);
18872 break;
18873 case 'b':
18874 if (type->isIntegerTy())
18875 weight = CW_Register;
18876 break;
18877 case 'f':
18878 if (type->isFloatTy())
18879 weight = CW_Register;
18880 break;
18881 case 'd':
18882 if (type->isDoubleTy())
18883 weight = CW_Register;
18884 break;
18885 case 'v':
18886 if (type->isVectorTy())
18887 weight = CW_Register;
18888 break;
18889 case 'y':
18890 weight = CW_Register;
18891 break;
18892 case 'Z':
18893 weight = CW_Memory;
18894 break;
18895 }
18896 return weight;
18897}
18898
18899std::pair<unsigned, const TargetRegisterClass *>
18900PPCTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
18901 StringRef Constraint,
18902 MVT VT) const {
18903 if (Constraint.size() == 1) {
18904 // GCC RS6000 Constraint Letters
18905 switch (Constraint[0]) {
18906 case 'b': // R1-R31
18907 if (VT == MVT::i64 && Subtarget.isPPC64())
18908 return std::make_pair(x: 0U, y: &PPC::G8RC_NOX0RegClass);
18909 return std::make_pair(x: 0U, y: &PPC::GPRC_NOR0RegClass);
18910 case 'r': // R0-R31
18911 if (VT == MVT::i64 && Subtarget.isPPC64())
18912 return std::make_pair(x: 0U, y: &PPC::G8RCRegClass);
18913 return std::make_pair(x: 0U, y: &PPC::GPRCRegClass);
18914 // 'd' and 'f' constraints are both defined to be "the floating point
18915 // registers", where one is for 32-bit and the other for 64-bit. We don't
18916 // really care overly much here so just give them all the same reg classes.
18917 case 'd':
18918 case 'f':
18919 if (Subtarget.hasSPE()) {
18920 if (VT == MVT::f32 || VT == MVT::i32)
18921 return std::make_pair(x: 0U, y: &PPC::GPRCRegClass);
18922 if (VT == MVT::f64 || VT == MVT::i64)
18923 return std::make_pair(x: 0U, y: &PPC::SPERCRegClass);
18924 } else {
18925 if (VT == MVT::f32 || VT == MVT::i32)
18926 return std::make_pair(x: 0U, y: &PPC::F4RCRegClass);
18927 if (VT == MVT::f64 || VT == MVT::i64)
18928 return std::make_pair(x: 0U, y: &PPC::F8RCRegClass);
18929 }
18930 break;
18931 case 'v':
18932 if (Subtarget.hasAltivec() && VT.isVector())
18933 return std::make_pair(x: 0U, y: &PPC::VRRCRegClass);
18934 else if (Subtarget.hasVSX())
18935 // Scalars in Altivec registers only make sense with VSX.
18936 return std::make_pair(x: 0U, y: &PPC::VFRCRegClass);
18937 break;
18938 case 'y': // crrc
18939 return std::make_pair(x: 0U, y: &PPC::CRRCRegClass);
18940 }
18941 } else if (Constraint == "wc" && Subtarget.useCRBits()) {
18942 // An individual CR bit.
18943 return std::make_pair(x: 0U, y: &PPC::CRBITRCRegClass);
18944 } else if ((Constraint == "wa" || Constraint == "wd" ||
18945 Constraint == "wf" || Constraint == "wi") &&
18946 Subtarget.hasVSX()) {
18947 // A VSX register for either a scalar (FP) or vector. There is no
18948 // support for single precision scalars on subtargets prior to Power8.
18949 if (VT.isVector())
18950 return std::make_pair(x: 0U, y: &PPC::VSRCRegClass);
18951 if (VT == MVT::f32 && Subtarget.hasP8Vector())
18952 return std::make_pair(x: 0U, y: &PPC::VSSRCRegClass);
18953 return std::make_pair(x: 0U, y: &PPC::VSFRCRegClass);
18954 } else if ((Constraint == "ws" || Constraint == "ww") && Subtarget.hasVSX()) {
18955 if (VT == MVT::f32 && Subtarget.hasP8Vector())
18956 return std::make_pair(x: 0U, y: &PPC::VSSRCRegClass);
18957 else
18958 return std::make_pair(x: 0U, y: &PPC::VSFRCRegClass);
18959 } else if (Constraint == "lr") {
18960 if (VT == MVT::i64)
18961 return std::make_pair(x: 0U, y: &PPC::LR8RCRegClass);
18962 else
18963 return std::make_pair(x: 0U, y: &PPC::LRRCRegClass);
18964 }
18965
18966 // Handle special cases of physical registers that are not properly handled
18967 // by the base class.
18968 if (Constraint[0] == '{' && Constraint[Constraint.size() - 1] == '}') {
18969 // If we name a VSX register, we can't defer to the base class because it
18970 // will not recognize the correct register (their names will be VSL{0-31}
18971 // and V{0-31} so they won't match). So we match them here.
18972 if (Constraint.size() > 3 && Constraint[1] == 'v' && Constraint[2] == 's') {
18973 int VSNum = atoi(nptr: Constraint.data() + 3);
18974 assert(VSNum >= 0 && VSNum <= 63 &&
18975 "Attempted to access a vsr out of range");
18976 if (VSNum < 32)
18977 return std::make_pair(x: PPC::VSL0 + VSNum, y: &PPC::VSRCRegClass);
18978 return std::make_pair(x: PPC::V0 + VSNum - 32, y: &PPC::VSRCRegClass);
18979 }
18980
18981 // For float registers, we can't defer to the base class as it will match
18982 // the SPILLTOVSRRC class.
18983 if (Constraint.size() > 3 && Constraint[1] == 'f') {
18984 int RegNum = atoi(nptr: Constraint.data() + 2);
18985 if (RegNum > 31 || RegNum < 0)
18986 report_fatal_error(reason: "Invalid floating point register number");
18987 if (VT == MVT::f32 || VT == MVT::i32)
18988 return Subtarget.hasSPE()
18989 ? std::make_pair(x: PPC::R0 + RegNum, y: &PPC::GPRCRegClass)
18990 : std::make_pair(x: PPC::F0 + RegNum, y: &PPC::F4RCRegClass);
18991 if (VT == MVT::f64 || VT == MVT::i64)
18992 return Subtarget.hasSPE()
18993 ? std::make_pair(x: PPC::S0 + RegNum, y: &PPC::SPERCRegClass)
18994 : std::make_pair(x: PPC::F0 + RegNum, y: &PPC::F8RCRegClass);
18995 }
18996 }
18997
18998 std::pair<unsigned, const TargetRegisterClass *> R =
18999 TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
19000
19001 // r[0-9]+ are used, on PPC64, to refer to the corresponding 64-bit registers
19002 // (which we call X[0-9]+). If a 64-bit value has been requested, and a
19003 // 32-bit GPR has been selected, then 'upgrade' it to the 64-bit parent
19004 // register.
19005 // FIXME: If TargetLowering::getRegForInlineAsmConstraint could somehow use
19006 // the AsmName field from *RegisterInfo.td, then this would not be necessary.
19007 if (R.first && VT == MVT::i64 && Subtarget.isPPC64() &&
19008 PPC::GPRCRegClass.contains(Reg: R.first))
19009 return std::make_pair(x: TRI->getMatchingSuperReg(Reg: R.first,
19010 SubIdx: PPC::sub_32, RC: &PPC::G8RCRegClass),
19011 y: &PPC::G8RCRegClass);
19012
19013 // GCC accepts 'cc' as an alias for 'cr0', and we need to do the same.
19014 if (!R.second && StringRef("{cc}").equals_insensitive(RHS: Constraint)) {
19015 R.first = PPC::CR0;
19016 R.second = &PPC::CRRCRegClass;
19017 }
19018 // FIXME: This warning should ideally be emitted in the front end.
19019 const auto &TM = getTargetMachine();
19020 if (Subtarget.isAIXABI() && !TM.getAIXExtendedAltivecABI()) {
19021 if (((R.first >= PPC::V20 && R.first <= PPC::V31) ||
19022 (R.first >= PPC::VF20 && R.first <= PPC::VF31)) &&
19023 (R.second == &PPC::VSRCRegClass || R.second == &PPC::VSFRCRegClass))
19024 errs() << "warning: vector registers 20 to 32 are reserved in the "
19025 "default AIX AltiVec ABI and cannot be used\n";
19026 }
19027
19028 return R;
19029}
19030
19031/// LowerAsmOperandForConstraint - Lower the specified operand into the Ops
19032/// vector. If it is invalid, don't add anything to Ops.
19033void PPCTargetLowering::LowerAsmOperandForConstraint(SDValue Op,
19034 StringRef Constraint,
19035 std::vector<SDValue> &Ops,
19036 SelectionDAG &DAG) const {
19037 SDValue Result;
19038
19039 // Only support length 1 constraints.
19040 if (Constraint.size() > 1)
19041 return;
19042
19043 char Letter = Constraint[0];
19044 switch (Letter) {
19045 default: break;
19046 case 'I':
19047 case 'J':
19048 case 'K':
19049 case 'L':
19050 case 'M':
19051 case 'N':
19052 case 'O':
19053 case 'P': {
19054 ConstantSDNode *CST = dyn_cast<ConstantSDNode>(Val&: Op);
19055 if (!CST) return; // Must be an immediate to match.
19056 SDLoc dl(Op);
19057 int64_t Value = CST->getSExtValue();
19058 EVT TCVT = MVT::i64; // All constants taken to be 64 bits so that negative
19059 // numbers are printed as such.
19060 switch (Letter) {
19061 default: llvm_unreachable("Unknown constraint letter!");
19062 case 'I': // "I" is a signed 16-bit constant.
19063 if (isInt<16>(x: Value))
19064 Result = DAG.getTargetConstant(Val: Value, DL: dl, VT: TCVT);
19065 break;
19066 case 'J': // "J" is a constant with only the high-order 16 bits nonzero.
19067 if (isShiftedUInt<16, 16>(x: Value))
19068 Result = DAG.getTargetConstant(Val: Value, DL: dl, VT: TCVT);
19069 break;
19070 case 'L': // "L" is a signed 16-bit constant shifted left 16 bits.
19071 if (isShiftedInt<16, 16>(x: Value))
19072 Result = DAG.getTargetConstant(Val: Value, DL: dl, VT: TCVT);
19073 break;
19074 case 'K': // "K" is a constant with only the low-order 16 bits nonzero.
19075 if (isUInt<16>(x: Value))
19076 Result = DAG.getTargetConstant(Val: Value, DL: dl, VT: TCVT);
19077 break;
19078 case 'M': // "M" is a constant that is greater than 31.
19079 if (Value > 31)
19080 Result = DAG.getTargetConstant(Val: Value, DL: dl, VT: TCVT);
19081 break;
19082 case 'N': // "N" is a positive constant that is an exact power of two.
19083 if (Value > 0 && isPowerOf2_64(Value))
19084 Result = DAG.getTargetConstant(Val: Value, DL: dl, VT: TCVT);
19085 break;
19086 case 'O': // "O" is the constant zero.
19087 if (Value == 0)
19088 Result = DAG.getTargetConstant(Val: Value, DL: dl, VT: TCVT);
19089 break;
19090 case 'P': // "P" is a constant whose negation is a signed 16-bit constant.
19091 if (isInt<16>(x: -Value))
19092 Result = DAG.getTargetConstant(Val: Value, DL: dl, VT: TCVT);
19093 break;
19094 }
19095 break;
19096 }
19097 }
19098
19099 if (Result.getNode()) {
19100 Ops.push_back(x: Result);
19101 return;
19102 }
19103
19104 // Handle standard constraint letters.
19105 TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
19106}
19107
19108void PPCTargetLowering::CollectTargetIntrinsicOperands(const CallInst &I,
19109 SmallVectorImpl<SDValue> &Ops,
19110 SelectionDAG &DAG) const {
19111 if (I.getNumOperands() <= 1)
19112 return;
19113 if (!isa<ConstantSDNode>(Val: Ops[1].getNode()))
19114 return;
19115 auto IntrinsicID = Ops[1].getNode()->getAsZExtVal();
19116 if (IntrinsicID != Intrinsic::ppc_tdw && IntrinsicID != Intrinsic::ppc_tw &&
19117 IntrinsicID != Intrinsic::ppc_trapd && IntrinsicID != Intrinsic::ppc_trap)
19118 return;
19119
19120 if (MDNode *MDN = I.getMetadata(KindID: LLVMContext::MD_annotation))
19121 Ops.push_back(Elt: DAG.getMDNode(MD: MDN));
19122}
19123
19124// isLegalAddressingMode - Return true if the addressing mode represented
19125// by AM is legal for this target, for a load/store of the specified type.
19126bool PPCTargetLowering::isLegalAddressingMode(const DataLayout &DL,
19127 const AddrMode &AM, Type *Ty,
19128 unsigned AS,
19129 Instruction *I) const {
19130 // Vector type r+i form is supported since power9 as DQ form. We don't check
19131 // the offset matching DQ form requirement(off % 16 == 0), because on PowerPC,
19132 // imm form is preferred and the offset can be adjusted to use imm form later
19133 // in pass PPCLoopInstrFormPrep. Also in LSR, for one LSRUse, it uses min and
19134 // max offset to check legal addressing mode, we should be a little aggressive
19135 // to contain other offsets for that LSRUse.
19136 if (Ty->isVectorTy() && AM.BaseOffs != 0 && !Subtarget.hasP9Vector())
19137 return false;
19138
19139 // PPC allows a sign-extended 16-bit immediate field.
19140 if (AM.BaseOffs <= -(1LL << 16) || AM.BaseOffs >= (1LL << 16)-1)
19141 return false;
19142
19143 // No global is ever allowed as a base.
19144 if (AM.BaseGV)
19145 return false;
19146
19147 // PPC only support r+r,
19148 switch (AM.Scale) {
19149 case 0: // "r+i" or just "i", depending on HasBaseReg.
19150 break;
19151 case 1:
19152 if (AM.HasBaseReg && AM.BaseOffs) // "r+r+i" is not allowed.
19153 return false;
19154 // Otherwise we have r+r or r+i.
19155 break;
19156 case 2:
19157 if (AM.HasBaseReg || AM.BaseOffs) // 2*r+r or 2*r+i is not allowed.
19158 return false;
19159 // Allow 2*r as r+r.
19160 break;
19161 default:
19162 // No other scales are supported.
19163 return false;
19164 }
19165
19166 return true;
19167}
19168
19169SDValue PPCTargetLowering::LowerRETURNADDR(SDValue Op,
19170 SelectionDAG &DAG) const {
19171 MachineFunction &MF = DAG.getMachineFunction();
19172 MachineFrameInfo &MFI = MF.getFrameInfo();
19173 MFI.setReturnAddressIsTaken(true);
19174
19175 SDLoc dl(Op);
19176 unsigned Depth = Op.getConstantOperandVal(i: 0);
19177
19178 // Make sure the function does not optimize away the store of the RA to
19179 // the stack.
19180 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
19181 FuncInfo->setLRStoreRequired();
19182 auto PtrVT = getPointerTy(DL: MF.getDataLayout());
19183
19184 if (Depth > 0) {
19185 // The link register (return address) is saved in the caller's frame
19186 // not the callee's stack frame. So we must get the caller's frame
19187 // address and load the return address at the LR offset from there.
19188 SDValue FrameAddr =
19189 DAG.getLoad(VT: Op.getValueType(), dl, Chain: DAG.getEntryNode(),
19190 Ptr: LowerFRAMEADDR(Op, DAG), PtrInfo: MachinePointerInfo());
19191 SDValue Offset =
19192 DAG.getConstant(Val: Subtarget.getFrameLowering()->getReturnSaveOffset(), DL: dl,
19193 VT: Subtarget.getScalarIntVT());
19194 return DAG.getLoad(VT: PtrVT, dl, Chain: DAG.getEntryNode(),
19195 Ptr: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: FrameAddr, N2: Offset),
19196 PtrInfo: MachinePointerInfo());
19197 }
19198
19199 // Just load the return address off the stack.
19200 SDValue RetAddrFI = getReturnAddrFrameIndex(DAG);
19201 return DAG.getLoad(VT: PtrVT, dl, Chain: DAG.getEntryNode(), Ptr: RetAddrFI,
19202 PtrInfo: MachinePointerInfo());
19203}
19204
19205SDValue PPCTargetLowering::LowerFRAMEADDR(SDValue Op,
19206 SelectionDAG &DAG) const {
19207 SDLoc dl(Op);
19208 unsigned Depth = Op.getConstantOperandVal(i: 0);
19209
19210 MachineFunction &MF = DAG.getMachineFunction();
19211 MachineFrameInfo &MFI = MF.getFrameInfo();
19212 MFI.setFrameAddressIsTaken(true);
19213
19214 EVT PtrVT = getPointerTy(DL: MF.getDataLayout());
19215 bool isPPC64 = PtrVT == MVT::i64;
19216
19217 // Naked functions never have a frame pointer, and so we use r1. For all
19218 // other functions, this decision must be delayed until during PEI.
19219 unsigned FrameReg;
19220 if (MF.getFunction().hasFnAttribute(Kind: Attribute::Naked))
19221 FrameReg = isPPC64 ? PPC::X1 : PPC::R1;
19222 else
19223 FrameReg = isPPC64 ? PPC::FP8 : PPC::FP;
19224
19225 SDValue FrameAddr = DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl, Reg: FrameReg,
19226 VT: PtrVT);
19227 while (Depth--)
19228 FrameAddr = DAG.getLoad(VT: Op.getValueType(), dl, Chain: DAG.getEntryNode(),
19229 Ptr: FrameAddr, PtrInfo: MachinePointerInfo());
19230 return FrameAddr;
19231}
19232
19233#define GET_REGISTER_MATCHER
19234#include "PPCGenAsmMatcher.inc"
19235
19236Register PPCTargetLowering::getRegisterByName(const char *RegName, LLT VT,
19237 const MachineFunction &MF) const {
19238 bool IsPPC64 = Subtarget.isPPC64();
19239
19240 bool Is64Bit = IsPPC64 && VT == LLT::scalar(SizeInBits: 64);
19241 if (!Is64Bit && VT != LLT::scalar(SizeInBits: 32))
19242 report_fatal_error(reason: "Invalid register global variable type");
19243
19244 Register Reg = MatchRegisterName(Name: RegName);
19245 if (!Reg)
19246 return Reg;
19247
19248 // FIXME: Unable to generate code for `-O2` but okay for `-O0`.
19249 // Need followup investigation as to why.
19250 if ((IsPPC64 && Reg == PPC::R2) || Reg == PPC::R0)
19251 report_fatal_error(reason: Twine("Trying to reserve an invalid register \"" +
19252 StringRef(RegName) + "\"."));
19253
19254 // Convert GPR to GP8R register for 64bit.
19255 if (Is64Bit && StringRef(RegName).starts_with_insensitive(Prefix: "r"))
19256 Reg = Reg.id() - PPC::R0 + PPC::X0;
19257
19258 return Reg;
19259}
19260
19261bool PPCTargetLowering::isAccessedAsGotIndirect(SDValue GA) const {
19262 // 32-bit SVR4 ABI access everything as got-indirect.
19263 if (Subtarget.is32BitELFABI())
19264 return true;
19265
19266 // AIX accesses everything indirectly through the TOC, which is similar to
19267 // the GOT.
19268 if (Subtarget.isAIXABI())
19269 return true;
19270
19271 CodeModel::Model CModel = getTargetMachine().getCodeModel();
19272 // If it is small or large code model, module locals are accessed
19273 // indirectly by loading their address from .toc/.got.
19274 if (CModel == CodeModel::Small || CModel == CodeModel::Large)
19275 return true;
19276
19277 // JumpTable and BlockAddress are accessed as got-indirect.
19278 if (isa<JumpTableSDNode>(Val: GA) || isa<BlockAddressSDNode>(Val: GA))
19279 return true;
19280
19281 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Val&: GA))
19282 return Subtarget.isGVIndirectSymbol(GV: G->getGlobal());
19283
19284 return false;
19285}
19286
19287bool
19288PPCTargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const {
19289 // The PowerPC target isn't yet aware of offsets.
19290 return false;
19291}
19292
19293void PPCTargetLowering::getTgtMemIntrinsic(
19294 SmallVectorImpl<IntrinsicInfo> &Infos, const CallBase &I,
19295 MachineFunction &MF, unsigned Intrinsic) const {
19296 IntrinsicInfo Info;
19297 switch (Intrinsic) {
19298 case Intrinsic::ppc_atomicrmw_xchg_i128:
19299 case Intrinsic::ppc_atomicrmw_add_i128:
19300 case Intrinsic::ppc_atomicrmw_sub_i128:
19301 case Intrinsic::ppc_atomicrmw_nand_i128:
19302 case Intrinsic::ppc_atomicrmw_and_i128:
19303 case Intrinsic::ppc_atomicrmw_or_i128:
19304 case Intrinsic::ppc_atomicrmw_xor_i128:
19305 case Intrinsic::ppc_cmpxchg_i128:
19306 Info.opc = ISD::INTRINSIC_W_CHAIN;
19307 Info.memVT = MVT::i128;
19308 Info.ptrVal = I.getArgOperand(i: 0);
19309 Info.offset = 0;
19310 Info.align = Align(16);
19311 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
19312 MachineMemOperand::MOVolatile;
19313 Infos.push_back(Elt: Info);
19314 return;
19315 case Intrinsic::ppc_atomic_load_i128:
19316 Info.opc = ISD::INTRINSIC_W_CHAIN;
19317 Info.memVT = MVT::i128;
19318 Info.ptrVal = I.getArgOperand(i: 0);
19319 Info.offset = 0;
19320 Info.align = Align(16);
19321 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile;
19322 Infos.push_back(Elt: Info);
19323 return;
19324 case Intrinsic::ppc_atomic_store_i128:
19325 Info.opc = ISD::INTRINSIC_VOID;
19326 Info.memVT = MVT::i128;
19327 Info.ptrVal = I.getArgOperand(i: 2);
19328 Info.offset = 0;
19329 Info.align = Align(16);
19330 Info.flags = MachineMemOperand::MOStore | MachineMemOperand::MOVolatile;
19331 Infos.push_back(Elt: Info);
19332 return;
19333 case Intrinsic::ppc_altivec_lvx:
19334 case Intrinsic::ppc_altivec_lvxl:
19335 case Intrinsic::ppc_altivec_lvebx:
19336 case Intrinsic::ppc_altivec_lvehx:
19337 case Intrinsic::ppc_altivec_lvewx:
19338 case Intrinsic::ppc_vsx_lxvd2x:
19339 case Intrinsic::ppc_vsx_lxvw4x:
19340 case Intrinsic::ppc_vsx_lxvd2x_be:
19341 case Intrinsic::ppc_vsx_lxvw4x_be:
19342 case Intrinsic::ppc_vsx_lxvl:
19343 case Intrinsic::ppc_vsx_lxvll: {
19344 EVT VT;
19345 switch (Intrinsic) {
19346 case Intrinsic::ppc_altivec_lvebx:
19347 VT = MVT::i8;
19348 break;
19349 case Intrinsic::ppc_altivec_lvehx:
19350 VT = MVT::i16;
19351 break;
19352 case Intrinsic::ppc_altivec_lvewx:
19353 VT = MVT::i32;
19354 break;
19355 case Intrinsic::ppc_vsx_lxvd2x:
19356 case Intrinsic::ppc_vsx_lxvd2x_be:
19357 VT = MVT::v2f64;
19358 break;
19359 default:
19360 VT = MVT::v4i32;
19361 break;
19362 }
19363
19364 Info.opc = ISD::INTRINSIC_W_CHAIN;
19365 Info.memVT = VT;
19366 Info.ptrVal = I.getArgOperand(i: 0);
19367 Info.offset = -VT.getStoreSize()+1;
19368 Info.size = 2*VT.getStoreSize()-1;
19369 Info.align = Align(1);
19370 Info.flags = MachineMemOperand::MOLoad;
19371 Infos.push_back(Elt: Info);
19372 return;
19373 }
19374 case Intrinsic::ppc_altivec_stvx:
19375 case Intrinsic::ppc_altivec_stvxl:
19376 case Intrinsic::ppc_altivec_stvebx:
19377 case Intrinsic::ppc_altivec_stvehx:
19378 case Intrinsic::ppc_altivec_stvewx:
19379 case Intrinsic::ppc_vsx_stxvd2x:
19380 case Intrinsic::ppc_vsx_stxvw4x:
19381 case Intrinsic::ppc_vsx_stxvd2x_be:
19382 case Intrinsic::ppc_vsx_stxvw4x_be:
19383 case Intrinsic::ppc_vsx_stxvl:
19384 case Intrinsic::ppc_vsx_stxvll: {
19385 EVT VT;
19386 switch (Intrinsic) {
19387 case Intrinsic::ppc_altivec_stvebx:
19388 VT = MVT::i8;
19389 break;
19390 case Intrinsic::ppc_altivec_stvehx:
19391 VT = MVT::i16;
19392 break;
19393 case Intrinsic::ppc_altivec_stvewx:
19394 VT = MVT::i32;
19395 break;
19396 case Intrinsic::ppc_vsx_stxvd2x:
19397 case Intrinsic::ppc_vsx_stxvd2x_be:
19398 VT = MVT::v2f64;
19399 break;
19400 default:
19401 VT = MVT::v4i32;
19402 break;
19403 }
19404
19405 Info.opc = ISD::INTRINSIC_VOID;
19406 Info.memVT = VT;
19407 Info.ptrVal = I.getArgOperand(i: 1);
19408 Info.offset = -VT.getStoreSize()+1;
19409 Info.size = 2*VT.getStoreSize()-1;
19410 Info.align = Align(1);
19411 Info.flags = MachineMemOperand::MOStore;
19412 Infos.push_back(Elt: Info);
19413 return;
19414 }
19415 case Intrinsic::ppc_stdcx:
19416 case Intrinsic::ppc_stwcx:
19417 case Intrinsic::ppc_sthcx:
19418 case Intrinsic::ppc_stbcx: {
19419 EVT VT;
19420 auto Alignment = Align(8);
19421 switch (Intrinsic) {
19422 case Intrinsic::ppc_stdcx:
19423 VT = MVT::i64;
19424 break;
19425 case Intrinsic::ppc_stwcx:
19426 VT = MVT::i32;
19427 Alignment = Align(4);
19428 break;
19429 case Intrinsic::ppc_sthcx:
19430 VT = MVT::i16;
19431 Alignment = Align(2);
19432 break;
19433 case Intrinsic::ppc_stbcx:
19434 VT = MVT::i8;
19435 Alignment = Align(1);
19436 break;
19437 }
19438 Info.opc = ISD::INTRINSIC_W_CHAIN;
19439 Info.memVT = VT;
19440 Info.ptrVal = I.getArgOperand(i: 0);
19441 Info.offset = 0;
19442 Info.align = Alignment;
19443 Info.flags = MachineMemOperand::MOStore | MachineMemOperand::MOVolatile;
19444 Infos.push_back(Elt: Info);
19445 return;
19446 }
19447 default:
19448 break;
19449 }
19450}
19451
19452/// It returns EVT::Other if the type should be determined using generic
19453/// target-independent logic.
19454EVT PPCTargetLowering::getOptimalMemOpType(
19455 LLVMContext &Context, const MemOp &Op,
19456 const AttributeList &FuncAttributes) const {
19457 if (getTargetMachine().getOptLevel() != CodeGenOptLevel::None) {
19458 // We should use Altivec/VSX loads and stores when available. For unaligned
19459 // addresses, unaligned VSX loads are only fast starting with the P8.
19460 if (Subtarget.hasAltivec() && Op.size() >= 16) {
19461 if (Op.isMemset() && Subtarget.hasVSX()) {
19462 uint64_t TailSize = Op.size() % 16;
19463 // For memset lowering, EXTRACT_VECTOR_ELT tries to return constant
19464 // element if vector element type matches tail store. For tail size
19465 // 3/4, the tail store is i32, v4i32 cannot be used, need a legal one.
19466 if (TailSize > 2 && TailSize <= 4) {
19467 return MVT::v8i16;
19468 }
19469 return MVT::v4i32;
19470 }
19471 if (Op.isAligned(AlignCheck: Align(16)) || Subtarget.hasP8Vector())
19472 return MVT::v4i32;
19473 }
19474 }
19475
19476 if (Subtarget.isPPC64()) {
19477 return MVT::i64;
19478 }
19479
19480 return MVT::i32;
19481}
19482
19483/// Returns true if it is beneficial to convert a load of a constant
19484/// to just the constant itself.
19485bool PPCTargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
19486 Type *Ty) const {
19487 assert(Ty->isIntegerTy());
19488
19489 unsigned BitSize = Ty->getPrimitiveSizeInBits();
19490 return !(BitSize == 0 || BitSize > 64);
19491}
19492
19493bool PPCTargetLowering::isTruncateFree(Type *Ty1, Type *Ty2) const {
19494 if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy())
19495 return false;
19496 unsigned NumBits1 = Ty1->getPrimitiveSizeInBits();
19497 unsigned NumBits2 = Ty2->getPrimitiveSizeInBits();
19498 return NumBits1 == 64 && NumBits2 == 32;
19499}
19500
19501bool PPCTargetLowering::isTruncateFree(EVT VT1, EVT VT2) const {
19502 if (!VT1.isInteger() || !VT2.isInteger())
19503 return false;
19504 unsigned NumBits1 = VT1.getSizeInBits();
19505 unsigned NumBits2 = VT2.getSizeInBits();
19506 return NumBits1 == 64 && NumBits2 == 32;
19507}
19508
19509bool PPCTargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
19510 // Generally speaking, zexts are not free, but they are free when they can be
19511 // folded with other operations.
19512 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Val)) {
19513 EVT MemVT = LD->getMemoryVT();
19514 if ((MemVT == MVT::i1 || MemVT == MVT::i8 || MemVT == MVT::i16 ||
19515 (Subtarget.isPPC64() && MemVT == MVT::i32)) &&
19516 (LD->getExtensionType() == ISD::NON_EXTLOAD ||
19517 LD->getExtensionType() == ISD::ZEXTLOAD))
19518 return true;
19519 }
19520
19521 // FIXME: Add other cases...
19522 // - 32-bit shifts with a zext to i64
19523 // - zext after ctlz, bswap, etc.
19524 // - zext after and by a constant mask
19525
19526 return TargetLowering::isZExtFree(Val, VT2);
19527}
19528
19529bool PPCTargetLowering::isFPExtFree(EVT DestVT, EVT SrcVT) const {
19530 assert(DestVT.isFloatingPoint() && SrcVT.isFloatingPoint() &&
19531 "invalid fpext types");
19532 // Extending to float128 is not free.
19533 if (DestVT == MVT::f128)
19534 return false;
19535 return true;
19536}
19537
19538bool PPCTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
19539 return isInt<16>(x: Imm) || isUInt<16>(x: Imm);
19540}
19541
19542bool PPCTargetLowering::isLegalAddImmediate(int64_t Imm) const {
19543 return isInt<16>(x: Imm) || isUInt<16>(x: Imm);
19544}
19545
19546bool PPCTargetLowering::allowsMisalignedMemoryAccesses(EVT VT, unsigned, Align,
19547 MachineMemOperand::Flags,
19548 unsigned *Fast) const {
19549 if (DisablePPCUnaligned)
19550 return false;
19551
19552 // PowerPC supports unaligned memory access for simple non-vector types.
19553 // Although accessing unaligned addresses is not as efficient as accessing
19554 // aligned addresses, it is generally more efficient than manual expansion,
19555 // and generally only traps for software emulation when crossing page
19556 // boundaries.
19557
19558 if (!VT.isSimple())
19559 return false;
19560
19561 if (VT.isFloatingPoint() && !VT.isVector() &&
19562 !Subtarget.allowsUnalignedFPAccess())
19563 return false;
19564
19565 if (VT.getSimpleVT().isVector()) {
19566 if (Subtarget.hasVSX()) {
19567 if (VT != MVT::v2f64 && VT != MVT::v2i64 &&
19568 VT != MVT::v4f32 && VT != MVT::v4i32)
19569 return false;
19570 } else {
19571 return false;
19572 }
19573 }
19574
19575 if (VT == MVT::ppcf128)
19576 return false;
19577
19578 if (Fast)
19579 *Fast = 1;
19580
19581 return true;
19582}
19583
19584bool PPCTargetLowering::decomposeMulByConstant(LLVMContext &Context, EVT VT,
19585 SDValue C) const {
19586 // Check integral scalar types.
19587 if (!VT.isScalarInteger())
19588 return false;
19589 if (auto *ConstNode = dyn_cast<ConstantSDNode>(Val: C.getNode())) {
19590 if (!ConstNode->getAPIntValue().isSignedIntN(N: 64))
19591 return false;
19592 // This transformation will generate >= 2 operations. But the following
19593 // cases will generate <= 2 instructions during ISEL. So exclude them.
19594 // 1. If the constant multiplier fits 16 bits, it can be handled by one
19595 // HW instruction, ie. MULLI
19596 // 2. If the multiplier after shifted fits 16 bits, an extra shift
19597 // instruction is needed than case 1, ie. MULLI and RLDICR
19598 int64_t Imm = ConstNode->getSExtValue();
19599 unsigned Shift = llvm::countr_zero<uint64_t>(Val: Imm);
19600 Imm >>= Shift;
19601 if (isInt<16>(x: Imm))
19602 return false;
19603 uint64_t UImm = static_cast<uint64_t>(Imm);
19604 if (isPowerOf2_64(Value: UImm + 1) || isPowerOf2_64(Value: UImm - 1) ||
19605 isPowerOf2_64(Value: 1 - UImm) || isPowerOf2_64(Value: -1 - UImm))
19606 return true;
19607 }
19608 return false;
19609}
19610
19611bool PPCTargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
19612 EVT VT) const {
19613 return isFMAFasterThanFMulAndFAdd(
19614 F: MF.getFunction(), Ty: VT.getTypeForEVT(Context&: MF.getFunction().getContext()));
19615}
19616
19617bool PPCTargetLowering::isFMAFasterThanFMulAndFAdd(const Function &F,
19618 Type *Ty) const {
19619 if (Subtarget.hasSPE() || Subtarget.useSoftFloat())
19620 return false;
19621 switch (Ty->getScalarType()->getTypeID()) {
19622 case Type::FloatTyID:
19623 case Type::DoubleTyID:
19624 return true;
19625 case Type::FP128TyID:
19626 return Subtarget.hasP9Vector();
19627 default:
19628 return false;
19629 }
19630}
19631
19632// FIXME: add more patterns which are not profitable to hoist.
19633bool PPCTargetLowering::isProfitableToHoist(Instruction *I) const {
19634 if (!I->hasOneUse())
19635 return true;
19636
19637 Instruction *User = I->user_back();
19638 assert(User && "A single use instruction with no uses.");
19639
19640 switch (I->getOpcode()) {
19641 case Instruction::FMul: {
19642 // Don't break FMA, PowerPC prefers FMA.
19643 if (User->getOpcode() != Instruction::FSub &&
19644 User->getOpcode() != Instruction::FAdd)
19645 return true;
19646
19647 const TargetOptions &Options = getTargetMachine().Options;
19648 const Function *F = I->getFunction();
19649 const DataLayout &DL = F->getDataLayout();
19650 Type *Ty = User->getOperand(i: 0)->getType();
19651 bool AllowContract = I->getFastMathFlags().allowContract() &&
19652 User->getFastMathFlags().allowContract();
19653
19654 return !(isFMAFasterThanFMulAndFAdd(F: *F, Ty) &&
19655 isOperationLegalOrCustom(Op: ISD::FMA, VT: getValueType(DL, Ty)) &&
19656 (AllowContract || Options.AllowFPOpFusion == FPOpFusion::Fast));
19657 }
19658 case Instruction::Load: {
19659 // Don't break "store (load float*)" pattern, this pattern will be combined
19660 // to "store (load int32)" in later InstCombine pass. See function
19661 // combineLoadToOperationType. On PowerPC, loading a float point takes more
19662 // cycles than loading a 32 bit integer.
19663 LoadInst *LI = cast<LoadInst>(Val: I);
19664 // For the loads that combineLoadToOperationType does nothing, like
19665 // ordered load, it should be profitable to hoist them.
19666 // For swifterror load, it can only be used for pointer to pointer type, so
19667 // later type check should get rid of this case.
19668 if (!LI->isUnordered())
19669 return true;
19670
19671 if (User->getOpcode() != Instruction::Store)
19672 return true;
19673
19674 if (I->getType()->getTypeID() != Type::FloatTyID)
19675 return true;
19676
19677 return false;
19678 }
19679 default:
19680 return true;
19681 }
19682 return true;
19683}
19684
19685const MCPhysReg *
19686PPCTargetLowering::getScratchRegisters(CallingConv::ID) const {
19687 // LR is a callee-save register, but we must treat it as clobbered by any call
19688 // site. Hence we include LR in the scratch registers, which are in turn added
19689 // as implicit-defs for stackmaps and patchpoints. The same reasoning applies
19690 // to CTR, which is used by any indirect call.
19691 static const MCPhysReg ScratchRegs[] = {
19692 PPC::X12, PPC::LR8, PPC::CTR8, 0
19693 };
19694
19695 return ScratchRegs;
19696}
19697
19698Register PPCTargetLowering::getExceptionPointerRegister(
19699 ExceptionHandling EH, const Constant *PersonalityFn) const {
19700 return Subtarget.isPPC64() ? PPC::X3 : PPC::R3;
19701}
19702
19703Register PPCTargetLowering::getExceptionSelectorRegister(
19704 ExceptionHandling EH, const Constant *PersonalityFn) const {
19705 return Subtarget.isPPC64() ? PPC::X4 : PPC::R4;
19706}
19707
19708bool
19709PPCTargetLowering::shouldExpandBuildVectorWithShuffles(
19710 EVT VT , unsigned DefinedValues) const {
19711 if (VT == MVT::v2i64)
19712 return Subtarget.hasDirectMove(); // Don't need stack ops with direct moves
19713
19714 if (Subtarget.hasVSX())
19715 return true;
19716
19717 return TargetLowering::shouldExpandBuildVectorWithShuffles(VT, DefinedValues);
19718}
19719
19720Sched::Preference PPCTargetLowering::getSchedulingPreference(SDNode *N) const {
19721 if (DisableILPPref || Subtarget.enableMachineScheduler())
19722 return TargetLowering::getSchedulingPreference(N);
19723
19724 return Sched::ILP;
19725}
19726
19727// Create a fast isel object.
19728FastISel *PPCTargetLowering::createFastISel(
19729 FunctionLoweringInfo &FuncInfo, const TargetLibraryInfo *LibInfo,
19730 const LibcallLoweringInfo *LibcallLowering) const {
19731 return PPC::createFastISel(FuncInfo, LibInfo, LibcallLowering);
19732}
19733
19734// 'Inverted' means the FMA opcode after negating one multiplicand.
19735// For example, (fma -a b c) = (fnmsub a b c)
19736static unsigned invertFMAOpcode(unsigned Opc) {
19737 switch (Opc) {
19738 default:
19739 llvm_unreachable("Invalid FMA opcode for PowerPC!");
19740 case ISD::FMA:
19741 return PPCISD::FNMSUB;
19742 case PPCISD::FNMSUB:
19743 return ISD::FMA;
19744 }
19745}
19746
19747SDValue PPCTargetLowering::getNegatedExpression(SDValue Op, SelectionDAG &DAG,
19748 bool LegalOps, bool OptForSize,
19749 NegatibleCost &Cost,
19750 unsigned Depth) const {
19751 if (Depth > SelectionDAG::MaxRecursionDepth)
19752 return SDValue();
19753
19754 unsigned Opc = Op.getOpcode();
19755 EVT VT = Op.getValueType();
19756 SDNodeFlags Flags = Op.getNode()->getFlags();
19757
19758 switch (Opc) {
19759 case PPCISD::FNMSUB:
19760 if (!Op.hasOneUse() || !isTypeLegal(VT))
19761 break;
19762
19763 SDValue N0 = Op.getOperand(i: 0);
19764 SDValue N1 = Op.getOperand(i: 1);
19765 SDValue N2 = Op.getOperand(i: 2);
19766 SDLoc Loc(Op);
19767
19768 NegatibleCost N2Cost = NegatibleCost::Expensive;
19769 SDValue NegN2 =
19770 getNegatedExpression(Op: N2, DAG, LegalOps, OptForSize, Cost&: N2Cost, Depth: Depth + 1);
19771
19772 if (!NegN2)
19773 return SDValue();
19774
19775 // (fneg (fnmsub a b c)) => (fnmsub (fneg a) b (fneg c))
19776 // (fneg (fnmsub a b c)) => (fnmsub a (fneg b) (fneg c))
19777 // These transformations may change sign of zeroes. For example,
19778 // -(-ab-(-c))=-0 while -(-(ab-c))=+0 when a=b=c=1.
19779 if (Flags.hasNoSignedZeros()) {
19780 // Try and choose the cheaper one to negate.
19781 NegatibleCost N0Cost = NegatibleCost::Expensive;
19782 SDValue NegN0 = getNegatedExpression(Op: N0, DAG, LegalOps, OptForSize,
19783 Cost&: N0Cost, Depth: Depth + 1);
19784
19785 NegatibleCost N1Cost = NegatibleCost::Expensive;
19786 SDValue NegN1 = getNegatedExpression(Op: N1, DAG, LegalOps, OptForSize,
19787 Cost&: N1Cost, Depth: Depth + 1);
19788
19789 if (NegN0 && N0Cost <= N1Cost) {
19790 Cost = std::min(a: N0Cost, b: N2Cost);
19791 return DAG.getNode(Opcode: Opc, DL: Loc, VT, N1: NegN0, N2: N1, N3: NegN2, Flags);
19792 } else if (NegN1) {
19793 Cost = std::min(a: N1Cost, b: N2Cost);
19794 return DAG.getNode(Opcode: Opc, DL: Loc, VT, N1: N0, N2: NegN1, N3: NegN2, Flags);
19795 }
19796 }
19797
19798 // (fneg (fnmsub a b c)) => (fma a b (fneg c))
19799 if (isOperationLegal(Op: ISD::FMA, VT)) {
19800 Cost = N2Cost;
19801 return DAG.getNode(Opcode: ISD::FMA, DL: Loc, VT, N1: N0, N2: N1, N3: NegN2, Flags);
19802 }
19803
19804 break;
19805 }
19806
19807 return TargetLowering::getNegatedExpression(Op, DAG, LegalOps, OptForSize,
19808 Cost, Depth);
19809}
19810
19811// Override to enable LOAD_STACK_GUARD lowering on Linux.
19812bool PPCTargetLowering::useLoadStackGuardNode(const Module &M) const {
19813 if (M.getStackProtectorGuard() == "tls" || Subtarget.isTargetLinux())
19814 return true;
19815 return TargetLowering::useLoadStackGuardNode(M);
19816}
19817
19818bool PPCTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
19819 bool ForCodeSize) const {
19820 if (!VT.isSimple() || !Subtarget.hasVSX())
19821 return false;
19822
19823 switch(VT.getSimpleVT().SimpleTy) {
19824 default:
19825 // For FP types that are currently not supported by PPC backend, return
19826 // false. Examples: f16, f80.
19827 return false;
19828 case MVT::f32:
19829 case MVT::f64: {
19830 if (Subtarget.hasPrefixInstrs() && Subtarget.hasP10Vector()) {
19831 // we can materialize all immediatess via XXSPLTI32DX and XXSPLTIDP.
19832 return true;
19833 }
19834 bool IsExact;
19835 APSInt IntResult(16, false);
19836 // The rounding mode doesn't really matter because we only care about floats
19837 // that can be converted to integers exactly.
19838 Imm.convertToInteger(Result&: IntResult, RM: APFloat::rmTowardZero, IsExact: &IsExact);
19839 // For exact values in the range [-16, 15] we can materialize the float.
19840 if (IsExact && IntResult <= 15 && IntResult >= -16)
19841 return true;
19842 return Imm.isZero();
19843 }
19844 case MVT::ppcf128:
19845 return Imm.isPosZero();
19846 }
19847}
19848
19849// For vector shift operation op, fold
19850// (op x, (and y, ((1 << numbits(x)) - 1))) -> (target op x, y)
19851static SDValue stripModuloOnShift(const TargetLowering &TLI, SDNode *N,
19852 SelectionDAG &DAG) {
19853 SDValue N0 = N->getOperand(Num: 0);
19854 SDValue N1 = N->getOperand(Num: 1);
19855 EVT VT = N0.getValueType();
19856 unsigned OpSizeInBits = VT.getScalarSizeInBits();
19857 unsigned Opcode = N->getOpcode();
19858 unsigned TargetOpcode;
19859
19860 switch (Opcode) {
19861 default:
19862 llvm_unreachable("Unexpected shift operation");
19863 case ISD::SHL:
19864 TargetOpcode = PPCISD::SHL;
19865 break;
19866 case ISD::SRL:
19867 TargetOpcode = PPCISD::SRL;
19868 break;
19869 case ISD::SRA:
19870 TargetOpcode = PPCISD::SRA;
19871 break;
19872 }
19873
19874 if (VT.isVector() && TLI.isOperationLegal(Op: Opcode, VT) &&
19875 N1->getOpcode() == ISD::AND)
19876 if (ConstantSDNode *Mask = isConstOrConstSplat(N: N1->getOperand(Num: 1)))
19877 if (Mask->getZExtValue() == OpSizeInBits - 1)
19878 return DAG.getNode(Opcode: TargetOpcode, DL: SDLoc(N), VT, N1: N0, N2: N1->getOperand(Num: 0));
19879
19880 return SDValue();
19881}
19882
19883SDValue PPCTargetLowering::combineVectorShift(SDNode *N,
19884 DAGCombinerInfo &DCI) const {
19885 EVT VT = N->getValueType(ResNo: 0);
19886 assert(VT.isVector() && "Vector type expected.");
19887
19888 unsigned Opc = N->getOpcode();
19889 assert((Opc == ISD::SHL || Opc == ISD::SRL || Opc == ISD::SRA) &&
19890 "Unexpected opcode.");
19891
19892 if (!isOperationLegal(Op: Opc, VT))
19893 return SDValue();
19894
19895 EVT EltTy = VT.getScalarType();
19896 unsigned EltBits = EltTy.getSizeInBits();
19897 if (EltTy != MVT::i64 && EltTy != MVT::i32)
19898 return SDValue();
19899
19900 SDValue N1 = N->getOperand(Num: 1);
19901 uint64_t SplatBits = 0;
19902 bool AddSplatCase = false;
19903 unsigned OpcN1 = N1.getOpcode();
19904 if (OpcN1 == PPCISD::VADD_SPLAT &&
19905 N1.getConstantOperandVal(i: 1) == VT.getVectorNumElements()) {
19906 AddSplatCase = true;
19907 SplatBits = N1.getConstantOperandVal(i: 0);
19908 }
19909
19910 if (!AddSplatCase) {
19911 if (OpcN1 != ISD::BUILD_VECTOR)
19912 return SDValue();
19913
19914 unsigned SplatBitSize;
19915 bool HasAnyUndefs;
19916 APInt APSplatBits, APSplatUndef;
19917 BuildVectorSDNode *BVN = cast<BuildVectorSDNode>(Val&: N1);
19918 bool BVNIsConstantSplat =
19919 BVN->isConstantSplat(SplatValue&: APSplatBits, SplatUndef&: APSplatUndef, SplatBitSize,
19920 HasAnyUndefs, MinSplatBits: 0, isBigEndian: !Subtarget.isLittleEndian());
19921 if (!BVNIsConstantSplat || SplatBitSize != EltBits)
19922 return SDValue();
19923 SplatBits = APSplatBits.getZExtValue();
19924 }
19925
19926 SDLoc DL(N);
19927 SDValue N0 = N->getOperand(Num: 0);
19928 // PPC vector shifts by word/double look at only the low 5/6 bits of the
19929 // shift vector, which means the max value is 31/63. A shift vector of all
19930 // 1s will be truncated to 31/63, which is useful as vspltiw is limited to
19931 // -16 to 15 range.
19932 if (SplatBits == (EltBits - 1)) {
19933 unsigned NewOpc;
19934 switch (Opc) {
19935 case ISD::SHL:
19936 NewOpc = PPCISD::SHL;
19937 break;
19938 case ISD::SRL:
19939 NewOpc = PPCISD::SRL;
19940 break;
19941 case ISD::SRA:
19942 NewOpc = PPCISD::SRA;
19943 break;
19944 }
19945 SDValue SplatOnes = getCanonicalConstSplat(Val: 255, SplatSize: 1, VT, DAG&: DCI.DAG, dl: DL);
19946 return DCI.DAG.getNode(Opcode: NewOpc, DL, VT, N1: N0, N2: SplatOnes);
19947 }
19948
19949 if (Opc != ISD::SHL || !isOperationLegal(Op: ISD::ADD, VT))
19950 return SDValue();
19951
19952 // For 64-bit there is no splat immediate so we want to catch shift by 1 here
19953 // before the BUILD_VECTOR is replaced by a load.
19954 if (EltTy != MVT::i64 || SplatBits != 1)
19955 return SDValue();
19956
19957 return DCI.DAG.getNode(Opcode: ISD::ADD, DL: SDLoc(N), VT, N1: N0, N2: N0);
19958}
19959
19960SDValue PPCTargetLowering::combineSHL(SDNode *N, DAGCombinerInfo &DCI) const {
19961 if (auto Value = stripModuloOnShift(TLI: *this, N, DAG&: DCI.DAG))
19962 return Value;
19963
19964 if (N->getValueType(ResNo: 0).isVector())
19965 return combineVectorShift(N, DCI);
19966
19967 SDValue N0 = N->getOperand(Num: 0);
19968 ConstantSDNode *CN1 = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
19969 if (!Subtarget.isISA3_0() || !Subtarget.isPPC64() ||
19970 N0.getOpcode() != ISD::SIGN_EXTEND ||
19971 N0.getOperand(i: 0).getValueType() != MVT::i32 || CN1 == nullptr ||
19972 N->getValueType(ResNo: 0) != MVT::i64)
19973 return SDValue();
19974
19975 // We can't save an operation here if the value is already extended, and
19976 // the existing shift is easier to combine.
19977 SDValue ExtsSrc = N0.getOperand(i: 0);
19978 if (ExtsSrc.getOpcode() == ISD::TRUNCATE &&
19979 ExtsSrc.getOperand(i: 0).getOpcode() == ISD::AssertSext)
19980 return SDValue();
19981
19982 SDLoc DL(N0);
19983 SDValue ShiftBy = SDValue(CN1, 0);
19984 // We want the shift amount to be i32 on the extswli, but the shift could
19985 // have an i64.
19986 if (ShiftBy.getValueType() == MVT::i64)
19987 ShiftBy = DCI.DAG.getConstant(Val: CN1->getZExtValue(), DL, VT: MVT::i32);
19988
19989 return DCI.DAG.getNode(Opcode: PPCISD::EXTSWSLI, DL, VT: MVT::i64, N1: N0->getOperand(Num: 0),
19990 N2: ShiftBy);
19991}
19992
19993SDValue PPCTargetLowering::combineSRA(SDNode *N, DAGCombinerInfo &DCI) const {
19994 if (auto Value = stripModuloOnShift(TLI: *this, N, DAG&: DCI.DAG))
19995 return Value;
19996
19997 if (N->getValueType(ResNo: 0).isVector())
19998 return combineVectorShift(N, DCI);
19999
20000 return SDValue();
20001}
20002
20003SDValue PPCTargetLowering::combineSRL(SDNode *N, DAGCombinerInfo &DCI) const {
20004 if (auto Value = stripModuloOnShift(TLI: *this, N, DAG&: DCI.DAG))
20005 return Value;
20006
20007 if (N->getValueType(ResNo: 0).isVector())
20008 return combineVectorShift(N, DCI);
20009
20010 return SDValue();
20011}
20012
20013// Transform (add X, (zext(setne Z, C))) -> (addze X, (addic (addi Z, -C), -1))
20014// Transform (add X, (zext(sete Z, C))) -> (addze X, (subfic (addi Z, -C), 0))
20015// When C is zero, the equation (addi Z, -C) can be simplified to Z
20016// Requirement: -C in [-32768, 32767], X and Z are MVT::i64 types
20017static SDValue combineADDToADDZE(SDNode *N, SelectionDAG &DAG,
20018 const PPCSubtarget &Subtarget) {
20019 if (!Subtarget.isPPC64())
20020 return SDValue();
20021
20022 SDValue LHS = N->getOperand(Num: 0);
20023 SDValue RHS = N->getOperand(Num: 1);
20024
20025 auto isZextOfCompareWithConstant = [](SDValue Op) {
20026 if (Op.getOpcode() != ISD::ZERO_EXTEND || !Op.hasOneUse() ||
20027 Op.getValueType() != MVT::i64)
20028 return false;
20029
20030 SDValue Cmp = Op.getOperand(i: 0);
20031 if (Cmp.getOpcode() != ISD::SETCC || !Cmp.hasOneUse() ||
20032 Cmp.getOperand(i: 0).getValueType() != MVT::i64)
20033 return false;
20034
20035 if (auto *Constant = dyn_cast<ConstantSDNode>(Val: Cmp.getOperand(i: 1))) {
20036 int64_t NegConstant = 0 - Constant->getSExtValue();
20037 // Due to the limitations of the addi instruction,
20038 // -C is required to be [-32768, 32767].
20039 return isInt<16>(x: NegConstant);
20040 }
20041
20042 return false;
20043 };
20044
20045 bool LHSHasPattern = isZextOfCompareWithConstant(LHS);
20046 bool RHSHasPattern = isZextOfCompareWithConstant(RHS);
20047
20048 // If there is a pattern, canonicalize a zext operand to the RHS.
20049 if (LHSHasPattern && !RHSHasPattern)
20050 std::swap(a&: LHS, b&: RHS);
20051 else if (!LHSHasPattern && !RHSHasPattern)
20052 return SDValue();
20053
20054 SDLoc DL(N);
20055 EVT CarryType = Subtarget.useCRBits() ? MVT::i1 : MVT::i32;
20056 SDVTList VTs = DAG.getVTList(VT1: MVT::i64, VT2: CarryType);
20057 SDValue Cmp = RHS.getOperand(i: 0);
20058 SDValue Z = Cmp.getOperand(i: 0);
20059 auto *Constant = cast<ConstantSDNode>(Val: Cmp.getOperand(i: 1));
20060 int64_t NegConstant = 0 - Constant->getSExtValue();
20061
20062 switch(cast<CondCodeSDNode>(Val: Cmp.getOperand(i: 2))->get()) {
20063 default: break;
20064 case ISD::SETNE: {
20065 // when C == 0
20066 // --> addze X, (addic Z, -1).carry
20067 // /
20068 // add X, (zext(setne Z, C))--
20069 // \ when -32768 <= -C <= 32767 && C != 0
20070 // --> addze X, (addic (addi Z, -C), -1).carry
20071 SDValue Add = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i64, N1: Z,
20072 N2: DAG.getConstant(Val: NegConstant, DL, VT: MVT::i64));
20073 SDValue AddOrZ = NegConstant != 0 ? Add : Z;
20074 SDValue Addc =
20075 DAG.getNode(Opcode: ISD::UADDO_CARRY, DL, VTList: DAG.getVTList(VT1: MVT::i64, VT2: CarryType),
20076 N1: AddOrZ, N2: DAG.getAllOnesConstant(DL, VT: MVT::i64),
20077 N3: DAG.getConstant(Val: 0, DL, VT: CarryType));
20078 return DAG.getNode(Opcode: ISD::UADDO_CARRY, DL, VTList: VTs, N1: LHS,
20079 N2: DAG.getConstant(Val: 0, DL, VT: MVT::i64),
20080 N3: SDValue(Addc.getNode(), 1));
20081 }
20082 case ISD::SETEQ: {
20083 // when C == 0
20084 // --> addze X, (subfic Z, 0).carry
20085 // /
20086 // add X, (zext(sete Z, C))--
20087 // \ when -32768 <= -C <= 32767 && C != 0
20088 // --> addze X, (subfic (addi Z, -C), 0).carry
20089 SDValue Add = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i64, N1: Z,
20090 N2: DAG.getConstant(Val: NegConstant, DL, VT: MVT::i64));
20091 SDValue AddOrZ = NegConstant != 0 ? Add : Z;
20092 SDValue Subc =
20093 DAG.getNode(Opcode: ISD::USUBO_CARRY, DL, VTList: DAG.getVTList(VT1: MVT::i64, VT2: CarryType),
20094 N1: DAG.getConstant(Val: 0, DL, VT: MVT::i64), N2: AddOrZ,
20095 N3: DAG.getConstant(Val: 0, DL, VT: CarryType));
20096 SDValue Invert = DAG.getNode(Opcode: ISD::XOR, DL, VT: CarryType, N1: Subc.getValue(R: 1),
20097 N2: DAG.getConstant(Val: 1UL, DL, VT: CarryType));
20098 return DAG.getNode(Opcode: ISD::UADDO_CARRY, DL, VTList: VTs, N1: LHS,
20099 N2: DAG.getConstant(Val: 0, DL, VT: MVT::i64), N3: Invert);
20100 }
20101 }
20102
20103 return SDValue();
20104}
20105
20106// Transform
20107// (add C1, (MAT_PCREL_ADDR GlobalAddr+C2)) to
20108// (MAT_PCREL_ADDR GlobalAddr+(C1+C2))
20109// In this case both C1 and C2 must be known constants.
20110// C1+C2 must fit into a 34 bit signed integer.
20111static SDValue combineADDToMAT_PCREL_ADDR(SDNode *N, SelectionDAG &DAG,
20112 const PPCSubtarget &Subtarget) {
20113 if (!Subtarget.isUsingPCRelativeCalls())
20114 return SDValue();
20115
20116 // Check both Operand 0 and Operand 1 of the ADD node for the PCRel node.
20117 // If we find that node try to cast the Global Address and the Constant.
20118 SDValue LHS = N->getOperand(Num: 0);
20119 SDValue RHS = N->getOperand(Num: 1);
20120
20121 if (LHS.getOpcode() != PPCISD::MAT_PCREL_ADDR)
20122 std::swap(a&: LHS, b&: RHS);
20123
20124 if (LHS.getOpcode() != PPCISD::MAT_PCREL_ADDR)
20125 return SDValue();
20126
20127 // Operand zero of PPCISD::MAT_PCREL_ADDR is the GA node.
20128 GlobalAddressSDNode *GSDN = dyn_cast<GlobalAddressSDNode>(Val: LHS.getOperand(i: 0));
20129 ConstantSDNode* ConstNode = dyn_cast<ConstantSDNode>(Val&: RHS);
20130
20131 // Check that both casts succeeded.
20132 if (!GSDN || !ConstNode)
20133 return SDValue();
20134
20135 int64_t NewOffset = GSDN->getOffset() + ConstNode->getSExtValue();
20136 SDLoc DL(GSDN);
20137
20138 // The signed int offset needs to fit in 34 bits.
20139 if (!isInt<34>(x: NewOffset))
20140 return SDValue();
20141
20142 // The new global address is a copy of the old global address except
20143 // that it has the updated Offset.
20144 SDValue GA =
20145 DAG.getTargetGlobalAddress(GV: GSDN->getGlobal(), DL, VT: GSDN->getValueType(ResNo: 0),
20146 offset: NewOffset, TargetFlags: GSDN->getTargetFlags());
20147 SDValue MatPCRel =
20148 DAG.getNode(Opcode: PPCISD::MAT_PCREL_ADDR, DL, VT: GSDN->getValueType(ResNo: 0), Operand: GA);
20149 return MatPCRel;
20150}
20151
20152// Transform (add X, (build_vector (T 1), (T 1), ...)) -> (sub X, (XXLEQVOnes))
20153// XXLEQVOnes creates an all-1s vector (0xFFFFFFFF...) efficiently via xxleqv
20154// Mathematical identity: X + 1 = X - (-1)
20155// Applies to v4i32, v2i64, v8i16, v16i8 where all elements are constant 1
20156// Requirement: VSX feature for efficient xxleqv generation
20157static SDValue combineADDToSUB(SDNode *N, SelectionDAG &DAG,
20158 const PPCSubtarget &Subtarget) {
20159
20160 EVT VT = N->getValueType(ResNo: 0);
20161 if (!Subtarget.hasVSX())
20162 return SDValue();
20163
20164 // Handle v2i64, v4i32, v8i16 and v16i8 types
20165 if (!(VT == MVT::v8i16 || VT == MVT::v16i8 || VT == MVT::v4i32 ||
20166 VT == MVT::v2i64))
20167 return SDValue();
20168
20169 SDValue LHS = N->getOperand(Num: 0);
20170 SDValue RHS = N->getOperand(Num: 1);
20171
20172 // Check if RHS is BUILD_VECTOR
20173 if (RHS.getOpcode() != ISD::BUILD_VECTOR)
20174 return SDValue();
20175
20176 // Check if all the elements are 1
20177 unsigned NumOfEles = RHS.getNumOperands();
20178 for (unsigned i = 0; i < NumOfEles; ++i) {
20179 auto *CN = dyn_cast<ConstantSDNode>(Val: RHS.getOperand(i));
20180 if (!CN || CN->getSExtValue() != 1)
20181 return SDValue();
20182 }
20183 SDLoc DL(N);
20184
20185 SDValue MinusOne = DAG.getConstant(Val: APInt::getAllOnes(numBits: 32), DL, VT: MVT::i32);
20186 SmallVector<SDValue, 4> Ops(4, MinusOne);
20187 SDValue AllOnesVec = DAG.getBuildVector(VT: MVT::v4i32, DL, Ops);
20188
20189 // Bitcast to the target vector type
20190 SDValue Bitcast = DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: AllOnesVec);
20191
20192 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: LHS, N2: Bitcast);
20193}
20194
20195SDValue PPCTargetLowering::combineADD(SDNode *N, DAGCombinerInfo &DCI) const {
20196 if (auto Value = combineADDToADDZE(N, DAG&: DCI.DAG, Subtarget))
20197 return Value;
20198
20199 if (auto Value = combineADDToMAT_PCREL_ADDR(N, DAG&: DCI.DAG, Subtarget))
20200 return Value;
20201
20202 if (auto Value = combineADDToSUB(N, DAG&: DCI.DAG, Subtarget))
20203 return Value;
20204 return SDValue();
20205}
20206
20207// Detect TRUNCATE operations on bitcasts of float128 values.
20208// What we are looking for here is the situtation where we extract a subset
20209// of bits from a 128 bit float.
20210// This can be of two forms:
20211// 1) BITCAST of f128 feeding TRUNCATE
20212// 2) BITCAST of f128 feeding SRL (a shift) feeding TRUNCATE
20213// The reason this is required is because we do not have a legal i128 type
20214// and so we want to prevent having to store the f128 and then reload part
20215// of it.
20216SDValue PPCTargetLowering::combineTRUNCATE(SDNode *N,
20217 DAGCombinerInfo &DCI) const {
20218 // If we are using CRBits then try that first.
20219 if (Subtarget.useCRBits()) {
20220 // Check if CRBits did anything and return that if it did.
20221 if (SDValue CRTruncValue = DAGCombineTruncBoolExt(N, DCI))
20222 return CRTruncValue;
20223 }
20224
20225 SDLoc dl(N);
20226 SDValue Op0 = N->getOperand(Num: 0);
20227
20228 // Looking for a truncate of i128 to i64.
20229 if (Op0.getValueType() != MVT::i128 || N->getValueType(ResNo: 0) != MVT::i64)
20230 return SDValue();
20231
20232 int EltToExtract = DCI.DAG.getDataLayout().isBigEndian() ? 1 : 0;
20233
20234 // SRL feeding TRUNCATE.
20235 if (Op0.getOpcode() == ISD::SRL) {
20236 ConstantSDNode *ConstNode = dyn_cast<ConstantSDNode>(Val: Op0.getOperand(i: 1));
20237 // The right shift has to be by 64 bits.
20238 if (!ConstNode || ConstNode->getZExtValue() != 64)
20239 return SDValue();
20240
20241 // Switch the element number to extract.
20242 EltToExtract = EltToExtract ? 0 : 1;
20243 // Update Op0 past the SRL.
20244 Op0 = Op0.getOperand(i: 0);
20245 }
20246
20247 // BITCAST feeding a TRUNCATE possibly via SRL.
20248 if (Op0.getOpcode() == ISD::BITCAST &&
20249 Op0.getValueType() == MVT::i128 &&
20250 Op0.getOperand(i: 0).getValueType() == MVT::f128) {
20251 SDValue Bitcast = DCI.DAG.getBitcast(VT: MVT::v2i64, V: Op0.getOperand(i: 0));
20252 return DCI.DAG.getNode(
20253 Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: MVT::i64, N1: Bitcast,
20254 N2: DCI.DAG.getTargetConstant(Val: EltToExtract, DL: dl, VT: MVT::i32));
20255 }
20256 return SDValue();
20257}
20258
20259SDValue PPCTargetLowering::combineMUL(SDNode *N, DAGCombinerInfo &DCI) const {
20260 SelectionDAG &DAG = DCI.DAG;
20261
20262 ConstantSDNode *ConstOpOrElement = isConstOrConstSplat(N: N->getOperand(Num: 1));
20263 if (!ConstOpOrElement)
20264 return SDValue();
20265
20266 // An imul is usually smaller than the alternative sequence for legal type.
20267 if (DAG.getMachineFunction().getFunction().hasMinSize() &&
20268 isOperationLegal(Op: ISD::MUL, VT: N->getValueType(ResNo: 0)))
20269 return SDValue();
20270
20271 auto IsProfitable = [this](bool IsNeg, bool IsAddOne, EVT VT) -> bool {
20272 switch (this->Subtarget.getCPUDirective()) {
20273 default:
20274 // TODO: enhance the condition for subtarget before pwr8
20275 return false;
20276 case PPC::DIR_PWR8:
20277 // type mul add shl
20278 // scalar 4 1 1
20279 // vector 7 2 2
20280 return true;
20281 case PPC::DIR_PWR9:
20282 case PPC::DIR_PWR10:
20283 case PPC::DIR_PWR11:
20284 case PPC::DIR_PWR_FUTURE:
20285 // type mul add shl
20286 // scalar 5 2 2
20287 // vector 7 2 2
20288
20289 // The cycle RATIO of related operations are showed as a table above.
20290 // Because mul is 5(scalar)/7(vector), add/sub/shl are all 2 for both
20291 // scalar and vector type. For 2 instrs patterns, add/sub + shl
20292 // are 4, it is always profitable; but for 3 instrs patterns
20293 // (mul x, -(2^N + 1)) => -(add (shl x, N), x), sub + add + shl are 6.
20294 // So we should only do it for vector type.
20295 return IsAddOne && IsNeg ? VT.isVector() : true;
20296 }
20297 };
20298
20299 EVT VT = N->getValueType(ResNo: 0);
20300 SDLoc DL(N);
20301
20302 const APInt &MulAmt = ConstOpOrElement->getAPIntValue();
20303 bool IsNeg = MulAmt.isNegative();
20304 APInt MulAmtAbs = MulAmt.abs();
20305
20306 if ((MulAmtAbs - 1).isPowerOf2()) {
20307 // (mul x, 2^N + 1) => (add (shl x, N), x)
20308 // (mul x, -(2^N + 1)) => -(add (shl x, N), x)
20309
20310 if (!IsProfitable(IsNeg, true, VT))
20311 return SDValue();
20312
20313 SDValue Op0 = N->getOperand(Num: 0);
20314 SDValue Op1 =
20315 DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: N->getOperand(Num: 0),
20316 N2: DAG.getConstant(Val: (MulAmtAbs - 1).logBase2(), DL, VT));
20317 SDValue Res = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Op0, N2: Op1);
20318
20319 if (!IsNeg)
20320 return Res;
20321
20322 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: DAG.getConstant(Val: 0, DL, VT), N2: Res);
20323 } else if ((MulAmtAbs + 1).isPowerOf2()) {
20324 // (mul x, 2^N - 1) => (sub (shl x, N), x)
20325 // (mul x, -(2^N - 1)) => (sub x, (shl x, N))
20326
20327 if (!IsProfitable(IsNeg, false, VT))
20328 return SDValue();
20329
20330 SDValue Op0 = N->getOperand(Num: 0);
20331 SDValue Op1 =
20332 DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: N->getOperand(Num: 0),
20333 N2: DAG.getConstant(Val: (MulAmtAbs + 1).logBase2(), DL, VT));
20334
20335 if (!IsNeg)
20336 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Op1, N2: Op0);
20337 else
20338 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Op0, N2: Op1);
20339
20340 } else {
20341 return SDValue();
20342 }
20343}
20344
20345// Combine fma-like op (like fnmsub) with fnegs to appropriate op. Do this
20346// in combiner since we need to check SD flags and other subtarget features.
20347SDValue PPCTargetLowering::combineFMALike(SDNode *N,
20348 DAGCombinerInfo &DCI) const {
20349 SDValue N0 = N->getOperand(Num: 0);
20350 SDValue N1 = N->getOperand(Num: 1);
20351 SDValue N2 = N->getOperand(Num: 2);
20352 SDNodeFlags Flags = N->getFlags();
20353 EVT VT = N->getValueType(ResNo: 0);
20354 SelectionDAG &DAG = DCI.DAG;
20355 unsigned Opc = N->getOpcode();
20356 bool CodeSize = DAG.getMachineFunction().getFunction().hasOptSize();
20357 bool LegalOps = !DCI.isBeforeLegalizeOps();
20358 SDLoc Loc(N);
20359
20360 if (!isOperationLegal(Op: ISD::FMA, VT))
20361 return SDValue();
20362
20363 // Allowing transformation to FNMSUB may change sign of zeroes when ab-c=0
20364 // since (fnmsub a b c)=-0 while c-ab=+0.
20365 if (!Flags.hasNoSignedZeros())
20366 return SDValue();
20367
20368 // (fma (fneg a) b c) => (fnmsub a b c)
20369 // (fnmsub (fneg a) b c) => (fma a b c)
20370 if (SDValue NegN0 = getCheaperNegatedExpression(Op: N0, DAG, LegalOps, OptForSize: CodeSize))
20371 return DAG.getNode(Opcode: invertFMAOpcode(Opc), DL: Loc, VT, N1: NegN0, N2: N1, N3: N2, Flags);
20372
20373 // (fma a (fneg b) c) => (fnmsub a b c)
20374 // (fnmsub a (fneg b) c) => (fma a b c)
20375 if (SDValue NegN1 = getCheaperNegatedExpression(Op: N1, DAG, LegalOps, OptForSize: CodeSize))
20376 return DAG.getNode(Opcode: invertFMAOpcode(Opc), DL: Loc, VT, N1: N0, N2: NegN1, N3: N2, Flags);
20377
20378 return SDValue();
20379}
20380
20381bool PPCTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
20382 // Only duplicate to increase tail-calls for the 64bit SysV ABIs.
20383 if (!Subtarget.is64BitELFABI())
20384 return false;
20385
20386 // If not a tail call then no need to proceed.
20387 if (!CI->isTailCall())
20388 return false;
20389
20390 // If sibling calls have been disabled and tail-calls aren't guaranteed
20391 // there is no reason to duplicate.
20392 auto &TM = getTargetMachine();
20393 if (!TM.Options.GuaranteedTailCallOpt && DisableSCO)
20394 return false;
20395
20396 // Can't tail call a function called indirectly, or if it has variadic args.
20397 const Function *Callee = CI->getCalledFunction();
20398 if (!Callee || Callee->isVarArg())
20399 return false;
20400
20401 // Make sure the callee and caller calling conventions are eligible for tco.
20402 const Function *Caller = CI->getParent()->getParent();
20403 if (!areCallingConvEligibleForTCO_64SVR4(CallerCC: Caller->getCallingConv(),
20404 CalleeCC: CI->getCallingConv()))
20405 return false;
20406
20407 // If the function is local then we have a good chance at tail-calling it
20408 return getTargetMachine().shouldAssumeDSOLocal(GV: Callee);
20409}
20410
20411bool PPCTargetLowering::
20412isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const {
20413 const Value *Mask = AndI.getOperand(i: 1);
20414 // If the mask is suitable for andi. or andis. we should sink the and.
20415 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Val: Mask)) {
20416 // Can't handle constants wider than 64-bits.
20417 if (CI->getBitWidth() > 64)
20418 return false;
20419 int64_t ConstVal = CI->getZExtValue();
20420 return isUInt<16>(x: ConstVal) ||
20421 (isUInt<16>(x: ConstVal >> 16) && !(ConstVal & 0xFFFF));
20422 }
20423
20424 // For non-constant masks, we can always use the record-form and.
20425 return true;
20426}
20427
20428/// getAddrModeForFlags - Based on the set of address flags, select the most
20429/// optimal instruction format to match by.
20430PPC::AddrMode PPCTargetLowering::getAddrModeForFlags(unsigned Flags) const {
20431 // This is not a node we should be handling here.
20432 if (Flags == PPC::MOF_None)
20433 return PPC::AM_None;
20434 // Unaligned D-Forms are tried first, followed by the aligned D-Forms.
20435 for (auto FlagSet : AddrModesMap.at(k: PPC::AM_DForm))
20436 if ((Flags & FlagSet) == FlagSet)
20437 return PPC::AM_DForm;
20438 for (auto FlagSet : AddrModesMap.at(k: PPC::AM_DSForm))
20439 if ((Flags & FlagSet) == FlagSet)
20440 return PPC::AM_DSForm;
20441 for (auto FlagSet : AddrModesMap.at(k: PPC::AM_DQForm))
20442 if ((Flags & FlagSet) == FlagSet)
20443 return PPC::AM_DQForm;
20444 for (auto FlagSet : AddrModesMap.at(k: PPC::AM_PrefixDForm))
20445 if ((Flags & FlagSet) == FlagSet)
20446 return PPC::AM_PrefixDForm;
20447 // If no other forms are selected, return an X-Form as it is the most
20448 // general addressing mode.
20449 return PPC::AM_XForm;
20450}
20451
20452/// Set alignment flags based on whether or not the Frame Index is aligned.
20453/// Utilized when computing flags for address computation when selecting
20454/// load and store instructions.
20455static void setAlignFlagsForFI(SDValue N, unsigned &FlagSet,
20456 SelectionDAG &DAG) {
20457 bool IsAdd = ((N.getOpcode() == ISD::ADD) || (N.getOpcode() == ISD::OR));
20458 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val: IsAdd ? N.getOperand(i: 0) : N);
20459 if (!FI)
20460 return;
20461 const MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
20462 unsigned FrameIndexAlign = MFI.getObjectAlign(ObjectIdx: FI->getIndex()).value();
20463 // If this is (add $FI, $S16Imm), the alignment flags are already set
20464 // based on the immediate. We just need to clear the alignment flags
20465 // if the FI alignment is weaker.
20466 if ((FrameIndexAlign % 4) != 0)
20467 FlagSet &= ~PPC::MOF_RPlusSImm16Mult4;
20468 if ((FrameIndexAlign % 16) != 0)
20469 FlagSet &= ~PPC::MOF_RPlusSImm16Mult16;
20470 // If the address is a plain FrameIndex, set alignment flags based on
20471 // FI alignment.
20472 if (!IsAdd) {
20473 if ((FrameIndexAlign % 4) == 0)
20474 FlagSet |= PPC::MOF_RPlusSImm16Mult4;
20475 if ((FrameIndexAlign % 16) == 0)
20476 FlagSet |= PPC::MOF_RPlusSImm16Mult16;
20477 }
20478}
20479
20480/// Given a node, compute flags that are used for address computation when
20481/// selecting load and store instructions. The flags computed are stored in
20482/// FlagSet. This function takes into account whether the node is a constant,
20483/// an ADD, OR, or a constant, and computes the address flags accordingly.
20484static void computeFlagsForAddressComputation(SDValue N, unsigned &FlagSet,
20485 SelectionDAG &DAG) {
20486 // Set the alignment flags for the node depending on if the node is
20487 // 4-byte or 16-byte aligned.
20488 auto SetAlignFlagsForImm = [&](uint64_t Imm) {
20489 if ((Imm & 0x3) == 0)
20490 FlagSet |= PPC::MOF_RPlusSImm16Mult4;
20491 if ((Imm & 0xf) == 0)
20492 FlagSet |= PPC::MOF_RPlusSImm16Mult16;
20493 };
20494
20495 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val&: N)) {
20496 // All 32-bit constants can be computed as LIS + Disp.
20497 const APInt &ConstImm = CN->getAPIntValue();
20498 if (ConstImm.isSignedIntN(N: 32)) { // Flag to handle 32-bit constants.
20499 FlagSet |= PPC::MOF_AddrIsSImm32;
20500 SetAlignFlagsForImm(ConstImm.getZExtValue());
20501 setAlignFlagsForFI(N, FlagSet, DAG);
20502 }
20503 if (ConstImm.isSignedIntN(N: 34)) // Flag to handle 34-bit constants.
20504 FlagSet |= PPC::MOF_RPlusSImm34;
20505 else // Let constant materialization handle large constants.
20506 FlagSet |= PPC::MOF_NotAddNorCst;
20507 } else if (N.getOpcode() == ISD::ADD || provablyDisjointOr(DAG, N)) {
20508 // This address can be represented as an addition of:
20509 // - Register + Imm16 (possibly a multiple of 4/16)
20510 // - Register + Imm34
20511 // - Register + PPCISD::Lo
20512 // - Register + Register
20513 // In any case, we won't have to match this as Base + Zero.
20514 SDValue RHS = N.getOperand(i: 1);
20515 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val&: RHS)) {
20516 const APInt &ConstImm = CN->getAPIntValue();
20517 if (ConstImm.isSignedIntN(N: 16)) {
20518 FlagSet |= PPC::MOF_RPlusSImm16; // Signed 16-bit immediates.
20519 SetAlignFlagsForImm(ConstImm.getZExtValue());
20520 setAlignFlagsForFI(N, FlagSet, DAG);
20521 }
20522 if (ConstImm.isSignedIntN(N: 34))
20523 FlagSet |= PPC::MOF_RPlusSImm34; // Signed 34-bit immediates.
20524 else
20525 FlagSet |= PPC::MOF_RPlusR; // Register.
20526 } else if (RHS.getOpcode() == PPCISD::Lo && !RHS.getConstantOperandVal(i: 1))
20527 FlagSet |= PPC::MOF_RPlusLo; // PPCISD::Lo.
20528 else
20529 FlagSet |= PPC::MOF_RPlusR;
20530 } else { // The address computation is not a constant or an addition.
20531 setAlignFlagsForFI(N, FlagSet, DAG);
20532 FlagSet |= PPC::MOF_NotAddNorCst;
20533 }
20534}
20535
20536static bool isPCRelNode(SDValue N) {
20537 return (N.getOpcode() == PPCISD::MAT_PCREL_ADDR ||
20538 isValidPCRelNode<ConstantPoolSDNode>(N) ||
20539 isValidPCRelNode<GlobalAddressSDNode>(N) ||
20540 isValidPCRelNode<JumpTableSDNode>(N) ||
20541 isValidPCRelNode<BlockAddressSDNode>(N));
20542}
20543
20544/// computeMOFlags - Given a node N and it's Parent (a MemSDNode), compute
20545/// the address flags of the load/store instruction that is to be matched.
20546unsigned PPCTargetLowering::computeMOFlags(const SDNode *Parent, SDValue N,
20547 SelectionDAG &DAG) const {
20548 unsigned FlagSet = PPC::MOF_None;
20549
20550 // Compute subtarget flags.
20551 if (!Subtarget.hasP9Vector())
20552 FlagSet |= PPC::MOF_SubtargetBeforeP9;
20553 else
20554 FlagSet |= PPC::MOF_SubtargetP9;
20555
20556 if (Subtarget.hasPrefixInstrs())
20557 FlagSet |= PPC::MOF_SubtargetP10;
20558
20559 if (Subtarget.hasSPE())
20560 FlagSet |= PPC::MOF_SubtargetSPE;
20561
20562 // Check if we have a PCRel node and return early.
20563 if ((FlagSet & PPC::MOF_SubtargetP10) && isPCRelNode(N))
20564 return FlagSet;
20565
20566 // If the node is the paired load/store intrinsics, compute flags for
20567 // address computation and return early.
20568 unsigned ParentOp = Parent->getOpcode();
20569 if (Subtarget.isISA3_1() && ((ParentOp == ISD::INTRINSIC_W_CHAIN) ||
20570 (ParentOp == ISD::INTRINSIC_VOID))) {
20571 unsigned ID = Parent->getConstantOperandVal(Num: 1);
20572 if ((ID == Intrinsic::ppc_vsx_lxvp) || (ID == Intrinsic::ppc_vsx_stxvp)) {
20573 SDValue IntrinOp = (ID == Intrinsic::ppc_vsx_lxvp)
20574 ? Parent->getOperand(Num: 2)
20575 : Parent->getOperand(Num: 3);
20576 computeFlagsForAddressComputation(N: IntrinOp, FlagSet, DAG);
20577 FlagSet |= PPC::MOF_Vector;
20578 return FlagSet;
20579 }
20580 }
20581
20582 // Mark this as something we don't want to handle here if it is atomic
20583 // or pre-increment instruction.
20584 if (const LSBaseSDNode *LSB = dyn_cast<LSBaseSDNode>(Val: Parent))
20585 if (LSB->isIndexed())
20586 return PPC::MOF_None;
20587
20588 // Compute in-memory type flags. This is based on if there are scalars,
20589 // floats or vectors.
20590 const MemSDNode *MN = dyn_cast<MemSDNode>(Val: Parent);
20591 assert(MN && "Parent should be a MemSDNode!");
20592 EVT MemVT = MN->getMemoryVT();
20593 unsigned Size = MemVT.getSizeInBits();
20594 if (MemVT.isScalarInteger()) {
20595 assert(Size <= 128 &&
20596 "Not expecting scalar integers larger than 16 bytes!");
20597 if (Size < 32)
20598 FlagSet |= PPC::MOF_SubWordInt;
20599 else if (Size == 32)
20600 FlagSet |= PPC::MOF_WordInt;
20601 else
20602 FlagSet |= PPC::MOF_DoubleWordInt;
20603 } else if (MemVT.isVector() && !MemVT.isFloatingPoint()) { // Integer vectors.
20604 if (Size == 128)
20605 FlagSet |= PPC::MOF_Vector;
20606 else if (Size == 256) {
20607 assert(Subtarget.pairedVectorMemops() &&
20608 "256-bit vectors are only available when paired vector memops is "
20609 "enabled!");
20610 FlagSet |= PPC::MOF_Vector;
20611 } else
20612 llvm_unreachable("Not expecting illegal vectors!");
20613 } else { // Floating point type: can be scalar, f128 or vector types.
20614 if (Size == 32 || Size == 64)
20615 FlagSet |= PPC::MOF_ScalarFloat;
20616 else if (MemVT == MVT::f128 || MemVT.isVector())
20617 FlagSet |= PPC::MOF_Vector;
20618 else
20619 llvm_unreachable("Not expecting illegal scalar floats!");
20620 }
20621
20622 // Compute flags for address computation.
20623 computeFlagsForAddressComputation(N, FlagSet, DAG);
20624
20625 // Compute type extension flags.
20626 if (const LoadSDNode *LN = dyn_cast<LoadSDNode>(Val: Parent)) {
20627 switch (LN->getExtensionType()) {
20628 case ISD::SEXTLOAD:
20629 FlagSet |= PPC::MOF_SExt;
20630 break;
20631 case ISD::EXTLOAD:
20632 case ISD::ZEXTLOAD:
20633 FlagSet |= PPC::MOF_ZExt;
20634 break;
20635 case ISD::NON_EXTLOAD:
20636 FlagSet |= PPC::MOF_NoExt;
20637 break;
20638 }
20639 } else
20640 FlagSet |= PPC::MOF_NoExt;
20641
20642 // For integers, no extension is the same as zero extension.
20643 // We set the extension mode to zero extension so we don't have
20644 // to add separate entries in AddrModesMap for loads and stores.
20645 if (MemVT.isScalarInteger() && (FlagSet & PPC::MOF_NoExt)) {
20646 FlagSet |= PPC::MOF_ZExt;
20647 FlagSet &= ~PPC::MOF_NoExt;
20648 }
20649
20650 // If we don't have prefixed instructions, 34-bit constants should be
20651 // treated as PPC::MOF_NotAddNorCst so they can match D-Forms.
20652 bool IsNonP1034BitConst =
20653 ((PPC::MOF_RPlusSImm34 | PPC::MOF_AddrIsSImm32 | PPC::MOF_SubtargetP10) &
20654 FlagSet) == PPC::MOF_RPlusSImm34;
20655 if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::OR &&
20656 IsNonP1034BitConst)
20657 FlagSet |= PPC::MOF_NotAddNorCst;
20658
20659 return FlagSet;
20660}
20661
20662/// SelectForceXFormMode - Given the specified address, force it to be
20663/// represented as an indexed [r+r] operation (an XForm instruction).
20664PPC::AddrMode PPCTargetLowering::SelectForceXFormMode(SDValue N, SDValue &Disp,
20665 SDValue &Base,
20666 SelectionDAG &DAG) const {
20667
20668 PPC::AddrMode Mode = PPC::AM_XForm;
20669 int16_t ForceXFormImm = 0;
20670 if (provablyDisjointOr(DAG, N) &&
20671 !isIntS16Immediate(Op: N.getOperand(i: 1), Imm&: ForceXFormImm)) {
20672 Disp = N.getOperand(i: 0);
20673 Base = N.getOperand(i: 1);
20674 return Mode;
20675 }
20676
20677 // If the address is the result of an add, we will utilize the fact that the
20678 // address calculation includes an implicit add. However, we can reduce
20679 // register pressure if we do not materialize a constant just for use as the
20680 // index register. We only get rid of the add if it is not an add of a
20681 // value and a 16-bit signed constant and both have a single use.
20682 if (N.getOpcode() == ISD::ADD &&
20683 (!isIntS16Immediate(Op: N.getOperand(i: 1), Imm&: ForceXFormImm) ||
20684 !N.getOperand(i: 1).hasOneUse() || !N.getOperand(i: 0).hasOneUse())) {
20685 Disp = N.getOperand(i: 0);
20686 Base = N.getOperand(i: 1);
20687 return Mode;
20688 }
20689
20690 // Otherwise, use R0 as the base register.
20691 Disp = DAG.getRegister(Reg: Subtarget.isPPC64() ? PPC::ZERO8 : PPC::ZERO,
20692 VT: N.getValueType());
20693 Base = N;
20694
20695 return Mode;
20696}
20697
20698bool PPCTargetLowering::splitValueIntoRegisterParts(
20699 SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
20700 unsigned NumParts, MVT PartVT, std::optional<CallingConv::ID> CC) const {
20701 EVT ValVT = Val.getValueType();
20702 // If we are splitting a scalar integer into f64 parts (i.e. so they
20703 // can be placed into VFRC registers), we need to zero extend and
20704 // bitcast the values. This will ensure the value is placed into a
20705 // VSR using direct moves or stack operations as needed.
20706 if (PartVT == MVT::f64 &&
20707 (ValVT == MVT::i32 || ValVT == MVT::i16 || ValVT == MVT::i8)) {
20708 Val = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i64, Operand: Val);
20709 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f64, Operand: Val);
20710 Parts[0] = Val;
20711 return true;
20712 }
20713 return false;
20714}
20715
20716SDValue PPCTargetLowering::lowerToLibCall(const char *LibCallName, SDValue Op,
20717 SelectionDAG &DAG) const {
20718 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
20719 TargetLowering::CallLoweringInfo CLI(DAG);
20720 EVT RetVT = Op.getValueType();
20721 Type *RetTy = RetVT.getTypeForEVT(Context&: *DAG.getContext());
20722 SDValue Callee =
20723 DAG.getExternalSymbol(Sym: LibCallName, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
20724 bool SignExtend = TLI.shouldSignExtendTypeInLibCall(Ty: RetTy, IsSigned: false);
20725 TargetLowering::ArgListTy Args;
20726 for (const SDValue &N : Op->op_values()) {
20727 EVT ArgVT = N.getValueType();
20728 Type *ArgTy = ArgVT.getTypeForEVT(Context&: *DAG.getContext());
20729 TargetLowering::ArgListEntry Entry(N, ArgTy);
20730 Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(Ty: ArgTy, IsSigned: SignExtend);
20731 Entry.IsZExt = !Entry.IsSExt;
20732 Args.push_back(x: Entry);
20733 }
20734
20735 SDValue InChain = DAG.getEntryNode();
20736 SDValue TCChain = InChain;
20737 const Function &F = DAG.getMachineFunction().getFunction();
20738 bool isTailCall =
20739 TLI.isInTailCallPosition(DAG, Node: Op.getNode(), Chain&: TCChain) &&
20740 (RetTy == F.getReturnType() || F.getReturnType()->isVoidTy());
20741 if (isTailCall)
20742 InChain = TCChain;
20743 CLI.setDebugLoc(SDLoc(Op))
20744 .setChain(InChain)
20745 .setLibCallee(CC: CallingConv::C, ResultType: RetTy, Target: Callee, ArgsList: std::move(Args))
20746 .setTailCall(isTailCall)
20747 .setSExtResult(SignExtend)
20748 .setZExtResult(!SignExtend)
20749 .setIsPostTypeLegalization(true);
20750 return TLI.LowerCallTo(CLI).first;
20751}
20752
20753SDValue PPCTargetLowering::lowerLibCallBasedOnType(
20754 const char *LibCallFloatName, const char *LibCallDoubleName, SDValue Op,
20755 SelectionDAG &DAG) const {
20756 if (Op.getValueType() == MVT::f32)
20757 return lowerToLibCall(LibCallName: LibCallFloatName, Op, DAG);
20758
20759 if (Op.getValueType() == MVT::f64)
20760 return lowerToLibCall(LibCallName: LibCallDoubleName, Op, DAG);
20761
20762 return SDValue();
20763}
20764
20765bool PPCTargetLowering::isLowringToMASSFiniteSafe(SDValue Op) const {
20766 SDNodeFlags Flags = Op.getNode()->getFlags();
20767 return isLowringToMASSSafe(Op) && Flags.hasNoSignedZeros() &&
20768 Flags.hasNoNaNs() && Flags.hasNoInfs();
20769}
20770
20771bool PPCTargetLowering::isLowringToMASSSafe(SDValue Op) const {
20772 return Op.getNode()->getFlags().hasApproximateFuncs();
20773}
20774
20775bool PPCTargetLowering::isScalarMASSConversionEnabled() const {
20776 return getTargetMachine().Options.PPCGenScalarMASSEntries;
20777}
20778
20779SDValue PPCTargetLowering::lowerLibCallBase(const char *LibCallDoubleName,
20780 const char *LibCallFloatName,
20781 const char *LibCallDoubleNameFinite,
20782 const char *LibCallFloatNameFinite,
20783 SDValue Op,
20784 SelectionDAG &DAG) const {
20785 if (!isScalarMASSConversionEnabled() || !isLowringToMASSSafe(Op))
20786 return SDValue();
20787
20788 if (!isLowringToMASSFiniteSafe(Op))
20789 return lowerLibCallBasedOnType(LibCallFloatName, LibCallDoubleName, Op,
20790 DAG);
20791
20792 return lowerLibCallBasedOnType(LibCallFloatName: LibCallFloatNameFinite,
20793 LibCallDoubleName: LibCallDoubleNameFinite, Op, DAG);
20794}
20795
20796SDValue PPCTargetLowering::lowerPow(SDValue Op, SelectionDAG &DAG) const {
20797 return lowerLibCallBase(LibCallDoubleName: "__xl_pow", LibCallFloatName: "__xl_powf", LibCallDoubleNameFinite: "__xl_pow_finite",
20798 LibCallFloatNameFinite: "__xl_powf_finite", Op, DAG);
20799}
20800
20801SDValue PPCTargetLowering::lowerSin(SDValue Op, SelectionDAG &DAG) const {
20802 return lowerLibCallBase(LibCallDoubleName: "__xl_sin", LibCallFloatName: "__xl_sinf", LibCallDoubleNameFinite: "__xl_sin_finite",
20803 LibCallFloatNameFinite: "__xl_sinf_finite", Op, DAG);
20804}
20805
20806SDValue PPCTargetLowering::lowerCos(SDValue Op, SelectionDAG &DAG) const {
20807 return lowerLibCallBase(LibCallDoubleName: "__xl_cos", LibCallFloatName: "__xl_cosf", LibCallDoubleNameFinite: "__xl_cos_finite",
20808 LibCallFloatNameFinite: "__xl_cosf_finite", Op, DAG);
20809}
20810
20811SDValue PPCTargetLowering::lowerLog(SDValue Op, SelectionDAG &DAG) const {
20812 return lowerLibCallBase(LibCallDoubleName: "__xl_log", LibCallFloatName: "__xl_logf", LibCallDoubleNameFinite: "__xl_log_finite",
20813 LibCallFloatNameFinite: "__xl_logf_finite", Op, DAG);
20814}
20815
20816SDValue PPCTargetLowering::lowerLog10(SDValue Op, SelectionDAG &DAG) const {
20817 return lowerLibCallBase(LibCallDoubleName: "__xl_log10", LibCallFloatName: "__xl_log10f", LibCallDoubleNameFinite: "__xl_log10_finite",
20818 LibCallFloatNameFinite: "__xl_log10f_finite", Op, DAG);
20819}
20820
20821SDValue PPCTargetLowering::lowerExp(SDValue Op, SelectionDAG &DAG) const {
20822 return lowerLibCallBase(LibCallDoubleName: "__xl_exp", LibCallFloatName: "__xl_expf", LibCallDoubleNameFinite: "__xl_exp_finite",
20823 LibCallFloatNameFinite: "__xl_expf_finite", Op, DAG);
20824}
20825
20826// If we happen to match to an aligned D-Form, check if the Frame Index is
20827// adequately aligned. If it is not, reset the mode to match to X-Form.
20828static void setXFormForUnalignedFI(SDValue N, unsigned Flags,
20829 PPC::AddrMode &Mode) {
20830 if (!isa<FrameIndexSDNode>(Val: N))
20831 return;
20832 if ((Mode == PPC::AM_DSForm && !(Flags & PPC::MOF_RPlusSImm16Mult4)) ||
20833 (Mode == PPC::AM_DQForm && !(Flags & PPC::MOF_RPlusSImm16Mult16)))
20834 Mode = PPC::AM_XForm;
20835}
20836
20837/// SelectOptimalAddrMode - Based on a node N and it's Parent (a MemSDNode),
20838/// compute the address flags of the node, get the optimal address mode based
20839/// on the flags, and set the Base and Disp based on the address mode.
20840PPC::AddrMode PPCTargetLowering::SelectOptimalAddrMode(const SDNode *Parent,
20841 SDValue N, SDValue &Disp,
20842 SDValue &Base,
20843 SelectionDAG &DAG,
20844 MaybeAlign Align) const {
20845 SDLoc DL(Parent);
20846
20847 // Compute the address flags.
20848 unsigned Flags = computeMOFlags(Parent, N, DAG);
20849
20850 // Get the optimal address mode based on the Flags.
20851 PPC::AddrMode Mode = getAddrModeForFlags(Flags);
20852
20853 // If the address mode is DS-Form or DQ-Form, check if the FI is aligned.
20854 // Select an X-Form load if it is not.
20855 setXFormForUnalignedFI(N, Flags, Mode);
20856
20857 // Set the mode to PC-Relative addressing mode if we have a valid PC-Rel node.
20858 if ((Mode == PPC::AM_XForm) && isPCRelNode(N)) {
20859 assert(Subtarget.isUsingPCRelativeCalls() &&
20860 "Must be using PC-Relative calls when a valid PC-Relative node is "
20861 "present!");
20862 Mode = PPC::AM_PCRel;
20863 }
20864
20865 // Set Base and Disp accordingly depending on the address mode.
20866 switch (Mode) {
20867 case PPC::AM_DForm:
20868 case PPC::AM_DSForm:
20869 case PPC::AM_DQForm: {
20870 // This is a register plus a 16-bit immediate. The base will be the
20871 // register and the displacement will be the immediate unless it
20872 // isn't sufficiently aligned.
20873 if (Flags & PPC::MOF_RPlusSImm16) {
20874 SDValue Op0 = N.getOperand(i: 0);
20875 SDValue Op1 = N.getOperand(i: 1);
20876 int16_t Imm = Op1->getAsZExtVal();
20877 if (!Align || isAligned(Lhs: *Align, SizeInBytes: Imm)) {
20878 Disp = DAG.getSignedTargetConstant(Val: Imm, DL, VT: N.getValueType());
20879 Base = Op0;
20880 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Op0)) {
20881 Base = DAG.getTargetFrameIndex(FI: FI->getIndex(), VT: N.getValueType());
20882 fixupFuncForFI(DAG, FrameIdx: FI->getIndex(), VT: N.getValueType());
20883 }
20884 break;
20885 }
20886 }
20887 // This is a register plus the @lo relocation. The base is the register
20888 // and the displacement is the global address.
20889 else if (Flags & PPC::MOF_RPlusLo) {
20890 Disp = N.getOperand(i: 1).getOperand(i: 0); // The global address.
20891 assert(Disp.getOpcode() == ISD::TargetGlobalAddress ||
20892 Disp.getOpcode() == ISD::TargetGlobalTLSAddress ||
20893 Disp.getOpcode() == ISD::TargetConstantPool ||
20894 Disp.getOpcode() == ISD::TargetJumpTable);
20895 Base = N.getOperand(i: 0);
20896 break;
20897 }
20898 // This is a constant address at most 32 bits. The base will be
20899 // zero or load-immediate-shifted and the displacement will be
20900 // the low 16 bits of the address.
20901 else if (Flags & PPC::MOF_AddrIsSImm32) {
20902 auto *CN = cast<ConstantSDNode>(Val&: N);
20903 EVT CNType = CN->getValueType(ResNo: 0);
20904 uint64_t CNImm = CN->getZExtValue();
20905 // If this address fits entirely in a 16-bit sext immediate field, codegen
20906 // this as "d, 0".
20907 int16_t Imm;
20908 if (isIntS16Immediate(N: CN, Imm) && (!Align || isAligned(Lhs: *Align, SizeInBytes: Imm))) {
20909 Disp = DAG.getSignedTargetConstant(Val: Imm, DL, VT: CNType);
20910 Base = DAG.getRegister(Reg: Subtarget.isPPC64() ? PPC::ZERO8 : PPC::ZERO,
20911 VT: CNType);
20912 break;
20913 }
20914 // Handle 32-bit sext immediate with LIS + Addr mode.
20915 if ((CNType == MVT::i32 || isInt<32>(x: CNImm)) &&
20916 (!Align || isAligned(Lhs: *Align, SizeInBytes: CNImm))) {
20917 int32_t Addr = (int32_t)CNImm;
20918 // Otherwise, break this down into LIS + Disp.
20919 Disp = DAG.getSignedTargetConstant(Val: (int16_t)Addr, DL, VT: MVT::i32);
20920 Base = DAG.getSignedTargetConstant(Val: (Addr - (int16_t)Addr) >> 16, DL,
20921 VT: MVT::i32);
20922 uint32_t LIS = CNType == MVT::i32 ? PPC::LIS : PPC::LIS8;
20923 Base = SDValue(DAG.getMachineNode(Opcode: LIS, dl: DL, VT: CNType, Op1: Base), 0);
20924 break;
20925 }
20926 }
20927 // Otherwise, the PPC:MOF_NotAdd flag is set. Load/Store is Non-foldable.
20928 Disp = DAG.getTargetConstant(Val: 0, DL, VT: getPointerTy(DL: DAG.getDataLayout()));
20929 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: N)) {
20930 Base = DAG.getTargetFrameIndex(FI: FI->getIndex(), VT: N.getValueType());
20931 fixupFuncForFI(DAG, FrameIdx: FI->getIndex(), VT: N.getValueType());
20932 } else
20933 Base = N;
20934 break;
20935 }
20936 case PPC::AM_PrefixDForm: {
20937 int64_t Imm34 = 0;
20938 unsigned Opcode = N.getOpcode();
20939 if (((Opcode == ISD::ADD) || (Opcode == ISD::OR)) &&
20940 (isIntS34Immediate(Op: N.getOperand(i: 1), Imm&: Imm34))) {
20941 // N is an Add/OR Node, and it's operand is a 34-bit signed immediate.
20942 Disp = DAG.getSignedTargetConstant(Val: Imm34, DL, VT: N.getValueType());
20943 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val: N.getOperand(i: 0)))
20944 Base = DAG.getTargetFrameIndex(FI: FI->getIndex(), VT: N.getValueType());
20945 else
20946 Base = N.getOperand(i: 0);
20947 } else if (isIntS34Immediate(Op: N, Imm&: Imm34)) {
20948 // The address is a 34-bit signed immediate.
20949 Disp = DAG.getSignedTargetConstant(Val: Imm34, DL, VT: N.getValueType());
20950 Base = DAG.getRegister(Reg: PPC::ZERO8, VT: N.getValueType());
20951 }
20952 break;
20953 }
20954 case PPC::AM_PCRel: {
20955 // When selecting PC-Relative instructions, "Base" is not utilized as
20956 // we select the address as [PC+imm].
20957 Disp = N;
20958 break;
20959 }
20960 case PPC::AM_None:
20961 break;
20962 default: { // By default, X-Form is always available to be selected.
20963 // When a frame index is not aligned, we also match by XForm.
20964 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: N);
20965 Base = FI ? N : N.getOperand(i: 1);
20966 Disp = FI ? DAG.getRegister(Reg: Subtarget.isPPC64() ? PPC::ZERO8 : PPC::ZERO,
20967 VT: N.getValueType())
20968 : N.getOperand(i: 0);
20969 break;
20970 }
20971 }
20972 return Mode;
20973}
20974
20975CCAssignFn *PPCTargetLowering::ccAssignFnForCall(CallingConv::ID CC,
20976 bool Return,
20977 bool IsVarArg) const {
20978 switch (CC) {
20979 case CallingConv::Cold:
20980 return (Return ? RetCC_PPC_Cold : CC_PPC64_ELF);
20981 default:
20982 return CC_PPC64_ELF;
20983 }
20984}
20985
20986bool PPCTargetLowering::shouldInlineQuadwordAtomics() const {
20987 return Subtarget.isPPC64() && Subtarget.hasQuadwordAtomics();
20988}
20989
20990TargetLowering::AtomicExpansionKind
20991PPCTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
20992 unsigned Size = AI->getType()->getPrimitiveSizeInBits();
20993 if (shouldInlineQuadwordAtomics() && Size == 128)
20994 return AtomicExpansionKind::MaskedIntrinsic;
20995
20996 switch (AI->getOperation()) {
20997 case AtomicRMWInst::UIncWrap:
20998 case AtomicRMWInst::UDecWrap:
20999 case AtomicRMWInst::USubCond:
21000 case AtomicRMWInst::USubSat:
21001 return AtomicExpansionKind::CmpXChg;
21002 default:
21003 return TargetLowering::shouldExpandAtomicRMWInIR(RMW: AI);
21004 }
21005
21006 llvm_unreachable("unreachable atomicrmw operation");
21007}
21008
21009TargetLowering::AtomicExpansionKind
21010PPCTargetLowering::shouldExpandAtomicCmpXchgInIR(
21011 const AtomicCmpXchgInst *AI) const {
21012 unsigned Size = AI->getNewValOperand()->getType()->getPrimitiveSizeInBits();
21013 if (shouldInlineQuadwordAtomics() && Size == 128)
21014 return AtomicExpansionKind::MaskedIntrinsic;
21015 return AtomicExpansionKind::LLSC;
21016}
21017
21018static Intrinsic::ID
21019getIntrinsicForAtomicRMWBinOp128(AtomicRMWInst::BinOp BinOp) {
21020 switch (BinOp) {
21021 default:
21022 llvm_unreachable("Unexpected AtomicRMW BinOp");
21023 case AtomicRMWInst::Xchg:
21024 return Intrinsic::ppc_atomicrmw_xchg_i128;
21025 case AtomicRMWInst::Add:
21026 return Intrinsic::ppc_atomicrmw_add_i128;
21027 case AtomicRMWInst::Sub:
21028 return Intrinsic::ppc_atomicrmw_sub_i128;
21029 case AtomicRMWInst::And:
21030 return Intrinsic::ppc_atomicrmw_and_i128;
21031 case AtomicRMWInst::Or:
21032 return Intrinsic::ppc_atomicrmw_or_i128;
21033 case AtomicRMWInst::Xor:
21034 return Intrinsic::ppc_atomicrmw_xor_i128;
21035 case AtomicRMWInst::Nand:
21036 return Intrinsic::ppc_atomicrmw_nand_i128;
21037 }
21038}
21039
21040Value *PPCTargetLowering::emitMaskedAtomicRMWIntrinsic(
21041 IRBuilderBase &Builder, AtomicRMWInst *AI, Value *AlignedAddr, Value *Incr,
21042 Value *Mask, Value *ShiftAmt, AtomicOrdering Ord) const {
21043 assert(shouldInlineQuadwordAtomics() && "Only support quadword now");
21044 Module *M = Builder.GetInsertBlock()->getParent()->getParent();
21045 Type *ValTy = Incr->getType();
21046 assert(ValTy->getPrimitiveSizeInBits() == 128);
21047 Type *Int64Ty = Type::getInt64Ty(C&: M->getContext());
21048 Value *IncrLo = Builder.CreateTrunc(V: Incr, DestTy: Int64Ty, Name: "incr_lo");
21049 Value *IncrHi =
21050 Builder.CreateTrunc(V: Builder.CreateLShr(LHS: Incr, RHS: 64), DestTy: Int64Ty, Name: "incr_hi");
21051 Value *LoHi = Builder.CreateIntrinsic(
21052 ID: getIntrinsicForAtomicRMWBinOp128(BinOp: AI->getOperation()), OverloadTypes: {},
21053 Args: {AlignedAddr, IncrLo, IncrHi});
21054 Value *Lo = Builder.CreateExtractValue(Agg: LoHi, Idxs: 0, Name: "lo");
21055 Value *Hi = Builder.CreateExtractValue(Agg: LoHi, Idxs: 1, Name: "hi");
21056 Lo = Builder.CreateZExt(V: Lo, DestTy: ValTy, Name: "lo64");
21057 Hi = Builder.CreateZExt(V: Hi, DestTy: ValTy, Name: "hi64");
21058 return Builder.CreateOr(
21059 LHS: Lo, RHS: Builder.CreateShl(LHS: Hi, RHS: ConstantInt::get(Ty: ValTy, V: 64)), Name: "val64");
21060}
21061
21062Value *PPCTargetLowering::emitMaskedAtomicCmpXchgIntrinsic(
21063 IRBuilderBase &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr,
21064 Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const {
21065 assert(shouldInlineQuadwordAtomics() && "Only support quadword now");
21066 Module *M = Builder.GetInsertBlock()->getParent()->getParent();
21067 Type *ValTy = CmpVal->getType();
21068 assert(ValTy->getPrimitiveSizeInBits() == 128);
21069 Function *IntCmpXchg =
21070 Intrinsic::getOrInsertDeclaration(M, id: Intrinsic::ppc_cmpxchg_i128);
21071 Type *Int64Ty = Type::getInt64Ty(C&: M->getContext());
21072 Value *CmpLo = Builder.CreateTrunc(V: CmpVal, DestTy: Int64Ty, Name: "cmp_lo");
21073 Value *CmpHi =
21074 Builder.CreateTrunc(V: Builder.CreateLShr(LHS: CmpVal, RHS: 64), DestTy: Int64Ty, Name: "cmp_hi");
21075 Value *NewLo = Builder.CreateTrunc(V: NewVal, DestTy: Int64Ty, Name: "new_lo");
21076 Value *NewHi =
21077 Builder.CreateTrunc(V: Builder.CreateLShr(LHS: NewVal, RHS: 64), DestTy: Int64Ty, Name: "new_hi");
21078 emitLeadingFence(Builder, Inst: CI, Ord);
21079 Value *LoHi =
21080 Builder.CreateCall(Callee: IntCmpXchg, Args: {AlignedAddr, CmpLo, CmpHi, NewLo, NewHi});
21081 emitTrailingFence(Builder, Inst: CI, Ord);
21082 Value *Lo = Builder.CreateExtractValue(Agg: LoHi, Idxs: 0, Name: "lo");
21083 Value *Hi = Builder.CreateExtractValue(Agg: LoHi, Idxs: 1, Name: "hi");
21084 Lo = Builder.CreateZExt(V: Lo, DestTy: ValTy, Name: "lo64");
21085 Hi = Builder.CreateZExt(V: Hi, DestTy: ValTy, Name: "hi64");
21086 return Builder.CreateOr(
21087 LHS: Lo, RHS: Builder.CreateShl(LHS: Hi, RHS: ConstantInt::get(Ty: ValTy, V: 64)), Name: "val64");
21088}
21089
21090bool PPCTargetLowering::hasMultipleConditionRegisters(EVT VT) const {
21091 return Subtarget.useCRBits();
21092}
21093
21094/// Shuffle masks for vectors of bits are not legal as such vectors are
21095/// reserved for MMA/DM.
21096bool PPCTargetLowering::isShuffleMaskLegal(ArrayRef<int> Mask, EVT VT) const {
21097 if (VT.getScalarType() == MVT::i1)
21098 return false;
21099 return TargetLowering::isShuffleMaskLegal(Mask, VT);
21100}
21101
21102// Optimize the following patterns using vbpermq/vbpermd:
21103// i16 = bitcast(v16i1 truncate(v16i8))
21104// i8 = bitcast(v8i1 truncate(v8i16))
21105// i8 = bitcast(v8i1 truncate(v8i8))
21106SDValue PPCTargetLowering::DAGCombineBitcast(SDNode *N,
21107 DAGCombinerInfo &DCI) const {
21108 SDValue Op0 = N->getOperand(Num: 0);
21109 if (Op0.getOpcode() != ISD::TRUNCATE)
21110 return SDValue();
21111 SDValue Src = Op0.getOperand(i: 0);
21112 EVT ResVT = N->getValueType(ResNo: 0);
21113 EVT TruncResVT = Op0.getValueType();
21114 EVT SrcVT = Src.getValueType();
21115 SDLoc dl(N);
21116 SelectionDAG &DAG = DCI.DAG;
21117 bool IsLittleEndian = Subtarget.isLittleEndian();
21118
21119 if (ResVT != MVT::i16 && ResVT != MVT::i8)
21120 return SDValue();
21121 SDValue VBPerm =
21122 GenerateVBPERM(DAG, dl, Src, SrcVT, ResVT: TruncResVT, IsLE: IsLittleEndian);
21123 if (!VBPerm)
21124 return SDValue();
21125 SDValue ForExtract = DAG.getBitcast(VT: MVT::v4i32, V: VBPerm);
21126 SDValue Extracted =
21127 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: MVT::i32, N1: ForExtract,
21128 N2: DAG.getIntPtrConstant(Val: IsLittleEndian ? 2 : 1, DL: dl));
21129 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: ResVT, Operand: Extracted);
21130}
21131
21132SDValue PPCTargetLowering::GenerateVBPERM(SelectionDAG &DAG, SDLoc dl,
21133 SDValue Src, EVT SrcVT, EVT ResVT,
21134 bool IsLE) const {
21135 bool IsV16i8 = (ResVT == MVT::v16i1 && SrcVT == MVT::v16i8);
21136 bool IsV8i16 = (ResVT == MVT::v8i1 && SrcVT == MVT::v8i16);
21137 bool IsV8i8 = (ResVT == MVT::v8i1 && SrcVT == MVT::v8i8);
21138
21139 if (!IsV16i8 && !IsV8i16 && !IsV8i8)
21140 return SDValue();
21141
21142 if (IsV8i8) {
21143 Src = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: MVT::v16i8,
21144 N1: DAG.getUNDEF(VT: MVT::v16i8), N2: Src,
21145 N3: DAG.getIntPtrConstant(Val: 0, DL: dl));
21146 }
21147 SmallVector<int, 16> BitIndices(16, 128);
21148 unsigned NumElts = SrcVT.getVectorNumElements();
21149 unsigned EltSize = SrcVT.getScalarType().getSizeInBits();
21150 for (int Idx = 0, End = SrcVT.getVectorNumElements(); Idx < End; Idx++) {
21151 BitIndices[Idx] = EltSize * (NumElts - Idx) - 1;
21152 if (IsV8i8 && IsLE)
21153 BitIndices[Idx] += 64;
21154 }
21155 if (!IsLE)
21156 std::reverse(first: BitIndices.begin(), last: BitIndices.end());
21157 SmallVector<SDValue, 16> BVOps;
21158 for (auto Idx : BitIndices)
21159 BVOps.push_back(Elt: DAG.getConstant(Val: Idx, DL: dl, VT: MVT::i8));
21160 SDValue VRB = DAG.getBuildVector(VT: MVT::v16i8, DL: dl, Ops: BVOps);
21161 return DAG.getNode(
21162 Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: MVT::v16i8,
21163 N1: DAG.getConstant(Val: Intrinsic::ppc_altivec_vbpermq, DL: dl, VT: MVT::i32),
21164 N2: DAG.getBitcast(VT: MVT::v16i8, V: Src), N3: VRB);
21165}
21166
21167// For Power8/9, optimize vec splats of small FP values that can be
21168// represented as integers. Use vspltisw + xvcvsxwdp/xvcvsxwsp instead of
21169// loading from constant pool.
21170SDValue PPCTargetLowering::LowerVecSplatSmallFP(SDValue Op, SelectionDAG &DAG,
21171 bool BVNIsConstantSplat,
21172 unsigned SplatBitSize) const {
21173
21174 if (!BVNIsConstantSplat || !Subtarget.hasVSX() || !Subtarget.hasP8Vector() ||
21175 Subtarget.hasP10Vector())
21176 return SDValue();
21177
21178 EVT VT = Op->getValueType(ResNo: 0);
21179 if (!((SplatBitSize == 64 && VT == MVT::v2f64) ||
21180 (SplatBitSize == 32 && VT == MVT::v4f32)))
21181 return SDValue();
21182
21183 auto *CN = dyn_cast<ConstantFPSDNode>(Val: Op.getOperand(i: 0));
21184 if (!CN)
21185 return SDValue();
21186
21187 APFloat APFloatVal = CN->getValueAPF();
21188 bool IsExact;
21189 APSInt IntResult(16, false);
21190 APFloatVal.convertToInteger(Result&: IntResult, RM: APFloat::rmTowardZero, IsExact: &IsExact);
21191
21192 if (!(IsExact && IntResult <= 15 && IntResult >= -16 && !APFloatVal.isZero()))
21193 return SDValue();
21194
21195 int64_t IntVal = IntResult.getSExtValue();
21196
21197 SDLoc dl(Op);
21198 SDValue IntSplat = getCanonicalConstSplat(Val: IntVal, SplatSize: 4, VT: MVT::v4i32, DAG, dl);
21199
21200 if (SplatBitSize == 64)
21201 return DAG.getNode(
21202 Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: MVT::v2f64,
21203 N1: DAG.getConstant(Val: Intrinsic::ppc_vsx_xvcvsxwdp, DL: dl, VT: MVT::i32), N2: IntSplat);
21204
21205 return DAG.getNode(Opcode: PPCISD::XVCVSXWSP, DL: dl, VT: MVT::v4f32, Operand: IntSplat);
21206}
21207